diff --git a/app/config/locale/translations/en.json b/app/config/locale/translations/en.json index 8e59c40123..3d667a36ad 100644 --- a/app/config/locale/translations/en.json +++ b/app/config/locale/translations/en.json @@ -57,21 +57,21 @@ "emails.recovery.thanks": "Thanks,", "emails.recovery.buttonText": "Reset password", "emails.recovery.signature": "{{project}} team", - "emails.csvExport.success.subject": "Your CSV export is ready", - "emails.csvExport.success.preview": "Your data export has been completed successfully.", - "emails.csvExport.success.hello": "Hello {{user}},", - "emails.csvExport.success.body": "Your CSV export is ready to download. Click the button below to download your data export.", - "emails.csvExport.success.footer": "This download link will expire in 1 hour.", - "emails.csvExport.success.thanks": "Thanks,", - "emails.csvExport.success.buttonText": "Download CSV", - "emails.csvExport.success.signature": "Appwrite team", - "emails.csvExport.failure.subject": "Your CSV export failed - file too large", - "emails.csvExport.failure.preview": "Your data export failed because the file size exceeds your plan limit.", - "emails.csvExport.failure.hello": "Hello {{user}},", - "emails.csvExport.failure.body": "Your CSV export could not be completed because the export file size ({{size}}MB) exceeds your plan limit. Please consider upgrading your plan or exporting a smaller dataset.", - "emails.csvExport.failure.footer": "If you have any questions, please contact our support team.", - "emails.csvExport.failure.thanks": "Thanks,", - "emails.csvExport.failure.signature": "{{project}} team", + "emails.dataExport.success.subject": "Your {{type}} export is ready", + "emails.dataExport.success.preview": "Your data export has been completed successfully.", + "emails.dataExport.success.hello": "Hello {{user}},", + "emails.dataExport.success.body": "Your {{type}} export is ready to download. Click the button below to download your data export.", + "emails.dataExport.success.footer": "This download link will expire in 1 hour.", + "emails.dataExport.success.thanks": "Thanks,", + "emails.dataExport.success.buttonText": "Download {{type}}", + "emails.dataExport.success.signature": "Appwrite team", + "emails.dataExport.failure.subject": "Your {{type}} export failed - file too large", + "emails.dataExport.failure.preview": "Your data export failed because the file size exceeds your plan limit.", + "emails.dataExport.failure.hello": "Hello {{user}},", + "emails.dataExport.failure.body": "Your {{type}} export could not be completed because the export file size ({{size}}MB) exceeds your plan limit. Please consider upgrading your plan or exporting a smaller dataset.", + "emails.dataExport.failure.footer": "If you have any questions, please contact our support team.", + "emails.dataExport.failure.thanks": "Thanks,", + "emails.dataExport.failure.signature": "{{project}} team", "emails.invitation.subject": "Invitation to {{team}} Team at {{project}}", "emails.invitation.preview": "{{owner}} invited you to join {{team}} at {{project}}", "emails.invitation.hello": "Hello {{user}},", diff --git a/app/controllers/api/migrations.php b/app/controllers/api/migrations.php index 5a87293b49..45a663fb56 100644 --- a/app/controllers/api/migrations.php +++ b/app/controllers/api/migrations.php @@ -29,6 +29,7 @@ use Utopia\Migration\Resource; use Utopia\Migration\Sources\Appwrite; use Utopia\Migration\Sources\CSV; use Utopia\Migration\Sources\Firebase; +use Utopia\Migration\Sources\JSON; use Utopia\Migration\Sources\NHost; use Utopia\Migration\Sources\Supabase; use Utopia\Migration\Transfer; @@ -53,6 +54,15 @@ function getDatabaseTransferResourceServices(string $databaseType) }; } +function getDatabaseResourceType(string $databaseType): string +{ + return match($databaseType) { + DATABASE_TYPE_VECTORSDB => Resource::TYPE_DATABASE_VECTORSDB, + DATABASE_TYPE_DOCUMENTSDB => Resource::TYPE_DATABASE_DOCUMENTSDB, + default => Resource::TYPE_DATABASE, + }; +} + Http::post('/v1/migrations/appwrite') ->groups(['api', 'migrations']) ->desc('Create Appwrite migration') @@ -447,6 +457,7 @@ Http::post('/v1/migrations/csv/imports') } $fileSize = $deviceForMigrations->getFileSize($newPath); $resources = Transfer::extractServices([getDatabaseTransferResourceServices($databaseType)]); + $resourceType = getDatabaseResourceType($databaseType); $migration = $dbForProject->createDocument('migrations', new Document([ '$id' => $migrationId, @@ -456,7 +467,7 @@ Http::post('/v1/migrations/csv/imports') 'destination' => Appwrite::getName(), 'resources' => $resources, 'resourceId' => $resourceId, - 'resourceType' => Resource::TYPE_DATABASE, + 'resourceType' => $resourceType, 'statusCounters' => '{}', 'resourceData' => '{}', 'errors' => [], @@ -565,16 +576,6 @@ Http::post('/v1/migrations/csv/exports') throw new Exception(Exception::COLLECTION_NOT_FOUND); } - $validator = new Documents( - attributes: $collection->getAttribute('attributes', []), - indexes: $collection->getAttribute('indexes', []), - idAttributeType: $dbForProject->getAdapter()->getIdAttributeType(), - ); - - if (!$validator->isValid($parsedQueries)) { - throw new Exception(Exception::GENERAL_QUERY_INVALID, $validator->getDescription()); - } - // getting databasetype $resources = explode(':', $resourceId); $databaseId = $resources[0]; @@ -583,7 +584,23 @@ Http::post('/v1/migrations/csv/exports') if (!in_array($databaseType, CSV_ALLOWED_DATABASE_TYPES)) { throw new Exception(Exception::MIGRATION_DATABASE_TYPE_UNSUPPORTED, 'Database type not supported for csv'); } + + // Schemaless databases (DocumentsDB, VectorsDB) allow queries on dynamic fields + $isSchemaless = in_array($databaseType, [DATABASE_TYPE_DOCUMENTSDB, DATABASE_TYPE_VECTORSDB]); + + $validator = new Documents( + attributes: $collection->getAttribute('attributes', []), + indexes: $collection->getAttribute('indexes', []), + idAttributeType: $dbForProject->getAdapter()->getIdAttributeType(), + supportForAttributes: !$isSchemaless, + ); + + if (!$validator->isValid($parsedQueries)) { + throw new Exception(Exception::GENERAL_QUERY_INVALID, $validator->getDescription()); + } + $resources = Transfer::extractServices([getDatabaseTransferResourceServices($databaseType)]); + $resourceType = getDatabaseResourceType($databaseType); $migration = $dbForProject->createDocument('migrations', new Document([ '$id' => ID::unique(), @@ -593,7 +610,7 @@ Http::post('/v1/migrations/csv/exports') 'destination' => CSV::getName(), 'resources' => $resources, 'resourceId' => $resourceId, - 'resourceType' => Resource::TYPE_DATABASE, + 'resourceType' => $resourceType, 'statusCounters' => '{}', 'resourceData' => '{}', 'errors' => [], @@ -624,6 +641,291 @@ Http::post('/v1/migrations/csv/exports') ->dynamic($migration, Response::MODEL_MIGRATION); }); +Http::post('/v1/migrations/json/imports') + ->groups(['api', 'migrations']) + ->desc('Import documents from a JSON') + ->label('scope', 'migrations.write') + ->label('event', 'migrations.[migrationId].create') + ->label('audits.event', 'migration.create') + ->label('sdk', new Method( + namespace: 'migrations', + group: null, + name: 'createJSONImport', + description: '/docs/references/migrations/migration-json-import.md', + auth: [AuthType::ADMIN], + responses: [ + new SDKResponse( + code: Response::STATUS_CODE_ACCEPTED, + model: Response::MODEL_MIGRATION, + ) + ] + )) + ->param('bucketId', '', new UID(), 'Storage bucket unique ID. You can create a new storage bucket using the Storage service [server integration](https://appwrite.io/docs/server/storage#createBucket).') + ->param('fileId', '', new UID(), 'File ID.') + ->param('resourceId', null, new CompoundUID(), 'Composite ID in the format {databaseId:collectionId}, identifying a collection within a database.') + ->param('internalFile', false, new Boolean(), 'Is the file stored in an internal bucket?', true) + ->inject('response') + ->inject('dbForProject') + ->inject('dbForPlatform') + ->inject('authorization') + ->inject('project') + ->inject('platform') + ->inject('deviceForFiles') + ->inject('deviceForMigrations') + ->inject('queueForEvents') + ->inject('queueForMigrations') + ->action(function ( + string $bucketId, + string $fileId, + string $resourceId, + bool $internalFile, + Response $response, + Database $dbForProject, + Database $dbForPlatform, + Authorization $authorization, + Document $project, + array $platform, + Device $deviceForFiles, + Device $deviceForMigrations, + Event $queueForEvents, + Migration $queueForMigrations + ) { + $bucket = $authorization->skip(function () use ($internalFile, $dbForPlatform, $dbForProject, $bucketId) { + if ($internalFile) { + return $dbForPlatform->getDocument('buckets', 'default'); + } + return $dbForProject->getDocument('buckets', $bucketId); + }); + + if ($bucket->isEmpty()) { + throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND); + } + + $file = $authorization->skip(fn () => $internalFile ? $dbForPlatform->getDocument('bucket_' . $bucket->getSequence(), $fileId) : $dbForProject->getDocument('bucket_' . $bucket->getSequence(), $fileId)); + if ($file->isEmpty()) { + throw new Exception(Exception::STORAGE_FILE_NOT_FOUND); + } + + $path = $file->getAttribute('path', ''); + if (!$deviceForFiles->exists($path)) { + throw new Exception(Exception::STORAGE_FILE_NOT_FOUND, 'File not found in ' . $path); + } + + // No encryption or compression on files above 20MB. + $hasEncryption = !empty($file->getAttribute('openSSLCipher')); + $compression = $file->getAttribute('algorithm', Compression::NONE); + $hasCompression = $compression !== Compression::NONE; + + $migrationId = ID::unique(); + $newPath = $deviceForMigrations->getPath($migrationId . '_' . $fileId . '.json'); + + if ($hasEncryption || $hasCompression) { + $source = $deviceForFiles->read($path); + + if ($hasEncryption) { + $source = OpenSSL::decrypt( + $source, + $file->getAttribute('openSSLCipher'), + System::getEnv('_APP_OPENSSL_KEY_V' . $file->getAttribute('openSSLVersion')), + 0, + hex2bin($file->getAttribute('openSSLIV')), + hex2bin($file->getAttribute('openSSLTag')) + ); + } + + if ($hasCompression) { + switch ($compression) { + case Compression::ZSTD: + $source = (new Zstd())->decompress($source); + break; + case Compression::GZIP: + $source = (new GZIP())->decompress($source); + break; + } + } + + // Manual write after decryption and/or decompression + if (!$deviceForMigrations->write($newPath, $source, 'application/json')) { + throw new \Exception('Unable to copy file'); + } + } elseif (!$deviceForFiles->transfer($path, $newPath, $deviceForMigrations)) { + throw new \Exception('Unable to copy file'); + } + + $fileSize = $deviceForMigrations->getFileSize($newPath); + + [$databaseId] = \explode(':', $resourceId, 2); + $database = $authorization->skip(fn () => $dbForProject->getDocument('databases', $databaseId)); + if ($database->isEmpty()) { + throw new Exception(Exception::DATABASE_NOT_FOUND); + } + $databaseType = $database->getAttribute('type'); + $resources = Transfer::extractServices([getDatabaseTransferResourceServices($databaseType)]); + $resourceType = getDatabaseResourceType($databaseType); + + $migration = $dbForProject->createDocument('migrations', new Document([ + '$id' => $migrationId, + 'status' => 'pending', + 'stage' => 'init', + 'source' => JSON::getName(), + 'destination' => Appwrite::getName(), + 'resources' => $resources, + 'resourceId' => $resourceId, + 'resourceType' => $resourceType, + 'statusCounters' => '{}', + 'resourceData' => '{}', + 'errors' => [], + 'options' => [ + 'path' => $newPath, + 'size' => $fileSize, + ], + ])); + + $queueForEvents->setParam('migrationId', $migration->getId()); + + $queueForMigrations + ->setMigration($migration) + ->setProject($project) + ->setPlatform($platform) + ->trigger(); + + $response + ->setStatusCode(Response::STATUS_CODE_ACCEPTED) + ->dynamic($migration, Response::MODEL_MIGRATION); + }); + +Http::post('/v1/migrations/json/exports') + ->groups(['api', 'migrations']) + ->desc('Export documents to JSON') + ->label('scope', 'migrations.write') + ->label('event', 'migrations.[migrationId].create') + ->label('audits.event', 'migration.create') + ->label('sdk', new Method( + namespace: 'migrations', + group: null, + name: 'createJSONExport', + description: '/docs/references/migrations/migration-json-export.md', + auth: [AuthType::ADMIN], + responses: [ + new SDKResponse( + code: Response::STATUS_CODE_ACCEPTED, + model: Response::MODEL_MIGRATION, + ) + ] + )) + ->param('resourceId', null, new CompoundUID(), 'Composite ID in the format {databaseId:collectionId}, identifying a collection within a database to export.') + ->param('filename', '', new Text(255), 'The name of the file to be created for the export, excluding the .json extension.') + ->param('columns', [], new ArrayList(new Text(Database::LENGTH_KEY)), 'List of attributes to export. If empty, all attributes will be exported. You can use the `*` wildcard to export all attributes from the collection.', true) + ->param('queries', [], new ArrayList(new Text(0)), 'Array of query strings generated using the Query class provided by the SDK to filter documents to export. [Learn more about queries](https://appwrite.io/docs/databases#querying-documents). Maximum of ' . APP_LIMIT_ARRAY_PARAMS_SIZE . ' queries are allowed, each ' . APP_LIMIT_ARRAY_ELEMENT_SIZE . ' characters long.', true) + ->param('notify', true, new Boolean(), 'Set to true to receive an email when the export is complete. Default is true.', true) + ->inject('user') + ->inject('response') + ->inject('dbForProject') + ->inject('dbForPlatform') + ->inject('authorization') + ->inject('project') + ->inject('platform') + ->inject('queueForEvents') + ->inject('queueForMigrations') + ->action(function ( + string $resourceId, + string $filename, + array $columns, + array $queries, + bool $notify, + Document $user, + Response $response, + Database $dbForProject, + Database $dbForPlatform, + Authorization $authorization, + Document $project, + array $platform, + Event $queueForEvents, + Migration $queueForMigrations + ) { + try { + $parsedQueries = Query::parseQueries($queries); + } catch (QueryException $e) { + throw new Exception(Exception::GENERAL_QUERY_INVALID, $e->getMessage()); + } + + $bucket = $authorization->skip(fn () => $dbForPlatform->getDocument('buckets', 'default')); + if ($bucket->isEmpty()) { + throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND); + } + + [$databaseId, $collectionId] = \explode(':', $resourceId, 2); + if (empty($databaseId)) { + throw new Exception(Exception::DATABASE_NOT_FOUND); + } + if (empty($collectionId)) { + throw new Exception(Exception::COLLECTION_NOT_FOUND); + } + + $database = $authorization->skip(fn () => $dbForProject->getDocument('databases', $databaseId)); + if ($database->isEmpty()) { + throw new Exception(Exception::DATABASE_NOT_FOUND); + } + + $collection = $authorization->skip(fn () => $dbForProject->getDocument('database_' . $database->getSequence(), $collectionId)); + if ($collection->isEmpty()) { + throw new Exception(Exception::COLLECTION_NOT_FOUND); + } + + $databaseType = $database->getAttribute('type'); + + // Schemaless databases (DocumentsDB, VectorsDB) allow queries on dynamic fields + $isSchemaless = in_array($databaseType, [DATABASE_TYPE_DOCUMENTSDB, DATABASE_TYPE_VECTORSDB]); + + $validator = new Documents( + attributes: $collection->getAttribute('attributes', []), + indexes: $collection->getAttribute('indexes', []), + idAttributeType: $dbForProject->getAdapter()->getIdAttributeType(), + supportForAttributes: !$isSchemaless, + ); + + if (!$validator->isValid($parsedQueries)) { + throw new Exception(Exception::GENERAL_QUERY_INVALID, $validator->getDescription()); + } + + $resources = Transfer::extractServices([getDatabaseTransferResourceServices($databaseType)]); + $resourceType = getDatabaseResourceType($databaseType); + + $migration = $dbForProject->createDocument('migrations', new Document([ + '$id' => ID::unique(), + 'status' => 'pending', + 'stage' => 'init', + 'source' => Appwrite::getName(), + 'destination' => JSON::getName(), + 'resources' => $resources, + 'resourceId' => $resourceId, + 'resourceType' => $resourceType, + 'statusCounters' => '{}', + 'resourceData' => '{}', + 'errors' => [], + 'options' => [ + 'bucketId' => 'default', // Always use internal bucket + 'filename' => $filename, + 'columns' => $columns, + 'queries' => $queries, + 'notify' => $notify, + 'userInternalId' => $user->getSequence(), + ], + ])); + + $queueForEvents->setParam('migrationId', $migration->getId()); + + $queueForMigrations + ->setMigration($migration) + ->setProject($project) + ->setPlatform($platform) + ->trigger(); + + $response + ->setStatusCode(Response::STATUS_CODE_ACCEPTED) + ->dynamic($migration, Response::MODEL_MIGRATION); + }); + Http::get('/v1/migrations') ->groups(['api', 'migrations']) ->desc('List migrations') diff --git a/composer.json b/composer.json index 65838a1615..1a530bfc5b 100644 --- a/composer.json +++ b/composer.json @@ -73,7 +73,7 @@ "utopia-php/locale": "0.8.*", "utopia-php/logger": "0.6.*", "utopia-php/messaging": "0.20.*", - "utopia-php/migration": "1.8.*", + "utopia-php/migration": "1.9.*", "utopia-php/platform": "0.7.*", "utopia-php/pools": "1.*", "utopia-php/span": "1.1.*", diff --git a/composer.lock b/composer.lock index 1441bf06d1..0c50b8cf34 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "f9225f2b580de0ccb796b2fb8c881384", + "content-hash": "b5261855586680e467168f527e0634ae", "packages": [ { "name": "adhocore/jwt", @@ -4518,16 +4518,16 @@ }, { "name": "utopia-php/migration", - "version": "1.8.3", + "version": "1.9.1", "source": { "type": "git", "url": "https://github.com/utopia-php/migration.git", - "reference": "8633523b3343d492427331b6eec53f020f6ab7a7" + "reference": "7a86aeadf182b63a9f4ceba7e137588b31c5d2e2" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/migration/zipball/8633523b3343d492427331b6eec53f020f6ab7a7", - "reference": "8633523b3343d492427331b6eec53f020f6ab7a7", + "url": "https://api.github.com/repos/utopia-php/migration/zipball/7a86aeadf182b63a9f4ceba7e137588b31c5d2e2", + "reference": "7a86aeadf182b63a9f4ceba7e137588b31c5d2e2", "shasum": "" }, "require": { @@ -4567,9 +4567,9 @@ ], "support": { "issues": "https://github.com/utopia-php/migration/issues", - "source": "https://github.com/utopia-php/migration/tree/1.8.3" + "source": "https://github.com/utopia-php/migration/tree/1.9.1" }, - "time": "2026-03-19T09:18:47+00:00" + "time": "2026-03-25T07:05:27+00:00" }, { "name": "utopia-php/mongo", diff --git a/docs/references/health/get-queue-audits.md b/docs/references/health/get-queue-audits.md index 75010cc2f4..bac075581f 100644 --- a/docs/references/health/get-queue-audits.md +++ b/docs/references/health/get-queue-audits.md @@ -1 +1 @@ -Get the number of audit logs that are waiting to be processed in the Appwrite internal queue server. \ No newline at end of file +Get the number of audit logs that are waiting to be processed in the Appwrite internal queue server. diff --git a/phpstan-baseline.neon b/phpstan-baseline.neon index 816ff0e5f5..5da64a1c97 100644 --- a/phpstan-baseline.neon +++ b/phpstan-baseline.neon @@ -1118,13 +1118,13 @@ parameters: - message: '#^Anonymous function has an unused use \$databaseId\.$#' identifier: closure.unusedUse - count: 5 + count: 6 path: tests/e2e/Services/Migrations/MigrationsConsoleClientTest.php - message: '#^Anonymous function has an unused use \$tableId\.$#' identifier: closure.unusedUse - count: 5 + count: 6 path: tests/e2e/Services/Migrations/MigrationsConsoleClientTest.php - diff --git a/src/Appwrite/Platform/Workers/Migrations.php b/src/Appwrite/Platform/Workers/Migrations.php index d96a25351f..1080ff066f 100644 --- a/src/Appwrite/Platform/Workers/Migrations.php +++ b/src/Appwrite/Platform/Workers/Migrations.php @@ -28,6 +28,7 @@ use Utopia\Locale\Locale; use Utopia\Migration\Destination; use Utopia\Migration\Destinations\Appwrite as DestinationAppwrite; use Utopia\Migration\Destinations\CSV as DestinationCSV; +use Utopia\Migration\Destinations\JSON as DestinationJSON; use Utopia\Migration\Exception as MigrationException; use Utopia\Migration\Resource; use Utopia\Migration\Resources\Database\Database as ResourceDatabase; @@ -37,6 +38,7 @@ use Utopia\Migration\Source; use Utopia\Migration\Sources\Appwrite as SourceAppwrite; use Utopia\Migration\Sources\CSV; use Utopia\Migration\Sources\Firebase; +use Utopia\Migration\Sources\JSON; use Utopia\Migration\Sources\NHost; use Utopia\Migration\Sources\Supabase; use Utopia\Migration\Transfer; @@ -208,8 +210,8 @@ class Migrations extends Action $getDatabasesDB = fn (Document $database): Database => $this->getDatabasesDBForProject($database); $queries = []; - if ($source === SourceAppwrite::getName() && $destination === DestinationCSV::getName()) { - $queries = Query::parseQueries($migrationOptions['queries']); + if ($source === SourceAppwrite::getName() && in_array($destination, [DestinationCSV::getName(), DestinationJSON::getName()])) { + $queries = Query::parseQueries($migrationOptions['queries'] ?? []); } $migrationSource = match ($source) { @@ -250,6 +252,12 @@ class Migrations extends Action $this->dbForProject, $getDatabasesDB ), + JSON::getName() => new JSON( + $resourceId, + $migrationOptions['path'], + $this->deviceForMigrations, + $this->dbForProject, + ), default => throw new \Exception('Invalid source type'), }; @@ -288,6 +296,13 @@ class Migrations extends Action $options['escape'], $options['header'], ), + DestinationJSON::getName() => new DestinationJSON( + $this->deviceForFiles, + $migration->getAttribute('resourceId'), + $options['bucketId'] ?? 'default', + $options['filename'], + $options['columns'] ?? [], + ), default => throw new \Exception('Invalid destination type'), }; } @@ -550,8 +565,9 @@ class Migrations extends Action $destination?->success(); $source?->success(); } - if ($migration->getAttribute('destination') === DestinationCSV::getName()) { - $this->handleCSVExportComplete($project, $migration, $queueForMails, $queueForRealtime, $platform, $authorization); + $destination_type = $migration->getAttribute('destination'); + if ($destination_type === DestinationCSV::getName() || $destination_type === DestinationJSON::getName()) { + $this->handleDataExportComplete($project, $migration, $queueForMails, $queueForRealtime, $platform, $authorization); } } finally { $source?->cleanup(); @@ -583,7 +599,7 @@ class Migrations extends Action * @param Authorization $authorization * @return void */ - protected function handleCSVExportComplete( + protected function handleDataExportComplete( Document $project, Document $migration, Mail $queueForMails, @@ -608,7 +624,8 @@ class Migrations extends Action throw new \Exception('Bucket not found'); } - $path = $this->deviceForFiles->getPath($bucketId . '/' . $this->sanitizeFilename($filename) . '.csv'); + $extension = $migration->getAttribute('destination') === DestinationJSON::getName() ? '.json' : '.csv'; + $path = $this->deviceForFiles->getPath($bucketId . '/' . $this->sanitizeFilename($filename) . $extension); $size = $this->deviceForFiles->getFileSize($path); $mime = $this->deviceForFiles->getFileMimeType($path); $hash = $this->deviceForFiles->getFileHash($path); @@ -632,13 +649,14 @@ class Migrations extends Action $migration->setAttribute('errors', $errors); $migration = $this->updateMigrationDocument($migration, $project, $queueForRealtime); - $this->sendCSVEmail( + $this->sendExportEmail( success: false, project: $project, user: $user, options: $options, queueForMails: $queueForMails, platform: $platform, + exportType: $migration->getAttribute('destination') === DestinationJSON::getName() ? 'JSON' : 'CSV', sizeMB: $sizeMB ); @@ -694,13 +712,14 @@ class Migrations extends Action $migration->setAttribute('options', $options); $this->updateMigrationDocument($migration, $project, $queueForRealtime); - $this->sendCSVEmail( + $this->sendExportEmail( success: true, project: $project, user: $user, options: $options, queueForMails: $queueForMails, platform: $platform, + exportType: $migration->getAttribute('destination') === DestinationJSON::getName() ? 'JSON' : 'CSV', downloadUrl: $downloadUrl ); } @@ -719,13 +738,14 @@ class Migrations extends Action * @return void * @throws \Exception */ - protected function sendCSVEmail( + protected function sendExportEmail( bool $success, Document $project, Document $user, array $options, Mail $queueForMails, array $platform, + string $exportType = 'CSV', string $downloadUrl = '', float $sizeMB = 0.0, ): void { @@ -745,15 +765,15 @@ class Migrations extends Action ? 'success' : 'failure'; - // Get localized email content - $subject = $locale->getText("emails.csvExport.{$emailType}.subject"); - $preview = $locale->getText("emails.csvExport.{$emailType}.preview"); - $hello = $locale->getText("emails.csvExport.{$emailType}.hello"); - $body = $locale->getText("emails.csvExport.{$emailType}.body"); - $footer = $locale->getText("emails.csvExport.{$emailType}.footer"); - $thanks = $locale->getText("emails.csvExport.{$emailType}.thanks"); - $signature = $locale->getText("emails.csvExport.{$emailType}.signature"); - $buttonText = $success ? $locale->getText("emails.csvExport.{$emailType}.buttonText") : ''; + // Get localized email content — replace {{type}} with export format (CSV/JSON) + $subject = \str_replace('{{type}}', $exportType, $locale->getText("emails.dataExport.{$emailType}.subject")); + $preview = \str_replace('{{type}}', $exportType, $locale->getText("emails.dataExport.{$emailType}.preview")); + $hello = $locale->getText("emails.dataExport.{$emailType}.hello"); + $body = $locale->getText("emails.dataExport.{$emailType}.body"); + $footer = $locale->getText("emails.dataExport.{$emailType}.footer"); + $thanks = $locale->getText("emails.dataExport.{$emailType}.thanks"); + $signature = $locale->getText("emails.dataExport.{$emailType}.signature"); + $buttonText = $success ? $locale->getText("emails.dataExport.{$emailType}.buttonText") : ''; // Build email body using appropriate template $templatePath = $success @@ -769,6 +789,7 @@ class Migrations extends Action ->setParam('{{direction}}', $locale->getText('settings.direction')) ->setParam('{{project}}', $project->getAttribute('name')) ->setParam('{{user}}', $user->getAttribute('name', $user->getAttribute('email'))) + ->setParam('{{type}}', $exportType) ->setParam('{{size}}', $success ? '' : (string)$sizeMB); if ($success) { @@ -789,6 +810,7 @@ class Migrations extends Action 'terms' => $platform['termsUrl'], 'privacy' => $platform['privacyUrl'], 'platform' => $platform['platformName'], + 'type' => $exportType, ]; $queueForMails diff --git a/tests/e2e/Services/Migrations/MigrationsBase.php b/tests/e2e/Services/Migrations/MigrationsBase.php index 1e8b1f5ad3..6b446145fe 100644 --- a/tests/e2e/Services/Migrations/MigrationsBase.php +++ b/tests/e2e/Services/Migrations/MigrationsBase.php @@ -3825,4 +3825,714 @@ trait MigrationsBase 'x-appwrite-key' => $sourceProject['apiKey'], ]); } + + public function testCreateJSONImport(): void + { + // Make a database + $response = $this->client->call(Client::METHOD_POST, '/databases', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'databaseId' => ID::unique(), + 'name' => 'Test Database' + ]); + + $this->assertNotEmpty($response['body']['$id']); + $this->assertEquals(201, $response['headers']['status-code']); + $this->assertEquals('Test Database', $response['body']['name']); + + $databaseId = $response['body']['$id']; + + // make a table + $response = $this->client->call(Client::METHOD_POST, '/tablesdb/' . $databaseId . '/tables', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'name' => 'Test table', + 'tableId' => ID::unique(), + ]); + + $this->assertEquals(201, $response['headers']['status-code']); + $this->assertEquals($response['body']['name'], 'Test table'); + + $tableId = $response['body']['$id']; + + // make columns + $response = $this->client->call(Client::METHOD_POST, '/tablesdb/' . $databaseId . '/tables/' . $tableId . '/columns/string', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'key' => 'name', + 'size' => 256, + 'required' => true, + ]); + + $this->assertEquals(202, $response['headers']['status-code']); + $this->assertEquals($response['body']['key'], 'name'); + $this->assertEquals($response['body']['type'], 'string'); + $this->assertEquals($response['body']['size'], 256); + $this->assertEquals($response['body']['required'], true); + + $response = $this->client->call(Client::METHOD_POST, '/tablesdb/' . $databaseId . '/tables/' . $tableId . '/columns/integer', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'key' => 'age', + 'min' => 18, + 'max' => 65, + 'required' => true, + ]); + + $this->assertEquals(202, $response['headers']['status-code']); + $this->assertEquals($response['body']['key'], 'age'); + $this->assertEquals($response['body']['type'], 'integer'); + $this->assertEquals($response['body']['min'], 18); + $this->assertEquals($response['body']['max'], 65); + $this->assertEquals($response['body']['required'], true); + + // make a bucket, upload a file to it! + $bucketOne = $this->client->call(Client::METHOD_POST, '/storage/buckets', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ], [ + 'bucketId' => ID::unique(), + 'name' => 'Test Bucket', + 'maximumFileSize' => 2000000, //2MB + 'allowedFileExtensions' => ['json'], + 'compression' => 'gzip', + 'encryption' => true + ]); + $this->assertEquals(201, $bucketOne['headers']['status-code']); + $this->assertNotEmpty($bucketOne['body']['$id']); + + $bucketOneId = $bucketOne['body']['$id']; + + $bucketIds = [ + 'default' => $bucketOneId, + 'missing-column' => $bucketOneId, + 'irrelevant-column' => $bucketOneId, + 'documents-internals' => $bucketOneId, + ]; + + $fileIds = []; + + foreach ($bucketIds as $label => $bucketId) { + $jsonFileName = match ($label) { + 'missing-column', + 'irrelevant-column', + 'documents-internals' => "$label.json", + default => 'documents.json', + }; + + $response = $this->client->call(Client::METHOD_POST, '/storage/buckets/' . $bucketId . '/files', array_merge([ + 'content-type' => 'multipart/form-data', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'fileId' => ID::unique(), + 'file' => new CURLFile(realpath(__DIR__ . '/../../../resources/json/'.$jsonFileName), 'application/json', $jsonFileName), + ]); + + $this->assertEquals(201, $response['headers']['status-code']); + $this->assertNotEmpty($response['body']['$id']); + $this->assertEquals($jsonFileName, $response['body']['name']); + $this->assertEquals('application/json', $response['body']['mimeType']); + + $fileIds[$label] = $response['body']['$id']; + } + + // missing column, fail in worker. + $missingColumn = $this->performJsonMigration( + [ + 'fileId' => $fileIds['missing-column'], + 'bucketId' => $bucketIds['missing-column'], + 'resourceId' => $databaseId . ':' . $tableId, + ] + ); + + $this->assertEventually(function () use ($missingColumn) { + $migrationId = $missingColumn['body']['$id']; + $migration = $this->client->call(Client::METHOD_GET, '/migrations/'.$migrationId, array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders())); + + $this->assertEquals(200, $migration['headers']['status-code']); + $this->assertEquals('finished', $migration['body']['stage']); + $this->assertEquals('failed', $migration['body']['status']); + $this->assertEquals('JSON', $migration['body']['source']); + $this->assertEquals('Appwrite', $migration['body']['destination']); + $this->assertContains(Resource::TYPE_ROW, $migration['body']['resources']); + + /* fails in batch create documents unlike csv which checks headers first! */ + $this->assertArrayHasKey(Resource::TYPE_ROW, $migration['body']['statusCounters']); + $this->assertGreaterThan(0, $migration['body']['statusCounters'][Resource::TYPE_ROW]['error']); + + $this->assertThat( + implode("\n", $migration['body']['errors']), + $this->stringContains('Missing required attribute') + ); + $this->assertThat( + implode("\n", $migration['body']['errors']), + $this->stringContains('age') + ); + }, 60_000, 500); + + // irrelevant column - email, success. + $irrelevantColumn = $this->performJsonMigration( + [ + 'fileId' => $fileIds['irrelevant-column'], + 'bucketId' => $bucketIds['irrelevant-column'], + 'resourceId' => $databaseId . ':' . $tableId, + ] + ); + + $this->assertEventually(function () use ($irrelevantColumn) { + $migrationId = $irrelevantColumn['body']['$id']; + $migration = $this->client->call(Client::METHOD_GET, '/migrations/'.$migrationId, array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders())); + + $this->assertEquals(200, $migration['headers']['status-code']); + $this->assertEquals('finished', $migration['body']['stage']); + $this->assertEquals('completed', $migration['body']['status']); + $this->assertEquals('JSON', $migration['body']['source']); + $this->assertEquals('Appwrite', $migration['body']['destination']); + $this->assertContains(Resource::TYPE_ROW, $migration['body']['resources']); + $this->assertArrayHasKey(Resource::TYPE_ROW, $migration['body']['statusCounters']); + $this->assertEquals(100, $migration['body']['statusCounters'][Resource::TYPE_ROW]['success']); + }, 10_000, 500); + + // all data exists, pass. + $migration = $this->performJsonMigration( + [ + 'endpoint' => $this->endpoint, + 'fileId' => $fileIds['default'], + 'bucketId' => $bucketIds['default'], + 'resourceId' => $databaseId . ':' . $tableId, + ] + ); + + $this->assertEventually(function () use ($migration, $databaseId, $tableId) { + $migrationId = $migration['body']['$id']; + $migration = $this->client->call(Client::METHOD_GET, '/migrations/'.$migrationId, array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders())); + + $this->assertEquals(200, $migration['headers']['status-code']); + $this->assertEquals('finished', $migration['body']['stage']); + $this->assertEquals('completed', $migration['body']['status']); + $this->assertEquals('JSON', $migration['body']['source']); + $this->assertEquals('Appwrite', $migration['body']['destination']); + $this->assertContains(Resource::TYPE_ROW, $migration['body']['resources']); + $this->assertArrayHasKey(Resource::TYPE_ROW, $migration['body']['statusCounters']); + $this->assertEquals(100, $migration['body']['statusCounters'][Resource::TYPE_ROW]['success']); + }, 10_000, 500); + + // get rows count + $rows = $this->client->call(Client::METHOD_GET, '/tablesdb/'.$databaseId.'/tables/'.$tableId.'/rows', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'queries' => [ + Query::limit(250)->toString() + ] + ]); + + $this->assertEquals(200, $rows['headers']['status-code']); + $this->assertIsArray($rows['body']['rows']); + $this->assertIsNumeric($rows['body']['total']); + $this->assertEquals(200, $rows['body']['total']); + + // all data exists and includes internals, pass. + $migration = $this->performJsonMigration( + [ + 'endpoint' => $this->endpoint, + 'fileId' => $fileIds['documents-internals'], + 'bucketId' => $bucketIds['documents-internals'], + 'resourceId' => $databaseId . ':' . $tableId, + ] + ); + + $this->assertEventually(function () use ($migration) { + $migrationId = $migration['body']['$id']; + $migration = $this->client->call(Client::METHOD_GET, '/migrations/'.$migrationId, array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders())); + + $this->assertEquals(200, $migration['headers']['status-code']); + $this->assertEquals('finished', $migration['body']['stage']); + $this->assertEquals('completed', $migration['body']['status']); + $this->assertEquals('JSON', $migration['body']['source']); + $this->assertEquals('Appwrite', $migration['body']['destination']); + $this->assertContains(Resource::TYPE_ROW, $migration['body']['resources']); + $this->assertArrayHasKey(Resource::TYPE_ROW, $migration['body']['statusCounters']); + $this->assertEquals(25, $migration['body']['statusCounters'][Resource::TYPE_ROW]['success']); + }, 10_000, 500); + } + + private function performJsonMigration(array $body): array + { + return $this->client->call(Client::METHOD_POST, '/migrations/json/imports', [ + 'content-type' => 'application/json', + 'x-appwrite-key' => $this->getProject()['apiKey'], + 'x-appwrite-project' => $this->getProject()['$id'], + ], $body); + } + + /** + * Test JSON export with email notification + */ + public function testCreateJSONExport(): void + { + // Create a database + $database = $this->client->call(Client::METHOD_POST, '/databases', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'databaseId' => ID::unique(), + 'name' => 'Test Export Database' + ]); + + $this->assertEquals(201, $database['headers']['status-code']); + $databaseId = $database['body']['$id']; + + // Create a collection + $collection = $this->client->call(Client::METHOD_POST, '/databases/' . $databaseId . '/collections', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'collectionId' => ID::unique(), + 'name' => 'Test Export Collection', + 'permissions' => [] + ]); + + $this->assertEquals(201, $collection['headers']['status-code']); + $collectionId = $collection['body']['$id']; + + // Create a simple attribute like the basic test + $name = $this->client->call(Client::METHOD_POST, '/databases/' . $databaseId . '/collections/' . $collectionId . '/attributes/string', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'key' => 'name', + 'size' => 255, + 'required' => true, + ]); + + $this->assertEquals(202, $name['headers']['status-code']); + + // Create a simple attribute like the basic test + $email = $this->client->call(Client::METHOD_POST, '/databases/' . $databaseId . '/collections/' . $collectionId . '/attributes/string', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'key' => 'email', + 'size' => 255, + 'required' => false, + ]); + + $this->assertEquals(202, $email['headers']['status-code']); + + \sleep(3); + + // Create sample documents + for ($i = 1; $i <= 10; $i++) { + $doc = $this->client->call(Client::METHOD_POST, '/databases/' . $databaseId . '/collections/' . $collectionId . '/documents', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'documentId' => ID::unique(), + 'data' => [ + 'name' => 'Test User ' . $i, + 'email' => 'user' . $i . '@appwrite.io' + ] + ]); + + $this->assertEquals(201, $doc['headers']['status-code'], 'Failed to create document ' . $i); + } + + // Verify documents were created + $docs = $this->client->call(Client::METHOD_GET, '/databases/' . $databaseId . '/collections/' . $collectionId . '/documents', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]); + + $this->assertEquals(200, $docs['headers']['status-code']); + $this->assertEquals(10, $docs['body']['total'], 'Expected 10 documents but got ' . $docs['body']['total']); + + // Perform JSON export with notification enabled (uses internal bucket) + $migration = $this->client->call(Client::METHOD_POST, '/migrations/json/exports', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'] + ], $this->getHeaders()), [ + 'resourceId' => $databaseId . ':' . $collectionId, + 'filename' => 'test-json-export', + 'columns' => [], + 'queries' => [], + 'notify' => true + ]); + + $this->assertEquals(202, $migration['headers']['status-code']); + $this->assertNotEmpty($migration['body']['$id']); + $migrationId = $migration['body']['$id']; + + $this->assertEventually(function () use ($migrationId) { + $response = $this->client->call(Client::METHOD_GET, '/migrations/' . $migrationId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ]); + + $this->assertEquals(200, $response['headers']['status-code']); + $this->assertEquals('finished', $response['body']['stage']); + $this->assertEquals('completed', $response['body']['status']); + $this->assertEquals('Appwrite', $response['body']['source']); + $this->assertEquals('JSON', $response['body']['destination']); + + return true; + }, 30_000, 500); + + // Check that email was sent with download link + $lastEmail = $this->getLastEmail(); + $this->assertNotEmpty($lastEmail); + $this->assertEquals('Your JSON export is ready', $lastEmail['subject']); + $this->assertStringContainsStringIgnoringCase('Your data export has been completed successfully', $lastEmail['text']); + + // Extract download URL from email HTML + \preg_match('/href="([^"]*\/storage\/buckets\/[^"]*\/push[^"]*)"/', $lastEmail['html'], $matches); + $this->assertNotEmpty($matches[1], 'Download URL not found in email'); + $downloadUrl = html_entity_decode($matches[1]); + + // Parse the URL to extract components + $components = \parse_url($downloadUrl); + $this->assertNotEmpty($components); + \parse_str($components['query'] ?? '', $queryParams); + $this->assertArrayHasKey('jwt', $queryParams, 'JWT not found in download URL'); + $this->assertNotEmpty($queryParams['jwt']); + $this->assertArrayHasKey('project', $queryParams, 'Project not found in download URL'); + $this->assertStringContainsString('/storage/buckets/default/files/', $downloadUrl); + + // Test download with JWT + $path = \str_replace('/v1', '', $components['path']); + $downloadWithJwt = $this->client->call(Client::METHOD_GET, $path . '?project=' . $queryParams['project'] . '&jwt=' . $queryParams['jwt']); + $this->assertEquals(200, $downloadWithJwt['headers']['status-code'], 'Failed to download file with JWT'); + + // Verify the downloaded content is valid JSON + $jsonData = $downloadWithJwt['body']; + $this->assertNotEmpty($jsonData, 'JSON export should not be empty'); + $decoded = json_decode($jsonData, true); + $this->assertIsArray($decoded, 'JSON should be valid and decodable'); + $this->assertCount(10, $decoded, 'JSON should contain 10 documents'); + $this->assertArrayHasKey('name', $decoded[0], 'JSON documents should contain name field'); + $this->assertArrayHasKey('email', $decoded[0], 'JSON documents should contain email field'); + $this->assertStringContainsString('Test User', $decoded[0]['name'], 'JSON should contain test data'); + + // Cleanup + $this->client->call(Client::METHOD_DELETE, '/databases/' . $databaseId, [ + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]); + } + + public function testCreateVectorsDBJSONExport(): void + { + $headers = [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]; + + // Create vectorsdb database + $database = $this->client->call(Client::METHOD_POST, '/vectorsdb', $headers, [ + 'databaseId' => ID::unique(), + 'name' => 'VectorsDB Export Test' + ]); + $this->assertEquals(201, $database['headers']['status-code']); + $databaseId = $database['body']['$id']; + + // Create collection with dimension 16 + $collection = $this->client->call(Client::METHOD_POST, '/vectorsdb/' . $databaseId . '/collections', $headers, [ + 'collectionId' => ID::unique(), + 'name' => 'VecExportCol', + 'dimension' => 16, + ]); + $this->assertEquals(201, $collection['headers']['status-code']); + $collectionId = $collection['body']['$id']; + + // Seed 5 documents + for ($i = 1; $i <= 5; $i++) { + $embeddings = array_map(fn () => round((mt_rand() / mt_getrandmax()) * 2 - 1, 6), range(1, 16)); + $doc = $this->client->call(Client::METHOD_POST, '/vectorsdb/' . $databaseId . '/collections/' . $collectionId . '/documents', $headers, [ + 'documentId' => ID::unique(), + 'data' => [ + 'embeddings' => $embeddings, + 'metadata' => ['title' => 'Doc ' . $i, 'score' => round($i * 0.2, 1)] + ] + ]); + $this->assertEquals(201, $doc['headers']['status-code'], 'Failed to create vector document ' . $i); + } + + // Trigger JSON export + $migration = $this->client->call(Client::METHOD_POST, '/migrations/json/exports', $headers, [ + 'resourceId' => $databaseId . ':' . $collectionId, + 'filename' => 'vectorsdb-export-test', + 'columns' => [], + 'queries' => [], + 'notify' => false, + ]); + $this->assertEquals(202, $migration['headers']['status-code']); + $migrationId = $migration['body']['$id']; + + // Poll until completed + $this->assertEventually(function () use ($migrationId, $headers) { + $migration = $this->client->call(Client::METHOD_GET, '/migrations/' . $migrationId, $headers); + + $this->assertEquals(200, $migration['headers']['status-code']); + $this->assertEquals('finished', $migration['body']['stage']); + $this->assertEquals('completed', $migration['body']['status']); + $this->assertEquals('Appwrite', $migration['body']['source']); + $this->assertEquals('JSON', $migration['body']['destination']); + }, 30_000, 500); + + // Cleanup + $this->client->call(Client::METHOD_DELETE, '/vectorsdb/' . $databaseId, $headers); + } + + public function testCreateVectorsDBJSONImport(): void + { + $headers = [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]; + + // Create vectorsdb database + $database = $this->client->call(Client::METHOD_POST, '/vectorsdb', $headers, [ + 'databaseId' => ID::unique(), + 'name' => 'VectorsDB Import Test' + ]); + $this->assertEquals(201, $database['headers']['status-code']); + $databaseId = $database['body']['$id']; + + // Create collection with dimension 16 + $collection = $this->client->call(Client::METHOD_POST, '/vectorsdb/' . $databaseId . '/collections', $headers, [ + 'collectionId' => ID::unique(), + 'name' => 'VecImportCol', + 'dimension' => 16, + ]); + $this->assertEquals(201, $collection['headers']['status-code']); + $collectionId = $collection['body']['$id']; + + // Create bucket and upload test file + $bucket = $this->client->call(Client::METHOD_POST, '/storage/buckets', $headers, [ + 'bucketId' => ID::unique(), + 'name' => 'VectorsDB Import Bucket', + 'maximumFileSize' => 2000000, + 'allowedFileExtensions' => ['json'], + ]); + $this->assertEquals(201, $bucket['headers']['status-code']); + $bucketId = $bucket['body']['$id']; + + $file = $this->client->call(Client::METHOD_POST, '/storage/buckets/' . $bucketId . '/files', array_merge([ + 'content-type' => 'multipart/form-data', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'fileId' => ID::unique(), + 'file' => new \CURLFile(realpath(__DIR__ . '/../../../resources/json/vectorsdb-documents.json'), 'application/json', 'vectorsdb-documents.json'), + ]); + $this->assertEquals(201, $file['headers']['status-code']); + $fileId = $file['body']['$id']; + + // Trigger import + $migration = $this->performJsonMigration([ + 'fileId' => $fileId, + 'bucketId' => $bucketId, + 'resourceId' => $databaseId . ':' . $collectionId, + ]); + $this->assertEquals(202, $migration['headers']['status-code']); + + // Poll until completed + $this->assertEventually(function () use ($migration, $headers) { + $migrationId = $migration['body']['$id']; + $result = $this->client->call(Client::METHOD_GET, '/migrations/' . $migrationId, $headers); + + $this->assertEquals(200, $result['headers']['status-code']); + $this->assertEquals('finished', $result['body']['stage']); + $this->assertEquals('completed', $result['body']['status']); + $this->assertEquals('JSON', $result['body']['source']); + $this->assertEquals('Appwrite', $result['body']['destination']); + }, 30_000, 500); + + // Verify documents were imported + $docs = $this->client->call(Client::METHOD_GET, '/vectorsdb/' . $databaseId . '/collections/' . $collectionId . '/documents', $headers); + $this->assertEquals(200, $docs['headers']['status-code']); + $this->assertEquals(10, $docs['body']['total'], 'Should have imported 10 vectorsdb documents'); + + // Verify first document structure + $firstDoc = $docs['body']['documents'][0]; + $this->assertArrayHasKey('embeddings', $firstDoc); + $this->assertCount(16, $firstDoc['embeddings'], 'Imported embeddings should have 16 dimensions'); + $this->assertArrayHasKey('metadata', $firstDoc); + + // Cleanup + $this->client->call(Client::METHOD_DELETE, '/vectorsdb/' . $databaseId, $headers); + } + + public function testCreateDocumentsDBJSONExport(): void + { + $headers = [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]; + + // Create documentsdb database + $database = $this->client->call(Client::METHOD_POST, '/documentsdb', $headers, [ + 'databaseId' => ID::unique(), + 'name' => 'DocumentsDB Export Test' + ]); + $this->assertEquals(201, $database['headers']['status-code']); + $databaseId = $database['body']['$id']; + + // Create collection (schemaless — no attributes needed) + $collection = $this->client->call(Client::METHOD_POST, '/documentsdb/' . $databaseId . '/collections', $headers, [ + 'collectionId' => ID::unique(), + 'name' => 'DocExportCol', + ]); + $this->assertEquals(201, $collection['headers']['status-code']); + $collectionId = $collection['body']['$id']; + + // Seed 5 documents + for ($i = 1; $i <= 5; $i++) { + $doc = $this->client->call(Client::METHOD_POST, '/documentsdb/' . $databaseId . '/collections/' . $collectionId . '/documents', $headers, [ + 'documentId' => ID::unique(), + 'data' => [ + 'name' => 'User ' . $i, + 'email' => 'user' . $i . '@test.com', + 'age' => 20 + $i, + 'address' => ['city' => 'City ' . $i, 'zip' => '1000' . $i] + ] + ]); + $this->assertEquals(201, $doc['headers']['status-code'], 'Failed to create document ' . $i); + } + + // Trigger JSON export + $migration = $this->client->call(Client::METHOD_POST, '/migrations/json/exports', $headers, [ + 'resourceId' => $databaseId . ':' . $collectionId, + 'filename' => 'documentsdb-export-test', + 'columns' => [], + 'queries' => [], + 'notify' => false, + ]); + $this->assertEquals(202, $migration['headers']['status-code']); + $migrationId = $migration['body']['$id']; + + // Poll until completed + $this->assertEventually(function () use ($migrationId, $headers) { + $migration = $this->client->call(Client::METHOD_GET, '/migrations/' . $migrationId, $headers); + + $this->assertEquals(200, $migration['headers']['status-code']); + $this->assertEquals('finished', $migration['body']['stage']); + $this->assertEquals('completed', $migration['body']['status']); + $this->assertEquals('Appwrite', $migration['body']['source']); + $this->assertEquals('JSON', $migration['body']['destination']); + }, 30_000, 500); + + // Cleanup + $this->client->call(Client::METHOD_DELETE, '/documentsdb/' . $databaseId, $headers); + } + + public function testCreateDocumentsDBJSONImport(): void + { + $headers = [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]; + + // Create documentsdb database + $database = $this->client->call(Client::METHOD_POST, '/documentsdb', $headers, [ + 'databaseId' => ID::unique(), + 'name' => 'DocumentsDB Import Test' + ]); + $this->assertEquals(201, $database['headers']['status-code']); + $databaseId = $database['body']['$id']; + + // Create collection (schemaless) + $collection = $this->client->call(Client::METHOD_POST, '/documentsdb/' . $databaseId . '/collections', $headers, [ + 'collectionId' => ID::unique(), + 'name' => 'DocImportCol', + ]); + $this->assertEquals(201, $collection['headers']['status-code']); + $collectionId = $collection['body']['$id']; + + // Create bucket and upload test file + $bucket = $this->client->call(Client::METHOD_POST, '/storage/buckets', $headers, [ + 'bucketId' => ID::unique(), + 'name' => 'DocumentsDB Import Bucket', + 'maximumFileSize' => 2000000, + 'allowedFileExtensions' => ['json'], + ]); + $this->assertEquals(201, $bucket['headers']['status-code']); + $bucketId = $bucket['body']['$id']; + + $file = $this->client->call(Client::METHOD_POST, '/storage/buckets/' . $bucketId . '/files', array_merge([ + 'content-type' => 'multipart/form-data', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'fileId' => ID::unique(), + 'file' => new \CURLFile(realpath(__DIR__ . '/../../../resources/json/documentsdb-documents.json'), 'application/json', 'documentsdb-documents.json'), + ]); + $this->assertEquals(201, $file['headers']['status-code']); + $fileId = $file['body']['$id']; + + // Trigger import + $migration = $this->performJsonMigration([ + 'fileId' => $fileId, + 'bucketId' => $bucketId, + 'resourceId' => $databaseId . ':' . $collectionId, + ]); + $this->assertEquals(202, $migration['headers']['status-code']); + + // Poll until completed + $this->assertEventually(function () use ($migration, $headers) { + $migrationId = $migration['body']['$id']; + $result = $this->client->call(Client::METHOD_GET, '/migrations/' . $migrationId, $headers); + + $this->assertEquals(200, $result['headers']['status-code']); + $this->assertEquals('finished', $result['body']['stage']); + $this->assertEquals('completed', $result['body']['status']); + $this->assertEquals('JSON', $result['body']['source']); + $this->assertEquals('Appwrite', $result['body']['destination']); + }, 30_000, 500); + + // Verify documents were imported + $docs = $this->client->call(Client::METHOD_GET, '/documentsdb/' . $databaseId . '/collections/' . $collectionId . '/documents', $headers); + $this->assertEquals(200, $docs['headers']['status-code']); + $this->assertEquals(10, $docs['body']['total'], 'Should have imported 10 documentsdb documents'); + + // Verify first document has nested data + $firstDoc = $docs['body']['documents'][0]; + $this->assertArrayHasKey('name', $firstDoc); + $this->assertArrayHasKey('address', $firstDoc); + $this->assertIsArray($firstDoc['address']); + + // Cleanup + $this->client->call(Client::METHOD_DELETE, '/documentsdb/' . $databaseId, $headers); + } + } diff --git a/tests/resources/json/documents-internals.json b/tests/resources/json/documents-internals.json new file mode 100644 index 0000000000..6fe6820cdf --- /dev/null +++ b/tests/resources/json/documents-internals.json @@ -0,0 +1,254 @@ +[ + { + "$id": "z1y2x3w4v5u6t7s8", + "$createdAt": "2022-10-23T10:33:01+00:00", + "$updatedAt": "2023-03-15T12:00:41+00:00", + "$permissions": [ + "read(\"any\")", + "update(\"user:123\")" + ], + "name": "Diamond Mendez", + "age": 56 + }, + { + "$id": "r9q0p1o2n3m4l5k6", + "$createdAt": "2021-08-11T21:05:13+00:00", + "$updatedAt": "2024-01-02T08:45:22+00:00", + "$permissions": [ + "read(\"any\")", + "update(\"user:456\")" + ], + "name": "Michael Huff", + "age": 20 + }, + { + "$id": "j7i8h9g0f1e2d3c4", + "$createdAt": "2020-05-29T14:22:56+00:00", + "$updatedAt": "2022-11-30T18:19:33+00:00", + "$permissions": [ + "read(\"any\")" + ], + "name": "Alyssa Rodriguez", + "age": 37 + }, + { + "$id": "b5a6z7y8x9w0v1u2", + "$createdAt": "2023-01-18T03:44:09+00:00", + "$updatedAt": "2023-09-07T23:50:17+00:00", + "$permissions": [ + "read(\"any\")" + ], + "name": "Barbara Smith", + "age": 26 + }, + { + "$id": "t3s4r5q6p7o8n9m0", + "$createdAt": "2020-11-02T09:12:45+00:00", + "$updatedAt": "2021-07-21T15:30:55+00:00", + "$permissions": [ + "read(\"any\")" + ], + "name": "Evelyn Edwards", + "age": 54 + }, + { + "$id": "l1k2j3i4h5g6f7e8", + "$createdAt": "2022-03-19T19:55:27+00:00", + "$updatedAt": "2024-05-14T06:28:11+00:00", + "$permissions": [ + "read(\"any\")" + ], + "name": "Tina Richardson", + "age": 41 + }, + { + "$id": "d9c0b1a2z3y4x5w6", + "$createdAt": "2021-04-07T01:18:34+00:00", + "$updatedAt": "2023-06-25T11:47:04+00:00", + "$permissions": [ + "read(\"any\")" + ], + "name": "Joel Hernandez", + "age": 49 + }, + { + "$id": "v7u8t9s0r1q2p3o4", + "$createdAt": "2023-08-22T16:40:18+00:00", + "$updatedAt": "2024-02-19T04:09:58+00:00", + "$permissions": [ + "read(\"any\")" + ], + "name": "Zachary Cooper", + "age": 59 + }, + { + "$id": "n5m6l7k8j9i0h1g2", + "$createdAt": "2020-02-12T07:59:01+00:00", + "$updatedAt": "2022-09-08T13:21:49+00:00", + "$permissions": [ + "read(\"any\")" + ], + "name": "Brittany Spears", + "age": 20 + }, + { + "$id": "f3e4d5c6b7a8z9y0", + "$createdAt": "2021-12-05T22:33:12+00:00", + "$updatedAt": "2023-11-11T02:55:37+00:00", + "$permissions": [ + "read(\"any\")" + ], + "name": "Holly White", + "age": 47 + }, + { + "$id": "x1w2v3u4t5s6r7q8", + "$createdAt": "2022-07-14T05:01:50+00:00", + "$updatedAt": "2024-04-01T20:10:26+00:00", + "$permissions": [ + "read(\"any\")" + ], + "name": "Kimberly Barnes", + "age": 27 + }, + { + "$id": "p9o0n1m2l3k4j5i6", + "$createdAt": "2020-09-28T11:27:36+00:00", + "$updatedAt": "2021-10-17T09:38:08+00:00", + "$permissions": [ + "read(\"any\")" + ], + "name": "Stephen Miller", + "age": 53 + }, + { + "$id": "h7g8f9e0d1c2b3a4", + "$createdAt": "2023-04-04T08:15:59+00:00", + "$updatedAt": "2024-06-29T17:03:14+00:00", + "$permissions": [ + "read(\"any\")" + ], + "name": "Yvonne Newman", + "age": 41 + }, + { + "$id": "y5x6w7v8u9t0s1r2", + "$createdAt": "2021-01-25T18:09:21+00:00", + "$updatedAt": "2022-08-16T22:44:51+00:00", + "$permissions": [ + "read(\"any\")" + ], + "name": "Carol Kane", + "age": 38 + }, + { + "$id": "q3p4o5n6m7l8k9j0", + "$createdAt": "2022-06-09T12:53:47+00:00", + "$updatedAt": "2023-12-24T01:16:05+00:00", + "$permissions": [ + "read(\"any\")" + ], + "name": "Doris Foster", + "age": 44 + }, + { + "$id": "i1h2g3f4e5d6c7b8", + "$createdAt": "2020-07-03T23:37:02+00:00", + "$updatedAt": "2021-05-09T05:52:43+00:00", + "$permissions": [ + "read(\"any\")" + ], + "name": "Joseph Stokes", + "age": 28 + }, + { + "$id": "a9z0y1x2w3v4u5t6", + "$createdAt": "2023-10-10T02:20:15+00:00", + "$updatedAt": "2024-03-28T14:33:29+00:00", + "$permissions": [ + "read(\"any\")" + ], + "name": "Steve Williams", + "age": 31 + }, + { + "$id": "s7r8q9p0o1n2m3l4", + "$createdAt": "2021-06-16T13:48:53+00:00", + "$updatedAt": "2022-04-22T07:07:19+00:00", + "$permissions": [ + "read(\"any\")" + ], + "name": "James Carey", + "age": 29 + }, + { + "$id": "k5j6i7h8g9f0e1d2", + "$createdAt": "2022-12-27T20:06:38+00:00", + "$updatedAt": "2023-08-03T10:25:57+00:00", + "$permissions": [ + "read(\"any\")" + ], + "name": "Kathryn Henry", + "age": 38 + }, + { + "$id": "c3b4a5z6y7x8w9v0", + "$createdAt": "2020-04-20T04:41:24+00:00", + "$updatedAt": "2021-02-13T19:14:06+00:00", + "$permissions": [ + "read(\"any\")" + ], + "name": "Christopher Landry", + "age": 23 + }, + { + "$id": "u1t2s3r4q5p6o7n8", + "$createdAt": "2023-05-08T00:58:10+00:00", + "$updatedAt": "2024-07-05T03:36:48+00:00", + "$permissions": [ + "read(\"any\")" + ], + "name": "Jennifer Mcgee", + "age": 62 + }, + { + "$id": "m9l0k1j2i3h4g5f6", + "$createdAt": "2021-09-01T06:11:42+00:00", + "$updatedAt": "2022-01-26T16:59:23+00:00", + "$permissions": [ + "read(\"any\")" + ], + "name": "Cathy Church", + "age": 35 + }, + { + "$id": "e7d8c9b0a1z2y3x4", + "$createdAt": "2022-02-18T15:24:07+00:00", + "$updatedAt": "2023-04-12T00:40:31+00:00", + "$permissions": [ + "read(\"any\")" + ], + "name": "Jose Lopez", + "age": 41 + }, + { + "$id": "w5v6u7t8s9r0q1p2", + "$createdAt": "2020-12-13T09:03:55+00:00", + "$updatedAt": "2021-11-06T11:23:16+00:00", + "$permissions": [ + "read(\"any\")" + ], + "name": "William Rose", + "age": 30 + }, + { + "$id": "o3n4m5l6k7j8i9h0", + "$createdAt": "2021-12-13T09:03:55+00:00", + "$updatedAt": "2022-11-06T11:23:16+00:00", + "$permissions": [ + "read(\"any\")" + ], + "name": "Charles Hammer", + "age": 61 + } +] \ No newline at end of file diff --git a/tests/resources/json/documents.json b/tests/resources/json/documents.json new file mode 100644 index 0000000000..1ce5533297 --- /dev/null +++ b/tests/resources/json/documents.json @@ -0,0 +1,502 @@ +[ + { + "$id": "hxfcwpcas5xokpwe", + "name": "Diamond Mendez", + "age": 56 + }, + { + "$id": "gw8nxwf6esn3tfwf", + "name": "Michael Huff", + "age": 20 + }, + { + "$id": "xb6bxg56lral1qy9", + "name": "Alyssa Rodriguez", + "age": 37 + }, + { + "$id": "imerjq5j36y3agh2", + "name": "Barbara Smith", + "age": 26 + }, + { + "$id": "07yq9qdlhmbzmr35", + "name": "Evelyn Edwards", + "age": 54 + }, + { + "$id": "ksqo631sbhwj5ltg", + "name": "Tina Richardson", + "age": 41 + }, + { + "$id": "j7zlndgu0gbshp15", + "name": "Joel Hernandez", + "age": 49 + }, + { + "$id": "mfntvnljrcmf7h6v", + "name": "Zachary Cooper", + "age": 59 + }, + { + "$id": "5f9b01nziqu2h8ed", + "name": "Brittany Spears", + "age": 20 + }, + { + "$id": "4vxzbnzraqznk5u8", + "name": "Holly White", + "age": 47 + }, + { + "$id": "d4ywy3mtphaatbpf", + "name": "Kimberly Barnes", + "age": 27 + }, + { + "$id": "88odnk6nthyyvbal", + "name": "Stephen Miller", + "age": 53 + }, + { + "$id": "08oekee3fn7mzaa5", + "name": "Yvonne Newman", + "age": 41 + }, + { + "$id": "quw55kn9895i5e4v", + "name": "Carol Kane", + "age": 38 + }, + { + "$id": "nge6bm8ykripei6f", + "name": "Doris Foster", + "age": 44 + }, + { + "$id": "4k16i33s0xl2ypx9", + "name": "Joseph Stokes", + "age": 28 + }, + { + "$id": "q0j5rxbgid66snyf", + "name": "Steve Williams", + "age": 31 + }, + { + "$id": "n1oxun7mqq3p103y", + "name": "James Carey", + "age": 29 + }, + { + "$id": "0dbvs840jkf8i0ye", + "name": "Kathryn Henry", + "age": 38 + }, + { + "$id": "5sfaidgs1h87v15v", + "name": "Christopher Landry", + "age": 23 + }, + { + "$id": "vg3punvfu5khmf41", + "name": "Jennifer Mcgee", + "age": 62 + }, + { + "$id": "f933qydr9u5b2r11", + "name": "Cathy Church", + "age": 35 + }, + { + "$id": "wjv87y1inf8yk32s", + "name": "Jose Lopez", + "age": 41 + }, + { + "$id": "uljysdvdlcyrbrwk", + "name": "William Rose", + "age": 30 + }, + { + "$id": "ot8xtzh77j55wq0s", + "name": "Sarah Ford", + "age": 26 + }, + { + "$id": "9t76vnsv2u36s43t", + "name": "Alisha Jones", + "age": 61 + }, + { + "$id": "66y4tnty62hw8c02", + "name": "Kristin Kelly", + "age": 61 + }, + { + "$id": "2punfblazi5v16ar", + "name": "Brendan Stout", + "age": 40 + }, + { + "$id": "sxhr4nf5w2gx4wbg", + "name": "Kelly Cruz", + "age": 18 + }, + { + "$id": "68dvrqfwqnkq5el9", + "name": "Samantha Martin", + "age": 50 + }, + { + "$id": "20192l6dbeinhkh0", + "name": "David Santos", + "age": 46 + }, + { + "$id": "si0l4dgay09ebfmf", + "name": "Elizabeth Carroll", + "age": 22 + }, + { + "$id": "lhse40vbldqb6ap1", + "name": "Corey Owens", + "age": 46 + }, + { + "$id": "h5t3pslykyx3kxfm", + "name": "Shelby Mueller", + "age": 65 + }, + { + "$id": "ldc0luydrw6jub0f", + "name": "Dr. Sylvia Myers", + "age": 29 + }, + { + "$id": "voc9628xg4dsgw2y", + "name": "Scott Freeman", + "age": 48 + }, + { + "$id": "o4y0gk3gqv1ax2fz", + "name": "Christopher Atkinson", + "age": 21 + }, + { + "$id": "u1n3x4e4u7e0vzj6", + "name": "Sean Diaz", + "age": 31 + }, + { + "$id": "s36eskwtm0w7lwr7", + "name": "Bobby Dyer", + "age": 57 + }, + { + "$id": "4hjnag1p5iwvtixd", + "name": "Daniel Hall", + "age": 62 + }, + { + "$id": "m91d80oxsa216zbh", + "name": "Jennifer Ramirez", + "age": 65 + }, + { + "$id": "5hj6858zo2g85n6v", + "name": "Angela Jackson", + "age": 57 + }, + { + "$id": "8m8oihv9a1e7nn92", + "name": "Kelly Lewis", + "age": 36 + }, + { + "$id": "7azy39la0no0mxi7", + "name": "Jessica Munoz", + "age": 55 + }, + { + "$id": "47pmjkhnnqhyit8c", + "name": "Kelly George", + "age": 65 + }, + { + "$id": "6j6cpy4kgneg1mmh", + "name": "Anthony Johnson", + "age": 65 + }, + { + "$id": "tnlmtvap1zz89km9", + "name": "Regina Fields", + "age": 61 + }, + { + "$id": "6cyuvnwwqdmrpfzh", + "name": "Sharon Schaefer", + "age": 30 + }, + { + "$id": "p1v4pyu2pqodc0ey", + "name": "Jacob French", + "age": 62 + }, + { + "$id": "6npynnhjt2jd05xo", + "name": "Jessica Costa", + "age": 23 + }, + { + "$id": "wcxedf13n2e9qi4l", + "name": "George Hardy", + "age": 53 + }, + { + "$id": "yf2xlcmszk2tqeig", + "name": "Andrea Allison", + "age": 20 + }, + { + "$id": "3bf2zzv7poststwa", + "name": "Kevin Ferguson", + "age": 32 + }, + { + "$id": "c2iataz0hhv39q63", + "name": "Joseph Johnson", + "age": 58 + }, + { + "$id": "3e8npxhov4a39pvq", + "name": "Ashley Martinez", + "age": 18 + }, + { + "$id": "t7dp41tysipytywq", + "name": "Charles Nixon", + "age": 23 + }, + { + "$id": "z8cztq7c47phyfhk", + "name": "Carol Dudley", + "age": 40 + }, + { + "$id": "2636f9d8r4ipm3h6", + "name": "David Weber", + "age": 51 + }, + { + "$id": "eh3f6wxtvkjq6ykq", + "name": "Scott Robinson", + "age": 32 + }, + { + "$id": "raskbwpsje69a59h", + "name": "Anthony Hardy", + "age": 38 + }, + { + "$id": "90hn1p0b4cs9e2og", + "name": "Mackenzie Owens", + "age": 52 + }, + { + "$id": "am3swwfbo076x0v1", + "name": "Brian Foster", + "age": 27 + }, + { + "$id": "5uw7utb9lq5cfncw", + "name": "Hannah Forbes", + "age": 56 + }, + { + "$id": "cs6mbfzkzifefx6r", + "name": "Lauren Reed", + "age": 26 + }, + { + "$id": "ftw3uvztziiz9x00", + "name": "Morgan Smith", + "age": 28 + }, + { + "$id": "uhrqseeo43mozpaq", + "name": "Samantha Alexander", + "age": 65 + }, + { + "$id": "pvvmzyfc1lxor11e", + "name": "Tiffany Roberts", + "age": 20 + }, + { + "$id": "jia7bdag4abz123s", + "name": "Emily Hayes", + "age": 34 + }, + { + "$id": "h6oozcngbz8o5x4y", + "name": "Rebecca Villegas", + "age": 52 + }, + { + "$id": "9v6z1pn2f9twcy12", + "name": "Donald Shah", + "age": 61 + }, + { + "$id": "wzz3jduioso77o7f", + "name": "Denise Cain", + "age": 59 + }, + { + "$id": "u51plhgvjodkswnr", + "name": "Kristine Ramirez", + "age": 53 + }, + { + "$id": "t1uhkmiytfyc13vc", + "name": "Stacey Adkins", + "age": 61 + }, + { + "$id": "iqaqnf0ybg2ct507", + "name": "Daniel Hunt", + "age": 20 + }, + { + "$id": "idwrwv2uu4hcpv2i", + "name": "Roberta Johnson", + "age": 48 + }, + { + "$id": "2yd2hd6auetjacyo", + "name": "Jason Williamson", + "age": 39 + }, + { + "$id": "egrmdbibnjhi914x", + "name": "Sandra Robinson", + "age": 50 + }, + { + "$id": "15m1pz2bb0ercgyk", + "name": "Steve Rice", + "age": 25 + }, + { + "$id": "0i21bhkxdagjurb7", + "name": "Kimberly Fritz", + "age": 53 + }, + { + "$id": "726ofi7h5snreq67", + "name": "Brianna Reynolds", + "age": 33 + }, + { + "$id": "csqxse3wym56eim6", + "name": "Alexander Williams", + "age": 50 + }, + { + "$id": "qeaoylnrsf8p3byg", + "name": "Andrew Thomas", + "age": 25 + }, + { + "$id": "edsswobumzyzbvhf", + "name": "Austin Williams", + "age": 57 + }, + { + "$id": "hdzhzpt0ahy5hkib", + "name": "Nicholas Williams", + "age": 24 + }, + { + "$id": "w1qmvmg4roa8xnwu", + "name": "Mrs. Michelle Cisneros", + "age": 48 + }, + { + "$id": "3z3o73x7adyuo6w0", + "name": "Stacey Smith", + "age": 39 + }, + { + "$id": "sse2u5zlgoqrgmcf", + "name": "Laura Beck", + "age": 20 + }, + { + "$id": "rvovijmvch58r4yx", + "name": "Molly Clark", + "age": 51 + }, + { + "$id": "doe06nrx8sg5mcuv", + "name": "Carmen Morris", + "age": 41 + }, + { + "$id": "jbjdwuvj5s4kw04y", + "name": "Amanda Munoz", + "age": 20 + }, + { + "$id": "6k2ewkla7js0yw23", + "name": "Rachel Collins", + "age": 44 + }, + { + "$id": "fcxuyr4kkhrnigu1", + "name": "John Alexander", + "age": 18 + }, + { + "$id": "d25fuwlos5mk07o0", + "name": "Stacy Hunter", + "age": 22 + }, + { + "$id": "1vdai2rxmwd57oet", + "name": "Eric Massey", + "age": 40 + }, + { + "$id": "pq4jnt9izu1wlrzd", + "name": "Scott Garcia", + "age": 20 + }, + { + "$id": "lz9kfc0lty5xcz14", + "name": "Cassandra Nelson", + "age": 35 + }, + { + "$id": "pu7w6tyab5jd4we9", + "name": "Aaron Johnson", + "age": 50 + }, + { + "$id": "8dupswd2kqwdyn8v", + "name": "Shannon Sherman", + "age": 45 + }, + { + "$id": "ye466l71jthiz2p6", + "name": "April Garcia", + "age": 60 + }, + { + "$id": "xogsmfwb73l16qdt", + "name": "Evan Lynn", + "age": 20 + } +] \ No newline at end of file diff --git a/tests/resources/json/documentsdb-documents.json b/tests/resources/json/documentsdb-documents.json new file mode 100644 index 0000000000..908c123e81 --- /dev/null +++ b/tests/resources/json/documentsdb-documents.json @@ -0,0 +1,162 @@ +[ + { + "$id": "doc01", + "name": "Alice", + "email": "alice@test.com", + "age": 35, + "active": false, + "tags": [ + "analyst", + "engineer" + ], + "address": { + "city": "New York", + "zip": "27436", + "country": "DE" + } + }, + { + "$id": "doc02", + "name": "Bob", + "email": "bob@test.com", + "age": 58, + "active": false, + "tags": [ + "engineer", + "manager" + ], + "address": { + "city": "London", + "zip": "49320", + "country": "JP" + } + }, + { + "$id": "doc03", + "name": "Charlie", + "email": "charlie@test.com", + "age": 23, + "active": true, + "tags": [ + "analyst", + "manager" + ], + "address": { + "city": "Tokyo", + "zip": "74758", + "country": "JP" + } + }, + { + "$id": "doc04", + "name": "Diana", + "email": "diana@test.com", + "age": 60, + "active": true, + "tags": [ + "analyst", + "manager" + ], + "address": { + "city": "Paris", + "zip": "50703", + "country": "JP" + } + }, + { + "$id": "doc05", + "name": "Eve", + "email": "eve@test.com", + "age": 59, + "active": false, + "tags": [ + "engineer", + "designer" + ], + "address": { + "city": "Berlin", + "zip": "98929", + "country": "UK" + } + }, + { + "$id": "doc06", + "name": "Frank", + "email": "frank@test.com", + "age": 23, + "active": true, + "tags": [ + "engineer", + "manager" + ], + "address": { + "city": "Sydney", + "zip": "82907", + "country": "US" + } + }, + { + "$id": "doc07", + "name": "Grace", + "email": "grace@test.com", + "age": 36, + "active": true, + "tags": [ + "analyst", + "developer" + ], + "address": { + "city": "Toronto", + "zip": "68710", + "country": "US" + } + }, + { + "$id": "doc08", + "name": "Hank", + "email": "hank@test.com", + "age": 28, + "active": false, + "tags": [ + "engineer", + "manager" + ], + "address": { + "city": "Mumbai", + "zip": "61026", + "country": "FR" + } + }, + { + "$id": "doc09", + "name": "Iris", + "email": "iris@test.com", + "age": 40, + "active": true, + "tags": [ + "developer", + "designer" + ], + "address": { + "city": "Seoul", + "zip": "48039", + "country": "FR" + } + }, + { + "$id": "doc10", + "name": "Jack", + "email": "jack@test.com", + "age": 45, + "active": false, + "tags": [ + "manager", + "developer" + ], + "address": { + "city": "Dubai", + "zip": "22733", + "country": "JP" + } + } +] \ No newline at end of file diff --git a/tests/resources/json/irrelevant-column.json b/tests/resources/json/irrelevant-column.json new file mode 100644 index 0000000000..d047620ab0 --- /dev/null +++ b/tests/resources/json/irrelevant-column.json @@ -0,0 +1,602 @@ +[ + { + "$id": "r5ctmrqwqn1m3rc0", + "name": "Diamond Mendez", + "age": 56, + "email": "diamond.mendez@example.com" + }, + { + "$id": "wxwp7e7q7nx3ltfx", + "name": "Michael Huff", + "age": 20, + "email": "michael.huff@example.com" + }, + { + "$id": "4ct0b38fwaojawlv", + "name": "Alyssa Rodriguez", + "age": 37, + "email": "alyssa.rodriguez@example.com" + }, + { + "$id": "o0jjcuygbta2zvga", + "name": "Barbara Smith", + "age": 26, + "email": "barbara.smith@example.com" + }, + { + "$id": "bdy6l2ofl8klb4pb", + "name": "Evelyn Edwards", + "age": 54, + "email": "evelyn.edwards@example.com" + }, + { + "$id": "rkccl72v7zwtbila", + "name": "Tina Richardson", + "age": 41, + "email": "tina.richardson@example.com" + }, + { + "$id": "cilw7um0cd927esj", + "name": "Joel Hernandez", + "age": 49, + "email": "joel.hernandez@example.com" + }, + { + "$id": "60povvz0votkve1j", + "name": "Zachary Cooper", + "age": 59, + "email": "zachary.cooper@example.com" + }, + { + "$id": "ayow5dzwktvbbtp2", + "name": "Brittany Spears", + "age": 20, + "email": "brittany.spears@example.com" + }, + { + "$id": "cfru98od0lab0b2n", + "name": "Holly White", + "age": 47, + "email": "holly.white@example.com" + }, + { + "$id": "vjxjldvu3r6uylq6", + "name": "Kimberly Barnes", + "age": 27, + "email": "kimberly.barnes@example.com" + }, + { + "$id": "d1p47hl97pw6xowb", + "name": "Stephen Miller", + "age": 53, + "email": "stephen.miller@example.com" + }, + { + "$id": "yxk6qaa5ryb3gqrb", + "name": "Yvonne Newman", + "age": 41, + "email": "yvonne.newman@example.com" + }, + { + "$id": "ifkeo7j8t7hfd7z8", + "name": "Carol Kane", + "age": 38, + "email": "carol.kane@example.com" + }, + { + "$id": "e1q4lq8vvpxp9ysb", + "name": "Doris Foster", + "age": 44, + "email": "doris.foster@example.com" + }, + { + "$id": "obec6d52dc6swzsf", + "name": "Joseph Stokes", + "age": 28, + "email": "joseph.stokes@example.com" + }, + { + "$id": "26v06la6ug8wkvim", + "name": "Steve Williams", + "age": 31, + "email": "steve.williams@example.com" + }, + { + "$id": "m4glre4ch12vkxp6", + "name": "James Carey", + "age": 29, + "email": "james.carey@example.com" + }, + { + "$id": "jee8fyfffnjugsd5", + "name": "Kathryn Henry", + "age": 38, + "email": "kathryn.henry@example.com" + }, + { + "$id": "miquc6ljb9l3a31r", + "name": "Christopher Landry", + "age": 23, + "email": "christopher.landry@example.com" + }, + { + "$id": "ghf7da7seeuj1zdl", + "name": "Jennifer Mcgee", + "age": 62, + "email": "jennifer.mcgee@example.com" + }, + { + "$id": "x7h11phjrz77w0q8", + "name": "Cathy Church", + "age": 35, + "email": "cathy.church@example.com" + }, + { + "$id": "dn8u5lsux4708z6j", + "name": "Jose Lopez", + "age": 41, + "email": "jose.lopez@example.com" + }, + { + "$id": "zb7fdlyohuyy5i9k", + "name": "William Rose", + "age": 30, + "email": "william.rose@example.com" + }, + { + "$id": "qyrj8m9krp4dt4wt", + "name": "Sarah Ford", + "age": 26, + "email": "sarah.ford@example.com" + }, + { + "$id": "t6t673zpfyhhz8pg", + "name": "Alisha Jones", + "age": 61, + "email": "alisha.jones@example.com" + }, + { + "$id": "0hfbo0iy1q9bwc2n", + "name": "Kristin Kelly", + "age": 61, + "email": "kristin.kelly@example.com" + }, + { + "$id": "8alv4e4xrpcj443z", + "name": "Brendan Stout", + "age": 40, + "email": "brendan.stout@example.com" + }, + { + "$id": "qxm7a0z32xdkzxdj", + "name": "Kelly Cruz", + "age": 18, + "email": "kelly.cruz@example.com" + }, + { + "$id": "885mti7j7oiz5p5g", + "name": "Samantha Martin", + "age": 50, + "email": "samantha.martin@example.com" + }, + { + "$id": "v8i7dvhby6711m66", + "name": "David Santos", + "age": 46, + "email": "david.santos@example.com" + }, + { + "$id": "rggc0ow8ccd2jgvp", + "name": "Elizabeth Carroll", + "age": 22, + "email": "elizabeth.carroll@example.com" + }, + { + "$id": "012472s64rvzq1c4", + "name": "Corey Owens", + "age": 46, + "email": "corey.owens@example.com" + }, + { + "$id": "0k2xrwj4g33ut14y", + "name": "Shelby Mueller", + "age": 65, + "email": "shelby.mueller@example.com" + }, + { + "$id": "s3y9rl4uzf3difiq", + "name": "Dr. Sylvia Myers", + "age": 29, + "email": "sylvia.myers@example.com" + }, + { + "$id": "ntpc2td892t7f6an", + "name": "Scott Freeman", + "age": 48, + "email": "scott.freeman@example.com" + }, + { + "$id": "7f703gibyr5ijdmt", + "name": "Christopher Atkinson", + "age": 21, + "email": "christopher.atkinson@example.com" + }, + { + "$id": "r2jdf2pivkxmqd0l", + "name": "Sean Diaz", + "age": 31, + "email": "sean.diaz@example.com" + }, + { + "$id": "fj98fji1lrxeigs9", + "name": "Bobby Dyer", + "age": 57, + "email": "bobby.dyer@example.com" + }, + { + "$id": "mehqmzp9u7xv1z3j", + "name": "Daniel Hall", + "age": 62, + "email": "daniel.hall@example.com" + }, + { + "$id": "4cd5ln65qjfv3h4j", + "name": "Jennifer Ramirez", + "age": 65, + "email": "jennifer.ramirez@example.com" + }, + { + "$id": "wdi6ap0oa7m1ab1d", + "name": "Angela Jackson", + "age": 57, + "email": "angela.jackson@example.com" + }, + { + "$id": "l2foqjhxvjhjzijb", + "name": "Kelly Lewis", + "age": 36, + "email": "kelly.lewis@example.com" + }, + { + "$id": "d963t5yu35uagwm4", + "name": "Jessica Munoz", + "age": 55, + "email": "jessica.munoz@example.com" + }, + { + "$id": "99ez9uxsim8zp64m", + "name": "Kelly George", + "age": 65, + "email": "kelly.george@example.com" + }, + { + "$id": "v7wl221gycftl63d", + "name": "Anthony Johnson", + "age": 65, + "email": "anthony.johnson@example.com" + }, + { + "$id": "p2zzj0lnmjvqzfc3", + "name": "Regina Fields", + "age": 61, + "email": "regina.fields@example.com" + }, + { + "$id": "fk655e243z2ivvx6", + "name": "Sharon Schaefer", + "age": 30, + "email": "sharon.schaefer@example.com" + }, + { + "$id": "4ywsv6fw8g2d8ncw", + "name": "Jacob French", + "age": 62, + "email": "jacob.french@example.com" + }, + { + "$id": "y61q9k6g4h0fxxz4", + "name": "Jessica Costa", + "age": 23, + "email": "jessica.costa@example.com" + }, + { + "$id": "knj4hfzsthk7vx5n", + "name": "George Hardy", + "age": 53, + "email": "george.hardy@example.com" + }, + { + "$id": "a88u9w2pct2nn8l6", + "name": "Andrea Allison", + "age": 20, + "email": "andrea.allison@example.com" + }, + { + "$id": "hw960v1ybycrwr5o", + "name": "Kevin Ferguson", + "age": 32, + "email": "kevin.ferguson@example.com" + }, + { + "$id": "j9garslpgx6jgzgb", + "name": "Joseph Johnson", + "age": 58, + "email": "joseph.johnson@example.com" + }, + { + "$id": "gv101bz36elm84cd", + "name": "Ashley Martinez", + "age": 18, + "email": "ashley.martinez@example.com" + }, + { + "$id": "xrvzgt3gc0c7g4cl", + "name": "Charles Nixon", + "age": 23, + "email": "charles.nixon@example.com" + }, + { + "$id": "awjlu7uk0eutcfpb", + "name": "Carol Dudley", + "age": 40, + "email": "carol.dudley@example.com" + }, + { + "$id": "95oi26p2zdudpime", + "name": "David Weber", + "age": 51, + "email": "david.weber@example.com" + }, + { + "$id": "h8x7pkhdvu5bcp89", + "name": "Scott Robinson", + "age": 32, + "email": "scott.robinson@example.com" + }, + { + "$id": "oj6cu4jm1z2afe7s", + "name": "Anthony Hardy", + "age": 38, + "email": "anthony.hardy@example.com" + }, + { + "$id": "hgsdi1g30poqqmf0", + "name": "Mackenzie Owens", + "age": 52, + "email": "mackenzie.owens@example.com" + }, + { + "$id": "8fzdz914bqlqk2tc", + "name": "Brian Foster", + "age": 27, + "email": "brian.foster@example.com" + }, + { + "$id": "fwlqoeiunjhczpl0", + "name": "Hannah Forbes", + "age": 56, + "email": "hannah.forbes@example.com" + }, + { + "$id": "rsv8156goe8z4j6j", + "name": "Lauren Reed", + "age": 26, + "email": "lauren.reed@example.com" + }, + { + "$id": "1fjqv3w7uwbswe2p", + "name": "Morgan Smith", + "age": 28, + "email": "morgan.smith@example.com" + }, + { + "$id": "soqrzmhhg05hhzn4", + "name": "Samantha Alexander", + "age": 65, + "email": "samantha.alexander@example.com" + }, + { + "$id": "8quy52cto9kjjokp", + "name": "Tiffany Roberts", + "age": 20, + "email": "tiffany.roberts@example.com" + }, + { + "$id": "e3i1g1lw04v7jd89", + "name": "Emily Hayes", + "age": 34, + "email": "emily.hayes@example.com" + }, + { + "$id": "s7n8lzb0sw7h93z1", + "name": "Rebecca Villegas", + "age": 52, + "email": "rebecca.villegas@example.com" + }, + { + "$id": "e2lc7i81tpkqs1rp", + "name": "Donald Shah", + "age": 61, + "email": "donald.shah@example.com" + }, + { + "$id": "3oe2mysup1xluiw0", + "name": "Denise Cain", + "age": 59, + "email": "denise.cain@example.com" + }, + { + "$id": "1vqypc37f85nuqz4", + "name": "Kristine Ramirez", + "age": 53, + "email": "kristine.ramirez@example.com" + }, + { + "$id": "m0uh7r3dc6z8ucb4", + "name": "Stacey Adkins", + "age": 61, + "email": "stacey.adkins@example.com" + }, + { + "$id": "jdofz6x1ahganmqf", + "name": "Daniel Hunt", + "age": 20, + "email": "daniel.hunt@example.com" + }, + { + "$id": "vbe903c2q4m4q97g", + "name": "Roberta Johnson", + "age": 48, + "email": "roberta.johnson@example.com" + }, + { + "$id": "sndngrxuwpd93pdb", + "name": "Jason Williamson", + "age": 39, + "email": "jason.williamson@example.com" + }, + { + "$id": "66hvaw2p5xwf07p8", + "name": "Sandra Robinson", + "age": 50, + "email": "sandra.robinson@example.com" + }, + { + "$id": "9pvingfsl8cmag5c", + "name": "Steve Rice", + "age": 25, + "email": "steve.rice@example.com" + }, + { + "$id": "qe154m5hh00u4iiz", + "name": "Kimberly Fritz", + "age": 53, + "email": "kimberly.fritz@example.com" + }, + { + "$id": "avqnbrco2f0tfupk", + "name": "Brianna Reynolds", + "age": 33, + "email": "brianna.reynolds@example.com" + }, + { + "$id": "cqs10gi2qu1r3ugb", + "name": "Alexander Williams", + "age": 50, + "email": "alexander.williams@example.com" + }, + { + "$id": "jrpmfi6hmm7pmegp", + "name": "Andrew Thomas", + "age": 25, + "email": "andrew.thomas@example.com" + }, + { + "$id": "heeab2qqf0zm446f", + "name": "Austin Williams", + "age": 57, + "email": "austin.williams@example.com" + }, + { + "$id": "bkhugvnil7kjchm6", + "name": "Nicholas Williams", + "age": 24, + "email": "nicholas.williams@example.com" + }, + { + "$id": "b045j302pvv8l1p4", + "name": "Mrs. Michelle Cisneros", + "age": 48, + "email": "michelle.cisneros@example.com" + }, + { + "$id": "aikhii5q210lrfpr", + "name": "Stacey Smith", + "age": 39, + "email": "stacey.smith@example.com" + }, + { + "$id": "x0zajitea1z2dfo0", + "name": "Laura Beck", + "age": 20, + "email": "laura.beck@example.com" + }, + { + "$id": "abeecki7mdff1tv0", + "name": "Molly Clark", + "age": 51, + "email": "molly.clark@example.com" + }, + { + "$id": "yizama8r3i1to548", + "name": "Carmen Morris", + "age": 41, + "email": "carmen.morris@example.com" + }, + { + "$id": "8690yh971g4rgspj", + "name": "Amanda Munoz", + "age": 20, + "email": "amanda.munoz@example.com" + }, + { + "$id": "cd9vk5v97t359ul2", + "name": "Rachel Collins", + "age": 44, + "email": "rachel.collins@example.com" + }, + { + "$id": "wrkgmx1v0w9ja4l8", + "name": "John Alexander", + "age": 18, + "email": "john.alexander@example.com" + }, + { + "$id": "kxp3ucqo6ped4ss7", + "name": "Stacy Hunter", + "age": 22, + "email": "stacy.hunter@example.com" + }, + { + "$id": "dbvv8okae2qgo0gm", + "name": "Eric Massey", + "age": 40, + "email": "eric.massey@example.com" + }, + { + "$id": "9tn3nm6ppnayisje", + "name": "Scott Garcia", + "age": 20, + "email": "scott.garcia@example.com" + }, + { + "$id": "1xuc5t60xpcvd4qi", + "name": "Cassandra Nelson", + "age": 35, + "email": "cassandra.nelson@example.com" + }, + { + "$id": "qao1nulwn0kqyfkc", + "name": "Aaron Johnson", + "age": 50, + "email": "aaron.johnson@example.com" + }, + { + "$id": "kd2q6owvuwsy5knx", + "name": "Shannon Sherman", + "age": 45, + "email": "shannon.sherman@example.com" + }, + { + "$id": "wsl37kjo0bib4wrc", + "name": "April Garcia", + "age": 60, + "email": "april.garcia@example.com" + }, + { + "$id": "ujlz7k84xzfx4khs", + "name": "Evan Lynn", + "age": 20, + "email": "evan.lynn@example.com" + } +] \ No newline at end of file diff --git a/tests/resources/json/missing-column.json b/tests/resources/json/missing-column.json new file mode 100644 index 0000000000..ac7a8e1a85 --- /dev/null +++ b/tests/resources/json/missing-column.json @@ -0,0 +1,402 @@ +[ + { + "$id": "hxfcwpcas5xokpwe", + "name": "Diamond Mendez" + }, + { + "$id": "gw8nxwf6esn3tfwf", + "name": "Michael Huff" + }, + { + "$id": "xb6bxg56lral1qy9", + "name": "Alyssa Rodriguez" + }, + { + "$id": "imerjq5j36y3agh2", + "name": "Barbara Smith" + }, + { + "$id": "07yq9qdlhmbzmr35", + "name": "Evelyn Edwards" + }, + { + "$id": "ksqo631sbhwj5ltg", + "name": "Tina Richardson" + }, + { + "$id": "j7zlndgu0gbshp15", + "name": "Joel Hernandez" + }, + { + "$id": "mfntvnljrcmf7h6v", + "name": "Zachary Cooper" + }, + { + "$id": "5f9b01nziqu2h8ed", + "name": "Brittany Spears" + }, + { + "$id": "4vxzbnzraqznk5u8", + "name": "Holly White" + }, + { + "$id": "d4ywy3mtphaatbpf", + "name": "Kimberly Barnes" + }, + { + "$id": "88odnk6nthyyvbal", + "name": "Stephen Miller" + }, + { + "$id": "08oekee3fn7mzaa5", + "name": "Yvonne Newman" + }, + { + "$id": "quw55kn9895i5e4v", + "name": "Carol Kane" + }, + { + "$id": "nge6bm8ykripei6f", + "name": "Doris Foster" + }, + { + "$id": "4k16i33s0xl2ypx9", + "name": "Joseph Stokes" + }, + { + "$id": "q0j5rxbgid66snyf", + "name": "Steve Williams" + }, + { + "$id": "n1oxun7mqq3p103y", + "name": "James Carey" + }, + { + "$id": "0dbvs840jkf8i0ye", + "name": "Kathryn Henry" + }, + { + "$id": "5sfaidgs1h87v15v", + "name": "Christopher Landry" + }, + { + "$id": "vg3punvfu5khmf41", + "name": "Jennifer Mcgee" + }, + { + "$id": "f933qydr9u5b2r11", + "name": "Cathy Church" + }, + { + "$id": "wjv87y1inf8yk32s", + "name": "Jose Lopez" + }, + { + "$id": "uljysdvdlcyrbrwk", + "name": "William Rose" + }, + { + "$id": "ot8xtzh77j55wq0s", + "name": "Sarah Ford" + }, + { + "$id": "9t76vnsv2u36s43t", + "name": "Alisha Jones" + }, + { + "$id": "66y4tnty62hw8c02", + "name": "Kristin Kelly" + }, + { + "$id": "2punfblazi5v16ar", + "name": "Brendan Stout" + }, + { + "$id": "sxhr4nf5w2gx4wbg", + "name": "Kelly Cruz" + }, + { + "$id": "68dvrqfwqnkq5el9", + "name": "Samantha Martin" + }, + { + "$id": "20192l6dbeinhkh0", + "name": "David Santos" + }, + { + "$id": "si0l4dgay09ebfmf", + "name": "Elizabeth Carroll" + }, + { + "$id": "lhse40vbldqb6ap1", + "name": "Corey Owens" + }, + { + "$id": "h5t3pslykyx3kxfm", + "name": "Shelby Mueller" + }, + { + "$id": "ldc0luydrw6jub0f", + "name": "Dr. Sylvia Myers" + }, + { + "$id": "voc9628xg4dsgw2y", + "name": "Scott Freeman" + }, + { + "$id": "o4y0gk3gqv1ax2fz", + "name": "Christopher Atkinson" + }, + { + "$id": "u1n3x4e4u7e0vzj6", + "name": "Sean Diaz" + }, + { + "$id": "s36eskwtm0w7lwr7", + "name": "Bobby Dyer" + }, + { + "$id": "4hjnag1p5iwvtixd", + "name": "Daniel Hall" + }, + { + "$id": "m91d80oxsa216zbh", + "name": "Jennifer Ramirez" + }, + { + "$id": "5hj6858zo2g85n6v", + "name": "Angela Jackson" + }, + { + "$id": "8m8oihv9a1e7nn92", + "name": "Kelly Lewis" + }, + { + "$id": "7azy39la0no0mxi7", + "name": "Jessica Munoz" + }, + { + "$id": "47pmjkhnnqhyit8c", + "name": "Kelly George" + }, + { + "$id": "6j6cpy4kgneg1mmh", + "name": "Anthony Johnson" + }, + { + "$id": "tnlmtvap1zz89km9", + "name": "Regina Fields" + }, + { + "$id": "6cyuvnwwqdmrpfzh", + "name": "Sharon Schaefer" + }, + { + "$id": "p1v4pyu2pqodc0ey", + "name": "Jacob French" + }, + { + "$id": "6npynnhjt2jd05xo", + "name": "Jessica Costa" + }, + { + "$id": "wcxedf13n2e9qi4l", + "name": "George Hardy" + }, + { + "$id": "yf2xlcmszk2tqeig", + "name": "Andrea Allison" + }, + { + "$id": "3bf2zzv7poststwa", + "name": "Kevin Ferguson" + }, + { + "$id": "c2iataz0hhv39q63", + "name": "Joseph Johnson" + }, + { + "$id": "3e8npxhov4a39pvq", + "name": "Ashley Martinez" + }, + { + "$id": "t7dp41tysipytywq", + "name": "Charles Nixon" + }, + { + "$id": "z8cztq7c47phyfhk", + "name": "Carol Dudley" + }, + { + "$id": "2636f9d8r4ipm3h6", + "name": "David Weber" + }, + { + "$id": "eh3f6wxtvkjq6ykq", + "name": "Scott Robinson" + }, + { + "$id": "raskbwpsje69a59h", + "name": "Anthony Hardy" + }, + { + "$id": "90hn1p0b4cs9e2og", + "name": "Mackenzie Owens" + }, + { + "$id": "am3swwfbo076x0v1", + "name": "Brian Foster" + }, + { + "$id": "5uw7utb9lq5cfncw", + "name": "Hannah Forbes" + }, + { + "$id": "cs6mbfzkzifefx6r", + "name": "Lauren Reed" + }, + { + "$id": "ftw3uvztziiz9x00", + "name": "Morgan Smith" + }, + { + "$id": "uhrqseeo43mozpaq", + "name": "Samantha Alexander" + }, + { + "$id": "pvvmzyfc1lxor11e", + "name": "Tiffany Roberts" + }, + { + "$id": "jia7bdag4abz123s", + "name": "Emily Hayes" + }, + { + "$id": "h6oozcngbz8o5x4y", + "name": "Rebecca Villegas" + }, + { + "$id": "9v6z1pn2f9twcy12", + "name": "Donald Shah" + }, + { + "$id": "wzz3jduioso77o7f", + "name": "Denise Cain" + }, + { + "$id": "u51plhgvjodkswnr", + "name": "Kristine Ramirez" + }, + { + "$id": "t1uhkmiytfyc13vc", + "name": "Stacey Adkins" + }, + { + "$id": "iqaqnf0ybg2ct507", + "name": "Daniel Hunt" + }, + { + "$id": "idwrwv2uu4hcpv2i", + "name": "Roberta Johnson" + }, + { + "$id": "2yd2hd6auetjacyo", + "name": "Jason Williamson" + }, + { + "$id": "egrmdbibnjhi914x", + "name": "Sandra Robinson" + }, + { + "$id": "15m1pz2bb0ercgyk", + "name": "Steve Rice" + }, + { + "$id": "0i21bhkxdagjurb7", + "name": "Kimberly Fritz" + }, + { + "$id": "726ofi7h5snreq67", + "name": "Brianna Reynolds" + }, + { + "$id": "csqxse3wym56eim6", + "name": "Alexander Williams" + }, + { + "$id": "qeaoylnrsf8p3byg", + "name": "Andrew Thomas" + }, + { + "$id": "edsswobumzyzbvhf", + "name": "Austin Williams" + }, + { + "$id": "hdzhzpt0ahy5hkib", + "name": "Nicholas Williams" + }, + { + "$id": "w1qmvmg4roa8xnwu", + "name": "Mrs. Michelle Cisneros" + }, + { + "$id": "3z3o73x7adyuo6w0", + "name": "Stacey Smith" + }, + { + "$id": "sse2u5zlgoqrgmcf", + "name": "Laura Beck" + }, + { + "$id": "rvovijmvch58r4yx", + "name": "Molly Clark" + }, + { + "$id": "doe06nrx8sg5mcuv", + "name": "Carmen Morris" + }, + { + "$id": "jbjdwuvj5s4kw04y", + "name": "Amanda Munoz" + }, + { + "$id": "6k2ewkla7js0yw23", + "name": "Rachel Collins" + }, + { + "$id": "fcxuyr4kkhrnigu1", + "name": "John Alexander" + }, + { + "$id": "d25fuwlos5mk07o0", + "name": "Stacy Hunter" + }, + { + "$id": "1vdai2rxmwd57oet", + "name": "Eric Massey" + }, + { + "$id": "pq4jnt9izu1wlrzd", + "name": "Scott Garcia" + }, + { + "$id": "lz9kfc0lty5xcz14", + "name": "Cassandra Nelson" + }, + { + "$id": "pu7w6tyab5jd4we9", + "name": "Aaron Johnson" + }, + { + "$id": "8dupswd2kqwdyn8v", + "name": "Shannon Sherman" + }, + { + "$id": "ye466l71jthiz2p6", + "name": "April Garcia" + }, + { + "$id": "xogsmfwb73l16qdt", + "name": "Evan Lynn" + } +] \ No newline at end of file diff --git a/tests/resources/json/vectorsdb-documents.json b/tests/resources/json/vectorsdb-documents.json new file mode 100644 index 0000000000..f1bada8a7e --- /dev/null +++ b/tests/resources/json/vectorsdb-documents.json @@ -0,0 +1,262 @@ +[ + { + "$id": "vec01", + "metadata": { + "title": "Vector Document 1", + "score": 0.643, + "category": "art" + }, + "embeddings": [ + -0.516361, + 0.261456, + -0.77323, + 0.909775, + -0.003065, + -0.626888, + 0.704171, + -0.366385, + 0.198681, + 0.854914, + -0.227548, + -0.14278, + -0.548405, + -0.822939, + 0.407267, + 0.570791 + ] + }, + { + "$id": "vec02", + "metadata": { + "title": "Vector Document 2", + "score": 0.322, + "category": "art" + }, + "embeddings": [ + 0.103211, + 0.028378, + 0.806001, + -0.412638, + -0.200009, + 0.448239, + -0.781144, + 0.621957, + -0.868352, + 0.217013, + 0.308217, + -0.851528, + -0.340766, + 0.805996, + -0.065638, + -0.363951 + ] + }, + { + "$id": "vec03", + "metadata": { + "title": "Vector Document 3", + "score": 0.503, + "category": "science" + }, + "embeddings": [ + 0.971901, + -0.475154, + 0.60526, + -0.151618, + -0.938199, + -0.139977, + 0.154581, + -0.868021, + 0.069567, + 0.233545, + -0.164826, + 0.239036, + 0.8875, + 0.488075, + 0.787231, + 0.090293 + ] + }, + { + "$id": "vec04", + "metadata": { + "title": "Vector Document 4", + "score": 0.943, + "category": "science" + }, + "embeddings": [ + 0.372035, + -0.868405, + -0.974987, + -0.836673, + 0.030057, + -0.667698, + 0.095189, + 0.518373, + -0.173732, + 0.966379, + 0.509861, + -0.172607, + 0.420855, + -0.534562, + 0.600872, + 0.194391 + ] + }, + { + "$id": "vec05", + "metadata": { + "title": "Vector Document 5", + "score": 0.924, + "category": "music" + }, + "embeddings": [ + -0.318783, + -0.876486, + 0.843971, + -0.472869, + -0.350164, + 0.936237, + -0.741591, + 0.298213, + -0.075429, + 0.243415, + 0.929625, + 0.96649, + 0.251843, + -0.371953, + 0.348079, + 0.090382 + ] + }, + { + "$id": "vec06", + "metadata": { + "title": "Vector Document 6", + "score": 0.385, + "category": "science" + }, + "embeddings": [ + -0.608111, + 0.549885, + 0.720346, + 0.260442, + -0.023344, + 0.800346, + -0.956586, + -0.407161, + 0.516883, + 0.230456, + 0.376704, + -0.347203, + 0.925657, + -0.486608, + 0.330032, + 0.323414 + ] + }, + { + "$id": "vec07", + "metadata": { + "title": "Vector Document 7", + "score": 0.61, + "category": "science" + }, + "embeddings": [ + 0.45451, + -0.251783, + -0.479459, + 0.167412, + 0.890343, + 0.415136, + -0.506263, + 0.318171, + -0.729294, + -0.907919, + 0.607033, + 0.652333, + 0.97129, + -0.118671, + 0.110045, + -0.514242 + ] + }, + { + "$id": "vec08", + "metadata": { + "title": "Vector Document 8", + "score": 0.107, + "category": "history" + }, + "embeddings": [ + -0.883255, + -0.294593, + -0.153187, + 0.595936, + 0.341818, + -0.410719, + 0.017348, + -0.350047, + -0.888906, + 0.344646, + 0.184335, + -0.429053, + 0.540542, + -0.408493, + -0.982748, + -0.255031 + ] + }, + { + "$id": "vec09", + "metadata": { + "title": "Vector Document 9", + "score": 0.914, + "category": "art" + }, + "embeddings": [ + 0.31891, + 0.502493, + -0.245712, + -0.325282, + -0.514772, + 0.097323, + 0.812412, + -0.762069, + 0.846688, + -0.391482, + 0.879953, + 0.986352, + -0.562588, + -0.566111, + 0.163109, + 0.119346 + ] + }, + { + "$id": "vec10", + "metadata": { + "title": "Vector Document 10", + "score": 0.169, + "category": "science" + }, + "embeddings": [ + -0.563645, + -0.19119, + -0.853168, + -0.526952, + -0.459635, + 0.850766, + -0.345261, + 0.547467, + 0.455409, + -0.507607, + -0.228436, + -0.587395, + 0.109146, + 0.190046, + 0.861559, + -0.724741 + ] + } +] \ No newline at end of file