From bc8683ab757cf0dc311c809d90b115f1a313628b Mon Sep 17 00:00:00 2001 From: Darshan Date: Wed, 16 Apr 2025 16:52:23 +0530 Subject: [PATCH] add: csv import tests! --- composer.json | 8 +- composer.lock | 44 ++-- .../Services/Migrations/MigrationsBase.php | 226 ++++++++++++++++++ tests/resources/documents.csv | 101 ++++++++ 4 files changed, 345 insertions(+), 34 deletions(-) create mode 100644 tests/resources/documents.csv diff --git a/composer.json b/composer.json index 4c26b19d1e..d0f3803210 100644 --- a/composer.json +++ b/composer.json @@ -60,7 +60,7 @@ "utopia-php/locale": "0.4.*", "utopia-php/logger": "0.6.*", "utopia-php/messaging": "0.16.*", - "utopia-php/migration": "dev-feat-csv", + "utopia-php/migration": "0.9.0", "utopia-php/orchestration": "0.9.*", "utopia-php/platform": "0.7.*", "utopia-php/pools": "0.8.*", @@ -91,12 +91,6 @@ "laravel/pint": "1.*", "phpbench/phpbench": "1.*" }, - "repositories": [ - { - "type": "git", - "url": "https://github.com/utopia-php/migration" - } - ], "provide": { "ext-phpiredis": "*" }, diff --git a/composer.lock b/composer.lock index c25ec33591..46353a4a11 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "cfb5c437126bf194a6fe7225961c1582", + "content-hash": "85afadfc660334537aaba2c355f98b9c", "packages": [ { "name": "adhocore/jwt", @@ -3951,11 +3951,17 @@ }, { "name": "utopia-php/migration", - "version": "dev-feat-csv", + "version": "0.9.0", "source": { "type": "git", - "url": "https://github.com/utopia-php/migration", - "reference": "9088ef1079da3fb13b8abc3821feee618475a0c3" + "url": "https://github.com/utopia-php/migration.git", + "reference": "545705e251b766940d2833893f267975d73abe32" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/utopia-php/migration/zipball/545705e251b766940d2833893f267975d73abe32", + "reference": "545705e251b766940d2833893f267975d73abe32", + "shasum": "" }, "require": { "appwrite/appwrite": "11.*", @@ -3981,25 +3987,7 @@ "Utopia\\Migration\\": "src/Migration" } }, - "autoload-dev": { - "psr-4": { - "Utopia\\Tests\\": "tests/Migration" - } - }, - "scripts": { - "test": [ - "./vendor/bin/phpunit" - ], - "lint": [ - "./vendor/bin/pint --test" - ], - "format": [ - "./vendor/bin/pint" - ], - "check": [ - "./vendor/bin/phpstan analyse --level 3 src tests --memory-limit 2G" - ] - }, + "notification-url": "https://packagist.org/downloads/", "license": [ "MIT" ], @@ -4011,7 +3999,11 @@ "upf", "utopia" ], - "time": "2025-04-16T06:21:15+00:00" + "support": { + "issues": "https://github.com/utopia-php/migration/issues", + "source": "https://github.com/utopia-php/migration/tree/0.9.0" + }, + "time": "2025-04-16T07:52:53+00:00" }, { "name": "utopia-php/orchestration", @@ -8134,9 +8126,7 @@ ], "aliases": [], "minimum-stability": "stable", - "stability-flags": { - "utopia-php/migration": 20 - }, + "stability-flags": {}, "prefer-stable": false, "prefer-lowest": false, "platform": { diff --git a/tests/e2e/Services/Migrations/MigrationsBase.php b/tests/e2e/Services/Migrations/MigrationsBase.php index 6c468ee730..f7e7d20e99 100644 --- a/tests/e2e/Services/Migrations/MigrationsBase.php +++ b/tests/e2e/Services/Migrations/MigrationsBase.php @@ -10,8 +10,10 @@ use Tests\E2E\Services\Functions\FunctionsBase; use Utopia\Database\Helpers\ID; use Utopia\Database\Helpers\Permission; use Utopia\Database\Helpers\Role; +use Utopia\Database\Query; use Utopia\Migration\Resource; use Utopia\Migration\Sources\Appwrite; +use Utopia\Migration\Sources\CSV; trait MigrationsBase { @@ -896,4 +898,228 @@ trait MigrationsBase 'x-appwrite-key' => $this->getDestinationProject()['apiKey'], ]); } + + /** + * Import documents from a CSV file. + */ + public function testCreateCsvMigration(): array + { + // make a database + $response = $this->client->call(Client::METHOD_POST, '/databases', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'databaseId' => ID::unique(), + 'name' => 'Test Database' + ]); + + $this->assertNotEmpty($response['body']['$id']); + $this->assertEquals(201, $response['headers']['status-code']); + $this->assertEquals('Test Database', $response['body']['name']); + + $databaseId = $response['body']['$id']; + + // make a collection + $response = $this->client->call(Client::METHOD_POST, '/databases/' . $databaseId . '/collections', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'name' => 'Test collection', + 'collectionId' => ID::unique(), + ]); + + $this->assertEquals(201, $response['headers']['status-code']); + $this->assertEquals($response['body']['name'], 'Test collection'); + + $collectionId = $response['body']['$id']; + + // make attributes + $response = $this->client->call(Client::METHOD_POST, '/databases/' . $databaseId . '/collections/' . $collectionId . '/attributes/string', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'key' => 'name', + 'size' => 256, + 'required' => true, + ]); + + $this->assertEquals(202, $response['headers']['status-code']); + $this->assertEquals($response['body']['key'], 'name'); + $this->assertEquals($response['body']['type'], 'string'); + $this->assertEquals($response['body']['size'], 256); + $this->assertEquals($response['body']['required'], true); + + $response = $this->client->call(Client::METHOD_POST, '/databases/' . $databaseId . '/collections/' . $collectionId . '/attributes/integer', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'key' => 'age', + 'min' => 18, + 'max' => 65, + 'required' => true, + ]); + + $this->assertEquals(202, $response['headers']['status-code']); + $this->assertEquals($response['body']['key'], 'age'); + $this->assertEquals($response['body']['type'], 'integer'); + $this->assertEquals($response['body']['min'], 18); + $this->assertEquals($response['body']['max'], 65); + $this->assertEquals($response['body']['required'], true); + + // make a bucket, upload a file to it! + // 1. enable compression, encryption + $bucketOne = $this->client->call(Client::METHOD_POST, '/storage/buckets', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ], [ + 'bucketId' => ID::unique(), + 'name' => 'Test Bucket', + 'maximumFileSize' => 2000000, //2MB + 'allowedFileExtensions' => ['csv'], + 'compression' => 'gzip', + 'encryption' => true + ]); + $this->assertEquals(201, $bucketOne['headers']['status-code']); + $this->assertNotEmpty($bucketOne['body']['$id']); + + $bucketOneId = $bucketOne['body']['$id']; + + // 2. no compression and encryption + $bucketTwo = $this->client->call(Client::METHOD_POST, '/storage/buckets', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ], [ + 'bucketId' => ID::unique(), + 'name' => 'Test Bucket 2', + 'maximumFileSize' => 2000000, //2MB + 'allowedFileExtensions' => ['csv'], + 'compression' => 'none', + 'encryption' => false + ]); + + $this->assertNotEmpty($bucketTwo['body']['$id']); + $this->assertEquals(201, $bucketTwo['headers']['status-code']); + + $bucketTwoId = $bucketTwo['body']['$id']; + + $bucketIds = [ + 'compressed' => $bucketOneId, + 'uncompressed' => $bucketTwoId, + ]; + + $fileIds = []; + + foreach ($bucketIds as $label => $bucketId) { + $response = $this->client->call(Client::METHOD_POST, '/storage/buckets/' . $bucketId . '/files', array_merge([ + 'content-type' => 'multipart/form-data', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'fileId' => ID::unique(), + 'file' => new CURLFile(realpath(__DIR__ . '/../../../resources/documents.csv'), 'text/csv', 'documents.csv'), + ]); + + $this->assertEquals(201, $response['headers']['status-code']); + $this->assertNotEmpty($response['body']['$id']); + $this->assertEquals('documents.csv', $response['body']['name']); + $this->assertEquals('text/csv', $response['body']['mimeType']); + + $fileIds[$label] = $response['body']['$id']; + } + + // compressed, fail. + $compressed = $this->performCsvMigration( + [ + 'fileId' => $fileIds['compressed'], + 'bucketId' => $bucketIds['compressed'], + 'resourceId' => $databaseId . ':' . $collectionId, + ] + ); + + // fail on compressed, encrypted buckets! + $this->assertEquals(400, $compressed['body']['code']); + $this->assertEquals('storage_file_type_unsupported', $compressed['body']['type']); + $this->assertEquals('Only uncompressed, unencrypted CSV files can be used for document import.', $compressed['body']['message']); + + // no compression, no encryption, pass. + $migration = $this->performCsvMigration( + [ + 'endpoint' => 'http://localhost/v1', + 'fileId' => $fileIds['uncompressed'], + 'bucketId' => $bucketIds['uncompressed'], + 'resourceId' => $databaseId . ':' . $collectionId, + ] + ); + + $this->assertEmpty($migration['body']['statusCounters']); + $this->assertEquals('CSV', $migration['body']['source']); + $this->assertEquals('pending', $migration['body']['status']); + $this->assertEquals('Appwrite', $migration['body']['destination']); + $this->assertContains(Resource::TYPE_DOCUMENT, $migration['body']['resources']); + + return [ + 'databaseId' => $databaseId, + 'collectionId' => $collectionId, + 'migrationId' => $migration['body']['$id'], + ]; + } + + /** + * @depends testCreateCsvMigration + */ + public function testImportSuccessful(array $response): void + { + $databaseId = $response['databaseId']; + $collectionId = $response['collectionId']; + $migrationId = $response['migrationId']; + + $documentsCountInCSV = 100; + + // get migration stats + $this->assertEventually(function () use ($migrationId, $databaseId, $collectionId, $documentsCountInCSV) { + $migration = $this->client->call(Client::METHOD_GET, '/migrations/'.$migrationId, array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders())); + + $this->assertEquals(200, $migration['headers']['status-code']); + $this->assertEquals('finished', $migration['body']['stage']); + $this->assertEquals('completed', $migration['body']['status']); + $this->assertEquals('CSV', $migration['body']['source']); + $this->assertEquals('Appwrite', $migration['body']['destination']); + $this->assertContains(Resource::TYPE_DOCUMENT, $migration['body']['resources']); + $this->assertArrayHasKey(Resource::TYPE_DOCUMENT, $migration['body']['statusCounters']); + $this->assertEquals($documentsCountInCSV, $migration['body']['statusCounters'][Resource::TYPE_DOCUMENT]['success']); + }, 60000, 500); + + // get documents count + $documents = $this->client->call(Client::METHOD_GET, '/databases/'.$databaseId.'/collections/'.$collectionId.'/documents', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'queries' => [ + // there should be only 100! + Query::limit(150)->toString() + ] + ]); + + $this->assertEquals(200, $documents['headers']['status-code']); + $this->assertIsArray($documents['body']['documents']); + $this->assertIsNumeric($documents['body']['total']); + $this->assertEquals($documentsCountInCSV, $documents['body']['total']); + } + + private function performCsvMigration(array $body): array + { + return $this->client->call(Client::METHOD_POST, '/migrations/csv', [ + 'content-type' => 'application/json', + 'x-appwrite-key' => $this->getProject()['apiKey'], + 'x-appwrite-project' => $this->getProject()['$id'], + ], $body); + } } diff --git a/tests/resources/documents.csv b/tests/resources/documents.csv new file mode 100644 index 0000000000..ea1e33b5bd --- /dev/null +++ b/tests/resources/documents.csv @@ -0,0 +1,101 @@ +$id,name,age +hxfcwpcas5xokpwe,Diamond Mendez,56 +gw8nxwf6esn3tfwf,Michael Huff,20 +xb6bxg56lral1qy9,Alyssa Rodriguez,37 +imerjq5j36y3agh2,Barbara Smith,26 +07yq9qdlhmbzmr35,Evelyn Edwards,54 +ksqo631sbhwj5ltg,Tina Richardson,41 +j7zlndgu0gbshp15,Joel Hernandez,49 +mfntvnljrcmf7h6v,Zachary Cooper,59 +5f9b01nziqu2h8ed,Brittany Spears,20 +4vxzbnzraqznk5u8,Holly White,47 +d4ywy3mtphaatbpf,Kimberly Barnes,27 +88odnk6nthyyvbal,Stephen Miller,53 +08oekee3fn7mzaa5,Yvonne Newman,41 +quw55kn9895i5e4v,Carol Kane,38 +nge6bm8ykripei6f,Doris Foster,44 +4k16i33s0xl2ypx9,Joseph Stokes,28 +q0j5rxbgid66snyf,Steve Williams,31 +n1oxun7mqq3p103y,James Carey,29 +0dbvs840jkf8i0ye,Kathryn Henry,38 +5sfaidgs1h87v15v,Christopher Landry,23 +vg3punvfu5khmf41,Jennifer Mcgee,62 +f933qydr9u5b2r11,Cathy Church,35 +wjv87y1inf8yk32s,Jose Lopez,41 +uljysdvdlcyrbrwk,William Rose,30 +ot8xtzh77j55wq0s,Sarah Ford,26 +9t76vnsv2u36s43t,Alisha Jones,61 +66y4tnty62hw8c02,Kristin Kelly,61 +2punfblazi5v16ar,Brendan Stout,40 +sxhr4nf5w2gx4wbg,Kelly Cruz,18 +68dvrqfwqnkq5el9,Samantha Martin,50 +20192l6dbeinhkh0,David Santos,46 +si0l4dgay09ebfmf,Elizabeth Carroll,22 +lhse40vbldqb6ap1,Corey Owens,46 +h5t3pslykyx3kxfm,Shelby Mueller,65 +ldc0luydrw6jub0f,Dr. Sylvia Myers,29 +voc9628xg4dsgw2y,Scott Freeman,48 +o4y0gk3gqv1ax2fz,Christopher Atkinson,21 +u1n3x4e4u7e0vzj6,Sean Diaz,31 +s36eskwtm0w7lwr7,Bobby Dyer,57 +4hjnag1p5iwvtixd,Daniel Hall,62 +m91d80oxsa216zbh,Jennifer Ramirez,65 +5hj6858zo2g85n6v,Angela Jackson,57 +8m8oihv9a1e7nn92,Kelly Lewis,36 +7azy39la0no0mxi7,Jessica Munoz,55 +47pmjkhnnqhyit8c,Kelly George,65 +6j6cpy4kgneg1mmh,Anthony Johnson,65 +tnlmtvap1zz89km9,Regina Fields,61 +6cyuvnwwqdmrpfzh,Sharon Schaefer,30 +p1v4pyu2pqodc0ey,Jacob French,62 +6npynnhjt2jd05xo,Jessica Costa,23 +wcxedf13n2e9qi4l,George Hardy,53 +yf2xlcmszk2tqeig,Andrea Allison,20 +3bf2zzv7poststwa,Kevin Ferguson,32 +c2iataz0hhv39q63,Joseph Johnson,58 +3e8npxhov4a39pvq,Ashley Martinez,18 +t7dp41tysipytywq,Charles Nixon,23 +z8cztq7c47phyfhk,Carol Dudley,40 +2636f9d8r4ipm3h6,David Weber,51 +eh3f6wxtvkjq6ykq,Scott Robinson,32 +raskbwpsje69a59h,Anthony Hardy,38 +90hn1p0b4cs9e2og,Mackenzie Owens,52 +am3swwfbo076x0v1,Brian Foster,27 +5uw7utb9lq5cfncw,Hannah Forbes,56 +cs6mbfzkzifefx6r,Lauren Reed,26 +ftw3uvztziiz9x00,Morgan Smith,28 +uhrqseeo43mozpaq,Samantha Alexander,65 +pvvmzyfc1lxor11e,Tiffany Roberts,20 +jia7bdag4abz123s,Emily Hayes,34 +h6oozcngbz8o5x4y,Rebecca Villegas,52 +9v6z1pn2f9twcy12,Donald Shah,61 +wzz3jduioso77o7f,Denise Cain,59 +u51plhgvjodkswnr,Kristine Ramirez,53 +t1uhkmiytfyc13vc,Stacey Adkins,61 +iqaqnf0ybg2ct507,Daniel Hunt,20 +idwrwv2uu4hcpv2i,Roberta Johnson,48 +2yd2hd6auetjacyo,Jason Williamson,39 +egrmdbibnjhi914x,Sandra Robinson,50 +15m1pz2bb0ercgyk,Steve Rice,25 +0i21bhkxdagjurb7,Kimberly Fritz,53 +726ofi7h5snreq67,Brianna Reynolds,33 +csqxse3wym56eim6,Alexander Williams,50 +qeaoylnrsf8p3byg,Andrew Thomas,25 +edsswobumzyzbvhf,Austin Williams,57 +hdzhzpt0ahy5hkib,Nicholas Williams,24 +w1qmvmg4roa8xnwu,Mrs. Michelle Cisneros,48 +3z3o73x7adyuo6w0,Stacey Smith,39 +sse2u5zlgoqrgmcf,Laura Beck,20 +rvovijmvch58r4yx,Molly Clark,51 +doe06nrx8sg5mcuv,Carmen Morris,41 +jbjdwuvj5s4kw04y,Amanda Munoz,20 +6k2ewkla7js0yw23,Rachel Collins,44 +fcxuyr4kkhrnigu1,John Alexander,18 +d25fuwlos5mk07o0,Stacy Hunter,22 +1vdai2rxmwd57oet,Eric Massey,40 +pq4jnt9izu1wlrzd,Scott Garcia,20 +lz9kfc0lty5xcz14,Cassandra Nelson,35 +pu7w6tyab5jd4we9,Aaron Johnson,50 +8dupswd2kqwdyn8v,Shannon Sherman,45 +ye466l71jthiz2p6,April Garcia,60 +xogsmfwb73l16qdt,Evan Lynn,20