Appwrite/appwrite
1
0
Fork 0
mirror of https://github.com/appwrite/appwrite.git synced 2026-05-26 13:51:13 +00:00
Code Issues Packages Projects Releases Wiki Activity

add: csv import tests!

Browse Source
This commit is contained in:
Darshan
2025-04-16 16:52:23 +05:30
parent 9d6cd8249a
commit bc8683ab75
4 changed files with 345 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files
composer.json
+1 -7
View File
@@ -60,7 +60,7 @@
"utopia-php/locale": "0.4.*",
"utopia-php/logger": "0.6.*",
"utopia-php/messaging": "0.16.*",
"utopia-php/migration": "dev-feat-csv",
"utopia-php/migration": "0.9.0",
"utopia-php/orchestration": "0.9.*",
"utopia-php/platform": "0.7.*",
"utopia-php/pools": "0.8.*",
@@ -91,12 +91,6 @@
"laravel/pint": "1.*",
"phpbench/phpbench": "1.*"
},
"repositories": [
{
"type": "git",
"url": "https://github.com/utopia-php/migration"
}
],
"provide": {
"ext-phpiredis": "*"
},
composer.lock
Generated
+17 -27
View File
@@ -4,7 +4,7 @@
"Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies",
"This file is @generated automatically"
],
"content-hash": "cfb5c437126bf194a6fe7225961c1582",
"content-hash": "85afadfc660334537aaba2c355f98b9c",
"packages": [
{
"name": "adhocore/jwt",
@@ -3951,11 +3951,17 @@
},
{
"name": "utopia-php/migration",
"version": "dev-feat-csv",
"version": "0.9.0",
"source": {
"type": "git",
"url": "https://github.com/utopia-php/migration",
"reference": "9088ef1079da3fb13b8abc3821feee618475a0c3"
"url": "https://github.com/utopia-php/migration.git",
"reference": "545705e251b766940d2833893f267975d73abe32"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/utopia-php/migration/zipball/545705e251b766940d2833893f267975d73abe32",
"reference": "545705e251b766940d2833893f267975d73abe32",
"shasum": ""
},
"require": {
"appwrite/appwrite": "11.*",
@@ -3981,25 +3987,7 @@
"Utopia\\Migration\\": "src/Migration"
}
},
"autoload-dev": {
"psr-4": {
"Utopia\\Tests\\": "tests/Migration"
}
},
"scripts": {
"test": [
"./vendor/bin/phpunit"
],
"lint": [
"./vendor/bin/pint --test"
],
"format": [
"./vendor/bin/pint"
],
"check": [
"./vendor/bin/phpstan analyse --level 3 src tests --memory-limit 2G"
]
},
"notification-url": "https://packagist.org/downloads/",
"license": [
"MIT"
],
@@ -4011,7 +3999,11 @@
"upf",
"utopia"
],
"time": "2025-04-16T06:21:15+00:00"
"support": {
"issues": "https://github.com/utopia-php/migration/issues",
"source": "https://github.com/utopia-php/migration/tree/0.9.0"
},
"time": "2025-04-16T07:52:53+00:00"
},
{
"name": "utopia-php/orchestration",
@@ -8134,9 +8126,7 @@
],
"aliases": [],
"minimum-stability": "stable",
"stability-flags": {
"utopia-php/migration": 20
},
"stability-flags": {},
"prefer-stable": false,
"prefer-lowest": false,
"platform": {
tests/e2e/Services/Migrations/MigrationsBase.php
+226
View File
@@ -10,8 +10,10 @@ use Tests\E2E\Services\Functions\FunctionsBase;
use Utopia\Database\Helpers\ID;
use Utopia\Database\Helpers\Permission;
use Utopia\Database\Helpers\Role;
use Utopia\Database\Query;
use Utopia\Migration\Resource;
use Utopia\Migration\Sources\Appwrite;
use Utopia\Migration\Sources\CSV;
trait MigrationsBase
{
@@ -896,4 +898,228 @@ trait MigrationsBase
'x-appwrite-key' => $this->getDestinationProject()['apiKey'],
]);
}
/**
* Import documents from a CSV file.
*/
public function testCreateCsvMigration(): array
{
// make a database
$response = $this->client->call(Client::METHOD_POST, '/databases', [
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
'x-appwrite-key' => $this->getProject()['apiKey']
], [
'databaseId' => ID::unique(),
'name' => 'Test Database'
]);
$this->assertNotEmpty($response['body']['$id']);
$this->assertEquals(201, $response['headers']['status-code']);
$this->assertEquals('Test Database', $response['body']['name']);
$databaseId = $response['body']['$id'];
// make a collection
$response = $this->client->call(Client::METHOD_POST, '/databases/' . $databaseId . '/collections', array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
'x-appwrite-key' => $this->getProject()['apiKey']
]), [
'name' => 'Test collection',
'collectionId' => ID::unique(),
]);
$this->assertEquals(201, $response['headers']['status-code']);
$this->assertEquals($response['body']['name'], 'Test collection');
$collectionId = $response['body']['$id'];
// make attributes
$response = $this->client->call(Client::METHOD_POST, '/databases/' . $databaseId . '/collections/' . $collectionId . '/attributes/string', array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
'x-appwrite-key' => $this->getProject()['apiKey']
]), [
'key' => 'name',
'size' => 256,
'required' => true,
]);
$this->assertEquals(202, $response['headers']['status-code']);
$this->assertEquals($response['body']['key'], 'name');
$this->assertEquals($response['body']['type'], 'string');
$this->assertEquals($response['body']['size'], 256);
$this->assertEquals($response['body']['required'], true);
$response = $this->client->call(Client::METHOD_POST, '/databases/' . $databaseId . '/collections/' . $collectionId . '/attributes/integer', array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
'x-appwrite-key' => $this->getProject()['apiKey']
]), [
'key' => 'age',
'min' => 18,
'max' => 65,
'required' => true,
]);
$this->assertEquals(202, $response['headers']['status-code']);
$this->assertEquals($response['body']['key'], 'age');
$this->assertEquals($response['body']['type'], 'integer');
$this->assertEquals($response['body']['min'], 18);
$this->assertEquals($response['body']['max'], 65);
$this->assertEquals($response['body']['required'], true);
// make a bucket, upload a file to it!
// 1. enable compression, encryption
$bucketOne = $this->client->call(Client::METHOD_POST, '/storage/buckets', [
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
'x-appwrite-key' => $this->getProject()['apiKey'],
], [
'bucketId' => ID::unique(),
'name' => 'Test Bucket',
'maximumFileSize' => 2000000, //2MB
'allowedFileExtensions' => ['csv'],
'compression' => 'gzip',
'encryption' => true
]);
$this->assertEquals(201, $bucketOne['headers']['status-code']);
$this->assertNotEmpty($bucketOne['body']['$id']);
$bucketOneId = $bucketOne['body']['$id'];
// 2. no compression and encryption
$bucketTwo = $this->client->call(Client::METHOD_POST, '/storage/buckets', [
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
'x-appwrite-key' => $this->getProject()['apiKey'],
], [
'bucketId' => ID::unique(),
'name' => 'Test Bucket 2',
'maximumFileSize' => 2000000, //2MB
'allowedFileExtensions' => ['csv'],
'compression' => 'none',
'encryption' => false
]);
$this->assertNotEmpty($bucketTwo['body']['$id']);
$this->assertEquals(201, $bucketTwo['headers']['status-code']);
$bucketTwoId = $bucketTwo['body']['$id'];
$bucketIds = [
'compressed' => $bucketOneId,
'uncompressed' => $bucketTwoId,
];
$fileIds = [];
foreach ($bucketIds as $label => $bucketId) {
$response = $this->client->call(Client::METHOD_POST, '/storage/buckets/' . $bucketId . '/files', array_merge([
'content-type' => 'multipart/form-data',
'x-appwrite-project' => $this->getProject()['$id'],
], $this->getHeaders()), [
'fileId' => ID::unique(),
'file' => new CURLFile(realpath(__DIR__ . '/../../../resources/documents.csv'), 'text/csv', 'documents.csv'),
]);
$this->assertEquals(201, $response['headers']['status-code']);
$this->assertNotEmpty($response['body']['$id']);
$this->assertEquals('documents.csv', $response['body']['name']);
$this->assertEquals('text/csv', $response['body']['mimeType']);
$fileIds[$label] = $response['body']['$id'];
}
// compressed, fail.
$compressed = $this->performCsvMigration(
[
'fileId' => $fileIds['compressed'],
'bucketId' => $bucketIds['compressed'],
'resourceId' => $databaseId . ':' . $collectionId,
]
);
// fail on compressed, encrypted buckets!
$this->assertEquals(400, $compressed['body']['code']);
$this->assertEquals('storage_file_type_unsupported', $compressed['body']['type']);
$this->assertEquals('Only uncompressed, unencrypted CSV files can be used for document import.', $compressed['body']['message']);
// no compression, no encryption, pass.
$migration = $this->performCsvMigration(
[
'endpoint' => 'http://localhost/v1',
'fileId' => $fileIds['uncompressed'],
'bucketId' => $bucketIds['uncompressed'],
'resourceId' => $databaseId . ':' . $collectionId,
]
);
$this->assertEmpty($migration['body']['statusCounters']);
$this->assertEquals('CSV', $migration['body']['source']);
$this->assertEquals('pending', $migration['body']['status']);
$this->assertEquals('Appwrite', $migration['body']['destination']);
$this->assertContains(Resource::TYPE_DOCUMENT, $migration['body']['resources']);
return [
'databaseId' => $databaseId,
'collectionId' => $collectionId,
'migrationId' => $migration['body']['$id'],
];
}
/**
* @depends testCreateCsvMigration
*/
public function testImportSuccessful(array $response): void
{
$databaseId = $response['databaseId'];
$collectionId = $response['collectionId'];
$migrationId = $response['migrationId'];
$documentsCountInCSV = 100;
// get migration stats
$this->assertEventually(function () use ($migrationId, $databaseId, $collectionId, $documentsCountInCSV) {
$migration = $this->client->call(Client::METHOD_GET, '/migrations/'.$migrationId, array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
], $this->getHeaders()));
$this->assertEquals(200, $migration['headers']['status-code']);
$this->assertEquals('finished', $migration['body']['stage']);
$this->assertEquals('completed', $migration['body']['status']);
$this->assertEquals('CSV', $migration['body']['source']);
$this->assertEquals('Appwrite', $migration['body']['destination']);
$this->assertContains(Resource::TYPE_DOCUMENT, $migration['body']['resources']);
$this->assertArrayHasKey(Resource::TYPE_DOCUMENT, $migration['body']['statusCounters']);
$this->assertEquals($documentsCountInCSV, $migration['body']['statusCounters'][Resource::TYPE_DOCUMENT]['success']);
}, 60000, 500);
// get documents count
$documents = $this->client->call(Client::METHOD_GET, '/databases/'.$databaseId.'/collections/'.$collectionId.'/documents', array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
], $this->getHeaders()), [
'queries' => [
// there should be only 100!
Query::limit(150)->toString()
]
]);
$this->assertEquals(200, $documents['headers']['status-code']);
$this->assertIsArray($documents['body']['documents']);
$this->assertIsNumeric($documents['body']['total']);
$this->assertEquals($documentsCountInCSV, $documents['body']['total']);
}
private function performCsvMigration(array $body): array
{
return $this->client->call(Client::METHOD_POST, '/migrations/csv', [
'content-type' => 'application/json',
'x-appwrite-key' => $this->getProject()['apiKey'],
'x-appwrite-project' => $this->getProject()['$id'],
], $body);
}
}
tests/resources/documents.csv
+101
View File
@@ -0,0 +1,101 @@
$id,name,age
hxfcwpcas5xokpwe,Diamond Mendez,56
gw8nxwf6esn3tfwf,Michael Huff,20
xb6bxg56lral1qy9,Alyssa Rodriguez,37
imerjq5j36y3agh2,Barbara Smith,26
07yq9qdlhmbzmr35,Evelyn Edwards,54
ksqo631sbhwj5ltg,Tina Richardson,41
j7zlndgu0gbshp15,Joel Hernandez,49
mfntvnljrcmf7h6v,Zachary Cooper,59
5f9b01nziqu2h8ed,Brittany Spears,20
4vxzbnzraqznk5u8,Holly White,47
d4ywy3mtphaatbpf,Kimberly Barnes,27
88odnk6nthyyvbal,Stephen Miller,53
08oekee3fn7mzaa5,Yvonne Newman,41
quw55kn9895i5e4v,Carol Kane,38
nge6bm8ykripei6f,Doris Foster,44
4k16i33s0xl2ypx9,Joseph Stokes,28
q0j5rxbgid66snyf,Steve Williams,31
n1oxun7mqq3p103y,James Carey,29
0dbvs840jkf8i0ye,Kathryn Henry,38
5sfaidgs1h87v15v,Christopher Landry,23
vg3punvfu5khmf41,Jennifer Mcgee,62
f933qydr9u5b2r11,Cathy Church,35
wjv87y1inf8yk32s,Jose Lopez,41
uljysdvdlcyrbrwk,William Rose,30
ot8xtzh77j55wq0s,Sarah Ford,26
9t76vnsv2u36s43t,Alisha Jones,61
66y4tnty62hw8c02,Kristin Kelly,61
2punfblazi5v16ar,Brendan Stout,40
sxhr4nf5w2gx4wbg,Kelly Cruz,18
68dvrqfwqnkq5el9,Samantha Martin,50
20192l6dbeinhkh0,David Santos,46
si0l4dgay09ebfmf,Elizabeth Carroll,22
lhse40vbldqb6ap1,Corey Owens,46
h5t3pslykyx3kxfm,Shelby Mueller,65
ldc0luydrw6jub0f,Dr. Sylvia Myers,29
voc9628xg4dsgw2y,Scott Freeman,48
o4y0gk3gqv1ax2fz,Christopher Atkinson,21
u1n3x4e4u7e0vzj6,Sean Diaz,31
s36eskwtm0w7lwr7,Bobby Dyer,57
4hjnag1p5iwvtixd,Daniel Hall,62
m91d80oxsa216zbh,Jennifer Ramirez,65
5hj6858zo2g85n6v,Angela Jackson,57
8m8oihv9a1e7nn92,Kelly Lewis,36
7azy39la0no0mxi7,Jessica Munoz,55
47pmjkhnnqhyit8c,Kelly George,65
6j6cpy4kgneg1mmh,Anthony Johnson,65
tnlmtvap1zz89km9,Regina Fields,61
6cyuvnwwqdmrpfzh,Sharon Schaefer,30
p1v4pyu2pqodc0ey,Jacob French,62
6npynnhjt2jd05xo,Jessica Costa,23
wcxedf13n2e9qi4l,George Hardy,53
yf2xlcmszk2tqeig,Andrea Allison,20
3bf2zzv7poststwa,Kevin Ferguson,32
c2iataz0hhv39q63,Joseph Johnson,58
3e8npxhov4a39pvq,Ashley Martinez,18
t7dp41tysipytywq,Charles Nixon,23
z8cztq7c47phyfhk,Carol Dudley,40
2636f9d8r4ipm3h6,David Weber,51
eh3f6wxtvkjq6ykq,Scott Robinson,32
raskbwpsje69a59h,Anthony Hardy,38
90hn1p0b4cs9e2og,Mackenzie Owens,52
am3swwfbo076x0v1,Brian Foster,27
5uw7utb9lq5cfncw,Hannah Forbes,56
cs6mbfzkzifefx6r,Lauren Reed,26
ftw3uvztziiz9x00,Morgan Smith,28
uhrqseeo43mozpaq,Samantha Alexander,65
pvvmzyfc1lxor11e,Tiffany Roberts,20
jia7bdag4abz123s,Emily Hayes,34
h6oozcngbz8o5x4y,Rebecca Villegas,52
9v6z1pn2f9twcy12,Donald Shah,61
wzz3jduioso77o7f,Denise Cain,59
u51plhgvjodkswnr,Kristine Ramirez,53
t1uhkmiytfyc13vc,Stacey Adkins,61
iqaqnf0ybg2ct507,Daniel Hunt,20
idwrwv2uu4hcpv2i,Roberta Johnson,48
2yd2hd6auetjacyo,Jason Williamson,39
egrmdbibnjhi914x,Sandra Robinson,50
15m1pz2bb0ercgyk,Steve Rice,25
0i21bhkxdagjurb7,Kimberly Fritz,53
726ofi7h5snreq67,Brianna Reynolds,33
csqxse3wym56eim6,Alexander Williams,50
qeaoylnrsf8p3byg,Andrew Thomas,25
edsswobumzyzbvhf,Austin Williams,57
hdzhzpt0ahy5hkib,Nicholas Williams,24
w1qmvmg4roa8xnwu,Mrs. Michelle Cisneros,48
3z3o73x7adyuo6w0,Stacey Smith,39
sse2u5zlgoqrgmcf,Laura Beck,20
rvovijmvch58r4yx,Molly Clark,51
doe06nrx8sg5mcuv,Carmen Morris,41
jbjdwuvj5s4kw04y,Amanda Munoz,20
6k2ewkla7js0yw23,Rachel Collins,44
fcxuyr4kkhrnigu1,John Alexander,18
d25fuwlos5mk07o0,Stacy Hunter,22
1vdai2rxmwd57oet,Eric Massey,40
pq4jnt9izu1wlrzd,Scott Garcia,20
lz9kfc0lty5xcz14,Cassandra Nelson,35
pu7w6tyab5jd4we9,Aaron Johnson,50
8dupswd2kqwdyn8v,Shannon Sherman,45
ye466l71jthiz2p6,April Garcia,60
xogsmfwb73l16qdt,Evan Lynn,20
1 $id name age
2 hxfcwpcas5xokpwe Diamond Mendez 56
3 gw8nxwf6esn3tfwf Michael Huff 20
4 xb6bxg56lral1qy9 Alyssa Rodriguez 37
5 imerjq5j36y3agh2 Barbara Smith 26
6 07yq9qdlhmbzmr35 Evelyn Edwards 54
7 ksqo631sbhwj5ltg Tina Richardson 41
8 j7zlndgu0gbshp15 Joel Hernandez 49
9 mfntvnljrcmf7h6v Zachary Cooper 59
10 5f9b01nziqu2h8ed Brittany Spears 20
11 4vxzbnzraqznk5u8 Holly White 47
12 d4ywy3mtphaatbpf Kimberly Barnes 27
13 88odnk6nthyyvbal Stephen Miller 53
14 08oekee3fn7mzaa5 Yvonne Newman 41
15 quw55kn9895i5e4v Carol Kane 38
16 nge6bm8ykripei6f Doris Foster 44
17 4k16i33s0xl2ypx9 Joseph Stokes 28
18 q0j5rxbgid66snyf Steve Williams 31
19 n1oxun7mqq3p103y James Carey 29
20 0dbvs840jkf8i0ye Kathryn Henry 38
21 5sfaidgs1h87v15v Christopher Landry 23
22 vg3punvfu5khmf41 Jennifer Mcgee 62
23 f933qydr9u5b2r11 Cathy Church 35
24 wjv87y1inf8yk32s Jose Lopez 41
25 uljysdvdlcyrbrwk William Rose 30
26 ot8xtzh77j55wq0s Sarah Ford 26
27 9t76vnsv2u36s43t Alisha Jones 61
28 66y4tnty62hw8c02 Kristin Kelly 61
29 2punfblazi5v16ar Brendan Stout 40
30 sxhr4nf5w2gx4wbg Kelly Cruz 18
31 68dvrqfwqnkq5el9 Samantha Martin 50
32 20192l6dbeinhkh0 David Santos 46
33 si0l4dgay09ebfmf Elizabeth Carroll 22
34 lhse40vbldqb6ap1 Corey Owens 46
35 h5t3pslykyx3kxfm Shelby Mueller 65
36 ldc0luydrw6jub0f Dr. Sylvia Myers 29
37 voc9628xg4dsgw2y Scott Freeman 48
38 o4y0gk3gqv1ax2fz Christopher Atkinson 21
39 u1n3x4e4u7e0vzj6 Sean Diaz 31
40 s36eskwtm0w7lwr7 Bobby Dyer 57
41 4hjnag1p5iwvtixd Daniel Hall 62
42 m91d80oxsa216zbh Jennifer Ramirez 65
43 5hj6858zo2g85n6v Angela Jackson 57
44 8m8oihv9a1e7nn92 Kelly Lewis 36
45 7azy39la0no0mxi7 Jessica Munoz 55
46 47pmjkhnnqhyit8c Kelly George 65
47 6j6cpy4kgneg1mmh Anthony Johnson 65
48 tnlmtvap1zz89km9 Regina Fields 61
49 6cyuvnwwqdmrpfzh Sharon Schaefer 30
50 p1v4pyu2pqodc0ey Jacob French 62
51 6npynnhjt2jd05xo Jessica Costa 23
52 wcxedf13n2e9qi4l George Hardy 53
53 yf2xlcmszk2tqeig Andrea Allison 20
54 3bf2zzv7poststwa Kevin Ferguson 32
55 c2iataz0hhv39q63 Joseph Johnson 58
56 3e8npxhov4a39pvq Ashley Martinez 18
57 t7dp41tysipytywq Charles Nixon 23
58 z8cztq7c47phyfhk Carol Dudley 40
59 2636f9d8r4ipm3h6 David Weber 51
60 eh3f6wxtvkjq6ykq Scott Robinson 32
61 raskbwpsje69a59h Anthony Hardy 38
62 90hn1p0b4cs9e2og Mackenzie Owens 52
63 am3swwfbo076x0v1 Brian Foster 27
64 5uw7utb9lq5cfncw Hannah Forbes 56
65 cs6mbfzkzifefx6r Lauren Reed 26
66 ftw3uvztziiz9x00 Morgan Smith 28
67 uhrqseeo43mozpaq Samantha Alexander 65
68 pvvmzyfc1lxor11e Tiffany Roberts 20
69 jia7bdag4abz123s Emily Hayes 34
70 h6oozcngbz8o5x4y Rebecca Villegas 52
71 9v6z1pn2f9twcy12 Donald Shah 61
72 wzz3jduioso77o7f Denise Cain 59
73 u51plhgvjodkswnr Kristine Ramirez 53
74 t1uhkmiytfyc13vc Stacey Adkins 61
75 iqaqnf0ybg2ct507 Daniel Hunt 20
76 idwrwv2uu4hcpv2i Roberta Johnson 48
77 2yd2hd6auetjacyo Jason Williamson 39
78 egrmdbibnjhi914x Sandra Robinson 50
79 15m1pz2bb0ercgyk Steve Rice 25
80 0i21bhkxdagjurb7 Kimberly Fritz 53
81 726ofi7h5snreq67 Brianna Reynolds 33
82 csqxse3wym56eim6 Alexander Williams 50
83 qeaoylnrsf8p3byg Andrew Thomas 25
84 edsswobumzyzbvhf Austin Williams 57
85 hdzhzpt0ahy5hkib Nicholas Williams 24
86 w1qmvmg4roa8xnwu Mrs. Michelle Cisneros 48
87 3z3o73x7adyuo6w0 Stacey Smith 39
88 sse2u5zlgoqrgmcf Laura Beck 20
89 rvovijmvch58r4yx Molly Clark 51
90 doe06nrx8sg5mcuv Carmen Morris 41
91 jbjdwuvj5s4kw04y Amanda Munoz 20
92 6k2ewkla7js0yw23 Rachel Collins 44
93 fcxuyr4kkhrnigu1 John Alexander 18
94 d25fuwlos5mk07o0 Stacy Hunter 22
95 1vdai2rxmwd57oet Eric Massey 40
96 pq4jnt9izu1wlrzd Scott Garcia 20
97 lz9kfc0lty5xcz14 Cassandra Nelson 35
98 pu7w6tyab5jd4we9 Aaron Johnson 50
99 8dupswd2kqwdyn8v Shannon Sherman 45
100 ye466l71jthiz2p6 April Garcia 60
101 xogsmfwb73l16qdt Evan Lynn 20