Pass overwrite and skip options from migration document to destination adapter

Merge remote-tracking branch 'origin/1.9.x' into csv-import-upsert
# Conflicts: # composer.lock
2026-05-26 13:51:13 +00:00 · 2026-04-09 02:09:58 +01:00 · 2026-04-09 02:08:52 +01:00 · 2026-04-09 01:58:30 +01:00 · 2026-04-09 01:48:09 +01:00 · 2026-04-08 11:48:16 +01:00
7 changed files with 480 additions and 22 deletions
@@ -86,6 +86,8 @@ Http::post('/v1/migrations/appwrite')
    ->param('endpoint', '', new URL(), 'Source Appwrite endpoint')
    ->param('projectId', '', fn (Database $dbForProject) => new UID($dbForProject->getAdapter()->getMaxUIDLength()), 'Source Project ID', false, ['dbForProject'])
    ->param('apiKey', '', new Text(512), 'Source API Key')
+    ->param('overwrite', false, new Boolean(), 'If true, existing documents with the same ID will be overwritten with the imported data. Cannot be used together with skip.', true)
+    ->param('skip', false, new Boolean(), 'If true, documents with duplicate IDs will be silently skipped instead of causing an error. Cannot be used together with overwrite.', true)
    ->inject('response')
    ->inject('dbForProject')
    ->inject('project')
@@ -93,7 +95,11 @@ Http::post('/v1/migrations/appwrite')
    ->inject('user')
    ->inject('queueForEvents')
    ->inject('queueForMigrations')
-    ->action(function (array $resources, string $endpoint, string $projectId, string $apiKey, Response $response, Database $dbForProject, Document $project, array $platform, Document $user, Event $queueForEvents, Migration $queueForMigrations) {
+    ->action(function (array $resources, string $endpoint, string $projectId, string $apiKey, bool $overwrite, bool $skip, Response $response, Database $dbForProject, Document $project, array $platform, Document $user, Event $queueForEvents, Migration $queueForMigrations) {
+        if ($overwrite && $skip) {
+            throw new Exception(Exception::GENERAL_ARGUMENT_INVALID, 'Cannot use both overwrite and skip at the same time.');
+        }
+
        $migration = $dbForProject->createDocument('migrations', new Document([
            '$id' => ID::unique(),
            'status' => 'pending',
@@ -109,6 +115,10 @@ Http::post('/v1/migrations/appwrite')
            'statusCounters' => '{}',
            'resourceData' => '{}',
            'errors' => [],
+            'options' => [
+                'overwrite' => $overwrite,
+                'skip' => $skip,
+            ],
        ]));

        $queueForEvents->setParam('migrationId', $migration->getId());
@@ -147,6 +157,8 @@ Http::post('/v1/migrations/firebase')
    ))
    ->param('resources', [], new ArrayList(new WhiteList(Firebase::getSupportedResources())), 'List of resources to migrate')
    ->param('serviceAccount', '', new Text(65536), 'JSON of the Firebase service account credentials')
+    ->param('overwrite', false, new Boolean(), 'If true, existing documents with the same ID will be overwritten with the imported data. Cannot be used together with skip.', true)
+    ->param('skip', false, new Boolean(), 'If true, documents with duplicate IDs will be silently skipped instead of causing an error. Cannot be used together with overwrite.', true)
    ->inject('response')
    ->inject('dbForProject')
    ->inject('project')
@@ -154,7 +166,11 @@ Http::post('/v1/migrations/firebase')
    ->inject('user')
    ->inject('queueForEvents')
    ->inject('queueForMigrations')
-    ->action(function (array $resources, string $serviceAccount, Response $response, Database $dbForProject, Document $project, array $platform, Document $user, Event $queueForEvents, Migration $queueForMigrations) {
+    ->action(function (array $resources, string $serviceAccount, bool $overwrite, bool $skip, Response $response, Database $dbForProject, Document $project, array $platform, Document $user, Event $queueForEvents, Migration $queueForMigrations) {
+        if ($overwrite && $skip) {
+            throw new Exception(Exception::GENERAL_ARGUMENT_INVALID, 'Cannot use both overwrite and skip at the same time.');
+        }
+
        $serviceAccountData = json_decode($serviceAccount, true);

        if (empty($serviceAccountData)) {
@@ -178,6 +194,10 @@ Http::post('/v1/migrations/firebase')
            'statusCounters' => '{}',
            'resourceData' => '{}',
            'errors' => [],
+            'options' => [
+                'overwrite' => $overwrite,
+                'skip' => $skip,
+            ],
        ]));

        $queueForEvents->setParam('migrationId', $migration->getId());
@@ -221,6 +241,8 @@ Http::post('/v1/migrations/supabase')
    ->param('username', '', new Text(512), 'Source\'s Database Username')
    ->param('password', '', new Text(512), 'Source\'s Database Password')
    ->param('port', 5432, new Integer(true), 'Source\'s Database Port', true)
+    ->param('overwrite', false, new Boolean(), 'If true, existing documents with the same ID will be overwritten with the imported data. Cannot be used together with skip.', true)
+    ->param('skip', false, new Boolean(), 'If true, documents with duplicate IDs will be silently skipped instead of causing an error. Cannot be used together with overwrite.', true)
    ->inject('response')
    ->inject('dbForProject')
    ->inject('project')
@@ -228,7 +250,11 @@ Http::post('/v1/migrations/supabase')
    ->inject('user')
    ->inject('queueForEvents')
    ->inject('queueForMigrations')
-    ->action(function (array $resources, string $endpoint, string $apiKey, string $databaseHost, string $username, string $password, int $port, Response $response, Database $dbForProject, Document $project, array $platform, Document $user, Event $queueForEvents, Migration $queueForMigrations) {
+    ->action(function (array $resources, string $endpoint, string $apiKey, string $databaseHost, string $username, string $password, int $port, bool $overwrite, bool $skip, Response $response, Database $dbForProject, Document $project, array $platform, Document $user, Event $queueForEvents, Migration $queueForMigrations) {
+        if ($overwrite && $skip) {
+            throw new Exception(Exception::GENERAL_ARGUMENT_INVALID, 'Cannot use both overwrite and skip at the same time.');
+        }
+
        $migration = $dbForProject->createDocument('migrations', new Document([
            '$id' => ID::unique(),
            'status' => 'pending',
@@ -247,6 +273,10 @@ Http::post('/v1/migrations/supabase')
            'statusCounters' => '{}',
            'resourceData' => '{}',
            'errors' => [],
+            'options' => [
+                'overwrite' => $overwrite,
+                'skip' => $skip,
+            ],
        ]));

        $queueForEvents->setParam('migrationId', $migration->getId());
@@ -291,6 +321,8 @@ Http::post('/v1/migrations/nhost')
    ->param('username', '', new Text(512), 'Source\'s Database Username')
    ->param('password', '', new Text(512), 'Source\'s Database Password')
    ->param('port', 5432, new Integer(true), 'Source\'s Database Port', true)
+    ->param('overwrite', false, new Boolean(), 'If true, existing documents with the same ID will be overwritten with the imported data. Cannot be used together with skip.', true)
+    ->param('skip', false, new Boolean(), 'If true, documents with duplicate IDs will be silently skipped instead of causing an error. Cannot be used together with overwrite.', true)
    ->inject('response')
    ->inject('dbForProject')
    ->inject('project')
@@ -298,7 +330,11 @@ Http::post('/v1/migrations/nhost')
    ->inject('user')
    ->inject('queueForEvents')
    ->inject('queueForMigrations')
-    ->action(function (array $resources, string $subdomain, string $region, string $adminSecret, string $database, string $username, string $password, int $port, Response $response, Database $dbForProject, Document $project, array $platform, Document $user, Event $queueForEvents, Migration $queueForMigrations) {
+    ->action(function (array $resources, string $subdomain, string $region, string $adminSecret, string $database, string $username, string $password, int $port, bool $overwrite, bool $skip, Response $response, Database $dbForProject, Document $project, array $platform, Document $user, Event $queueForEvents, Migration $queueForMigrations) {
+        if ($overwrite && $skip) {
+            throw new Exception(Exception::GENERAL_ARGUMENT_INVALID, 'Cannot use both overwrite and skip at the same time.');
+        }
+
        $migration = $dbForProject->createDocument('migrations', new Document([
            '$id' => ID::unique(),
            'status' => 'pending',
@@ -318,6 +354,10 @@ Http::post('/v1/migrations/nhost')
            'statusCounters' => '{}',
            'resourceData' => '{}',
            'errors' => [],
+            'options' => [
+                'overwrite' => $overwrite,
+                'skip' => $skip,
+            ],
        ]));

        $queueForEvents->setParam('migrationId', $migration->getId());
@@ -359,6 +399,8 @@ Http::post('/v1/migrations/csv/imports')
    ->param('fileId', '', fn (Database $dbForProject) => new UID($dbForProject->getAdapter()->getMaxUIDLength()), 'File ID.', false, ['dbForProject'])
    ->param('resourceId', null, new CompoundUID(), 'Composite ID in the format {databaseId:collectionId}, identifying a collection within a database.')
    ->param('internalFile', false, new Boolean(), 'Is the file stored in an internal bucket?', true)
+    ->param('overwrite', false, new Boolean(), 'If true, existing documents with the same ID will be overwritten with the imported data. Cannot be used together with skip.', true)
+    ->param('skip', false, new Boolean(), 'If true, documents with duplicate IDs will be silently skipped instead of causing an error. Cannot be used together with overwrite.', true)
    ->inject('response')
    ->inject('dbForProject')
    ->inject('dbForPlatform')
@@ -374,6 +416,8 @@ Http::post('/v1/migrations/csv/imports')
        string $fileId,
        string $resourceId,
        bool $internalFile,
+        bool $overwrite,
+        bool $skip,
        Response $response,
        Database $dbForProject,
        Database $dbForPlatform,
@@ -385,6 +429,9 @@ Http::post('/v1/migrations/csv/imports')
        Event $queueForEvents,
        Migration $queueForMigrations
    ) {
+        if ($overwrite && $skip) {
+            throw new Exception(Exception::GENERAL_ARGUMENT_INVALID, 'Cannot use both overwrite and skip at the same time.');
+        }
        $bucket = $authorization->skip(function () use ($internalFile, $dbForPlatform, $dbForProject, $bucketId) {
            if ($internalFile) {
                return $dbForPlatform->getDocument('buckets', 'default');
@@ -474,6 +521,8 @@ Http::post('/v1/migrations/csv/imports')
            'options' => [
                'path' => $newPath,
                'size' => $fileSize,
+                'overwrite' => $overwrite,
+                'skip' => $skip,
            ],
        ]));

@@ -664,6 +713,8 @@ Http::post('/v1/migrations/json/imports')
    ->param('fileId', '', new UID(), 'File ID.')
    ->param('resourceId', null, new CompoundUID(), 'Composite ID in the format {databaseId:collectionId}, identifying a collection within a database.')
    ->param('internalFile', false, new Boolean(), 'Is the file stored in an internal bucket?', true)
+    ->param('overwrite', false, new Boolean(), 'If true, existing documents with the same ID will be overwritten with the imported data. Cannot be used together with skip.', true)
+    ->param('skip', false, new Boolean(), 'If true, documents with duplicate IDs will be silently skipped instead of causing an error. Cannot be used together with overwrite.', true)
    ->inject('response')
    ->inject('dbForProject')
    ->inject('dbForPlatform')
@@ -679,6 +730,8 @@ Http::post('/v1/migrations/json/imports')
        string $fileId,
        string $resourceId,
        bool $internalFile,
+        bool $overwrite,
+        bool $skip,
        Response $response,
        Database $dbForProject,
        Database $dbForPlatform,
@@ -690,6 +743,10 @@ Http::post('/v1/migrations/json/imports')
        Event $queueForEvents,
        Migration $queueForMigrations
    ) {
+        if ($overwrite && $skip) {
+            throw new Exception(Exception::GENERAL_ARGUMENT_INVALID, 'Cannot use both overwrite and skip at the same time.');
+        }
+
        $bucket = $authorization->skip(function () use ($internalFile, $dbForPlatform, $dbForProject, $bucketId) {
            if ($internalFile) {
                return $dbForPlatform->getDocument('buckets', 'default');
@@ -778,6 +835,8 @@ Http::post('/v1/migrations/json/imports')
            'options' => [
                'path' => $newPath,
                'size' => $fileSize,
+                'overwrite' => $overwrite,
+                'skip' => $skip,
            ],
        ]));

@@ -61,7 +61,7 @@
        "utopia-php/compression": "0.1.*",
        "utopia-php/config": "1.*",
        "utopia-php/console": "0.1.*",
-        "utopia-php/database": "5.*",
+        "utopia-php/database": "dev-csv-import-upsert-v2 as 5.0.0",
        "utopia-php/detector": "0.2.*",
        "utopia-php/domains": "1.*",
        "utopia-php/emails": "0.6.*",
@@ -73,7 +73,7 @@
        "utopia-php/locale": "0.8.*",
        "utopia-php/logger": "0.6.*",
        "utopia-php/messaging": "0.22.*",
-        "utopia-php/migration": "1.9.*",
+        "utopia-php/migration": "dev-csv-import-upsert as 1.9.0",
        "utopia-php/platform": "0.12.*",
        "utopia-php/pools": "1.*",
        "utopia-php/span": "1.1.*",
@@ -4,7 +4,7 @@
        "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies",
        "This file is @generated automatically"
    ],
-    "content-hash": "4fb974e9843f6104e40396e7cad4a833",
+    "content-hash": "4163406e181d863265fc6bbba652476b",
    "packages": [
        {
            "name": "adhocore/jwt",
@@ -3850,16 +3850,16 @@
        },
        {
            "name": "utopia-php/database",
-            "version": "5.3.19",
+            "version": "dev-csv-import-upsert-v2",
            "source": {
                "type": "git",
                "url": "https://github.com/utopia-php/database.git",
-                "reference": "72ee1614c37e37c7fdd9d4dc87f1f7cdfa1ca691"
+                "reference": "bb419ba6a5da1975373bbd958f9fd4d61fb3ee48"
            },
            "dist": {
                "type": "zip",
-                "url": "https://api.github.com/repos/utopia-php/database/zipball/72ee1614c37e37c7fdd9d4dc87f1f7cdfa1ca691",
-                "reference": "72ee1614c37e37c7fdd9d4dc87f1f7cdfa1ca691",
+                "url": "https://api.github.com/repos/utopia-php/database/zipball/bb419ba6a5da1975373bbd958f9fd4d61fb3ee48",
+                "reference": "bb419ba6a5da1975373bbd958f9fd4d61fb3ee48",
                "shasum": ""
            },
            "require": {
@@ -3903,9 +3903,9 @@
            ],
            "support": {
                "issues": "https://github.com/utopia-php/database/issues",
-                "source": "https://github.com/utopia-php/database/tree/5.3.19"
+                "source": "https://github.com/utopia-php/database/tree/csv-import-upsert-v2"
            },
-            "time": "2026-03-31T15:52:08+00:00"
+            "time": "2026-04-09T00:55:20+00:00"
        },
        {
            "name": "utopia-php/detector",
@@ -4526,16 +4526,16 @@
        },
        {
            "name": "utopia-php/migration",
-            "version": "1.9.1",
+            "version": "dev-csv-import-upsert",
            "source": {
                "type": "git",
                "url": "https://github.com/utopia-php/migration.git",
-                "reference": "7a86aeadf182b63a9f4ceba7e137588b31c5d2e2"
+                "reference": "8c85cbebcb98724c03c5e50af4defeabe020ab9e"
            },
            "dist": {
                "type": "zip",
-                "url": "https://api.github.com/repos/utopia-php/migration/zipball/7a86aeadf182b63a9f4ceba7e137588b31c5d2e2",
-                "reference": "7a86aeadf182b63a9f4ceba7e137588b31c5d2e2",
+                "url": "https://api.github.com/repos/utopia-php/migration/zipball/8c85cbebcb98724c03c5e50af4defeabe020ab9e",
+                "reference": "8c85cbebcb98724c03c5e50af4defeabe020ab9e",
                "shasum": ""
            },
            "require": {
@@ -4544,7 +4544,7 @@
                "ext-openssl": "*",
                "halaxa/json-machine": "^1.2",
                "php": ">=8.1",
-                "utopia-php/database": "5.*",
+                "utopia-php/database": "dev-csv-import-upsert-v2 as 5.0.0",
                "utopia-php/dsn": "0.2.*",
                "utopia-php/storage": "1.0.*"
            },
@@ -4575,9 +4575,9 @@
            ],
            "support": {
                "issues": "https://github.com/utopia-php/migration/issues",
-                "source": "https://github.com/utopia-php/migration/tree/1.9.1"
+                "source": "https://github.com/utopia-php/migration/tree/csv-import-upsert"
            },
-            "time": "2026-03-25T07:05:27+00:00"
+            "time": "2026-04-09T00:59:50+00:00"
        },
        {
            "name": "utopia-php/mongo",
@@ -8426,9 +8426,25 @@
            "time": "2024-11-07T12:36:22+00:00"
        }
    ],
-    "aliases": [],
+    "aliases": [
+        {
+            "package": "utopia-php/database",
+            "version": "dev-csv-import-upsert-v2",
+            "alias": "5.0.0",
+            "alias_normalized": "5.0.0.0"
+        },
+        {
+            "package": "utopia-php/migration",
+            "version": "dev-csv-import-upsert",
+            "alias": "1.9.0",
+            "alias_normalized": "1.9.0.0"
+        }
+    ],
    "minimum-stability": "dev",
-    "stability-flags": {},
+    "stability-flags": {
+        "utopia-php/database": 20,
+        "utopia-php/migration": 20
+    },
    "prefer-stable": true,
    "prefer-lowest": false,
    "platform": {
@@ -444,6 +444,13 @@ class Migrations extends Action
            $source = $this->processSource($migration);
            $destination = $this->processDestination($migration);

+            $options = $migration->getAttribute('options', []);
+
+            if ($destination instanceof DestinationAppwrite) {
+                $destination->setOverwrite($options['overwrite'] ?? false);
+                $destination->setSkip($options['skip'] ?? false);
+            }
+
            $transfer = new Transfer(
                $source,
                $destination
@@ -1482,6 +1482,355 @@ trait MigrationsBase
        }, 10_000, 500);
    }

+    public function testCSVImportOverwriteAndSkip(): void
+    {
+        $headers = [
+            'content-type' => 'application/json',
+            'x-appwrite-project' => $this->getProject()['$id'],
+            'x-appwrite-key' => $this->getProject()['apiKey'],
+        ];
+
+        // Create database and table
+        $database = $this->client->call(Client::METHOD_POST, '/databases', $headers, [
+            'databaseId' => ID::unique(),
+            'name' => 'Overwrite Skip DB',
+        ]);
+        $this->assertEquals(201, $database['headers']['status-code']);
+        $databaseId = $database['body']['$id'];
+
+        $table = $this->client->call(Client::METHOD_POST, '/tablesdb/' . $databaseId . '/tables', $headers, [
+            'tableId' => ID::unique(),
+            'name' => 'Test Table',
+        ]);
+        $this->assertEquals(201, $table['headers']['status-code']);
+        $tableId = $table['body']['$id'];
+
+        $response = $this->client->call(Client::METHOD_POST, '/tablesdb/' . $databaseId . '/tables/' . $tableId . '/columns/string', $headers, [
+            'key' => 'name',
+            'size' => 256,
+            'required' => true,
+        ]);
+        $this->assertEquals(202, $response['headers']['status-code']);
+
+        $response = $this->client->call(Client::METHOD_POST, '/tablesdb/' . $databaseId . '/tables/' . $tableId . '/columns/integer', $headers, [
+            'key' => 'age',
+            'min' => 18,
+            'max' => 65,
+            'required' => true,
+        ]);
+        $this->assertEquals(202, $response['headers']['status-code']);
+
+        // Create bucket
+        $bucket = $this->client->call(Client::METHOD_POST, '/storage/buckets', $headers, [
+            'bucketId' => ID::unique(),
+            'name' => 'CSV Overwrite Skip Bucket',
+            'maximumFileSize' => 2000000,
+            'allowedFileExtensions' => ['csv'],
+        ]);
+        $this->assertEquals(201, $bucket['headers']['status-code']);
+        $bucketId = $bucket['body']['$id'];
+
+        // Upload initial CSV
+        $initialFile = $this->client->call(Client::METHOD_POST, '/storage/buckets/' . $bucketId . '/files', array_merge([
+            'content-type' => 'multipart/form-data',
+            'x-appwrite-project' => $this->getProject()['$id'],
+        ], $this->getHeaders()), [
+            'fileId' => ID::unique(),
+            'file' => new CURLFile(realpath(__DIR__ . '/../../../resources/csv/documents.csv'), 'text/csv', 'documents.csv'),
+        ]);
+        $this->assertEquals(201, $initialFile['headers']['status-code']);
+
+        // Upload duplicates CSV (2 existing IDs with new data + 2 new IDs)
+        $dupFile = $this->client->call(Client::METHOD_POST, '/storage/buckets/' . $bucketId . '/files', array_merge([
+            'content-type' => 'multipart/form-data',
+            'x-appwrite-project' => $this->getProject()['$id'],
+        ], $this->getHeaders()), [
+            'fileId' => ID::unique(),
+            'file' => new CURLFile(realpath(__DIR__ . '/../../../resources/csv/documents-duplicates.csv'), 'text/csv', 'documents-duplicates.csv'),
+        ]);
+        $this->assertEquals(201, $dupFile['headers']['status-code']);
+
+        $resourceId = $databaseId . ':' . $tableId;
+
+        // 1. Import initial 100 rows
+        $migration = $this->performCsvMigration([
+            'fileId' => $initialFile['body']['$id'],
+            'bucketId' => $bucketId,
+            'resourceId' => $resourceId,
+        ]);
+
+        $this->assertEventually(function () use ($migration) {
+            $m = $this->client->call(Client::METHOD_GET, '/migrations/' . $migration['body']['$id'], array_merge([
+                'content-type' => 'application/json',
+                'x-appwrite-project' => $this->getProject()['$id'],
+            ], $this->getHeaders()));
+            $this->assertEquals('finished', $m['body']['stage']);
+            $this->assertEquals('completed', $m['body']['status']);
+            $this->assertEquals(100, $m['body']['statusCounters'][Resource::TYPE_ROW]['success']);
+        }, 30_000, 500);
+
+        // 2. Test overwrite+skip mutual exclusion
+        $invalid = $this->performCsvMigration([
+            'fileId' => $dupFile['body']['$id'],
+            'bucketId' => $bucketId,
+            'resourceId' => $resourceId,
+            'overwrite' => true,
+            'skip' => true,
+        ]);
+        $this->assertEquals(400, $invalid['headers']['status-code']);
+
+        // 3. Import duplicates with skip=true — duplicates skipped, new rows added
+        $skipMigration = $this->performCsvMigration([
+            'fileId' => $dupFile['body']['$id'],
+            'bucketId' => $bucketId,
+            'resourceId' => $resourceId,
+            'skip' => true,
+        ]);
+
+        $this->assertEventually(function () use ($skipMigration) {
+            $m = $this->client->call(Client::METHOD_GET, '/migrations/' . $skipMigration['body']['$id'], array_merge([
+                'content-type' => 'application/json',
+                'x-appwrite-project' => $this->getProject()['$id'],
+            ], $this->getHeaders()));
+            $this->assertEquals('finished', $m['body']['stage']);
+            $this->assertEquals('completed', $m['body']['status']);
+        }, 30_000, 500);
+
+        // Verify total is 102 (100 original + 2 new), not 104
+        $rows = $this->client->call(Client::METHOD_GET, '/tablesdb/' . $databaseId . '/tables/' . $tableId . '/rows', array_merge([
+            'content-type' => 'application/json',
+            'x-appwrite-project' => $this->getProject()['$id'],
+        ], $this->getHeaders()), [
+            'queries' => [Query::limit(150)->toString()]
+        ]);
+        $this->assertEquals(102, $rows['body']['total']);
+
+        // Verify original row was NOT overwritten
+        $row = $this->client->call(Client::METHOD_GET, '/tablesdb/' . $databaseId . '/tables/' . $tableId . '/rows/hxfcwpcas5xokpwe', array_merge([
+            'content-type' => 'application/json',
+            'x-appwrite-project' => $this->getProject()['$id'],
+        ], $this->getHeaders()));
+        $this->assertEquals('Diamond Mendez', $row['body']['name']);
+
+        // 4. Import duplicates with overwrite=true — existing rows updated
+        // Re-upload file since migration consumes it
+        $dupFile2 = $this->client->call(Client::METHOD_POST, '/storage/buckets/' . $bucketId . '/files', array_merge([
+            'content-type' => 'multipart/form-data',
+            'x-appwrite-project' => $this->getProject()['$id'],
+        ], $this->getHeaders()), [
+            'fileId' => ID::unique(),
+            'file' => new CURLFile(realpath(__DIR__ . '/../../../resources/csv/documents-duplicates.csv'), 'text/csv', 'documents-duplicates.csv'),
+        ]);
+        $this->assertEquals(201, $dupFile2['headers']['status-code']);
+
+        $overwriteMigration = $this->performCsvMigration([
+            'fileId' => $dupFile2['body']['$id'],
+            'bucketId' => $bucketId,
+            'resourceId' => $resourceId,
+            'overwrite' => true,
+        ]);
+
+        $this->assertEventually(function () use ($overwriteMigration) {
+            $m = $this->client->call(Client::METHOD_GET, '/migrations/' . $overwriteMigration['body']['$id'], array_merge([
+                'content-type' => 'application/json',
+                'x-appwrite-project' => $this->getProject()['$id'],
+            ], $this->getHeaders()));
+            $this->assertEquals('finished', $m['body']['stage']);
+            $this->assertEquals('completed', $m['body']['status']);
+        }, 30_000, 500);
+
+        // Total should still be 102 (no new rows added)
+        $rows = $this->client->call(Client::METHOD_GET, '/tablesdb/' . $databaseId . '/tables/' . $tableId . '/rows', array_merge([
+            'content-type' => 'application/json',
+            'x-appwrite-project' => $this->getProject()['$id'],
+        ], $this->getHeaders()), [
+            'queries' => [Query::limit(150)->toString()]
+        ]);
+        $this->assertEquals(102, $rows['body']['total']);
+
+        // Verify row was overwritten
+        $row = $this->client->call(Client::METHOD_GET, '/tablesdb/' . $databaseId . '/tables/' . $tableId . '/rows/hxfcwpcas5xokpwe', array_merge([
+            'content-type' => 'application/json',
+            'x-appwrite-project' => $this->getProject()['$id'],
+        ], $this->getHeaders()));
+        $this->assertEquals('Updated Diamond', $row['body']['name']);
+        $this->assertEquals(30, $row['body']['age']);
+    }
+
+    public function testJSONImportOverwriteAndSkip(): void
+    {
+        $headers = [
+            'content-type' => 'application/json',
+            'x-appwrite-project' => $this->getProject()['$id'],
+            'x-appwrite-key' => $this->getProject()['apiKey'],
+        ];
+
+        // Create database and table
+        $database = $this->client->call(Client::METHOD_POST, '/databases', $headers, [
+            'databaseId' => ID::unique(),
+            'name' => 'JSON Overwrite Skip DB',
+        ]);
+        $this->assertEquals(201, $database['headers']['status-code']);
+        $databaseId = $database['body']['$id'];
+
+        $table = $this->client->call(Client::METHOD_POST, '/tablesdb/' . $databaseId . '/tables', $headers, [
+            'tableId' => ID::unique(),
+            'name' => 'Test Table',
+        ]);
+        $this->assertEquals(201, $table['headers']['status-code']);
+        $tableId = $table['body']['$id'];
+
+        $response = $this->client->call(Client::METHOD_POST, '/tablesdb/' . $databaseId . '/tables/' . $tableId . '/columns/string', $headers, [
+            'key' => 'name',
+            'size' => 256,
+            'required' => true,
+        ]);
+        $this->assertEquals(202, $response['headers']['status-code']);
+
+        $response = $this->client->call(Client::METHOD_POST, '/tablesdb/' . $databaseId . '/tables/' . $tableId . '/columns/integer', $headers, [
+            'key' => 'age',
+            'min' => 18,
+            'max' => 65,
+            'required' => true,
+        ]);
+        $this->assertEquals(202, $response['headers']['status-code']);
+
+        // Create bucket
+        $bucket = $this->client->call(Client::METHOD_POST, '/storage/buckets', $headers, [
+            'bucketId' => ID::unique(),
+            'name' => 'JSON Overwrite Skip Bucket',
+            'maximumFileSize' => 2000000,
+            'allowedFileExtensions' => ['json'],
+        ]);
+        $this->assertEquals(201, $bucket['headers']['status-code']);
+        $bucketId = $bucket['body']['$id'];
+
+        // Upload initial JSON
+        $initialFile = $this->client->call(Client::METHOD_POST, '/storage/buckets/' . $bucketId . '/files', array_merge([
+            'content-type' => 'multipart/form-data',
+            'x-appwrite-project' => $this->getProject()['$id'],
+        ], $this->getHeaders()), [
+            'fileId' => ID::unique(),
+            'file' => new CURLFile(realpath(__DIR__ . '/../../../resources/json/documents.json'), 'application/json', 'documents.json'),
+        ]);
+        $this->assertEquals(201, $initialFile['headers']['status-code']);
+
+        // Upload duplicates JSON
+        $dupFile = $this->client->call(Client::METHOD_POST, '/storage/buckets/' . $bucketId . '/files', array_merge([
+            'content-type' => 'multipart/form-data',
+            'x-appwrite-project' => $this->getProject()['$id'],
+        ], $this->getHeaders()), [
+            'fileId' => ID::unique(),
+            'file' => new CURLFile(realpath(__DIR__ . '/../../../resources/json/documents-duplicates.json'), 'application/json', 'documents-duplicates.json'),
+        ]);
+        $this->assertEquals(201, $dupFile['headers']['status-code']);
+
+        $resourceId = $databaseId . ':' . $tableId;
+
+        // 1. Import initial 100 rows
+        $migration = $this->performJsonMigration([
+            'fileId' => $initialFile['body']['$id'],
+            'bucketId' => $bucketId,
+            'resourceId' => $resourceId,
+        ]);
+
+        $this->assertEventually(function () use ($migration) {
+            $m = $this->client->call(Client::METHOD_GET, '/migrations/' . $migration['body']['$id'], array_merge([
+                'content-type' => 'application/json',
+                'x-appwrite-project' => $this->getProject()['$id'],
+            ], $this->getHeaders()));
+            $this->assertEquals('finished', $m['body']['stage']);
+            $this->assertEquals('completed', $m['body']['status']);
+            $this->assertEquals(100, $m['body']['statusCounters'][Resource::TYPE_ROW]['success']);
+        }, 30_000, 500);
+
+        // 2. Test overwrite+skip mutual exclusion
+        $invalid = $this->performJsonMigration([
+            'fileId' => $dupFile['body']['$id'],
+            'bucketId' => $bucketId,
+            'resourceId' => $resourceId,
+            'overwrite' => true,
+            'skip' => true,
+        ]);
+        $this->assertEquals(400, $invalid['headers']['status-code']);
+
+        // 3. Import duplicates with skip=true
+        $skipMigration = $this->performJsonMigration([
+            'fileId' => $dupFile['body']['$id'],
+            'bucketId' => $bucketId,
+            'resourceId' => $resourceId,
+            'skip' => true,
+        ]);
+
+        $this->assertEventually(function () use ($skipMigration) {
+            $m = $this->client->call(Client::METHOD_GET, '/migrations/' . $skipMigration['body']['$id'], array_merge([
+                'content-type' => 'application/json',
+                'x-appwrite-project' => $this->getProject()['$id'],
+            ], $this->getHeaders()));
+            $this->assertEquals('finished', $m['body']['stage']);
+            $this->assertEquals('completed', $m['body']['status']);
+        }, 30_000, 500);
+
+        // Verify total is 102 (100 original + 2 new)
+        $rows = $this->client->call(Client::METHOD_GET, '/tablesdb/' . $databaseId . '/tables/' . $tableId . '/rows', array_merge([
+            'content-type' => 'application/json',
+            'x-appwrite-project' => $this->getProject()['$id'],
+        ], $this->getHeaders()), [
+            'queries' => [Query::limit(150)->toString()]
+        ]);
+        $this->assertEquals(102, $rows['body']['total']);
+
+        // Verify original row was NOT overwritten
+        $row = $this->client->call(Client::METHOD_GET, '/tablesdb/' . $databaseId . '/tables/' . $tableId . '/rows/hxfcwpcas5xokpwe', array_merge([
+            'content-type' => 'application/json',
+            'x-appwrite-project' => $this->getProject()['$id'],
+        ], $this->getHeaders()));
+        $this->assertEquals('Diamond Mendez', $row['body']['name']);
+
+        // 4. Import duplicates with overwrite=true
+        $dupFile2 = $this->client->call(Client::METHOD_POST, '/storage/buckets/' . $bucketId . '/files', array_merge([
+            'content-type' => 'multipart/form-data',
+            'x-appwrite-project' => $this->getProject()['$id'],
+        ], $this->getHeaders()), [
+            'fileId' => ID::unique(),
+            'file' => new CURLFile(realpath(__DIR__ . '/../../../resources/json/documents-duplicates.json'), 'application/json', 'documents-duplicates.json'),
+        ]);
+        $this->assertEquals(201, $dupFile2['headers']['status-code']);
+
+        $overwriteMigration = $this->performJsonMigration([
+            'fileId' => $dupFile2['body']['$id'],
+            'bucketId' => $bucketId,
+            'resourceId' => $resourceId,
+            'overwrite' => true,
+        ]);
+
+        $this->assertEventually(function () use ($overwriteMigration) {
+            $m = $this->client->call(Client::METHOD_GET, '/migrations/' . $overwriteMigration['body']['$id'], array_merge([
+                'content-type' => 'application/json',
+                'x-appwrite-project' => $this->getProject()['$id'],
+            ], $this->getHeaders()));
+            $this->assertEquals('finished', $m['body']['stage']);
+            $this->assertEquals('completed', $m['body']['status']);
+        }, 30_000, 500);
+
+        // Total should still be 102
+        $rows = $this->client->call(Client::METHOD_GET, '/tablesdb/' . $databaseId . '/tables/' . $tableId . '/rows', array_merge([
+            'content-type' => 'application/json',
+            'x-appwrite-project' => $this->getProject()['$id'],
+        ], $this->getHeaders()), [
+            'queries' => [Query::limit(150)->toString()]
+        ]);
+        $this->assertEquals(102, $rows['body']['total']);
+
+        // Verify row was overwritten
+        $row = $this->client->call(Client::METHOD_GET, '/tablesdb/' . $databaseId . '/tables/' . $tableId . '/rows/hxfcwpcas5xokpwe', array_merge([
+            'content-type' => 'application/json',
+            'x-appwrite-project' => $this->getProject()['$id'],
+        ], $this->getHeaders()));
+        $this->assertEquals('Updated Diamond', $row['body']['name']);
+        $this->assertEquals(30, $row['body']['age']);
+    }
+
    private function performCsvMigration(array $body): array
    {
        return $this->client->call(Client::METHOD_POST, '/migrations/csv', [
@@ -0,0 +1,5 @@
+$id,name,age
+hxfcwpcas5xokpwe,Updated Diamond,30
+gw8nxwf6esn3tfwf,Updated Michael,25
+newdoc001abcdefg,New Person One,40
+newdoc002abcdefg,New Person Two,35
@@ -0,0 +1,22 @@
+[
+    {
+        "$id": "hxfcwpcas5xokpwe",
+        "name": "Updated Diamond",
+        "age": 30
+    },
+    {
+        "$id": "gw8nxwf6esn3tfwf",
+        "name": "Updated Michael",
+        "age": 25
+    },
+    {
+        "$id": "newdoc001abcdefg",
+        "name": "New Person One",
+        "age": 40
+    },
+    {
+        "$id": "newdoc002abcdefg",
+        "name": "New Person Two",
+        "age": 35
+    }
+]
Author	SHA1	Message	Date
Prem Palanisamy	fcd32e1cd9	Pass overwrite and skip options from migration document to destination adapter	2026-04-09 02:09:58 +01:00
Prem Palanisamy	24907848e3	Merge remote-tracking branch 'origin/1.9.x' into csv-import-upsert # Conflicts: # composer.lock	2026-04-09 02:08:52 +01:00
Prem Palanisamy	be464d6e37	Update composer.lock for database csv-import-upsert-v2 and migration branches	2026-04-09 01:58:30 +01:00
Prem Palanisamy	eac9858447	Point utopia-php/database to csv-import-upsert-v2 branch	2026-04-09 01:48:09 +01:00
Prem Palanisamy	34ce1cd223	Add overwrite and skip params to all migration endpoints	2026-04-08 11:48:16 +01:00
Prem Palanisamy	5ec0ed724c	Assert column creation responses in overwrite/skip tests	2026-04-08 08:47:29 +01:00
Prem Palanisamy	6e47274d1a	Merge 1.9.x and resolve composer conflicts	2026-04-08 08:21:48 +01:00
Prem Palanisamy	bd6b90e8ca	Add E2E tests for CSV/JSON import overwrite and skip options - Add testCSVImportOverwriteAndSkip and testJSONImportOverwriteAndSkip - Test mutual exclusion validation (overwrite+skip returns 400) - Test skip: duplicates silently skipped, originals unchanged - Test overwrite: existing rows updated with new data - Point database and migration deps to csv-import-upsert branches	2026-04-08 08:12:33 +01:00