Merge pull request #11910 from appwrite/feat/skip-duplicates

feat(migrations): add onDuplicate option to CSV/JSON/Appwrite imports

src/Appwrite/Platform/Modules/Migrations/Http/Migrations/Appwrite/Create.php

@@ -13,6 +13,7 @@ use Utopia\Database\Database;
 use Utopia\Database\Document;
 use Utopia\Database\Helpers\ID;
 use Utopia\Database\Validator\UID;
+use Utopia\Migration\Destinations\OnDuplicate;
 use Utopia\Migration\Sources\Appwrite as AppwriteSource;
 use Utopia\Platform\Action;
 use Utopia\Platform\Scope\HTTP;
@@ -57,6 +58,7 @@ class Create extends Action
             ->param('endpoint', '', new URL(), 'Source Appwrite endpoint')
             ->param('projectId', '', fn (Database $dbForProject) => new UID($dbForProject->getAdapter()->getMaxUIDLength()), 'Source Project ID', false, ['dbForProject'])
             ->param('apiKey', '', new Text(512), 'Source API Key')
+            ->param('onDuplicate', OnDuplicate::Fail->value, new WhiteList(OnDuplicate::values()), 'Behavior when a row with an existing $id is encountered. "fail" (default): abort on first conflict. "skip": silently ignore. "overwrite": replace existing row.', true)
             ->inject('response')
             ->inject('dbForProject')
             ->inject('project')
@@ -71,6 +73,7 @@ class Create extends Action
         string $endpoint,
         string $projectId,
         string $apiKey,
+        string $onDuplicate,
         Response $response,
         Database $dbForProject,
         Document $project,
@@ -93,6 +96,9 @@ class Create extends Action
             'statusCounters' => '{}',
             'resourceData' => '{}',
             'errors' => [],
+            'options' => [
+                'onDuplicate' => $onDuplicate,
+            ],
         ]));
 
         $queueForEvents->setParam('migrationId', $migration->getId());

src/Appwrite/Platform/Modules/Migrations/Http/Migrations/CSV/Imports/Create.php

@@ -20,6 +20,7 @@ use Utopia\Database\Document;
 use Utopia\Database\Helpers\ID;
 use Utopia\Database\Validator\Authorization;
 use Utopia\Database\Validator\UID;
+use Utopia\Migration\Destinations\OnDuplicate;
 use Utopia\Migration\Resource;
 use Utopia\Migration\Sources\Appwrite as AppwriteSource;
 use Utopia\Migration\Sources\CSV;
@@ -29,6 +30,7 @@ use Utopia\Platform\Scope\HTTP;
 use Utopia\Storage\Device;
 use Utopia\System\System;
 use Utopia\Validator\Boolean;
+use Utopia\Validator\WhiteList;
 
 class Create extends Action
 {
@@ -67,6 +69,7 @@ class Create extends Action
             ->param('fileId', '', fn (Database $dbForProject) => new UID($dbForProject->getAdapter()->getMaxUIDLength()), 'File ID.', false, ['dbForProject'])
             ->param('resourceId', null, new CompoundUID(), 'Composite ID in the format {databaseId:collectionId}, identifying a collection within a database.')
             ->param('internalFile', false, new Boolean(), 'Is the file stored in an internal bucket?', true)
+            ->param('onDuplicate', OnDuplicate::Fail->value, new WhiteList(OnDuplicate::values()), 'Behavior when a row with an existing $id is encountered. "fail" (default): abort on first conflict. "skip": silently ignore. "overwrite": replace existing row.', true)
             ->inject('response')
             ->inject('dbForProject')
             ->inject('dbForPlatform')
@@ -85,6 +88,7 @@ class Create extends Action
         string $fileId,
         string $resourceId,
         bool $internalFile,
+        string $onDuplicate,
         Response $response,
         Database $dbForProject,
         Database $dbForPlatform,
@@ -183,6 +187,7 @@ class Create extends Action
             'options' => [
                 'path' => $newPath,
                 'size' => $fileSize,
+                'onDuplicate' => $onDuplicate,
             ],
         ]));
 

src/Appwrite/Platform/Modules/Migrations/Http/Migrations/JSON/Imports/Create.php

@@ -20,6 +20,7 @@ use Utopia\Database\Document;
 use Utopia\Database\Helpers\ID;
 use Utopia\Database\Validator\Authorization;
 use Utopia\Database\Validator\UID;
+use Utopia\Migration\Destinations\OnDuplicate;
 use Utopia\Migration\Resource;
 use Utopia\Migration\Sources\Appwrite as AppwriteSource;
 use Utopia\Migration\Sources\JSON as JSONSource;
@@ -29,6 +30,7 @@ use Utopia\Platform\Scope\HTTP;
 use Utopia\Storage\Device;
 use Utopia\System\System;
 use Utopia\Validator\Boolean;
+use Utopia\Validator\WhiteList;
 
 class Create extends Action
 {
@@ -66,6 +68,7 @@ class Create extends Action
             ->param('fileId', '', new UID(), 'File ID.')
             ->param('resourceId', null, new CompoundUID(), 'Composite ID in the format {databaseId:collectionId}, identifying a collection within a database.')
             ->param('internalFile', false, new Boolean(), 'Is the file stored in an internal bucket?', true)
+            ->param('onDuplicate', OnDuplicate::Fail->value, new WhiteList(OnDuplicate::values()), 'Behavior when a row with an existing $id is encountered. "fail" (default): abort on first conflict. "skip": silently ignore. "overwrite": replace existing row.', true)
             ->inject('response')
             ->inject('dbForProject')
             ->inject('dbForPlatform')
@@ -84,6 +87,7 @@ class Create extends Action
         string $fileId,
         string $resourceId,
         bool $internalFile,
+        string $onDuplicate,
         Response $response,
         Database $dbForProject,
         Database $dbForPlatform,
@@ -183,6 +187,7 @@ class Create extends Action
             'options' => [
                 'path' => $newPath,
                 'size' => $fileSize,
+                'onDuplicate' => $onDuplicate,
             ],
         ]));
 

src/Appwrite/Platform/Workers/Migrations.php

@@ -30,6 +30,7 @@ use Utopia\Migration\Destination;
 use Utopia\Migration\Destinations\Appwrite as DestinationAppwrite;
 use Utopia\Migration\Destinations\CSV as DestinationCSV;
 use Utopia\Migration\Destinations\JSON as DestinationJSON;
+use Utopia\Migration\Destinations\OnDuplicate;
 use Utopia\Migration\Exception as MigrationException;
 use Utopia\Migration\Resource;
 use Utopia\Migration\Resources\Database\Database as ResourceDatabase;
@@ -291,6 +292,7 @@ class Migrations extends Action
                 $this->dbForProject,
                 $this->getDatabasesDB,
                 Config::getParam('collections', [])['databases']['collections'],
+                OnDuplicate::tryFrom($options['onDuplicate'] ?? '') ?? OnDuplicate::Fail,
             ),
             DestinationCSV::getName() => new DestinationCSV(
                 $this->deviceForFiles,