From 8dfbc128acc173e741c92f7954bc2fdf717aec27 Mon Sep 17 00:00:00 2001 From: Darshan Date: Tue, 8 Apr 2025 11:19:39 +0530 Subject: [PATCH] feat: import csv. --- Dockerfile | 2 +- app/config/collections/projects.php | 24 +++++- app/controllers/api/migrations.php | 90 ++++++++++++++++++++- docs/references/migrations/migration-csv.md | 1 + 4 files changed, 114 insertions(+), 3 deletions(-) create mode 100644 docs/references/migrations/migration-csv.md diff --git a/Dockerfile b/Dockerfile index 88d5ed030b..1fdaaf2f0e 100755 --- a/Dockerfile +++ b/Dockerfile @@ -38,7 +38,7 @@ COPY --from=composer /usr/local/src/vendor /usr/src/code/vendor COPY ./app /usr/src/code/app COPY ./public /usr/src/code/public COPY ./bin /usr/local/bin -COPY ./docs /usr/src/code/docs +#COPY ./docs /usr/src/code/docs COPY ./src /usr/src/code/src COPY ./dev /usr/src/code/dev diff --git a/app/config/collections/projects.php b/app/config/collections/projects.php index ee2c6c0e71..851b467159 100644 --- a/app/config/collections/projects.php +++ b/app/config/collections/projects.php @@ -1848,7 +1848,29 @@ return [ 'default' => null, 'array' => false, 'filters' => [], - ] + ], + [ + '$id' => ID::custom('resourceId'), + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => Database::LENGTH_KEY, + 'signed' => true, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => ID::custom('resourceType'), + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => Database::LENGTH_KEY, + 'signed' => true, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => [], + ], ], 'indexes' => [ [ diff --git a/app/controllers/api/migrations.php b/app/controllers/api/migrations.php index 75afc7ed2c..10fc48bd33 100644 --- a/app/controllers/api/migrations.php +++ b/app/controllers/api/migrations.php @@ -1,5 +1,6 @@ dynamic($migration, Response::MODEL_MIGRATION); }); - App::post('/v1/migrations/firebase') ->groups(['api', 'migrations']) ->desc('Migrate Firebase data') @@ -290,6 +294,90 @@ App::post('/v1/migrations/nhost') ->dynamic($migration, Response::MODEL_MIGRATION); }); +App::post('/v1/migrations/csv') + ->groups(['api', 'migrations']) + ->desc('Import documents from a CSV') + ->label('scope', 'migrations.write') + ->label('event', 'migrations.[migrationId].create') + ->label('audits.event', 'migration.create') + ->label('sdk', new Method( + namespace: 'migrations', + name: 'createCsvMigration', + description: '/docs/references/migrations/migration-csv.md', + auth: [AuthType::ADMIN], + responses: [ + new SDKResponse( + code: Response::STATUS_CODE_ACCEPTED, + model: Response::MODEL_MIGRATION, + ) + ] + )) + ->param('bucketId', '', new UID(), 'Storage bucket unique ID. You can create a new storage bucket using the Storage service [server integration](https://appwrite.io/docs/server/storage#createBucket).') + ->param('fileId', '', new UID(), 'File ID.') + ->param('resourceId', null, new UID(), 'Composite ID in the format {databaseId:collectionId}, identifying a collection within a database.') + ->inject('request') + ->inject('response') + ->inject('dbForProject') + ->inject('project') + ->inject('deviceForFiles') + ->inject('deviceForLocal') + ->inject('$queueForEvents') + ->inject('queueForMigrations') + ->action(function (string $bucketId, string $fileId, string $resourceId, Request $request, Response $response, Database $dbForProject, Document $project, Device $deviceForFiles, Migration $queueForEvents, Migration $queueForMigrations) { + + // TODO: Check if there's already a migrations worker process running for CSV Import for the same collection. + // If so, short-circuit and cancel the task early on because console may not allow it but API will. + // if (inProgress(resourceId)) { + // throw some exception. + //} + + $bucket = Authorization::skip(fn () => $dbForProject->getDocument('buckets', $bucketId)); + + $isAPIKey = Auth::isAppUser(Authorization::getRoles()); + $isPrivilegedUser = Auth::isPrivilegedUser(Authorization::getRoles()); + + if ($bucket->isEmpty() || (!$isAPIKey && !$isPrivilegedUser)) { + throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND); + } + + $file = Authorization::skip(fn () => $dbForProject->getDocument('bucket_' . $bucket->getInternalId(), $fileId)); + if ($file->isEmpty()) { + throw new Exception(Exception::STORAGE_FILE_NOT_FOUND); + } + + $path = $file->getAttribute('path', ''); + + if (!$deviceForFiles->exists($path)) { + throw new Exception(Exception::STORAGE_FILE_NOT_FOUND, 'File not found in ' . $path); + } + + // TODO: send path migrations/csv worker + $migration = $dbForProject->createDocument('migrations', new Document([ + '$id' => ID::unique(), + 'status' => 'pending', + 'stage' => 'init', + // TODO: add stuff to migration library + 'source' => SourcesCSV::getName(), + 'destination' => DestinationsCSV::getName(), + 'resources' => [Resource::TYPE_DOCUMENT], + 'resourceId' => $resourceId, + 'resourceType' => Resource::TYPE_DATABASE, + 'statusCounters' => [], + 'resourceData' => [], + 'errors' => [], + 'credentials' => [], + ])); + + // TODO: use migrationId or importId? + $queueForEvents->setParam('migrationId', $migration->getId()); + + // Trigger Import + $queueForMigrations + ->setMigration($migration) + ->setProject($project) + ->trigger(); + }); + App::get('/v1/migrations') ->groups(['api', 'migrations']) ->desc('List migrations') diff --git a/docs/references/migrations/migration-csv.md b/docs/references/migrations/migration-csv.md new file mode 100644 index 0000000000..7a32d5ff6e --- /dev/null +++ b/docs/references/migrations/migration-csv.md @@ -0,0 +1 @@ +Import documents from a CSV file into your Appwrite database. This endpoint allows you to import documents from a CSV file uploaded to Appwrite Storage bucket. \ No newline at end of file