From bf1af094c139f7de04c26d6ea4ecf3f400e63b41 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Wed, 6 Aug 2025 00:40:39 +1200 Subject: [PATCH] Add specific column selection --- app/controllers/api/migrations.php | 17 ++++++++++------- app/init/resources.php | 2 +- app/worker.php | 2 +- composer.lock | 12 ++++++------ src/Appwrite/Platform/Workers/Migrations.php | 18 ++++++++---------- 5 files changed, 26 insertions(+), 25 deletions(-) diff --git a/app/controllers/api/migrations.php b/app/controllers/api/migrations.php index 51f305d16e..f6dc56a5eb 100644 --- a/app/controllers/api/migrations.php +++ b/app/controllers/api/migrations.php @@ -307,6 +307,7 @@ App::post('/v1/migrations/nhost') }); App::post('/v1/migrations/csv/imports') + ->alias('/v1/migrations/csv') ->groups(['api', 'migrations']) ->desc('Import documents from a CSV') ->label('scope', 'migrations.write') @@ -332,10 +333,10 @@ App::post('/v1/migrations/csv/imports') ->inject('dbForProject') ->inject('project') ->inject('deviceForFiles') - ->inject('deviceForImports') + ->inject('deviceForMigrations') ->inject('queueForEvents') ->inject('queueForMigrations') - ->action(function (string $bucketId, string $fileId, string $resourceId, Response $response, Database $dbForProject, Document $project, Device $deviceForFiles, Device $deviceForImports, Event $queueForEvents, Migration $queueForMigrations) { + ->action(function (string $bucketId, string $fileId, string $resourceId, Response $response, Database $dbForProject, Document $project, Device $deviceForFiles, Device $deviceForMigrations, Event $queueForEvents, Migration $queueForMigrations) { $isAPIKey = Auth::isAppUser(Authorization::getRoles()); $isPrivilegedUser = Auth::isPrivilegedUser(Authorization::getRoles()); @@ -361,7 +362,7 @@ App::post('/v1/migrations/csv/imports') $hasCompression = $compression !== Compression::NONE; $migrationId = ID::unique(); - $newPath = $deviceForImports->getPath($migrationId . '_' . $fileId . '.csv'); + $newPath = $deviceForMigrations->getPath($migrationId . '_' . $fileId . '.csv'); if ($hasEncryption || $hasCompression) { $source = $deviceForFiles->read($path); @@ -391,14 +392,14 @@ App::post('/v1/migrations/csv/imports') } // manual write after decryption and/or decompression - if (! $deviceForImports->write($newPath, $source, 'text/csv')) { + if (! $deviceForMigrations->write($newPath, $source, 'text/csv')) { throw new \Exception("Unable to copy file"); } - } elseif (! $deviceForFiles->transfer($path, $newPath, $deviceForImports)) { + } elseif (! $deviceForFiles->transfer($path, $newPath, $deviceForMigrations)) { throw new \Exception("Unable to copy file"); } - $fileSize = $deviceForImports->getFileSize($newPath); + $fileSize = $deviceForMigrations->getFileSize($newPath); $resources = Transfer::extractServices([Transfer::GROUP_DATABASES]); $migration = $dbForProject->createDocument('migrations', new Document([ @@ -452,12 +453,13 @@ App::post('/v1/migrations/csv/exports') )) ->param('bucketId', '', new UID(), 'Storage bucket unique ID where the exported CSV will be stored.') ->param('resourceId', null, new CompoundUID(), 'Composite ID in the format {databaseId:collectionId}, identifying a collection within a database to export.') + ->param('columns', [], new ArrayList(new Text(255)), 'List of attributes to export. If empty, all attributes will be exported. You can use the `*` wildcard to export all attributes from the collection.') ->inject('response') ->inject('dbForProject') ->inject('project') ->inject('queueForEvents') ->inject('queueForMigrations') - ->action(function (string $bucketId, string $resourceId, Response $response, Database $dbForProject, Document $project, Event $queueForEvents, Migration $queueForMigrations) { + ->action(function (string $bucketId, string $resourceId, array $columns, Response $response, Database $dbForProject, Document $project, Event $queueForEvents, Migration $queueForMigrations) { $isAPIKey = Auth::isAppUser(Authorization::getRoles()); $isPrivilegedUser = Auth::isPrivilegedUser(Authorization::getRoles()); @@ -484,6 +486,7 @@ App::post('/v1/migrations/csv/exports') 'errors' => [], 'options' => [ 'bucketId' => $bucketId, + 'columns' => $columns, ], ])); diff --git a/app/init/resources.php b/app/init/resources.php index 162eab1973..8b7ec941f8 100644 --- a/app/init/resources.php +++ b/app/init/resources.php @@ -514,7 +514,7 @@ App::setResource('deviceForFiles', function ($project, Telemetry $telemetry) { App::setResource('deviceForSites', function ($project, Telemetry $telemetry) { return new Device\Telemetry($telemetry, getDevice(APP_STORAGE_SITES . '/app-' . $project->getId())); }, ['project', 'telemetry']); -App::setResource('deviceForImports', function ($project, Telemetry $telemetry) { +App::setResource('deviceForMigrations', function ($project, Telemetry $telemetry) { return new Device\Telemetry($telemetry, getDevice(APP_STORAGE_IMPORTS . '/app-' . $project->getId())); }, ['project', 'telemetry']); App::setResource('deviceForFunctions', function ($project, Telemetry $telemetry) { diff --git a/app/worker.php b/app/worker.php index 90f3368fe7..9429cb853f 100644 --- a/app/worker.php +++ b/app/worker.php @@ -341,7 +341,7 @@ Server::setResource('deviceForSites', function (Document $project, Telemetry $te return new TelemetryDevice($telemetry, getDevice(APP_STORAGE_SITES . '/app-' . $project->getId())); }, ['project', 'telemetry']); -Server::setResource('deviceForImports', function (Document $project, Telemetry $telemetry) { +Server::setResource('deviceForMigrations', function (Document $project, Telemetry $telemetry) { return new TelemetryDevice($telemetry, getDevice(APP_STORAGE_IMPORTS . '/app-' . $project->getId())); }, ['project', 'telemetry']); diff --git a/composer.lock b/composer.lock index e3b3d2208f..2cd331ff6d 100644 --- a/composer.lock +++ b/composer.lock @@ -3542,16 +3542,16 @@ }, { "name": "utopia-php/database", - "version": "0.71.11", + "version": "0.71.12", "source": { "type": "git", "url": "https://github.com/utopia-php/database.git", - "reference": "644ed827aace63cbdf8c6c64a3998c11b43e3383" + "reference": "72c2a9c185f0f606e4792913a071f744cca21d42" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/database/zipball/644ed827aace63cbdf8c6c64a3998c11b43e3383", - "reference": "644ed827aace63cbdf8c6c64a3998c11b43e3383", + "url": "https://api.github.com/repos/utopia-php/database/zipball/72c2a9c185f0f606e4792913a071f744cca21d42", + "reference": "72c2a9c185f0f606e4792913a071f744cca21d42", "shasum": "" }, "require": { @@ -3592,9 +3592,9 @@ ], "support": { "issues": "https://github.com/utopia-php/database/issues", - "source": "https://github.com/utopia-php/database/tree/0.71.11" + "source": "https://github.com/utopia-php/database/tree/0.71.12" }, - "time": "2025-08-05T08:35:29+00:00" + "time": "2025-08-05T09:38:25+00:00" }, { "name": "utopia-php/detector", diff --git a/src/Appwrite/Platform/Workers/Migrations.php b/src/Appwrite/Platform/Workers/Migrations.php index f878853ed6..4e20c966b1 100644 --- a/src/Appwrite/Platform/Workers/Migrations.php +++ b/src/Appwrite/Platform/Workers/Migrations.php @@ -4,6 +4,7 @@ namespace Appwrite\Platform\Workers; use Ahc\Jwt\JWT; use Appwrite\Event\Realtime; +use Appwrite\Migration\CSV as DestinationCSV; use Exception; use Utopia\CLI\Console; use Utopia\Config\Config; @@ -34,7 +35,7 @@ class Migrations extends Action protected Database $dbForPlatform; - protected Device $deviceForImports; + protected Device $deviceForMigrations; protected Document $project; @@ -68,17 +69,17 @@ class Migrations extends Action ->inject('dbForPlatform') ->inject('logError') ->inject('queueForRealtime') - ->inject('deviceForImports') + ->inject('deviceForMigrations') ->callback($this->action(...)); } /** * @throws Exception */ - public function action(Message $message, Document $project, Database $dbForProject, Database $dbForPlatform, callable $logError, Realtime $queueForRealtime, Device $deviceForImports): void + public function action(Message $message, Document $project, Database $dbForProject, Database $dbForPlatform, callable $logError, Realtime $queueForRealtime, Device $deviceForMigrations): void { $payload = $message->getPayload() ?? []; - $this->deviceForImports = $deviceForImports; + $this->deviceForMigrations = $deviceForMigrations; if (empty($payload)) { throw new Exception('Missing payload'); @@ -146,7 +147,7 @@ class Migrations extends Action CSV::getName() => new CSV( $resourceId, $migrationOptions['path'], - $this->deviceForImports, + $this->deviceForMigrations, $this->dbForProject ), default => throw new \Exception('Invalid source type'), @@ -173,11 +174,9 @@ class Migrations extends Action Config::getParam('collections', [])['databases']['collections'], ), DestinationCSV::getName() => new DestinationCSV( - $this->project, - $this->dbForProject, + $this->deviceForMigrations, $migration->getAttribute('resourceId'), - $migration->getAttribute('options', []), - $this->deviceForImports + $migration->getAttribute('options', [])['columns'] ?? [], ), default => throw new \Exception('Invalid destination type'), }; @@ -211,7 +210,6 @@ class Migrations extends Action // set the errors back without trace $clonedMigrationDocument->setAttribute('errors', $errorMessages); - /** Trigger Realtime Events */ $queueForRealtime ->setProject($project)