From c70915306d6cbf6aaca7c9b1e20a7f3a0920b448 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Tue, 5 Aug 2025 17:04:31 +1200 Subject: [PATCH 01/94] WIP export --- app/controllers/api/migrations.php | 74 +++++++++++++++++++- src/Appwrite/Platform/Workers/Migrations.php | 7 ++ 2 files changed, 78 insertions(+), 3 deletions(-) diff --git a/app/controllers/api/migrations.php b/app/controllers/api/migrations.php index 85751811ba..51f305d16e 100644 --- a/app/controllers/api/migrations.php +++ b/app/controllers/api/migrations.php @@ -306,7 +306,7 @@ App::post('/v1/migrations/nhost') ->dynamic($migration, Response::MODEL_MIGRATION); }); -App::post('/v1/migrations/csv') +App::post('/v1/migrations/csv/imports') ->groups(['api', 'migrations']) ->desc('Import documents from a CSV') ->label('scope', 'migrations.write') @@ -315,8 +315,8 @@ App::post('/v1/migrations/csv') ->label('sdk', new Method( namespace: 'migrations', group: null, - name: 'createCsvMigration', - description: '/docs/references/migrations/migration-csv.md', + name: 'createCsvImportMigration', + description: '/docs/references/migrations/migration-csv-import.md', auth: [AuthType::ADMIN], responses: [ new SDKResponse( @@ -431,6 +431,74 @@ App::post('/v1/migrations/csv') ->dynamic($migration, Response::MODEL_MIGRATION); }); +App::post('/v1/migrations/csv/exports') + ->groups(['api', 'migrations']) + ->desc('Export documents to CSV') + ->label('scope', 'migrations.write') + ->label('event', 'migrations.[migrationId].create') + ->label('audits.event', 'migration.create') + ->label('sdk', new Method( + namespace: 'migrations', + group: null, + name: 'createCsvExportMigration', + description: '/docs/references/migrations/migration-csv-export.md', + auth: [AuthType::ADMIN], + responses: [ + new SDKResponse( + code: Response::STATUS_CODE_ACCEPTED, + model: Response::MODEL_MIGRATION, + ) + ] + )) + ->param('bucketId', '', new UID(), 'Storage bucket unique ID where the exported CSV will be stored.') + ->param('resourceId', null, new CompoundUID(), 'Composite ID in the format {databaseId:collectionId}, identifying a collection within a database to export.') + ->inject('response') + ->inject('dbForProject') + ->inject('project') + ->inject('queueForEvents') + ->inject('queueForMigrations') + ->action(function (string $bucketId, string $resourceId, Response $response, Database $dbForProject, Document $project, Event $queueForEvents, Migration $queueForMigrations) { + $isAPIKey = Auth::isAppUser(Authorization::getRoles()); + $isPrivilegedUser = Auth::isPrivilegedUser(Authorization::getRoles()); + + $bucket = Authorization::skip(fn () => $dbForProject->getDocument('buckets', $bucketId)); + + if ($bucket->isEmpty() || (!$isAPIKey && !$isPrivilegedUser)) { + throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND); + } + + $migrationId = ID::unique(); + $resources = Transfer::extractServices([Transfer::GROUP_DATABASES]); + + $migration = $dbForProject->createDocument('migrations', new Document([ + '$id' => $migrationId, + 'status' => 'pending', + 'stage' => 'init', + 'source' => Appwrite::getName(), + 'destination' => CSV::getName(), + 'resources' => $resources, + 'resourceId' => $resourceId, + 'resourceType' => Resource::TYPE_DATABASE, + 'statusCounters' => '{}', + 'resourceData' => '{}', + 'errors' => [], + 'options' => [ + 'bucketId' => $bucketId, + ], + ])); + + $queueForEvents->setParam('migrationId', $migration->getId()); + + $queueForMigrations + ->setMigration($migration) + ->setProject($project) + ->trigger(); + + $response + ->setStatusCode(Response::STATUS_CODE_ACCEPTED) + ->dynamic($migration, Response::MODEL_MIGRATION); + }); + App::get('/v1/migrations') ->groups(['api', 'migrations']) ->desc('List migrations') diff --git a/src/Appwrite/Platform/Workers/Migrations.php b/src/Appwrite/Platform/Workers/Migrations.php index 807cf5ec9d..f878853ed6 100644 --- a/src/Appwrite/Platform/Workers/Migrations.php +++ b/src/Appwrite/Platform/Workers/Migrations.php @@ -172,6 +172,13 @@ class Migrations extends Action $this->dbForProject, Config::getParam('collections', [])['databases']['collections'], ), + DestinationCSV::getName() => new DestinationCSV( + $this->project, + $this->dbForProject, + $migration->getAttribute('resourceId'), + $migration->getAttribute('options', []), + $this->deviceForImports + ), default => throw new \Exception('Invalid destination type'), }; } From bf1af094c139f7de04c26d6ea4ecf3f400e63b41 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Wed, 6 Aug 2025 00:40:39 +1200 Subject: [PATCH 02/94] Add specific column selection --- app/controllers/api/migrations.php | 17 ++++++++++------- app/init/resources.php | 2 +- app/worker.php | 2 +- composer.lock | 12 ++++++------ src/Appwrite/Platform/Workers/Migrations.php | 18 ++++++++---------- 5 files changed, 26 insertions(+), 25 deletions(-) diff --git a/app/controllers/api/migrations.php b/app/controllers/api/migrations.php index 51f305d16e..f6dc56a5eb 100644 --- a/app/controllers/api/migrations.php +++ b/app/controllers/api/migrations.php @@ -307,6 +307,7 @@ App::post('/v1/migrations/nhost') }); App::post('/v1/migrations/csv/imports') + ->alias('/v1/migrations/csv') ->groups(['api', 'migrations']) ->desc('Import documents from a CSV') ->label('scope', 'migrations.write') @@ -332,10 +333,10 @@ App::post('/v1/migrations/csv/imports') ->inject('dbForProject') ->inject('project') ->inject('deviceForFiles') - ->inject('deviceForImports') + ->inject('deviceForMigrations') ->inject('queueForEvents') ->inject('queueForMigrations') - ->action(function (string $bucketId, string $fileId, string $resourceId, Response $response, Database $dbForProject, Document $project, Device $deviceForFiles, Device $deviceForImports, Event $queueForEvents, Migration $queueForMigrations) { + ->action(function (string $bucketId, string $fileId, string $resourceId, Response $response, Database $dbForProject, Document $project, Device $deviceForFiles, Device $deviceForMigrations, Event $queueForEvents, Migration $queueForMigrations) { $isAPIKey = Auth::isAppUser(Authorization::getRoles()); $isPrivilegedUser = Auth::isPrivilegedUser(Authorization::getRoles()); @@ -361,7 +362,7 @@ App::post('/v1/migrations/csv/imports') $hasCompression = $compression !== Compression::NONE; $migrationId = ID::unique(); - $newPath = $deviceForImports->getPath($migrationId . '_' . $fileId . '.csv'); + $newPath = $deviceForMigrations->getPath($migrationId . '_' . $fileId . '.csv'); if ($hasEncryption || $hasCompression) { $source = $deviceForFiles->read($path); @@ -391,14 +392,14 @@ App::post('/v1/migrations/csv/imports') } // manual write after decryption and/or decompression - if (! $deviceForImports->write($newPath, $source, 'text/csv')) { + if (! $deviceForMigrations->write($newPath, $source, 'text/csv')) { throw new \Exception("Unable to copy file"); } - } elseif (! $deviceForFiles->transfer($path, $newPath, $deviceForImports)) { + } elseif (! $deviceForFiles->transfer($path, $newPath, $deviceForMigrations)) { throw new \Exception("Unable to copy file"); } - $fileSize = $deviceForImports->getFileSize($newPath); + $fileSize = $deviceForMigrations->getFileSize($newPath); $resources = Transfer::extractServices([Transfer::GROUP_DATABASES]); $migration = $dbForProject->createDocument('migrations', new Document([ @@ -452,12 +453,13 @@ App::post('/v1/migrations/csv/exports') )) ->param('bucketId', '', new UID(), 'Storage bucket unique ID where the exported CSV will be stored.') ->param('resourceId', null, new CompoundUID(), 'Composite ID in the format {databaseId:collectionId}, identifying a collection within a database to export.') + ->param('columns', [], new ArrayList(new Text(255)), 'List of attributes to export. If empty, all attributes will be exported. You can use the `*` wildcard to export all attributes from the collection.') ->inject('response') ->inject('dbForProject') ->inject('project') ->inject('queueForEvents') ->inject('queueForMigrations') - ->action(function (string $bucketId, string $resourceId, Response $response, Database $dbForProject, Document $project, Event $queueForEvents, Migration $queueForMigrations) { + ->action(function (string $bucketId, string $resourceId, array $columns, Response $response, Database $dbForProject, Document $project, Event $queueForEvents, Migration $queueForMigrations) { $isAPIKey = Auth::isAppUser(Authorization::getRoles()); $isPrivilegedUser = Auth::isPrivilegedUser(Authorization::getRoles()); @@ -484,6 +486,7 @@ App::post('/v1/migrations/csv/exports') 'errors' => [], 'options' => [ 'bucketId' => $bucketId, + 'columns' => $columns, ], ])); diff --git a/app/init/resources.php b/app/init/resources.php index 162eab1973..8b7ec941f8 100644 --- a/app/init/resources.php +++ b/app/init/resources.php @@ -514,7 +514,7 @@ App::setResource('deviceForFiles', function ($project, Telemetry $telemetry) { App::setResource('deviceForSites', function ($project, Telemetry $telemetry) { return new Device\Telemetry($telemetry, getDevice(APP_STORAGE_SITES . '/app-' . $project->getId())); }, ['project', 'telemetry']); -App::setResource('deviceForImports', function ($project, Telemetry $telemetry) { +App::setResource('deviceForMigrations', function ($project, Telemetry $telemetry) { return new Device\Telemetry($telemetry, getDevice(APP_STORAGE_IMPORTS . '/app-' . $project->getId())); }, ['project', 'telemetry']); App::setResource('deviceForFunctions', function ($project, Telemetry $telemetry) { diff --git a/app/worker.php b/app/worker.php index 90f3368fe7..9429cb853f 100644 --- a/app/worker.php +++ b/app/worker.php @@ -341,7 +341,7 @@ Server::setResource('deviceForSites', function (Document $project, Telemetry $te return new TelemetryDevice($telemetry, getDevice(APP_STORAGE_SITES . '/app-' . $project->getId())); }, ['project', 'telemetry']); -Server::setResource('deviceForImports', function (Document $project, Telemetry $telemetry) { +Server::setResource('deviceForMigrations', function (Document $project, Telemetry $telemetry) { return new TelemetryDevice($telemetry, getDevice(APP_STORAGE_IMPORTS . '/app-' . $project->getId())); }, ['project', 'telemetry']); diff --git a/composer.lock b/composer.lock index e3b3d2208f..2cd331ff6d 100644 --- a/composer.lock +++ b/composer.lock @@ -3542,16 +3542,16 @@ }, { "name": "utopia-php/database", - "version": "0.71.11", + "version": "0.71.12", "source": { "type": "git", "url": "https://github.com/utopia-php/database.git", - "reference": "644ed827aace63cbdf8c6c64a3998c11b43e3383" + "reference": "72c2a9c185f0f606e4792913a071f744cca21d42" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/database/zipball/644ed827aace63cbdf8c6c64a3998c11b43e3383", - "reference": "644ed827aace63cbdf8c6c64a3998c11b43e3383", + "url": "https://api.github.com/repos/utopia-php/database/zipball/72c2a9c185f0f606e4792913a071f744cca21d42", + "reference": "72c2a9c185f0f606e4792913a071f744cca21d42", "shasum": "" }, "require": { @@ -3592,9 +3592,9 @@ ], "support": { "issues": "https://github.com/utopia-php/database/issues", - "source": "https://github.com/utopia-php/database/tree/0.71.11" + "source": "https://github.com/utopia-php/database/tree/0.71.12" }, - "time": "2025-08-05T08:35:29+00:00" + "time": "2025-08-05T09:38:25+00:00" }, { "name": "utopia-php/detector", diff --git a/src/Appwrite/Platform/Workers/Migrations.php b/src/Appwrite/Platform/Workers/Migrations.php index f878853ed6..4e20c966b1 100644 --- a/src/Appwrite/Platform/Workers/Migrations.php +++ b/src/Appwrite/Platform/Workers/Migrations.php @@ -4,6 +4,7 @@ namespace Appwrite\Platform\Workers; use Ahc\Jwt\JWT; use Appwrite\Event\Realtime; +use Appwrite\Migration\CSV as DestinationCSV; use Exception; use Utopia\CLI\Console; use Utopia\Config\Config; @@ -34,7 +35,7 @@ class Migrations extends Action protected Database $dbForPlatform; - protected Device $deviceForImports; + protected Device $deviceForMigrations; protected Document $project; @@ -68,17 +69,17 @@ class Migrations extends Action ->inject('dbForPlatform') ->inject('logError') ->inject('queueForRealtime') - ->inject('deviceForImports') + ->inject('deviceForMigrations') ->callback($this->action(...)); } /** * @throws Exception */ - public function action(Message $message, Document $project, Database $dbForProject, Database $dbForPlatform, callable $logError, Realtime $queueForRealtime, Device $deviceForImports): void + public function action(Message $message, Document $project, Database $dbForProject, Database $dbForPlatform, callable $logError, Realtime $queueForRealtime, Device $deviceForMigrations): void { $payload = $message->getPayload() ?? []; - $this->deviceForImports = $deviceForImports; + $this->deviceForMigrations = $deviceForMigrations; if (empty($payload)) { throw new Exception('Missing payload'); @@ -146,7 +147,7 @@ class Migrations extends Action CSV::getName() => new CSV( $resourceId, $migrationOptions['path'], - $this->deviceForImports, + $this->deviceForMigrations, $this->dbForProject ), default => throw new \Exception('Invalid source type'), @@ -173,11 +174,9 @@ class Migrations extends Action Config::getParam('collections', [])['databases']['collections'], ), DestinationCSV::getName() => new DestinationCSV( - $this->project, - $this->dbForProject, + $this->deviceForMigrations, $migration->getAttribute('resourceId'), - $migration->getAttribute('options', []), - $this->deviceForImports + $migration->getAttribute('options', [])['columns'] ?? [], ), default => throw new \Exception('Invalid destination type'), }; @@ -211,7 +210,6 @@ class Migrations extends Action // set the errors back without trace $clonedMigrationDocument->setAttribute('errors', $errorMessages); - /** Trigger Realtime Events */ $queueForRealtime ->setProject($project) From 892cfe01b2ebb34e144caccb95f729ee4d607377 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Thu, 7 Aug 2025 00:47:11 +1200 Subject: [PATCH 03/94] Optional columns --- app/controllers/api/migrations.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/controllers/api/migrations.php b/app/controllers/api/migrations.php index f6dc56a5eb..36a920a56e 100644 --- a/app/controllers/api/migrations.php +++ b/app/controllers/api/migrations.php @@ -453,7 +453,7 @@ App::post('/v1/migrations/csv/exports') )) ->param('bucketId', '', new UID(), 'Storage bucket unique ID where the exported CSV will be stored.') ->param('resourceId', null, new CompoundUID(), 'Composite ID in the format {databaseId:collectionId}, identifying a collection within a database to export.') - ->param('columns', [], new ArrayList(new Text(255)), 'List of attributes to export. If empty, all attributes will be exported. You can use the `*` wildcard to export all attributes from the collection.') + ->param('columns', [], new ArrayList(new Text(255)), 'List of attributes to export. If empty, all attributes will be exported. You can use the `*` wildcard to export all attributes from the collection.', true) ->inject('response') ->inject('dbForProject') ->inject('project') From 57a46b98ecd92e630b70d81986da92db917f97eb Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Fri, 22 Aug 2025 18:01:38 +1200 Subject: [PATCH 04/94] Use utopia adapter --- composer.json | 2 +- composer.lock | 27 +++++++++++++------- src/Appwrite/Platform/Workers/Migrations.php | 2 +- 3 files changed, 20 insertions(+), 11 deletions(-) diff --git a/composer.json b/composer.json index 0c662c775f..fcc0acb213 100644 --- a/composer.json +++ b/composer.json @@ -63,7 +63,7 @@ "utopia-php/locale": "0.8.*", "utopia-php/logger": "0.6.*", "utopia-php/messaging": "0.18.*", - "utopia-php/migration": "1.*", + "utopia-php/migration": "dev-feat-csv-export as 1.0.0", "utopia-php/orchestration": "0.9.*", "utopia-php/platform": "0.7.*", "utopia-php/pools": "0.8.*", diff --git a/composer.lock b/composer.lock index b7e9a76088..0dffee1bde 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "0da713ee5642eba1d30bc51c1a04a723", + "content-hash": "954529a36566209d6687df9f41a0f2e6", "packages": [ { "name": "adhocore/jwt", @@ -4109,16 +4109,16 @@ }, { "name": "utopia-php/migration", - "version": "1.0.0", + "version": "dev-feat-csv-export", "source": { "type": "git", "url": "https://github.com/utopia-php/migration.git", - "reference": "0e4499d9dd2c90c2be188cc5fb7a32d9a892b569" + "reference": "8435f1db0db4854ca27cb4c9cf275b905fcb3b41" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/migration/zipball/0e4499d9dd2c90c2be188cc5fb7a32d9a892b569", - "reference": "0e4499d9dd2c90c2be188cc5fb7a32d9a892b569", + "url": "https://api.github.com/repos/utopia-php/migration/zipball/8435f1db0db4854ca27cb4c9cf275b905fcb3b41", + "reference": "8435f1db0db4854ca27cb4c9cf275b905fcb3b41", "shasum": "" }, "require": { @@ -4159,9 +4159,9 @@ ], "support": { "issues": "https://github.com/utopia-php/migration/issues", - "source": "https://github.com/utopia-php/migration/tree/1.0.0" + "source": "https://github.com/utopia-php/migration/tree/feat-csv-export" }, - "time": "2025-08-13T09:15:53+00:00" + "time": "2025-08-21T12:56:18+00:00" }, { "name": "utopia-php/orchestration", @@ -8425,9 +8425,18 @@ "time": "2024-03-07T20:33:40+00:00" } ], - "aliases": [], + "aliases": [ + { + "package": "utopia-php/migration", + "version": "dev-feat-csv-export", + "alias": "1.0.0", + "alias_normalized": "1.0.0.0" + } + ], "minimum-stability": "stable", - "stability-flags": {}, + "stability-flags": { + "utopia-php/migration": 20 + }, "prefer-stable": false, "prefer-lowest": false, "platform": { diff --git a/src/Appwrite/Platform/Workers/Migrations.php b/src/Appwrite/Platform/Workers/Migrations.php index 23b8cf7ba3..33af785ab5 100644 --- a/src/Appwrite/Platform/Workers/Migrations.php +++ b/src/Appwrite/Platform/Workers/Migrations.php @@ -4,7 +4,7 @@ namespace Appwrite\Platform\Workers; use Ahc\Jwt\JWT; use Appwrite\Event\Realtime; -use Appwrite\Migration\CSV as DestinationCSV; +use Utopia\Migration\Destinations\CSV as DestinationCSV; use Exception; use Utopia\CLI\Console; use Utopia\Config\Config; From 1722e9e416ad998fed889c098526a01f76493848 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Fri, 22 Aug 2025 18:03:10 +1200 Subject: [PATCH 05/94] Fix DB read source --- src/Appwrite/Platform/Workers/Migrations.php | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/Appwrite/Platform/Workers/Migrations.php b/src/Appwrite/Platform/Workers/Migrations.php index 33af785ab5..ce9b7e2881 100644 --- a/src/Appwrite/Platform/Workers/Migrations.php +++ b/src/Appwrite/Platform/Workers/Migrations.php @@ -18,6 +18,7 @@ use Utopia\Migration\Destination; use Utopia\Migration\Destinations\Appwrite as DestinationAppwrite; use Utopia\Migration\Exception as MigrationException; use Utopia\Migration\Source; +use Utopia\Migration\Sources\Appwrite; use Utopia\Migration\Sources\Appwrite as SourceAppwrite; use Utopia\Migration\Sources\CSV; use Utopia\Migration\Sources\Firebase; @@ -113,9 +114,17 @@ class Migrations extends Action protected function processSource(Document $migration): Source { $source = $migration->getAttribute('source'); + $destination = $migration->getAttribute('destination'); $resourceId = $migration->getAttribute('resourceId'); $credentials = $migration->getAttribute('credentials'); $migrationOptions = $migration->getAttribute('options'); + $dataSource = Appwrite::SOURCE_API; + $database = null; + + if ($source === Appwrite::getName() && $destination === DestinationCSV::getName()) { + $dataSource = Appwrite::SOURCE_DATABASE; + $database = $this->dbForProject; + } $migrationSource = match ($source) { Firebase::getName() => new Firebase( @@ -143,6 +152,8 @@ class Migrations extends Action $credentials['projectId'], $credentials['endpoint'] === 'http://localhost/v1' ? 'http://appwrite/v1' : $credentials['endpoint'], $credentials['apiKey'], + $dataSource, + $database ), CSV::getName() => new CSV( $resourceId, @@ -251,7 +262,9 @@ class Migrations extends Action 'functions.write', 'databases.read', 'collections.read', + 'collections.write', 'tables.read', + 'tables.write', 'documents.read', 'documents.write', 'rows.read', From 679de3574f23cbd091405f5c99150332bac72b89 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 15 Sep 2025 19:11:33 +0000 Subject: [PATCH 06/94] Initial plan From fea3544d4bd7e89f4e3fd5dddccbc662a6ad5c43 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 15 Sep 2025 19:18:07 +0000 Subject: [PATCH 07/94] Implement dynamic specification defaults for Functions and Sites Co-authored-by: stnguyen90 <1477010+stnguyen90@users.noreply.github.com> --- .../Platform/Modules/Compute/Base.php | 44 +++++++++++++++++++ .../Functions/Http/Functions/Create.php | 2 +- .../Functions/Http/Functions/Update.php | 2 +- .../Modules/Sites/Http/Sites/Create.php | 2 +- .../Modules/Sites/Http/Sites/Update.php | 2 +- 5 files changed, 48 insertions(+), 4 deletions(-) diff --git a/src/Appwrite/Platform/Modules/Compute/Base.php b/src/Appwrite/Platform/Modules/Compute/Base.php index 92805fbaf8..7115747fb8 100644 --- a/src/Appwrite/Platform/Modules/Compute/Base.php +++ b/src/Appwrite/Platform/Modules/Compute/Base.php @@ -5,6 +5,7 @@ namespace Appwrite\Platform\Modules\Compute; use Appwrite\Event\Build; use Appwrite\Extend\Exception; use Appwrite\Platform\Action; +use Utopia\Config\Config; use Utopia\Database\Database; use Utopia\Database\Document; use Utopia\Database\Exception\Duplicate; @@ -19,6 +20,49 @@ use Utopia\VCS\Exception\RepositoryNotFound; class Base extends Action { + /** + * Get default specification based on plan and available specifications. + * + * @param array $plan The billing plan configuration + * @return string The appropriate default specification + */ + protected function getDefaultSpecification(array $plan): string + { + $specifications = Config::getParam('specifications', []); + + if (empty($specifications)) { + return APP_COMPUTE_SPECIFICATION_DEFAULT; + } + + // If there's a plan with runtime specifications, use the highest one from the plan + if (!empty($plan) && array_key_exists('runtimeSpecifications', $plan) && !empty($plan['runtimeSpecifications'])) { + $planSpecifications = $plan['runtimeSpecifications']; + // Find the highest specification in the plan + foreach (array_reverse(array_keys($specifications)) as $specKey) { + if (in_array($specKey, $planSpecifications)) { + return $specKey; + } + } + } + + // If no plan or plan-based specification, use the highest available specification + $maxCpus = (float) System::getEnv('_APP_COMPUTE_CPUS', 0); + $maxMemory = (int) System::getEnv('_APP_COMPUTE_MEMORY', 0); + + $highestSpec = APP_COMPUTE_SPECIFICATION_DEFAULT; + foreach (array_reverse(array_keys($specifications)) as $specKey) { + $spec = $specifications[$specKey]; + $withinCpuLimit = empty($maxCpus) || $spec['cpus'] <= $maxCpus; + $withinMemoryLimit = empty($maxMemory) || $spec['memory'] <= $maxMemory; + + if ($withinCpuLimit && $withinMemoryLimit) { + $highestSpec = $specKey; + break; + } + } + + return $highestSpec; + } public function redeployVcsFunction(Request $request, Document $function, Document $project, Document $installation, Database $dbForProject, Build $queueForBuilds, Document $template, GitHub $github, bool $activate, string $referenceType = 'branch', string $reference = ''): Document { $deploymentId = ID::unique(); diff --git a/src/Appwrite/Platform/Modules/Functions/Http/Functions/Create.php b/src/Appwrite/Platform/Modules/Functions/Http/Functions/Create.php index 21a74f9a81..8e3e0c3772 100644 --- a/src/Appwrite/Platform/Modules/Functions/Http/Functions/Create.php +++ b/src/Appwrite/Platform/Modules/Functions/Http/Functions/Create.php @@ -92,7 +92,7 @@ class Create extends Base ->param('providerBranch', '', new Text(128, 0), 'Production branch for the repo linked to the function.', true) ->param('providerSilentMode', false, new Boolean(), 'Is the VCS (Version Control System) connection in silent mode for the repo linked to the function? In silent mode, comments will not be made on commits and pull requests.', true) ->param('providerRootDirectory', '', new Text(128, 0), 'Path to function code in the linked repo.', true) - ->param('specification', APP_COMPUTE_SPECIFICATION_DEFAULT, fn (array $plan) => new Specification( + ->param('specification', fn (array $plan) => $this->getDefaultSpecification($plan), fn (array $plan) => new Specification( $plan, Config::getParam('specifications', []), System::getEnv('_APP_COMPUTE_CPUS', 0), diff --git a/src/Appwrite/Platform/Modules/Functions/Http/Functions/Update.php b/src/Appwrite/Platform/Modules/Functions/Http/Functions/Update.php index aaff953af0..318c2a2032 100644 --- a/src/Appwrite/Platform/Modules/Functions/Http/Functions/Update.php +++ b/src/Appwrite/Platform/Modules/Functions/Http/Functions/Update.php @@ -89,7 +89,7 @@ class Update extends Base ->param('providerBranch', '', new Text(128, 0), 'Production branch for the repo linked to the function', true) ->param('providerSilentMode', false, new Boolean(), 'Is the VCS (Version Control System) connection in silent mode for the repo linked to the function? In silent mode, comments will not be made on commits and pull requests.', true) ->param('providerRootDirectory', '', new Text(128, 0), 'Path to function code in the linked repo.', true) - ->param('specification', APP_COMPUTE_SPECIFICATION_DEFAULT, fn (array $plan) => new Specification( + ->param('specification', fn (array $plan) => $this->getDefaultSpecification($plan), fn (array $plan) => new Specification( $plan, Config::getParam('specifications', []), System::getEnv('_APP_COMPUTE_CPUS', 0), diff --git a/src/Appwrite/Platform/Modules/Sites/Http/Sites/Create.php b/src/Appwrite/Platform/Modules/Sites/Http/Sites/Create.php index 9be95441cb..a1633b8eba 100644 --- a/src/Appwrite/Platform/Modules/Sites/Http/Sites/Create.php +++ b/src/Appwrite/Platform/Modules/Sites/Http/Sites/Create.php @@ -78,7 +78,7 @@ class Create extends Base ->param('providerBranch', '', new Text(128, 0), 'Production branch for the repo linked to the site.', true) ->param('providerSilentMode', false, new Boolean(), 'Is the VCS (Version Control System) connection in silent mode for the repo linked to the site? In silent mode, comments will not be made on commits and pull requests.', true) ->param('providerRootDirectory', '', new Text(128, 0), 'Path to site code in the linked repo.', true) - ->param('specification', APP_COMPUTE_SPECIFICATION_DEFAULT, fn (array $plan) => new Specification( + ->param('specification', fn (array $plan) => $this->getDefaultSpecification($plan), fn (array $plan) => new Specification( $plan, Config::getParam('specifications', []), System::getEnv('_APP_COMPUTE_CPUS', 0), diff --git a/src/Appwrite/Platform/Modules/Sites/Http/Sites/Update.php b/src/Appwrite/Platform/Modules/Sites/Http/Sites/Update.php index 80354d5067..72ec04a2a5 100644 --- a/src/Appwrite/Platform/Modules/Sites/Http/Sites/Update.php +++ b/src/Appwrite/Platform/Modules/Sites/Http/Sites/Update.php @@ -82,7 +82,7 @@ class Update extends Base ->param('providerBranch', '', new Text(128, 0), 'Production branch for the repo linked to the site.', true) ->param('providerSilentMode', false, new Boolean(), 'Is the VCS (Version Control System) connection in silent mode for the repo linked to the site? In silent mode, comments will not be made on commits and pull requests.', true) ->param('providerRootDirectory', '', new Text(128, 0), 'Path to site code in the linked repo.', true) - ->param('specification', APP_COMPUTE_SPECIFICATION_DEFAULT, fn (array $plan) => new Specification( + ->param('specification', fn (array $plan) => $this->getDefaultSpecification($plan), fn (array $plan) => new Specification( $plan, Config::getParam('specifications', []), System::getEnv('_APP_COMPUTE_CPUS', 0), From 62014fda26633c1c77c4129ccf8f28ade1d85b93 Mon Sep 17 00:00:00 2001 From: Steven Nguyen <1477010+stnguyen90@users.noreply.github.com> Date: Mon, 15 Sep 2025 21:29:44 +0000 Subject: [PATCH 08/94] fix: implement dynamic specification defaults for Functions and Sites Prior to this, self-hosted instances would defualt to the lowest specification. Now, if there is no plan (ie. on self-hosted), the highest specification is used. If there is a plan, the lowest specification available in the plan is used. --- .../Platform/Modules/Compute/Base.php | 43 +++++++------------ 1 file changed, 16 insertions(+), 27 deletions(-) diff --git a/src/Appwrite/Platform/Modules/Compute/Base.php b/src/Appwrite/Platform/Modules/Compute/Base.php index 7115747fb8..a538eb1497 100644 --- a/src/Appwrite/Platform/Modules/Compute/Base.php +++ b/src/Appwrite/Platform/Modules/Compute/Base.php @@ -5,6 +5,7 @@ namespace Appwrite\Platform\Modules\Compute; use Appwrite\Event\Build; use Appwrite\Extend\Exception; use Appwrite\Platform\Action; +use Appwrite\Platform\Modules\Compute\Validator\Specification as SpecificationValidator; use Utopia\Config\Config; use Utopia\Database\Database; use Utopia\Database\Document; @@ -29,40 +30,28 @@ class Base extends Action protected function getDefaultSpecification(array $plan): string { $specifications = Config::getParam('specifications', []); - + if (empty($specifications)) { return APP_COMPUTE_SPECIFICATION_DEFAULT; } - // If there's a plan with runtime specifications, use the highest one from the plan - if (!empty($plan) && array_key_exists('runtimeSpecifications', $plan) && !empty($plan['runtimeSpecifications'])) { - $planSpecifications = $plan['runtimeSpecifications']; - // Find the highest specification in the plan - foreach (array_reverse(array_keys($specifications)) as $specKey) { - if (in_array($specKey, $planSpecifications)) { - return $specKey; - } - } + $specificationValidator = new SpecificationValidator( + $plan, + $specifications, + System::getEnv('_APP_COMPUTE_CPUS', 0), + System::getEnv('_APP_COMPUTE_MEMORY', 0) + ); + $allowedSpecifications = $specificationValidator->getAllowedSpecifications(); + + // If there is no plan use the highest specification + if (empty($plan)) { + return end($allowedSpecifications) ?? APP_COMPUTE_SPECIFICATION_DEFAULT; } - // If no plan or plan-based specification, use the highest available specification - $maxCpus = (float) System::getEnv('_APP_COMPUTE_CPUS', 0); - $maxMemory = (int) System::getEnv('_APP_COMPUTE_MEMORY', 0); - - $highestSpec = APP_COMPUTE_SPECIFICATION_DEFAULT; - foreach (array_reverse(array_keys($specifications)) as $specKey) { - $spec = $specifications[$specKey]; - $withinCpuLimit = empty($maxCpus) || $spec['cpus'] <= $maxCpus; - $withinMemoryLimit = empty($maxMemory) || $spec['memory'] <= $maxMemory; - - if ($withinCpuLimit && $withinMemoryLimit) { - $highestSpec = $specKey; - break; - } - } - - return $highestSpec; + // Otherwise, use the lowest specification available in the plan + return $allowedSpecifications[0] ?? APP_COMPUTE_SPECIFICATION_DEFAULT; } + public function redeployVcsFunction(Request $request, Document $function, Document $project, Document $installation, Database $dbForProject, Build $queueForBuilds, Document $template, GitHub $github, bool $activate, string $referenceType = 'branch', string $reference = ''): Document { $deploymentId = ID::unique(); From 536cff9cc65a8836d0b5821c1bec231b44913845 Mon Sep 17 00:00:00 2001 From: Steven Nguyen <1477010+stnguyen90@users.noreply.github.com> Date: Mon, 15 Sep 2025 21:30:18 +0000 Subject: [PATCH 09/94] chore: add specification validator test --- .../Compute/Validator/SpecificationTest.php | 75 +++++++++++++++++++ 1 file changed, 75 insertions(+) create mode 100644 tests/unit/Platform/Modules/Compute/Validator/SpecificationTest.php diff --git a/tests/unit/Platform/Modules/Compute/Validator/SpecificationTest.php b/tests/unit/Platform/Modules/Compute/Validator/SpecificationTest.php new file mode 100644 index 0000000000..0505494e17 --- /dev/null +++ b/tests/unit/Platform/Modules/Compute/Validator/SpecificationTest.php @@ -0,0 +1,75 @@ +specifications = Config::getParam('specifications', []); + } + + public function testGetAllowedSpecificationsNoLimits(): void + { + $validator = new Specification( + plan: [], + specifications: $this->specifications, + maxCpus: 0, + maxMemory: 0 + ); + + $allowed = $validator->getAllowedSpecifications(); + $this->assertCount(count($this->specifications), $allowed); + $this->assertEquals( + $this->specifications[array_key_last($this->specifications)]['slug'], + $allowed[array_key_last($allowed)] + ); + } + + public function testGetAllowedSpecificationsWithMaxCpusAndMemory(): void + { + $validator = new Specification( + plan: [], + specifications: $this->specifications, + maxCpus: 2, + maxMemory: 2048 + ); + + $allowed = $validator->getAllowedSpecifications(); + $this->assertCount(4, $allowed); + $this->assertEquals( + SpecificationConstants::S_2VCPU_2GB, + $allowed[array_key_last($allowed)] + ); + } + + public function testGetAllowedSpecificationsWithPlanLimits(): void + { + $plan = [ + 'runtimeSpecifications' => [ + SpecificationConstants::S_05VCPU_512MB, + SpecificationConstants::S_1VCPU_512MB + ] + ]; + $validator = new Specification( + plan: $plan, + specifications: $this->specifications, + maxCpus: 0, + maxMemory: 0 + ); + + $allowed = $validator->getAllowedSpecifications(); + $this->assertCount(2, $allowed); + $this->assertContains(SpecificationConstants::S_05VCPU_512MB, $allowed); + $this->assertContains(SpecificationConstants::S_1VCPU_512MB, $allowed); + } +} From 5d5c1d8f43286c5baec29183ed77f2583c5d9bcb Mon Sep 17 00:00:00 2001 From: Khushboo Verma Date: Mon, 22 Sep 2025 20:29:25 +0530 Subject: [PATCH 10/94] Fix Author URL in template deployments --- app/init/constants.php | 1 + src/Appwrite/Platform/Modules/Functions/Workers/Builds.php | 5 ++--- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/app/init/constants.php b/app/init/constants.php index 28cf8a4052..6d723e9182 100644 --- a/app/init/constants.php +++ b/app/init/constants.php @@ -85,6 +85,7 @@ const APP_PLATFORM_CLIENT = 'client'; const APP_PLATFORM_CONSOLE = 'console'; const APP_VCS_GITHUB_USERNAME = 'Appwrite'; const APP_VCS_GITHUB_EMAIL = 'team@appwrite.io'; +const APP_VCS_GITHUB_URL = 'https://github.com/TeamAppwrite'; // Database Reconnect const DATABASE_RECONNECT_SLEEP = 2; diff --git a/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php b/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php index 9547a752ef..c0d55aa7fb 100644 --- a/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php +++ b/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php @@ -467,11 +467,10 @@ class Builds extends Action } $providerCommitHash = \trim($stdout); - $authorUrl = "https://github.com/$cloneOwner"; $deployment->setAttribute('providerCommitHash', $providerCommitHash ?? ''); - $deployment->setAttribute('providerCommitAuthorUrl', $authorUrl); - $deployment->setAttribute('providerCommitAuthor', 'Appwrite'); + $deployment->setAttribute('providerCommitAuthorUrl', APP_VCS_GITHUB_URL); + $deployment->setAttribute('providerCommitAuthor', APP_VCS_GITHUB_USERNAME); $deployment->setAttribute('providerCommitMessage', "Create '" . $resource->getAttribute('name', '') . "' function"); $deployment->setAttribute('providerCommitUrl', "https://github.com/$cloneOwner/$cloneRepository/commit/$providerCommitHash"); $deployment = $dbForProject->updateDocument('deployments', $deployment->getId(), $deployment); From c8993d7f713a073ead1bfca7c54ee130863ea30c Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Wed, 24 Sep 2025 22:37:12 +1200 Subject: [PATCH 11/94] Add additional export params --- app/controllers/api/migrations.php | 101 +++++++++++++++++++++-------- 1 file changed, 74 insertions(+), 27 deletions(-) diff --git a/app/controllers/api/migrations.php b/app/controllers/api/migrations.php index d5bcfc2fd7..6e0ecd218d 100644 --- a/app/controllers/api/migrations.php +++ b/app/controllers/api/migrations.php @@ -339,12 +339,20 @@ App::post('/v1/migrations/csv/imports') ->inject('deviceForMigrations') ->inject('queueForEvents') ->inject('queueForMigrations') - ->action(function (string $bucketId, string $fileId, string $resourceId, bool $internalFile, Response $response, Database $dbForProject, Document $project, Device $deviceForFiles, Device $deviceForMigrations, Event $queueForEvents, Migration $queueForMigrations) { - $isAPIKey = Auth::isAppUser(Authorization::getRoles()); - $isPrivilegedUser = Auth::isPrivilegedUser(Authorization::getRoles()); - if ($internalFile && !$isPrivilegedUser) { - throw new Exception(Exception::USER_UNAUTHORIZED); - } + ->action(function ( + string $bucketId, + string $fileId, + string $resourceId, + bool $internalFile, + Response $response, + Database $dbForProject, + Database $dbForPlatform, + Document $project, + Device $deviceForFiles, + Device $deviceForMigrations, + Event $queueForEvents, + Migration $queueForMigrations + ) { $bucket = Authorization::skip(function () use ($internalFile, $dbForPlatform, $dbForProject, $bucketId) { if ($internalFile) { return $dbForPlatform->getDocument('buckets', 'default'); @@ -352,7 +360,7 @@ App::post('/v1/migrations/csv/imports') return $dbForProject->getDocument('buckets', $bucketId); }); - if ($bucket->isEmpty() || (!$isAPIKey && !$isPrivilegedUser)) { + if ($bucket->isEmpty()) { throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND); } @@ -366,7 +374,7 @@ App::post('/v1/migrations/csv/imports') throw new Exception(Exception::STORAGE_FILE_NOT_FOUND, 'File not found in ' . $path); } - // no encryption, compression on files above 20MB. + // No encryption or compression on files above 20MB. $hasEncryption = !empty($file->getAttribute('openSSLCipher')); $compression = $file->getAttribute('algorithm', Compression::NONE); $hasCompression = $compression !== Compression::NONE; @@ -377,7 +385,6 @@ App::post('/v1/migrations/csv/imports') if ($hasEncryption || $hasCompression) { $source = $deviceForFiles->read($path); - // 1. decrypt if ($hasEncryption) { $source = OpenSSL::decrypt( $source, @@ -389,7 +396,6 @@ App::post('/v1/migrations/csv/imports') ); } - // 2. decompress if ($hasCompression) { switch ($compression) { case Compression::ZSTD: @@ -401,12 +407,12 @@ App::post('/v1/migrations/csv/imports') } } - // manual write after decryption and/or decompression - if (! $deviceForMigrations->write($newPath, $source, 'text/csv')) { - throw new \Exception("Unable to copy file"); + // Manual write after decryption and/or decompression + if (!$deviceForMigrations->write($newPath, $source, 'text/csv')) { + throw new \Exception('Unable to copy file'); } - } elseif (! $deviceForFiles->transfer($path, $newPath, $deviceForMigrations)) { - throw new \Exception("Unable to copy file"); + } elseif (!$deviceForFiles->transfer($path, $newPath, $deviceForMigrations)) { + throw new \Exception('Unable to copy file'); } $fileSize = $deviceForMigrations->getFileSize($newPath); @@ -461,34 +467,68 @@ App::post('/v1/migrations/csv/exports') ) ] )) - ->param('bucketId', '', new UID(), 'Storage bucket unique ID where the exported CSV will be stored.') ->param('resourceId', null, new CompoundUID(), 'Composite ID in the format {databaseId:collectionId}, identifying a collection within a database to export.') - ->param('columns', [], new ArrayList(new Text(255)), 'List of attributes to export. If empty, all attributes will be exported. You can use the `*` wildcard to export all attributes from the collection.', true) + ->param('bucketId', '', new UID(), 'Storage bucket unique ID where the exported CSV will be stored.') + ->param('filename', '', new Text(255), 'The name of the file to be created for the export, excluding the .csv extension.') + ->param('columns', [], new ArrayList(new Text(Database::LENGTH_KEY)), 'List of attributes to export. If empty, all attributes will be exported. You can use the `*` wildcard to export all attributes from the collection.', true) + ->param('delimiter', ',', new Text(1), 'The character that separates each column value. Default is comma ",".', true) + ->param('enclosure', '"', new Text(1), 'The character that encloses each column value. Default is double quotes \'"\'.', true) + ->param('escape', '\\', new Text(1), 'The escape character for the enclosure character. Default is backslash "\\".', true) + ->param('header', true, new Boolean(), 'Whether to include the header row with column names. Default is true.', true) + ->param('notify', true, new Boolean(), 'Set to true to receive an email when the export is complete. Default is true.', true) + ->inject('user') ->inject('response') ->inject('dbForProject') ->inject('project') ->inject('queueForEvents') ->inject('queueForMigrations') - ->action(function (string $bucketId, string $resourceId, array $columns, Response $response, Database $dbForProject, Document $project, Event $queueForEvents, Migration $queueForMigrations) { - $isAPIKey = Auth::isAppUser(Authorization::getRoles()); - $isPrivilegedUser = Auth::isPrivilegedUser(Authorization::getRoles()); - + ->action(function ( + string $resourceId, + string $bucketId, + string $filename, + array $columns, + string $delimiter, + string $enclosure, + string $escape, + bool $header, + bool $notify, + Document $user, + Response $response, + Database $dbForProject, + Document $project, + Event $queueForEvents, + Migration $queueForMigrations + ) { $bucket = Authorization::skip(fn () => $dbForProject->getDocument('buckets', $bucketId)); - - if ($bucket->isEmpty() || (!$isAPIKey && !$isPrivilegedUser)) { + if ($bucket->isEmpty()) { throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND); } - $migrationId = ID::unique(); - $resources = Transfer::extractServices([Transfer::GROUP_DATABASES]); + [$databaseId, $collectionId] = \explode(':', $resourceId, 2); + if (empty($databaseId)) { + throw new Exception(Exception::DATABASE_NOT_FOUND); + } + if (empty($collectionId)) { + throw new Exception(Exception::COLLECTION_NOT_FOUND); + } + + $database = Authorization::skip(fn () => $dbForProject->getDocument('databases', $databaseId)); + if ($database->isEmpty()) { + throw new Exception(Exception::DATABASE_NOT_FOUND); + } + + $collection = Authorization::skip(fn () => $dbForProject->getDocument('database_' . $database->getSequence(), $collectionId)); + if ($collection->isEmpty()) { + throw new Exception(Exception::COLLECTION_NOT_FOUND); + } $migration = $dbForProject->createDocument('migrations', new Document([ - '$id' => $migrationId, + '$id' => ID::unique(), 'status' => 'pending', 'stage' => 'init', 'source' => Appwrite::getName(), 'destination' => CSV::getName(), - 'resources' => $resources, + 'resources' => Transfer::extractServices([Transfer::GROUP_DATABASES]), 'resourceId' => $resourceId, 'resourceType' => Resource::TYPE_DATABASE, 'statusCounters' => '{}', @@ -496,7 +536,14 @@ App::post('/v1/migrations/csv/exports') 'errors' => [], 'options' => [ 'bucketId' => $bucketId, + 'filename' => $filename, 'columns' => $columns, + 'delimiter' => $delimiter, + 'enclosure' => $enclosure, + 'escape' => $escape, + 'header' => $header, + 'notify' => $notify, + 'userInternalId' => $user->getSequence(), ], ])); From 5e8951fbe02dce5ac3f7841b45ff2328ca868794 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Wed, 24 Sep 2025 22:37:49 +1200 Subject: [PATCH 12/94] Save export to bucket on complete --- src/Appwrite/Platform/Workers/Migrations.php | 224 +++++++++++++------ 1 file changed, 153 insertions(+), 71 deletions(-) diff --git a/src/Appwrite/Platform/Workers/Migrations.php b/src/Appwrite/Platform/Workers/Migrations.php index ce9b7e2881..bbdf537157 100644 --- a/src/Appwrite/Platform/Workers/Migrations.php +++ b/src/Appwrite/Platform/Workers/Migrations.php @@ -3,19 +3,23 @@ namespace Appwrite\Platform\Workers; use Ahc\Jwt\JWT; +use Appwrite\Event\Mail; use Appwrite\Event\Realtime; -use Utopia\Migration\Destinations\CSV as DestinationCSV; +use Appwrite\Template\Template; use Exception; use Utopia\CLI\Console; use Utopia\Config\Config; use Utopia\Database\Database; use Utopia\Database\Document; +use Utopia\Locale\Locale; use Utopia\Database\Exception\Authorization; use Utopia\Database\Exception\Conflict; use Utopia\Database\Exception\Restricted; use Utopia\Database\Exception\Structure; +use Utopia\Database\Query; use Utopia\Migration\Destination; use Utopia\Migration\Destinations\Appwrite as DestinationAppwrite; +use Utopia\Migration\Destinations\CSV as DestinationCSV; use Utopia\Migration\Exception as MigrationException; use Utopia\Migration\Source; use Utopia\Migration\Sources\Appwrite; @@ -27,6 +31,7 @@ use Utopia\Migration\Sources\Supabase; use Utopia\Migration\Transfer; use Utopia\Platform\Action; use Utopia\Queue\Message; +use Utopia\Storage\Compression\Compression; use Utopia\Storage\Device; use Utopia\System\System; @@ -37,6 +42,7 @@ class Migrations extends Action protected Database $dbForPlatform; protected Device $deviceForMigrations; + protected Device $deviceForFiles; protected Document $project; @@ -71,16 +77,28 @@ class Migrations extends Action ->inject('logError') ->inject('queueForRealtime') ->inject('deviceForMigrations') + ->inject('deviceForFiles') + ->inject('queueForMails') ->callback($this->action(...)); } /** * @throws Exception */ - public function action(Message $message, Document $project, Database $dbForProject, Database $dbForPlatform, callable $logError, Realtime $queueForRealtime, Device $deviceForMigrations): void - { + public function action( + Message $message, + Document $project, + Database $dbForProject, + Database $dbForPlatform, + callable $logError, + Realtime $queueForRealtime, + Device $deviceForMigrations, + Device $deviceForFiles, + Mail $queueForMails, + ): void { $payload = $message->getPayload() ?? []; $this->deviceForMigrations = $deviceForMigrations; + $this->deviceForFiles = $deviceForFiles; if (empty($payload)) { throw new Exception('Missing payload'); @@ -105,7 +123,7 @@ class Migrations extends Action return; } - $this->processMigration($migration, $queueForRealtime); + $this->processMigration($migration, $queueForRealtime, $queueForMails); } /** @@ -175,6 +193,7 @@ class Migrations extends Action protected function processDestination(Document $migration, string $apiKey): Destination { $destination = $migration->getAttribute('destination'); + $options = $migration->getAttribute('options', []); return match ($destination) { DestinationAppwrite::getName() => new DestinationAppwrite( @@ -185,14 +204,32 @@ class Migrations extends Action Config::getParam('collections', [])['databases']['collections'], ), DestinationCSV::getName() => new DestinationCSV( - $this->deviceForMigrations, + $this->deviceForFiles, $migration->getAttribute('resourceId'), - $migration->getAttribute('options', [])['columns'] ?? [], + $options['bucketId'], + $options['filename'], + $options['columns'], + $options['delimiter'], + $options['enclosure'], + $options['escape'], + $options['header'], ), default => throw new \Exception('Invalid destination type'), }; } + /** + * Sanitize a filename to make it filesystem-safe + */ + protected function sanitizeFilename(string $filename): string + { + // Replace problematic characters with underscores + $sanitized = \preg_replace('/[:\/<>"|*?]/', '_', $filename); + $sanitized = \preg_replace('/[^\x20-\x7E]/', '_', $sanitized); + $sanitized = \trim($sanitized); + return empty($sanitized) ? 'export' : $sanitized; + } + /** * @throws Authorization * @throws Structure @@ -202,24 +239,18 @@ class Migrations extends Action */ protected function updateMigrationDocument(Document $migration, Document $project, Realtime $queueForRealtime): Document { - $errorMessages = []; - $clonedMigrationDocument = clone $migration; - - // we cannot use #sensitive because - // `errors` is nested which requires an override. - $errors = $clonedMigrationDocument->getAttribute('errors', []); + $messages = []; + $errors = $migration->getAttribute('errors', []); foreach ($errors as $error) { - $decoded = json_decode($error, true); - - if (is_array($decoded) && isset($decoded['trace'])) { + $decoded = \json_decode($error, true); + if (\is_array($decoded) && isset($decoded['trace'])) { unset($decoded['trace']); - $errorMessages[] = json_encode($decoded); + $messages[] = json_encode($decoded); } } - // set the errors back without trace - $clonedMigrationDocument->setAttribute('errors', $errorMessages); + $migration->setAttribute('errors', $messages); /** Trigger Realtime Events */ $queueForRealtime @@ -227,10 +258,14 @@ class Migrations extends Action ->setSubscribers(['console', $project->getId()]) ->setEvent('migrations.[migrationId].update') ->setParam('migrationId', $migration->getId()) - ->setPayload($clonedMigrationDocument->getArrayCopy(), ['options', 'credentials']) + ->setPayload($migration->getArrayCopy(), sensitive: ['options', 'credentials']) ->trigger(); - return $this->dbForProject->updateDocument('migrations', $migration->getId(), $migration); + return $this->dbForProject->updateDocument( + 'migrations', + $migration->getId(), + $migration + ); } /** @@ -285,11 +320,13 @@ class Migrations extends Action * @throws \Utopia\Database\Exception * @throws Exception */ - protected function processMigration(Document $migration, Realtime $queueForRealtime): void - { - $project = $this->project; - $projectDocument = $this->dbForPlatform->getDocument('projects', $project->getId()); - $tempAPIKey = $this->generateAPIKey($projectDocument); + protected function processMigration( + Document $migration, + Realtime $queueForRealtime, + Mail $queueForMails, + ): void { + $project = $this->dbForPlatform->getDocument('projects', $this->project->getId()); + $tempAPIKey = $this->generateAPIKey($project); $transfer = $source = $destination = null; @@ -299,17 +336,15 @@ class Migrations extends Action empty($migration->getAttribute('credentials', [])) ) { $credentials = $migration->getAttribute('credentials', []); - - $credentials['projectId'] = $credentials['projectId'] ?? $projectDocument->getId(); + $credentials['projectId'] = $credentials['projectId'] ?? $project->getId(); $credentials['endpoint'] = $credentials['endpoint'] ?? 'http://appwrite/v1'; $credentials['apiKey'] = $credentials['apiKey'] ?? $tempAPIKey; - $migration->setAttribute('credentials', $credentials); } $migration->setAttribute('stage', 'processing'); $migration->setAttribute('status', 'processing'); - $this->updateMigrationDocument($migration, $projectDocument, $queueForRealtime); + $this->updateMigrationDocument($migration, $project, $queueForRealtime); $source = $this->processSource($migration); $destination = $this->processDestination($migration, $tempAPIKey); @@ -322,40 +357,44 @@ class Migrations extends Action /** Start Transfer */ if (empty($source->getErrors())) { $migration->setAttribute('stage', 'migrating'); - $this->updateMigrationDocument($migration, $projectDocument, $queueForRealtime); + $this->updateMigrationDocument($migration, $project, $queueForRealtime); $transfer->run( $migration->getAttribute('resources'), - function () use ($migration, $transfer, $projectDocument, $queueForRealtime) { + function () use ($migration, $transfer, $project, $queueForRealtime) { $migration->setAttribute('resourceData', json_encode($transfer->getCache())); $migration->setAttribute('statusCounters', json_encode($transfer->getStatusCounters())); - $this->updateMigrationDocument($migration, $projectDocument, $queueForRealtime); + $this->updateMigrationDocument($migration, $project, $queueForRealtime); }, $migration->getAttribute('resourceId'), $migration->getAttribute('resourceType') ); } - $destination->shutDown(); - $source->shutDown(); + // Debug logging for CSV exports before shutdown + if ($migration->getAttribute('destination') === DestinationCSV::getName()) { + $statusCounters = $transfer->getStatusCounters(); + Console::info('CSV export transfer completed. Status counters: ' . json_encode($statusCounters)); + Console::info('CSV export options: ' . json_encode($migration->getAttribute('options'))); + Console::info('CSV export errors: ' . json_encode($destination->getErrors())); + } + + $destination->shutdown(); + $source->shutdown(); $sourceErrors = $source->getErrors(); $destinationErrors = $destination->getErrors(); - if (! empty($sourceErrors) || ! empty($destinationErrors)) { + if (!empty($sourceErrors) || ! empty($destinationErrors)) { $migration->setAttribute('status', 'failed'); $migration->setAttribute('stage', 'finished'); - $errorMessages = []; - foreach ($sourceErrors as $error) { - $errorMessages[] = json_encode($error); - } - foreach ($destinationErrors as $error) { - $errorMessages[] = json_encode($error); + $errors = []; + foreach ([...$sourceErrors, ...$destinationErrors] as $error) { + $errors[] = \json_encode($error); } - $migration->setAttribute('errors', $errorMessages); - + $migration->setAttribute('errors', $errors); return; } @@ -382,57 +421,100 @@ class Migrations extends Action $sourceErrors = $source->getErrors(); $destinationErrors = $destination->getErrors(); - $errorMessages = []; - foreach ($sourceErrors as $error) { - $errorMessages[] = json_encode($error); - } - foreach ($destinationErrors as $error) { - $errorMessages[] = json_encode($error); + $errors = []; + foreach ([...$sourceErrors, ...$destinationErrors] as $error) { + $errors[] = \json_encode($error); } - $migration->setAttribute('errors', $errorMessages); + $migration->setAttribute('errors', $errors); } } finally { - $this->updateMigrationDocument($migration, $projectDocument, $queueForRealtime); + $this->updateMigrationDocument($migration, $project, $queueForRealtime); if ($migration->getAttribute('status', '') === 'failed') { Console::error('Migration('.$migration->getSequence().':'.$migration->getId().') failed, Project('.$this->project->getSequence().':'.$this->project->getId().')'); - if ($destination) { - $destination->error(); + $sourceErrors = $source?->getErrors() ?? []; + $destinationErrors = $destination?->getErrors() ?? []; - foreach ($destination->getErrors() as $error) { - /** @var MigrationException $error */ - call_user_func($this->logError, $error, 'appwrite-worker', 'appwrite-queue-' . self::getName(), [ + foreach ([...$sourceErrors, ...$destinationErrors] as $error) { + /** @var MigrationException $error */ + if ($error->getCode() === 0 || $error->getCode() >= 500) { + ($this->logError)($error, 'appwrite-worker', 'appwrite-queue-' . self::getName(), [ 'migrationId' => $migration->getId(), 'source' => $migration->getAttribute('source') ?? '', 'destination' => $migration->getAttribute('destination') ?? '', 'resourceName' => $error->getResourceName(), - 'resourceGroup' => $error->getResourceGroup() + 'resourceGroup' => $error->getResourceGroup(), ]); } } - if ($source) { - $source->error(); - - foreach ($source->getErrors() as $error) { - /** @var MigrationException $error */ - call_user_func($this->logError, $error, 'appwrite-worker', 'appwrite-queue-' . self::getName(), [ - 'migrationId' => $migration->getId(), - 'source' => $migration->getAttribute('source') ?? '', - 'destination' => $migration->getAttribute('destination') ?? '', - 'resourceName' => $error->getResourceName(), - 'resourceGroup' => $error->getResourceGroup() - ]); - } - } + $source?->error(); + $destination?->error(); } if ($migration->getAttribute('status', '') === 'completed') { $destination?->success(); $source?->success(); + + if ($migration->getAttribute('destination') === DestinationCSV::getName()) { + $this->handleCSVExportComplete($project, $migration, $queueForMails); + } } } } + + protected function handleCSVExportComplete(Document $project, Document $migration, Mail $queueForMails): void + { + $options = $migration->getAttribute('options', []); + $bucketId = $options['bucketId'] ?? null; + $filename = $options['filename'] ?? 'export.csv'; + $userInternalId = $options['userInternalId'] ?? ''; + $resourceId = $migration->getAttribute('resourceId'); + + // Save file to bucket + $bucket = $this->dbForProject->getDocument('buckets', $bucketId); + if ($bucket->isEmpty()) { + throw new \Exception("Bucket not found: $bucketId"); + } + + $path = $this->deviceForFiles->getPath($bucketId . '/' . $this->sanitizeFilename($filename) . '.csv'); + $size = $this->deviceForFiles->getFileSize($path); + $mime = $this->deviceForFiles->getFileMimeType($path); + $hash = $this->deviceForFiles->getFileHash($path); + $algorithm = Compression::NONE; + $fileId = \md5($resourceId); + + $this->dbForProject->createDocument('bucket_' . $bucket->getSequence(), new Document([ + '$id' => $fileId, + '$permissions' => [], + 'bucketId' => $bucket->getId(), + 'bucketInternalId' => $bucket->getSequence(), + 'name' => $filename, + 'path' => $path, + 'signature' => $hash, + 'mimeType' => $mime, + 'sizeOriginal' => $size, + 'sizeActual' => $size, + 'algorithm' => $algorithm, + 'comment' => '', + 'chunksTotal' => 1, + 'chunksUploaded' => 1, + 'openSSLVersion' => null, + 'openSSLCipher' => null, + 'openSSLTag' => null, + 'openSSLIV' => null, + 'search' => \implode(' ', [$fileId, $filename]), + 'metadata' => ['content_type' => $mime] + ])); + + Console::info("Created file document in bucket: $fileId"); + + // No notification required, skip email sending + if (!($options['notify'] ?? false)) { + return; + } + + } } From 4970aa7426925e3c4d0567e5d8d3bc4c37a9ce33 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Wed, 24 Sep 2025 22:38:12 +1200 Subject: [PATCH 13/94] Send email if notify on complete is set --- app/config/locale/translations/en.json | 8 +++ src/Appwrite/Platform/Workers/Migrations.php | 66 ++++++++++++++++++++ 2 files changed, 74 insertions(+) diff --git a/app/config/locale/translations/en.json b/app/config/locale/translations/en.json index e2ee20b2d7..c194819744 100644 --- a/app/config/locale/translations/en.json +++ b/app/config/locale/translations/en.json @@ -54,6 +54,14 @@ "emails.recovery.thanks": "Thanks,", "emails.recovery.buttonText": "Reset password", "emails.recovery.signature": "{{project}} team", + "emails.csvExport.subject": "Your CSV export is ready", + "emails.csvExport.preview": "Your data export has been completed successfully.", + "emails.csvExport.hello": "Hello {{user}},", + "emails.csvExport.body": "Your CSV export is ready for download. Click the link below to download your data export.", + "emails.csvExport.footer": "This download link will expire in 1 hour.", + "emails.csvExport.thanks": "Thanks,", + "emails.csvExport.buttonText": "Download CSV", + "emails.csvExport.signature": "{{project}} team", "emails.invitation.subject": "Invitation to {{team}} Team at {{project}}", "emails.invitation.preview": "{{owner}} invited you to join {{team}} at {{project}}", "emails.invitation.hello": "Hello {{user}},", diff --git a/src/Appwrite/Platform/Workers/Migrations.php b/src/Appwrite/Platform/Workers/Migrations.php index bbdf537157..ab8c7091cb 100644 --- a/src/Appwrite/Platform/Workers/Migrations.php +++ b/src/Appwrite/Platform/Workers/Migrations.php @@ -516,5 +516,71 @@ class Migrations extends Action return; } + $user = $this->dbForPlatform->findOne('users', [ + Query::equal('$sequence', [$userInternalId]) + ]); + + // Set up locale + $locale = new Locale(System::getEnv('_APP_LOCALE', 'en')); + $locale->setFallback(System::getEnv('_APP_LOCALE', 'en')); + + // Generate JWT + $expiry = (new \DateTime())->add(new \DateInterval('PT1H'))->format('U'); + $encoder = new JWT(System::getEnv('_APP_OPENSSL_KEY_V1'), 'HS256', \intval($expiry), 0); + $jwt = $encoder->encode([ + 'bucketId' => $bucketId, + 'fileId' => $fileId, + 'projectId' => $project->getId(), + ]); + + // Generate download URL with JWT + $endpoint = System::getEnv('_APP_DOMAIN', ''); + $protocol = System::getEnv('_APP_OPTIONS_FORCE_HTTPS', 'disabled') === 'enabled' ? 'https' : 'http'; + $downloadUrl = "{$protocol}://{$endpoint}/v1/storage/buckets/{$bucketId}/files/{$fileId}/push?project={$project->getId()}&jwt={$jwt}"; + + // Get localized email content + $subject = $locale->getText('emails.csvExport.subject'); + $preview = $locale->getText('emails.csvExport.preview'); + $hello = $locale->getText('emails.csvExport.hello'); + $body = $locale->getText('emails.csvExport.body'); + $footer = $locale->getText('emails.csvExport.footer'); + $thanks = $locale->getText('emails.csvExport.thanks'); + $buttonText = $locale->getText('emails.csvExport.buttonText'); + $signature = $locale->getText('emails.csvExport.signature'); + + // Build email body using inner template + $message = Template::fromFile(__DIR__ . '/../../../../app/config/locale/templates/email-inner-base.tpl'); + $message + ->setParam('{{body}}', $body, escapeHtml: false) + ->setParam('{{hello}}', $hello) + ->setParam('{{footer}}', $footer) + ->setParam('{{thanks}}', $thanks) + ->setParam('{{buttonText}}', $buttonText) + ->setParam('{{signature}}', $signature) + ->setParam('{{direction}}', $locale->getText('settings.direction')) + ->setParam('{{project}}', $project->getAttribute('name')) + ->setParam('{{user}}', $user->getAttribute('name', $user->getAttribute('email'))) + ->setParam('{{redirect}}', $downloadUrl); + + $emailBody = $message->render(); + + $emailVariables = [ + 'direction' => $locale->getText('settings.direction'), + 'project' => $project->getAttribute('name'), + 'user' => $user->getAttribute('name', $user->getAttribute('email')), + 'redirect' => $downloadUrl, + ]; + + $queueForMails + ->setSubject($subject) + ->setPreview($preview) + ->setBody($emailBody) + ->setBodyTemplate(__DIR__ . '/../../../../app/config/locale/templates/email-base-styled.tpl') + ->setVariables($emailVariables) + ->setName($user->getAttribute('name', $user->getAttribute('email'))) + ->setRecipient($user->getAttribute('email')) + ->trigger(); + + Console::info('CSV export notification email sent to ' . $user->getAttribute('email')); } } From 61d9db8c67500fadda6bf65805ae47168a5e265e Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Wed, 24 Sep 2025 22:38:35 +1200 Subject: [PATCH 14/94] Cover export + notify in tests --- .../Services/Migrations/MigrationsBase.php | 231 +++++++++++++++++- 1 file changed, 230 insertions(+), 1 deletion(-) diff --git a/tests/e2e/Services/Migrations/MigrationsBase.php b/tests/e2e/Services/Migrations/MigrationsBase.php index 7a57b7f8f9..4d27cf2828 100644 --- a/tests/e2e/Services/Migrations/MigrationsBase.php +++ b/tests/e2e/Services/Migrations/MigrationsBase.php @@ -900,7 +900,7 @@ trait MigrationsBase /** * Import documents from a CSV file. */ - public function testCreateCsvMigration(): void + public function testCreateCSVImport(): void { // Make a database $response = $this->client->call(Client::METHOD_POST, '/databases', [ @@ -1194,4 +1194,233 @@ trait MigrationsBase 'x-appwrite-project' => $this->getProject()['$id'], ], $body); } + + /** + * Test CSV export with email notification + */ + public function testCreateCSVExport(): void + { + // Create a database + $database = $this->client->call(Client::METHOD_POST, '/databases', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'databaseId' => ID::unique(), + 'name' => 'Test Export Database' + ]); + + $this->assertEquals(201, $database['headers']['status-code']); + $databaseId = $database['body']['$id']; + + // Create a collection + $collection = $this->client->call(Client::METHOD_POST, '/databases/' . $databaseId . '/collections', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'collectionId' => ID::unique(), + 'name' => 'Test Export Collection', + 'permissions' => [] + ]); + + $this->assertEquals(201, $collection['headers']['status-code']); + $collectionId = $collection['body']['$id']; + + // Create a simple attribute like the basic test + $name = $this->client->call(Client::METHOD_POST, '/databases/' . $databaseId . '/collections/' . $collectionId . '/attributes/string', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'key' => 'name', + 'size' => 255, + 'required' => true, + ]); + + $this->assertEquals(202, $name['headers']['status-code']); + + // Create a simple attribute like the basic test + $email = $this->client->call(Client::METHOD_POST, '/databases/' . $databaseId . '/collections/' . $collectionId . '/attributes/string', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'key' => 'email', + 'size' => 255, + 'required' => false, + ]); + + $this->assertEquals(202, $email['headers']['status-code']); + + \sleep(3); + + // Create sample documents + for ($i = 1; $i <= 10; $i++) { + $doc = $this->client->call(Client::METHOD_POST, '/databases/' . $databaseId . '/collections/' . $collectionId . '/documents', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'documentId' => ID::unique(), + 'data' => [ + 'name' => 'Test User ' . $i, + 'email' => 'user' . $i . '@appwrite.io' + ] + ]); + + $this->assertEquals(201, $doc['headers']['status-code'], 'Failed to create document ' . $i); + } + + // Verify documents were created + $docs = $this->client->call(Client::METHOD_GET, '/databases/' . $databaseId . '/collections/' . $collectionId . '/documents', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]); + + $this->assertEquals(200, $docs['headers']['status-code']); + $this->assertEquals(10, $docs['body']['total'], 'Expected 10 documents but got ' . $docs['body']['total']); + + // Create a storage bucket for the export + $bucketIdUnique = ID::unique(); + $bucket = $this->client->call(Client::METHOD_POST, '/storage/buckets', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'bucketId' => $bucketIdUnique, + 'name' => 'Test Export Bucket', + 'permissions' => [ + Permission::read(Role::any()), + Permission::create(Role::any()), + Permission::update(Role::any()), + Permission::delete(Role::any()), + ], + 'fileSecurity' => false, + 'enabled' => true, + 'maximumFileSize' => 10485760, // 10MB + 'allowedFileExtensions' => ['csv'], + 'compression' => 'none', + 'encryption' => false, + 'antivirus' => false + ]); + + $this->assertEquals(201, $bucket['headers']['status-code']); + $bucketId = $bucket['body']['$id']; + + // Perform CSV export with notification enabled + $migration = $this->client->call(Client::METHOD_POST, '/migrations/csv/exports', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'] + ], $this->getHeaders()), [ + 'bucketId' => $bucketId, + 'resourceId' => $databaseId . ':' . $collectionId, + 'filename' => 'test-export', + 'columns' => [], + 'delimiter' => ',', + 'enclosure' => '"', + 'escape' => '\\', + 'header' => true, + 'notify' => true + ]); + + $this->assertEquals(202, $migration['headers']['status-code']); + $this->assertNotEmpty($migration['body']['$id']); + $migrationId = $migration['body']['$id']; + + $this->assertEventually(function () use ($migrationId) { + $response = $this->client->call(Client::METHOD_GET, '/migrations/' . $migrationId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ]); + + $this->assertEquals(200, $response['headers']['status-code']); + $this->assertEquals('finished', $response['body']['stage']); + $this->assertEquals('completed', $response['body']['status']); + $this->assertEquals('Appwrite', $response['body']['source']); + $this->assertEquals('CSV', $response['body']['destination']); + + return true; + }, 30000, 500); + + // Check that the file was created in the bucket + // File ID is MD5 of the resourceId (not the filename) + $fileId = \md5($databaseId . ':' . $collectionId); + + $file = $this->client->call(Client::METHOD_GET, '/storage/buckets/' . $bucketId . '/files/' . $fileId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]); + + $this->assertEquals(200, $file['headers']['status-code']); + $this->assertEquals($fileId, $file['body']['$id']); + $this->assertEquals($bucketId, $file['body']['bucketId']); + $this->assertEquals('test-export', $file['body']['name']); + $this->assertEquals('text/csv', $file['body']['mimeType']); + $this->assertGreaterThan(0, $file['body']['sizeOriginal']); + + // Download and verify CSV content + $download = $this->client->call(Client::METHOD_GET, '/storage/buckets/' . $bucketId . '/files/' . $fileId . '/download', \array_merge([ + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders())); + + $this->assertEquals(200, $download['headers']['status-code']); + + $csvContent = $download['body']; + $lines = explode("\n", trim($csvContent)); + $this->assertCount(11, $lines); + $this->assertStringContainsString('$id', $lines[0]); + $this->assertStringContainsString('$permissions', $lines[0]); + $this->assertStringContainsString('$createdAt', $lines[0]); + $this->assertStringContainsString('$updatedAt', $lines[0]); + $this->assertStringContainsString('name', $lines[0]); + $this->assertStringContainsString('email', $lines[0]); + + $this->assertStringContainsString('Test User 1', $lines[1]); + $this->assertStringContainsString('user1@appwrite.io', $lines[1]); + + // Check that email was sent with download link + $lastEmail = $this->getLastEmail(); + $this->assertNotEmpty($lastEmail); + $this->assertEquals('Your CSV export is ready', $lastEmail['subject']); + $this->assertStringContainsStringIgnoringCase('Your data export has been completed successfully', $lastEmail['text']); + + // Extract download URL from email HTML + \preg_match('/href="([^"]*\/storage\/buckets\/[^"]*\/push[^"]*)"/', $lastEmail['html'], $matches); + $this->assertNotEmpty($matches[1], 'Download URL not found in email'); + $downloadUrl = html_entity_decode($matches[1]); + + // Parse the URL to extract components + $components = \parse_url($downloadUrl); + $this->assertNotEmpty($components); + \parse_str($components['query'] ?? '', $queryParams); + $this->assertArrayHasKey('jwt', $queryParams, 'JWT not found in download URL'); + $this->assertNotEmpty($queryParams['jwt']); + + // Test download with JWT + $path = \str_replace('/v1', '', $components['path']); + $downloadWithJwt = $this->client->call(Client::METHOD_GET, $path . '?project=' . $queryParams['project'] . '&jwt=' . $queryParams['jwt']); + $this->assertEquals(200, $downloadWithJwt['headers']['status-code'], 'Failed to download file with JWT'); + $this->assertEquals($csvContent, $downloadWithJwt['body'], 'Downloaded content differs from original'); + + // Test that download without JWT fails + $downloadWithoutJwt = $this->client->call(Client::METHOD_GET, '/storage/buckets/' . $bucketId . '/files/' . $fileId . '/download'); + $this->assertEquals(404, $downloadWithoutJwt['headers']['status-code'], 'File should not be downloadable without JWT'); + + $this->client->call(Client::METHOD_DELETE, '/storage/buckets/' . $bucketId . '/files/' . $fileId, [ + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]); + $this->client->call(Client::METHOD_DELETE, '/databases/' . $databaseId, [ + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]); + $this->client->call(Client::METHOD_DELETE, '/storage/buckets/' . $bucketId, [ + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]); + } } From 223523f722f4a5b34ccc93e475d364334d1dd5ae Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Wed, 24 Sep 2025 22:38:55 +1200 Subject: [PATCH 15/94] Mount uploads to migrations worker --- docker-compose.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/docker-compose.yml b/docker-compose.yml index da6362b4c4..0c187dd762 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -698,6 +698,7 @@ services: - appwrite volumes: - appwrite-imports:/storage/imports:rw + - appwrite-uploads:/storage/uploads:rw - ./app:/usr/src/code/app - ./src:/usr/src/code/src - ./tests:/usr/src/code/tests From 2494c9afb376e89311fd11e4904465b7c40f228c Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Wed, 24 Sep 2025 22:39:06 +1200 Subject: [PATCH 16/94] Update to release migrations --- composer.json | 2 +- composer.lock | 133 ++++++++++++++++++++++++++------------------------ 2 files changed, 69 insertions(+), 66 deletions(-) diff --git a/composer.json b/composer.json index 8a1e325f7f..6f89312223 100644 --- a/composer.json +++ b/composer.json @@ -63,7 +63,7 @@ "utopia-php/locale": "0.8.*", "utopia-php/logger": "0.6.*", "utopia-php/messaging": "0.18.*", - "utopia-php/migration": "dev-feat-csv-export as 1.0.0", + "utopia-php/migration": "1.2.*", "utopia-php/orchestration": "0.9.*", "utopia-php/platform": "0.7.*", "utopia-php/pools": "0.8.*", diff --git a/composer.lock b/composer.lock index 73fb57f6f8..0fc58749f5 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "954529a36566209d6687df9f41a0f2e6", + "content-hash": "5150254cf0d6aa361a31244b7f7d1eb7", "packages": [ { "name": "adhocore/jwt", @@ -1159,20 +1159,20 @@ }, { "name": "open-telemetry/api", - "version": "1.5.0", + "version": "1.6.0", "source": { "type": "git", "url": "https://github.com/opentelemetry-php/api.git", - "reference": "7692075f486c14d8cfd37fba98a08a5667f089e5" + "reference": "ee17d937652eca06c2341b6fadc0f74c1c1a5af2" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/opentelemetry-php/api/zipball/7692075f486c14d8cfd37fba98a08a5667f089e5", - "reference": "7692075f486c14d8cfd37fba98a08a5667f089e5", + "url": "https://api.github.com/repos/opentelemetry-php/api/zipball/ee17d937652eca06c2341b6fadc0f74c1c1a5af2", + "reference": "ee17d937652eca06c2341b6fadc0f74c1c1a5af2", "shasum": "" }, "require": { - "open-telemetry/context": "^1.0", + "open-telemetry/context": "^1.4", "php": "^8.1", "psr/log": "^1.1|^2.0|^3.0", "symfony/polyfill-php82": "^1.26" @@ -1225,20 +1225,20 @@ "issues": "https://github.com/open-telemetry/opentelemetry-php/issues", "source": "https://github.com/open-telemetry/opentelemetry-php" }, - "time": "2025-08-07T23:07:38+00:00" + "time": "2025-09-19T00:05:49+00:00" }, { "name": "open-telemetry/context", - "version": "1.3.1", + "version": "1.4.0", "source": { "type": "git", "url": "https://github.com/opentelemetry-php/context.git", - "reference": "438f71812242db3f196fb4c717c6f92cbc819be6" + "reference": "d4c4470b541ce72000d18c339cfee633e4c8e0cf" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/opentelemetry-php/context/zipball/438f71812242db3f196fb4c717c6f92cbc819be6", - "reference": "438f71812242db3f196fb4c717c6f92cbc819be6", + "url": "https://api.github.com/repos/opentelemetry-php/context/zipball/d4c4470b541ce72000d18c339cfee633e4c8e0cf", + "reference": "d4c4470b541ce72000d18c339cfee633e4c8e0cf", "shasum": "" }, "require": { @@ -1284,7 +1284,7 @@ "issues": "https://github.com/open-telemetry/opentelemetry-php/issues", "source": "https://github.com/open-telemetry/opentelemetry-php" }, - "time": "2025-08-13T01:12:00+00:00" + "time": "2025-09-19T00:05:49+00:00" }, { "name": "open-telemetry/exporter-otlp", @@ -1415,23 +1415,23 @@ }, { "name": "open-telemetry/sdk", - "version": "1.7.1", + "version": "1.8.0", "source": { "type": "git", "url": "https://github.com/opentelemetry-php/sdk.git", - "reference": "52690d4b37ae4f091af773eef3c238ed2bc0aa06" + "reference": "105c6e81e3d86150bd5704b00c7e4e165e957b89" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/opentelemetry-php/sdk/zipball/52690d4b37ae4f091af773eef3c238ed2bc0aa06", - "reference": "52690d4b37ae4f091af773eef3c238ed2bc0aa06", + "url": "https://api.github.com/repos/opentelemetry-php/sdk/zipball/105c6e81e3d86150bd5704b00c7e4e165e957b89", + "reference": "105c6e81e3d86150bd5704b00c7e4e165e957b89", "shasum": "" }, "require": { "ext-json": "*", "nyholm/psr7-server": "^1.1", - "open-telemetry/api": "^1.4", - "open-telemetry/context": "^1.0", + "open-telemetry/api": "^1.6", + "open-telemetry/context": "^1.4", "open-telemetry/sem-conv": "^1.0", "php": "^8.1", "php-http/discovery": "^1.14", @@ -1508,7 +1508,7 @@ "issues": "https://github.com/open-telemetry/opentelemetry-php/issues", "source": "https://github.com/open-telemetry/opentelemetry-php" }, - "time": "2025-09-05T07:17:06+00:00" + "time": "2025-09-19T00:05:49+00:00" }, { "name": "open-telemetry/sem-conv", @@ -1569,16 +1569,16 @@ }, { "name": "paragonie/constant_time_encoding", - "version": "v2.7.0", + "version": "v2.8.1", "source": { "type": "git", "url": "https://github.com/paragonie/constant_time_encoding.git", - "reference": "52a0d99e69f56b9ec27ace92ba56897fe6993105" + "reference": "e6352b9f43318821f148c1e8c2d9e944aa9accb5" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/paragonie/constant_time_encoding/zipball/52a0d99e69f56b9ec27ace92ba56897fe6993105", - "reference": "52a0d99e69f56b9ec27ace92ba56897fe6993105", + "url": "https://api.github.com/repos/paragonie/constant_time_encoding/zipball/e6352b9f43318821f148c1e8c2d9e944aa9accb5", + "reference": "e6352b9f43318821f148c1e8c2d9e944aa9accb5", "shasum": "" }, "require": { @@ -1632,7 +1632,7 @@ "issues": "https://github.com/paragonie/constant_time_encoding/issues", "source": "https://github.com/paragonie/constant_time_encoding" }, - "time": "2024-05-08T12:18:48+00:00" + "time": "2025-09-24T01:40:13+00:00" }, { "name": "paragonie/random_compat", @@ -4187,16 +4187,16 @@ }, { "name": "utopia-php/migration", - "version": "dev-feat-csv-export", + "version": "1.2.0", "source": { "type": "git", "url": "https://github.com/utopia-php/migration.git", - "reference": "8435f1db0db4854ca27cb4c9cf275b905fcb3b41" + "reference": "42ff497c5231f5a727d1e229419ff1d2195d8093" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/migration/zipball/8435f1db0db4854ca27cb4c9cf275b905fcb3b41", - "reference": "8435f1db0db4854ca27cb4c9cf275b905fcb3b41", + "url": "https://api.github.com/repos/utopia-php/migration/zipball/42ff497c5231f5a727d1e229419ff1d2195d8093", + "reference": "42ff497c5231f5a727d1e229419ff1d2195d8093", "shasum": "" }, "require": { @@ -4237,9 +4237,9 @@ ], "support": { "issues": "https://github.com/utopia-php/migration/issues", - "source": "https://github.com/utopia-php/migration/tree/feat-csv-export" + "source": "https://github.com/utopia-php/migration/tree/1.2.0" }, - "time": "2025-08-21T12:56:18+00:00" + "time": "2025-09-24T10:32:24+00:00" }, { "name": "utopia-php/orchestration", @@ -5004,16 +5004,16 @@ "packages-dev": [ { "name": "appwrite/sdk-generator", - "version": "1.3.5", + "version": "1.4.0", "source": { "type": "git", "url": "https://github.com/appwrite/sdk-generator.git", - "reference": "6fda9e58b37c9872c1a2a424e5467de8de1bc567" + "reference": "3583fa6fddb1d1a902b37ff2048527a5827fc008" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/appwrite/sdk-generator/zipball/6fda9e58b37c9872c1a2a424e5467de8de1bc567", - "reference": "6fda9e58b37c9872c1a2a424e5467de8de1bc567", + "url": "https://api.github.com/repos/appwrite/sdk-generator/zipball/3583fa6fddb1d1a902b37ff2048527a5827fc008", + "reference": "3583fa6fddb1d1a902b37ff2048527a5827fc008", "shasum": "" }, "require": { @@ -5049,9 +5049,9 @@ "description": "Appwrite PHP library for generating API SDKs for multiple programming languages and platforms", "support": { "issues": "https://github.com/appwrite/sdk-generator/issues", - "source": "https://github.com/appwrite/sdk-generator/tree/1.3.5" + "source": "https://github.com/appwrite/sdk-generator/tree/1.4.0" }, - "time": "2025-09-15T04:19:40+00:00" + "time": "2025-09-23T02:27:10+00:00" }, { "name": "doctrine/annotations", @@ -5278,16 +5278,16 @@ }, { "name": "laravel/pint", - "version": "v1.25.0", + "version": "v1.25.1", "source": { "type": "git", "url": "https://github.com/laravel/pint.git", - "reference": "595de38458c6b0ab4cae4bcc769c2e5c5d5b8e96" + "reference": "5016e263f95d97670d71b9a987bd8996ade6d8d9" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/laravel/pint/zipball/595de38458c6b0ab4cae4bcc769c2e5c5d5b8e96", - "reference": "595de38458c6b0ab4cae4bcc769c2e5c5d5b8e96", + "url": "https://api.github.com/repos/laravel/pint/zipball/5016e263f95d97670d71b9a987bd8996ade6d8d9", + "reference": "5016e263f95d97670d71b9a987bd8996ade6d8d9", "shasum": "" }, "require": { @@ -5340,7 +5340,7 @@ "issues": "https://github.com/laravel/pint/issues", "source": "https://github.com/laravel/pint" }, - "time": "2025-09-17T01:36:44+00:00" + "time": "2025-09-19T02:57:12+00:00" }, { "name": "matthiasmullie/minify", @@ -6230,16 +6230,16 @@ }, { "name": "phpunit/phpunit", - "version": "9.6.27", + "version": "9.6.29", "source": { "type": "git", "url": "https://github.com/sebastianbergmann/phpunit.git", - "reference": "0a9aa4440b6a9528cf360071502628d717af3e0a" + "reference": "9ecfec57835a5581bc888ea7e13b51eb55ab9dd3" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/phpunit/zipball/0a9aa4440b6a9528cf360071502628d717af3e0a", - "reference": "0a9aa4440b6a9528cf360071502628d717af3e0a", + "url": "https://api.github.com/repos/sebastianbergmann/phpunit/zipball/9ecfec57835a5581bc888ea7e13b51eb55ab9dd3", + "reference": "9ecfec57835a5581bc888ea7e13b51eb55ab9dd3", "shasum": "" }, "require": { @@ -6264,7 +6264,7 @@ "sebastian/comparator": "^4.0.9", "sebastian/diff": "^4.0.6", "sebastian/environment": "^5.1.5", - "sebastian/exporter": "^4.0.6", + "sebastian/exporter": "^4.0.8", "sebastian/global-state": "^5.0.8", "sebastian/object-enumerator": "^4.0.4", "sebastian/resource-operations": "^3.0.4", @@ -6313,7 +6313,7 @@ "support": { "issues": "https://github.com/sebastianbergmann/phpunit/issues", "security": "https://github.com/sebastianbergmann/phpunit/security/policy", - "source": "https://github.com/sebastianbergmann/phpunit/tree/9.6.27" + "source": "https://github.com/sebastianbergmann/phpunit/tree/9.6.29" }, "funding": [ { @@ -6337,7 +6337,7 @@ "type": "tidelift" } ], - "time": "2025-09-14T06:18:03+00:00" + "time": "2025-09-24T06:29:11+00:00" }, { "name": "psr/cache", @@ -6829,16 +6829,16 @@ }, { "name": "sebastian/exporter", - "version": "4.0.6", + "version": "4.0.8", "source": { "type": "git", "url": "https://github.com/sebastianbergmann/exporter.git", - "reference": "78c00df8f170e02473b682df15bfcdacc3d32d72" + "reference": "14c6ba52f95a36c3d27c835d65efc7123c446e8c" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/exporter/zipball/78c00df8f170e02473b682df15bfcdacc3d32d72", - "reference": "78c00df8f170e02473b682df15bfcdacc3d32d72", + "url": "https://api.github.com/repos/sebastianbergmann/exporter/zipball/14c6ba52f95a36c3d27c835d65efc7123c446e8c", + "reference": "14c6ba52f95a36c3d27c835d65efc7123c446e8c", "shasum": "" }, "require": { @@ -6894,15 +6894,27 @@ ], "support": { "issues": "https://github.com/sebastianbergmann/exporter/issues", - "source": "https://github.com/sebastianbergmann/exporter/tree/4.0.6" + "source": "https://github.com/sebastianbergmann/exporter/tree/4.0.8" }, "funding": [ { "url": "https://github.com/sebastianbergmann", "type": "github" + }, + { + "url": "https://liberapay.com/sebastianbergmann", + "type": "liberapay" + }, + { + "url": "https://thanks.dev/u/gh/sebastianbergmann", + "type": "thanks_dev" + }, + { + "url": "https://tidelift.com/funding/github/packagist/sebastian/exporter", + "type": "tidelift" } ], - "time": "2024-03-02T06:33:00+00:00" + "time": "2025-09-24T06:03:27+00:00" }, { "name": "sebastian/global-state", @@ -8504,18 +8516,9 @@ "time": "2024-03-07T20:33:40+00:00" } ], - "aliases": [ - { - "package": "utopia-php/migration", - "version": "dev-feat-csv-export", - "alias": "1.0.0", - "alias_normalized": "1.0.0.0" - } - ], + "aliases": [], "minimum-stability": "stable", - "stability-flags": { - "utopia-php/migration": 20 - }, + "stability-flags": {}, "prefer-stable": false, "prefer-lowest": false, "platform": { From 16821a28fcad97655b94c48b02caa6970adc0ab9 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Wed, 24 Sep 2025 22:53:13 +1200 Subject: [PATCH 17/94] Format --- app/controllers/api/migrations.php | 1 - src/Appwrite/Platform/Workers/Migrations.php | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/app/controllers/api/migrations.php b/app/controllers/api/migrations.php index 6e0ecd218d..c69185d0e8 100644 --- a/app/controllers/api/migrations.php +++ b/app/controllers/api/migrations.php @@ -1,6 +1,5 @@ Date: Fri, 10 Oct 2025 23:01:26 +0530 Subject: [PATCH 18/94] Dark mode styles for emails --- .../locale/templates/email-base-styled.tpl | 33 ++++++++++++++++++- app/config/locale/templates/email-base.tpl | 32 ++++++++++++++++++ 2 files changed, 64 insertions(+), 1 deletion(-) diff --git a/app/config/locale/templates/email-base-styled.tpl b/app/config/locale/templates/email-base-styled.tpl index 37ca630d43..c47d276ea8 100644 --- a/app/config/locale/templates/email-base-styled.tpl +++ b/app/config/locale/templates/email-base-styled.tpl @@ -2,6 +2,38 @@ + + +