From 3376bed6f797a8d48f0107e9a0d8e40745d6914b Mon Sep 17 00:00:00 2001 From: Prem Palanisamy Date: Thu, 14 May 2026 20:00:51 +0100 Subject: [PATCH] Add project variable migration support (under new GROUP_SETTINGS) Introduces Transfer::GROUP_SETTINGS for project-level configuration resources that aren't external integrations. Project variables are the first inhabitant; webhooks will follow in a stacked PR. - New abstract Source::exportGroupSettings() with no-op stubs in CSV/JSON/Firebase/NHost sources. - Appwrite source: project variables exported via Project SDK listVariables() with cursor pagination. - Appwrite destination: createProjectVariable writes directly to dbForProject 'variables' collection, mirroring upstream Project Variables Create payload (resourceType 'project', empty resource id). --- src/Migration/Destinations/Appwrite.php | 62 +++++++++++++ src/Migration/Resource.php | 7 +- .../Resources/Settings/ProjectVariable.php | 78 +++++++++++++++++ src/Migration/Source.php | 17 ++++ src/Migration/Sources/Appwrite.php | 87 +++++++++++++++++++ src/Migration/Sources/CSV.php | 5 ++ src/Migration/Sources/Firebase.php | 5 ++ src/Migration/Sources/JSON.php | 5 ++ src/Migration/Sources/NHost.php | 5 ++ src/Migration/Transfer.php | 9 +- 10 files changed, 278 insertions(+), 2 deletions(-) create mode 100644 src/Migration/Resources/Settings/ProjectVariable.php diff --git a/src/Migration/Destinations/Appwrite.php b/src/Migration/Destinations/Appwrite.php index 1b7c4d2362..f3b7d335f1 100644 --- a/src/Migration/Destinations/Appwrite.php +++ b/src/Migration/Destinations/Appwrite.php @@ -52,6 +52,7 @@ use Utopia\Migration\Resources\Functions\EnvVar; use Utopia\Migration\Resources\Functions\Func; use Utopia\Migration\Resources\Integrations\ApiKey; use Utopia\Migration\Resources\Integrations\Platform; +use Utopia\Migration\Resources\Settings\ProjectVariable; use Utopia\Migration\Resources\Messaging\Message; use Utopia\Migration\Resources\Messaging\Provider; use Utopia\Migration\Resources\Messaging\Subscriber; @@ -278,6 +279,9 @@ class Appwrite extends Destination Resource::TYPE_PLATFORM, Resource::TYPE_API_KEY, + // Settings + Resource::TYPE_PROJECT_VARIABLE, + // Backups Resource::TYPE_BACKUP_POLICY, ]; @@ -438,6 +442,7 @@ class Appwrite extends Destination Transfer::GROUP_SITES => $this->importSiteResource($resource), Transfer::GROUP_INTEGRATIONS => $this->importIntegrationsResource($resource), Transfer::GROUP_BACKUPS => $this->importBackupResource($resource), + Transfer::GROUP_SETTINGS => $this->importSettingsResource($resource), default => throw new \Exception('Invalid resource group', Exception::CODE_VALIDATION), }; } catch (\Throwable $e) { @@ -3089,6 +3094,63 @@ class Appwrite extends Destination return $resource; } + public function importSettingsResource(Resource $resource): Resource + { + switch ($resource->getName()) { + case Resource::TYPE_PROJECT_VARIABLE: + /** @var ProjectVariable $resource */ + $this->createProjectVariable($resource); + break; + } + + if ($resource->getStatus() !== Resource::STATUS_SKIPPED) { + $resource->setStatus(Resource::STATUS_SUCCESS); + } + + return $resource; + } + + protected function createProjectVariable(ProjectVariable $resource): bool + { + // Variables live in dbForProject (not dbForPlatform). resourceType is 'project' (singular) + // for project-scope variables; function/site-scope variables use 'function'/'site'. + $existing = $this->dbForProject->findOne('variables', [ + Query::equal('resourceType', ['project']), + Query::equal('key', [$resource->getKey()]), + ]); + + if ($existing !== false && !$existing->isEmpty()) { + $resource->setStatus(Resource::STATUS_SKIPPED, 'Project variable already exists'); + return false; + } + + $createdAt = $this->normalizeDateTime($resource->getCreatedAt()); + $updatedAt = $this->normalizeDateTime($resource->getUpdatedAt(), $createdAt); + $variableId = ID::unique(); + $key = $resource->getKey(); + + try { + $this->dbForProject->createDocument('variables', new UtopiaDocument([ + '$id' => $variableId, + '$permissions' => $resource->getPermissions(), + 'resourceInternalId' => '', + 'resourceId' => '', + 'resourceType' => 'project', + 'key' => $key, + 'value' => $resource->getValue(), + 'secret' => $resource->isSecret(), + 'search' => \implode(' ', [$variableId, $key, 'project']), + '$createdAt' => $createdAt, + '$updatedAt' => $updatedAt, + ])); + } catch (DuplicateException) { + $resource->setStatus(Resource::STATUS_SKIPPED, 'Project variable already exists'); + return false; + } + + return true; + } + /** * @throws \Throwable */ diff --git a/src/Migration/Resource.php b/src/Migration/Resource.php index 144525704b..6dd1df78ca 100644 --- a/src/Migration/Resource.php +++ b/src/Migration/Resource.php @@ -74,8 +74,12 @@ abstract class Resource implements \JsonSerializable // Integrations public const TYPE_PLATFORM = 'platform'; public const TYPE_API_KEY = 'api-key'; - public const TYPE_SUBSCRIBER = 'subscriber'; + // Settings + public const TYPE_PROJECT_VARIABLE = 'project-variable'; + + // Messaging + public const TYPE_SUBSCRIBER = 'subscriber'; public const TYPE_MESSAGE = 'message'; // Backups @@ -114,6 +118,7 @@ abstract class Resource implements \JsonSerializable self::TYPE_MEMBERSHIP, self::TYPE_PLATFORM, self::TYPE_API_KEY, + self::TYPE_PROJECT_VARIABLE, self::TYPE_PROVIDER, self::TYPE_TOPIC, self::TYPE_SUBSCRIBER, diff --git a/src/Migration/Resources/Settings/ProjectVariable.php b/src/Migration/Resources/Settings/ProjectVariable.php new file mode 100644 index 0000000000..1585695dfb --- /dev/null +++ b/src/Migration/Resources/Settings/ProjectVariable.php @@ -0,0 +1,78 @@ +id = $id; + $this->createdAt = $createdAt; + $this->updatedAt = $updatedAt; + } + + /** + * @param array $array + * @return self + */ + public static function fromArray(array $array): self + { + return new self( + $array['id'], + $array['key'], + $array['value'] ?? '', + (bool) ($array['secret'] ?? false), + createdAt: $array['createdAt'] ?? '', + updatedAt: $array['updatedAt'] ?? '', + ); + } + + /** + * @return array + */ + public function jsonSerialize(): array + { + return [ + 'id' => $this->id, + 'key' => $this->key, + 'value' => $this->value, + 'secret' => $this->secret, + 'createdAt' => $this->createdAt, + 'updatedAt' => $this->updatedAt, + ]; + } + + public static function getName(): string + { + return Resource::TYPE_PROJECT_VARIABLE; + } + + public function getGroup(): string + { + return Transfer::GROUP_SETTINGS; + } + + public function getKey(): string + { + return $this->key; + } + + public function getValue(): string + { + return $this->value; + } + + public function isSecret(): bool + { + return $this->secret; + } +} diff --git a/src/Migration/Source.php b/src/Migration/Source.php index bf129b2bc9..fe53f2cd13 100644 --- a/src/Migration/Source.php +++ b/src/Migration/Source.php @@ -56,6 +56,11 @@ abstract class Source extends Target return static::$defaultBatchSize; } + public function getSettingsBatchSize(): int + { + return static::$defaultBatchSize; + } + /** * @param array $resources * @return void @@ -121,6 +126,7 @@ abstract class Source extends Target Transfer::GROUP_SITES => Transfer::GROUP_SITES_RESOURCES, Transfer::GROUP_INTEGRATIONS => Transfer::GROUP_INTEGRATIONS_RESOURCES, Transfer::GROUP_BACKUPS => Transfer::GROUP_BACKUPS_RESOURCES, + Transfer::GROUP_SETTINGS => Transfer::GROUP_SETTINGS_RESOURCES, ]; foreach ($mapping as $group => $resources) { @@ -161,6 +167,9 @@ abstract class Source extends Target case Transfer::GROUP_BACKUPS: $this->exportGroupBackups($this->getBackupsBatchSize(), $resources); break; + case Transfer::GROUP_SETTINGS: + $this->exportGroupSettings($this->getSettingsBatchSize(), $resources); + break; } } } @@ -228,4 +237,12 @@ abstract class Source extends Target * @param array $resources Resources to export */ abstract protected function exportGroupBackups(int $batchSize, array $resources): void; + + /** + * Export Settings Group + * + * @param int $batchSize + * @param array $resources Resources to export + */ + abstract protected function exportGroupSettings(int $batchSize, array $resources): void; } diff --git a/src/Migration/Sources/Appwrite.php b/src/Migration/Sources/Appwrite.php index a219652045..28e6341641 100644 --- a/src/Migration/Sources/Appwrite.php +++ b/src/Migration/Sources/Appwrite.php @@ -56,6 +56,7 @@ use Utopia\Migration\Resources\Functions\Deployment; use Utopia\Migration\Resources\Functions\EnvVar; use Utopia\Migration\Resources\Functions\Func; use Utopia\Migration\Resources\Integrations\ApiKey; +use Utopia\Migration\Resources\Settings\ProjectVariable; use Utopia\Migration\Resources\Integrations\Platform; use Utopia\Migration\Resources\Messaging\Message; use Utopia\Migration\Resources\Messaging\Provider; @@ -214,6 +215,7 @@ class Appwrite extends Source Resource::TYPE_BACKUP_POLICY, // Settings + Resource::TYPE_PROJECT_VARIABLE, ]; } @@ -254,6 +256,7 @@ class Appwrite extends Source $this->reportSites($resources, $report, $resourceIds); $this->reportIntegrations($resources, $report, $resourceIds); $this->reportBackups($resources, $report, $resourceIds); + $this->reportSettings($resources, $report, $resourceIds); $report['version'] = $this->call( 'GET', @@ -1449,6 +1452,90 @@ class Appwrite extends Source } } + private function reportSettings(array $resources, array &$report, array $resourceIds = []): void + { + if (\in_array(Resource::TYPE_PROJECT_VARIABLE, $resources)) { + $variableQueries = $this->buildQueries( + resourceType: Resource::TYPE_PROJECT_VARIABLE, + resourceIds: $resourceIds, + limit: 1 + ); + try { + $report[Resource::TYPE_PROJECT_VARIABLE] = $this->project->listVariables($variableQueries)->total; + } catch (\Throwable) { + $report[Resource::TYPE_PROJECT_VARIABLE] = 0; + } + } + } + + /** + * @param int $batchSize + * @param array $resources + */ + protected function exportGroupSettings(int $batchSize, array $resources): void + { + if (\in_array(Resource::TYPE_PROJECT_VARIABLE, $resources)) { + try { + $this->exportProjectVariables($batchSize); + } catch (\Throwable $e) { + $this->addError(new Exception( + Resource::TYPE_PROJECT_VARIABLE, + Transfer::GROUP_SETTINGS, + message: $e->getMessage(), + code: $e->getCode(), + previous: $e + )); + } + } + } + + /** + * @throws AppwriteException + */ + private function exportProjectVariables(int $batchSize): void + { + $lastId = null; + + while (true) { + $queries = [Query::limit($batchSize)]; + + if ($this->rootResourceId !== '' && $this->rootResourceType === Resource::TYPE_PROJECT_VARIABLE) { + $queries[] = Query::equal('$id', $this->rootResourceId); + $queries[] = Query::limit(1); + } + + if ($lastId !== null) { + $queries[] = Query::cursorAfter($lastId); + } + + $response = $this->project->listVariables($queries); + if ($response->total === 0) { + break; + } + + $variables = []; + + foreach ($response->variables as $variable) { + $variables[] = new ProjectVariable( + $variable->id, + $variable->key, + $variable->value, + $variable->secret, + createdAt: $variable->createdAt, + updatedAt: $variable->updatedAt, + ); + + $lastId = $variable->id; + } + + $this->callback($variables); + + if (\count($response->variables) < $batchSize) { + break; + } + } + } + /** * @throws AppwriteException */ diff --git a/src/Migration/Sources/CSV.php b/src/Migration/Sources/CSV.php index 678b3b9eec..22c9399b86 100644 --- a/src/Migration/Sources/CSV.php +++ b/src/Migration/Sources/CSV.php @@ -440,6 +440,11 @@ class CSV extends Source throw new \Exception('Not Implemented'); } + protected function exportGroupSettings(int $batchSize, array $resources): void + { + throw new \Exception('Not Implemented'); + } + /** * @param callable(resource $stream, string $delimiter): void $callback * @return void diff --git a/src/Migration/Sources/Firebase.php b/src/Migration/Sources/Firebase.php index 007797b5b6..2bea50de0d 100644 --- a/src/Migration/Sources/Firebase.php +++ b/src/Migration/Sources/Firebase.php @@ -827,4 +827,9 @@ class Firebase extends Source { throw new \Exception('Not implemented'); } + + protected function exportGroupSettings(int $batchSize, array $resources): void + { + throw new \Exception('Not implemented'); + } } diff --git a/src/Migration/Sources/JSON.php b/src/Migration/Sources/JSON.php index 0b3e7829f1..149ad66f97 100644 --- a/src/Migration/Sources/JSON.php +++ b/src/Migration/Sources/JSON.php @@ -214,6 +214,11 @@ class JSON extends Source throw new \Exception('Not Implemented'); } + protected function exportGroupSettings(int $batchSize, array $resources): void + { + throw new \Exception('Not Implemented'); + } + /** * @throws \Exception */ diff --git a/src/Migration/Sources/NHost.php b/src/Migration/Sources/NHost.php index c72ca81223..6c8fe0ac88 100644 --- a/src/Migration/Sources/NHost.php +++ b/src/Migration/Sources/NHost.php @@ -962,4 +962,9 @@ class NHost extends Source { throw new \Exception('Not Implemented'); } + + protected function exportGroupSettings(int $batchSize, array $resources): void + { + throw new \Exception('Not Implemented'); + } } diff --git a/src/Migration/Transfer.php b/src/Migration/Transfer.php index 15a0c8aeb2..069a780bc8 100644 --- a/src/Migration/Transfer.php +++ b/src/Migration/Transfer.php @@ -28,6 +28,8 @@ class Transfer public const GROUP_BACKUPS = 'backups'; + public const GROUP_SETTINGS = 'settings'; + public const GROUP_AUTH_RESOURCES = [ Resource::TYPE_USER, Resource::TYPE_TEAM, @@ -92,7 +94,9 @@ class Transfer Resource::TYPE_ATTRIBUTE ]; - public const GROUP_SETTINGS_RESOURCES = []; + public const GROUP_SETTINGS_RESOURCES = [ + Resource::TYPE_PROJECT_VARIABLE, + ]; public const GROUP_BACKUPS_RESOURCES = [ Resource::TYPE_BACKUP_POLICY, @@ -132,6 +136,9 @@ class Transfer Resource::TYPE_PLATFORM, Resource::TYPE_API_KEY, + // Settings + Resource::TYPE_PROJECT_VARIABLE, + // legacy Resource::TYPE_DOCUMENT, Resource::TYPE_ATTRIBUTE,