From d64d7ece1262acf9ce4026625cfb734937e55627 Mon Sep 17 00:00:00 2001 From: Prem Palanisamy Date: Thu, 14 May 2026 17:24:32 +0100 Subject: [PATCH 1/6] Add project variable migration support Reports project-variable counts in MigrationReport and points the migration lib pin at the add-project-variable-migration dev branch. --- composer.json | 14 +++++-- composer.lock | 38 ++++++++++++++----- .../Utopia/Response/Model/MigrationReport.php | 6 +++ 3 files changed, 44 insertions(+), 14 deletions(-) diff --git a/composer.json b/composer.json index 37e7a1ae0e..67b0dfcf59 100644 --- a/composer.json +++ b/composer.json @@ -74,7 +74,7 @@ "utopia-php/locale": "0.8.*", "utopia-php/logger": "0.8.*", "utopia-php/messaging": "0.22.*", - "utopia-php/migration": "dev-add-api-key-migration as 1.12.0", + "utopia-php/migration": "dev-add-project-variable-migration as 1.12.0", "utopia-php/platform": "^1.0@RC", "utopia-php/pools": "1.*", "utopia-php/span": "1.1.*", @@ -112,11 +112,17 @@ "ext-phpiredis": "*" }, "config": { - "platform": { - }, + "platform": {}, "allow-plugins": { "php-http/discovery": true, "tbachert/spi": true } - } + }, + "repositories": [ + { + "name": "migration", + "type": "vcs", + "url": "https://github.com/utopia-php/migration" + } + ] } diff --git a/composer.lock b/composer.lock index 199710b788..02bc082abf 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "70b7a6b3803660b7a8aa1a542f630b02", + "content-hash": "5bf93473859f86fbf9438b17cb660331", "packages": [ { "name": "adhocore/jwt", @@ -4606,16 +4606,16 @@ }, { "name": "utopia-php/migration", - "version": "dev-add-api-key-migration", + "version": "dev-add-project-variable-migration", "source": { "type": "git", "url": "https://github.com/utopia-php/migration.git", - "reference": "93715c256bf57cd93a1ef92e366c19f9a53ff21a" + "reference": "c040472abd3c9874017f18cb1589c3c26677451c" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/migration/zipball/93715c256bf57cd93a1ef92e366c19f9a53ff21a", - "reference": "93715c256bf57cd93a1ef92e366c19f9a53ff21a", + "url": "https://api.github.com/repos/utopia-php/migration/zipball/c040472abd3c9874017f18cb1589c3c26677451c", + "reference": "c040472abd3c9874017f18cb1589c3c26677451c", "shasum": "" }, "require": { @@ -4641,7 +4641,25 @@ "Utopia\\Migration\\": "src/Migration" } }, - "notification-url": "https://packagist.org/downloads/", + "autoload-dev": { + "psr-4": { + "Utopia\\Tests\\": "tests/Migration" + } + }, + "scripts": { + "test": [ + "./vendor/bin/phpunit" + ], + "lint": [ + "./vendor/bin/pint --test" + ], + "format": [ + "./vendor/bin/pint" + ], + "check": [ + "./vendor/bin/phpstan analyse --level 3 src tests --memory-limit 2G" + ] + }, "license": [ "MIT" ], @@ -4654,10 +4672,10 @@ "utopia" ], "support": { - "issues": "https://github.com/utopia-php/migration/issues", - "source": "https://github.com/utopia-php/migration/tree/add-api-key-migration" + "source": "https://github.com/utopia-php/migration/tree/add-project-variable-migration", + "issues": "https://github.com/utopia-php/migration/issues" }, - "time": "2026-05-14T14:53:20+00:00" + "time": "2026-05-14T16:18:34+00:00" }, { "name": "utopia-php/mongo", @@ -8567,7 +8585,7 @@ "aliases": [ { "package": "utopia-php/migration", - "version": "dev-add-api-key-migration", + "version": "dev-add-project-variable-migration", "alias": "1.12.0", "alias_normalized": "1.12.0.0" } diff --git a/src/Appwrite/Utopia/Response/Model/MigrationReport.php b/src/Appwrite/Utopia/Response/Model/MigrationReport.php index 09380d8628..9de39d75bf 100644 --- a/src/Appwrite/Utopia/Response/Model/MigrationReport.php +++ b/src/Appwrite/Utopia/Response/Model/MigrationReport.php @@ -65,6 +65,12 @@ class MigrationReport extends Model 'default' => 0, 'example' => 5, ]) + ->addRule(Resource::TYPE_PROJECT_VARIABLE, [ + 'type' => self::TYPE_INTEGER, + 'description' => 'Number of project variables to be migrated.', + 'default' => 0, + 'example' => 5, + ]) ->addRule(Resource::TYPE_SITE, [ 'type' => self::TYPE_INTEGER, 'description' => 'Number of sites to be migrated.', From 176d916dc3ed3442af321c9473afd457b60634d4 Mon Sep 17 00:00:00 2001 From: Prem Palanisamy Date: Thu, 14 May 2026 20:11:04 +0100 Subject: [PATCH 2/6] Bump utopia-php/migration lock to GROUP_SETTINGS refactor --- composer.lock | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/composer.lock b/composer.lock index 02bc082abf..9b6291a94d 100644 --- a/composer.lock +++ b/composer.lock @@ -4610,12 +4610,12 @@ "source": { "type": "git", "url": "https://github.com/utopia-php/migration.git", - "reference": "c040472abd3c9874017f18cb1589c3c26677451c" + "reference": "3376bed6f797a8d48f0107e9a0d8e40745d6914b" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/migration/zipball/c040472abd3c9874017f18cb1589c3c26677451c", - "reference": "c040472abd3c9874017f18cb1589c3c26677451c", + "url": "https://api.github.com/repos/utopia-php/migration/zipball/3376bed6f797a8d48f0107e9a0d8e40745d6914b", + "reference": "3376bed6f797a8d48f0107e9a0d8e40745d6914b", "shasum": "" }, "require": { @@ -4675,7 +4675,7 @@ "source": "https://github.com/utopia-php/migration/tree/add-project-variable-migration", "issues": "https://github.com/utopia-php/migration/issues" }, - "time": "2026-05-14T16:18:34+00:00" + "time": "2026-05-14T19:00:51+00:00" }, { "name": "utopia-php/mongo", From 2c124171d6fd8828620933ccf101fe5a3517c2fa Mon Sep 17 00:00:00 2001 From: Prem Palanisamy Date: Fri, 15 May 2026 07:22:44 +0100 Subject: [PATCH 3/6] Add E2E test for project variable migration Covers plain + secret variables, asserts secret value is not exfiltrated through the migration (source SDK strips it, destination writes empty). --- .../Services/Migrations/MigrationsBase.php | 92 +++++++++++++++++++ 1 file changed, 92 insertions(+) diff --git a/tests/e2e/Services/Migrations/MigrationsBase.php b/tests/e2e/Services/Migrations/MigrationsBase.php index 1d3c93d704..e3c8318ed0 100644 --- a/tests/e2e/Services/Migrations/MigrationsBase.php +++ b/tests/e2e/Services/Migrations/MigrationsBase.php @@ -2616,6 +2616,98 @@ trait MigrationsBase $this->client->call(Client::METHOD_DELETE, '/project/keys/' . $apiKey['$id'], $sourceHeaders); } + public function testAppwriteMigrationProjectVariable(): void + { + $sourceHeaders = [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ]; + + $destinationHeaders = [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getDestinationProject()['$id'], + 'x-appwrite-key' => $this->getDestinationProject()['apiKey'], + ]; + + // Source-side variable IDs and keys are uniquified so re-runs and parallel suites + // can't trip the source-side findOne('variables', [key=...]) skip path. + $plainKey = 'TEST_PLAIN_' . \strtoupper(ID::unique()); + $secretKey = 'TEST_SECRET_' . \strtoupper(ID::unique()); + + // Non-secret variable: value should round-trip exactly. + $plainResp = $this->client->call(Client::METHOD_POST, '/project/variables', $sourceHeaders, [ + 'variableId' => ID::unique(), + 'key' => $plainKey, + 'value' => 'plain-value', + 'secret' => false, + ]); + $this->assertEquals(201, $plainResp['headers']['status-code']); + $plainVariable = $plainResp['body']; + + // Secret variable: SDK strips `value` on subsequent reads, so the migration + // source sees empty and the destination writes empty. Test asserts that. + $secretResp = $this->client->call(Client::METHOD_POST, '/project/variables', $sourceHeaders, [ + 'variableId' => ID::unique(), + 'key' => $secretKey, + 'value' => 'real-secret-value', + 'secret' => true, + ]); + $this->assertEquals(201, $secretResp['headers']['status-code']); + $secretVariable = $secretResp['body']; + + $result = $this->performMigrationSync([ + 'resources' => [ + Resource::TYPE_PROJECT_VARIABLE, + ], + 'endpoint' => $this->webEndpoint, + 'projectId' => $this->getProject()['$id'], + 'apiKey' => $this->getProject()['apiKey'], + ]); + + $this->assertEquals('completed', $result['status']); + $this->assertEquals([Resource::TYPE_PROJECT_VARIABLE], $result['resources']); + $this->assertArrayHasKey(Resource::TYPE_PROJECT_VARIABLE, $result['statusCounters']); + $this->assertEquals(0, $result['statusCounters'][Resource::TYPE_PROJECT_VARIABLE]['error']); + $this->assertEquals(0, $result['statusCounters'][Resource::TYPE_PROJECT_VARIABLE]['pending']); + $this->assertGreaterThanOrEqual(2, $result['statusCounters'][Resource::TYPE_PROJECT_VARIABLE]['success']); + $this->assertEquals(0, $result['statusCounters'][Resource::TYPE_PROJECT_VARIABLE]['processing']); + $this->assertEquals(0, $result['statusCounters'][Resource::TYPE_PROJECT_VARIABLE]['warning']); + + $response = $this->client->call(Client::METHOD_GET, '/project/variables', $destinationHeaders); + $this->assertEquals(200, $response['headers']['status-code']); + + $foundPlain = null; + $foundSecret = null; + foreach ($response['body']['variables'] as $v) { + if ($v['key'] === $plainKey) { + $foundPlain = $v; + } elseif ($v['key'] === $secretKey) { + $foundSecret = $v; + } + } + + $this->assertNotNull($foundPlain, 'Plain variable not found on destination'); + $this->assertEquals($plainKey, $foundPlain['key']); + $this->assertEquals('plain-value', $foundPlain['value']); + $this->assertFalse($foundPlain['secret']); + + $this->assertNotNull($foundSecret, 'Secret variable not found on destination'); + $this->assertEquals($secretKey, $foundSecret['key']); + // Secret variables: source SDK never returned the real value, so the destination + // also stores empty. The original 'real-secret-value' must not have leaked. + $this->assertNotEquals('real-secret-value', $foundSecret['value']); + $this->assertTrue($foundSecret['secret']); + + // Cleanup on destination + $this->client->call(Client::METHOD_DELETE, '/project/variables/' . $foundPlain['$id'], $destinationHeaders); + $this->client->call(Client::METHOD_DELETE, '/project/variables/' . $foundSecret['$id'], $destinationHeaders); + + // Cleanup on source + $this->client->call(Client::METHOD_DELETE, '/project/variables/' . $plainVariable['$id'], $sourceHeaders); + $this->client->call(Client::METHOD_DELETE, '/project/variables/' . $secretVariable['$id'], $sourceHeaders); + } + /** * Import documents from a CSV file. */ From 02a09aa5e78dbf940ee6cf40d48ded15dbbb01bf Mon Sep 17 00:00:00 2001 From: Prem Palanisamy Date: Wed, 27 May 2026 15:01:41 +0100 Subject: [PATCH 4/6] Tighten secret project-variable assertion (Greptile P1) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit assertNotEquals('real-secret-value', $value) only rules out that exact string — a partial/corrupted leak (e.g. 'real-secret', 'REAL-SECRET-VALUE', hash) would still pass. The documented behavior is the destination stores '' because the source SDK strips secret values from list responses, so assertEmpty is the strictly tighter assertion. --- tests/e2e/Services/Migrations/MigrationsBase.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/e2e/Services/Migrations/MigrationsBase.php b/tests/e2e/Services/Migrations/MigrationsBase.php index d9181262fc..0c66cf8762 100644 --- a/tests/e2e/Services/Migrations/MigrationsBase.php +++ b/tests/e2e/Services/Migrations/MigrationsBase.php @@ -2707,7 +2707,7 @@ trait MigrationsBase $this->assertEquals($secretKey, $foundSecret['key']); // Secret variables: source SDK never returned the real value, so the destination // also stores empty. The original 'real-secret-value' must not have leaked. - $this->assertNotEquals('real-secret-value', $foundSecret['value']); + $this->assertEmpty($foundSecret['value']); $this->assertTrue($foundSecret['secret']); // Cleanup on destination From acbb15260b3d2d2918626b15eda1c0d1f1c0a03f Mon Sep 17 00:00:00 2001 From: Prem Palanisamy Date: Wed, 27 May 2026 21:40:13 +0100 Subject: [PATCH 5/6] Bump migration pin to pull in GROUP_SETTINGS extractServices fix --- composer.lock | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/composer.lock b/composer.lock index 830c4fd8e3..2a4f517de6 100644 --- a/composer.lock +++ b/composer.lock @@ -4661,12 +4661,12 @@ "source": { "type": "git", "url": "https://github.com/utopia-php/migration.git", - "reference": "5c96d2e0c54fdfabf4dc7468bcc14f2d53c0f928" + "reference": "0ecc00c1456328fb8e61514795f6fc631242e0a3" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/migration/zipball/5c96d2e0c54fdfabf4dc7468bcc14f2d53c0f928", - "reference": "5c96d2e0c54fdfabf4dc7468bcc14f2d53c0f928", + "url": "https://api.github.com/repos/utopia-php/migration/zipball/0ecc00c1456328fb8e61514795f6fc631242e0a3", + "reference": "0ecc00c1456328fb8e61514795f6fc631242e0a3", "shasum": "" }, "require": { @@ -4726,7 +4726,7 @@ "source": "https://github.com/utopia-php/migration/tree/add-project-variable-migration", "issues": "https://github.com/utopia-php/migration/issues" }, - "time": "2026-05-14T22:03:48+00:00" + "time": "2026-05-27T20:34:11+00:00" }, { "name": "utopia-php/mongo", From 9a7b6f6f035493ac62e64977f3263e5b7efb6300 Mon Sep 17 00:00:00 2001 From: Prem Palanisamy Date: Wed, 3 Jun 2026 10:16:38 +0100 Subject: [PATCH 6/6] Clean migrated project variables in E2E --- .../Services/Migrations/MigrationsBase.php | 45 ++++++++++++++++--- 1 file changed, 39 insertions(+), 6 deletions(-) diff --git a/tests/e2e/Services/Migrations/MigrationsBase.php b/tests/e2e/Services/Migrations/MigrationsBase.php index a0c6de3e59..0cafd64975 100644 --- a/tests/e2e/Services/Migrations/MigrationsBase.php +++ b/tests/e2e/Services/Migrations/MigrationsBase.php @@ -11,6 +11,7 @@ use Tests\E2E\Scopes\ProjectCustom; use Tests\E2E\Services\Functions\FunctionsBase; use Utopia\Console; use Utopia\Database\Database; +use Utopia\Database\Document; use Utopia\Database\Helpers\ID; use Utopia\Database\Helpers\Permission; use Utopia\Database\Helpers\Role; @@ -2641,6 +2642,35 @@ trait MigrationsBase 'x-appwrite-key' => $this->getDestinationProject()['apiKey'], ]; + $listDestinationVariables = function () use ($destinationHeaders): array { + $variables = []; + $cursorId = null; + + do { + $queries = [ + Query::limit(100)->toString(), + ]; + + if ($cursorId !== null) { + $queries[] = Query::cursorAfter(new Document(['$id' => $cursorId]))->toString(); + } + + $response = $this->client->call(Client::METHOD_GET, '/project/variables', $destinationHeaders, [ + 'queries' => $queries, + 'total' => false, + ]); + $this->assertEquals(200, $response['headers']['status-code']); + + $batch = $response['body']['variables']; + array_push($variables, ...$batch); + $cursorId = !empty($batch) ? $batch[array_key_last($batch)]['$id'] : null; + } while (\count($batch) === 100); + + return $variables; + }; + + $existingDestinationVariableIds = \array_flip(\array_column($listDestinationVariables(), '$id')); + // Source-side variable IDs and keys are uniquified so re-runs and parallel suites // can't trip the source-side findOne('variables', [key=...]) skip path. $plainKey = 'TEST_PLAIN_' . \strtoupper(ID::unique()); @@ -2685,12 +2715,11 @@ trait MigrationsBase $this->assertEquals(0, $result['statusCounters'][Resource::TYPE_PROJECT_VARIABLE]['processing']); $this->assertEquals(0, $result['statusCounters'][Resource::TYPE_PROJECT_VARIABLE]['warning']); - $response = $this->client->call(Client::METHOD_GET, '/project/variables', $destinationHeaders); - $this->assertEquals(200, $response['headers']['status-code']); + $destinationVariables = $listDestinationVariables(); $foundPlain = null; $foundSecret = null; - foreach ($response['body']['variables'] as $v) { + foreach ($destinationVariables as $v) { if ($v['key'] === $plainKey) { $foundPlain = $v; } elseif ($v['key'] === $secretKey) { @@ -2710,9 +2739,13 @@ trait MigrationsBase $this->assertEmpty($foundSecret['value']); $this->assertTrue($foundSecret['secret']); - // Cleanup on destination - $this->client->call(Client::METHOD_DELETE, '/project/variables/' . $foundPlain['$id'], $destinationHeaders); - $this->client->call(Client::METHOD_DELETE, '/project/variables/' . $foundSecret['$id'], $destinationHeaders); + // Cleanup every destination variable this migration added, including any + // unrelated source variables copied by the resource-level migration. + foreach ($destinationVariables as $variable) { + if (!isset($existingDestinationVariableIds[$variable['$id']])) { + $this->client->call(Client::METHOD_DELETE, '/project/variables/' . $variable['$id'], $destinationHeaders); + } + } // Cleanup on source $this->client->call(Client::METHOD_DELETE, '/project/variables/' . $plainVariable['$id'], $sourceHeaders);