From fb4930741eb94eb4fa514e670833c1b9f3d9e36b Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Fri, 1 May 2026 12:37:01 +1200 Subject: [PATCH] refactor(migrations): split composite resourceId into parent/leaf The migrations collection used a composite "{databaseId}:{collectionId}" in resourceId for CSV/JSON imports and exports, which mixed two identifiers in one column and broke queryability. Split it into parent (database) and leaf (collection) attributes following the convention used by other collections. Schema (collection bumped to V25): - Adds resourceInternalId, parentResourceId, parentResourceInternalId, parentResourceType to the migrations collection - Drops _key_resource_id; adds per-attribute indexes for every resource-related attribute so all of them are queryable - V25 backfills existing migration documents by splitting the legacy composite resourceId, mapping the legacy resourceType (database type) to parentResourceType, and looking up sequence IDs API: - /v1/migrations/csv|json/imports|exports replace the CompoundUID resourceId param with separate UID databaseId + collectionId params - Worker now reads parent/leaf fields directly and recomposes the legacy "id:id" shape only at the boundary to utopia-php/migration Backwards compatibility: - Request V25 filter splits old-SDK resourceId into databaseId/collectionId - Response V25 filter recomposes resourceId for old-SDK consumers and strips the new fields - Query validator now allows querying every resource-related attribute Co-Authored-By: Claude Opus 4.7 (1M context) --- app/config/collections/projects.php | 81 +++++++++++- app/controllers/general.php | 8 ++ app/init/constants.php | 2 +- src/Appwrite/Migration/Migration.php | 1 + src/Appwrite/Migration/Version/V25.php | 116 ++++++++++++++++++ .../Http/Migrations/CSV/Exports/Create.php | 26 ++-- .../Http/Migrations/CSV/Imports/Create.php | 28 +++-- .../Http/Migrations/JSON/Exports/Create.php | 26 ++-- .../Http/Migrations/JSON/Imports/Create.php | 24 ++-- src/Appwrite/Platform/Workers/Migrations.php | 35 ++++-- .../Database/Validator/Queries/Migrations.php | 6 +- src/Appwrite/Utopia/Request/Filters/V25.php | 40 ++++++ src/Appwrite/Utopia/Response/Filters/V25.php | 38 ++++++ .../Utopia/Response/Model/Migration.php | 39 +++++- .../Services/Migrations/MigrationsBase.php | 51 +++++--- 15 files changed, 448 insertions(+), 73 deletions(-) create mode 100644 src/Appwrite/Migration/Version/V25.php create mode 100644 src/Appwrite/Utopia/Request/Filters/V25.php create mode 100644 src/Appwrite/Utopia/Response/Filters/V25.php diff --git a/app/config/collections/projects.php b/app/config/collections/projects.php index 9568c59369..45fd61095d 100644 --- a/app/config/collections/projects.php +++ b/app/config/collections/projects.php @@ -2483,6 +2483,17 @@ return [ 'array' => false, 'filters' => [], ], + [ + '$id' => ID::custom('resourceInternalId'), + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => Database::LENGTH_KEY, + 'signed' => true, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => [], + ], [ '$id' => ID::custom('resourceType'), 'type' => Database::VAR_STRING, @@ -2494,6 +2505,39 @@ return [ 'array' => false, 'filters' => [], ], + [ + '$id' => ID::custom('parentResourceId'), + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => Database::LENGTH_KEY, + 'signed' => true, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => ID::custom('parentResourceInternalId'), + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => Database::LENGTH_KEY, + 'signed' => true, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => ID::custom('parentResourceType'), + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => Database::LENGTH_KEY, + 'signed' => true, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => [], + ], ], 'indexes' => [ [ @@ -2518,12 +2562,47 @@ return [ 'orders' => [Database::ORDER_ASC], ], [ - '$id' => '_key_resource_id', + '$id' => '_key_resourceId', 'type' => Database::INDEX_KEY, 'attributes' => ['resourceId'], 'lengths' => [Database::LENGTH_KEY], 'orders' => [Database::ORDER_DESC], ], + [ + '$id' => '_key_resourceType', + 'type' => Database::INDEX_KEY, + 'attributes' => ['resourceType'], + 'lengths' => [Database::LENGTH_KEY], + 'orders' => [Database::ORDER_ASC], + ], + [ + '$id' => '_key_resourceInternalId', + 'type' => Database::INDEX_KEY, + 'attributes' => ['resourceInternalId'], + 'lengths' => [Database::LENGTH_KEY], + 'orders' => [Database::ORDER_ASC], + ], + [ + '$id' => '_key_parentResourceId', + 'type' => Database::INDEX_KEY, + 'attributes' => ['parentResourceId'], + 'lengths' => [Database::LENGTH_KEY], + 'orders' => [Database::ORDER_ASC], + ], + [ + '$id' => '_key_parentResourceType', + 'type' => Database::INDEX_KEY, + 'attributes' => ['parentResourceType'], + 'lengths' => [Database::LENGTH_KEY], + 'orders' => [Database::ORDER_ASC], + ], + [ + '$id' => '_key_parentResourceInternalId', + 'type' => Database::INDEX_KEY, + 'attributes' => ['parentResourceInternalId'], + 'lengths' => [Database::LENGTH_KEY], + 'orders' => [Database::ORDER_ASC], + ], [ '$id' => ID::custom('_fulltext_search'), 'type' => Database::INDEX_FULLTEXT, diff --git a/app/controllers/general.php b/app/controllers/general.php index eb4899a3d8..21bcded22c 100644 --- a/app/controllers/general.php +++ b/app/controllers/general.php @@ -28,6 +28,7 @@ use Appwrite\Utopia\Request\Filters\V21 as RequestV21; use Appwrite\Utopia\Request\Filters\V22 as RequestV22; use Appwrite\Utopia\Request\Filters\V23 as RequestV23; use Appwrite\Utopia\Request\Filters\V24 as RequestV24; +use Appwrite\Utopia\Request\Filters\V25 as RequestV25; use Appwrite\Utopia\Response; use Appwrite\Utopia\Response\Filters\V16 as ResponseV16; use Appwrite\Utopia\Response\Filters\V17 as ResponseV17; @@ -38,6 +39,7 @@ use Appwrite\Utopia\Response\Filters\V21 as ResponseV21; use Appwrite\Utopia\Response\Filters\V22 as ResponseV22; use Appwrite\Utopia\Response\Filters\V23 as ResponseV23; use Appwrite\Utopia\Response\Filters\V24 as ResponseV24; +use Appwrite\Utopia\Response\Filters\V25 as ResponseV25; use Appwrite\Utopia\View; use Executor\Executor; use MaxMind\Db\Reader; @@ -904,6 +906,9 @@ Http::init() if (version_compare($requestFormat, '1.9.3', '<')) { $request->addFilter(new RequestV24()); } + if (version_compare($requestFormat, '1.9.4', '<')) { + $request->addFilter(new RequestV25()); + } } $localeParam = (string) $request->getParam('locale', $request->getHeader('x-appwrite-locale', '')); @@ -928,6 +933,9 @@ Http::init() */ $responseFormat = $request->getHeader('x-appwrite-response-format', System::getEnv('_APP_SYSTEM_RESPONSE_FORMAT', '')); if ($responseFormat) { + if (version_compare($responseFormat, '1.9.4', '<')) { + $response->addFilter(new ResponseV25()); + } if (version_compare($responseFormat, '1.9.3', '<')) { $response->addFilter(new ResponseV24()); } diff --git a/app/init/constants.php b/app/init/constants.php index f27d0c7c70..64635bab2a 100644 --- a/app/init/constants.php +++ b/app/init/constants.php @@ -45,7 +45,7 @@ const APP_RESOURCE_TOKEN_ACCESS = 24 * 60 * 60; // 24 hours const APP_FILE_ACCESS = 24 * 60 * 60; // 24 hours const APP_CACHE_UPDATE = 24 * 60 * 60; // 24 hours const APP_CACHE_BUSTER = 4324; -const APP_VERSION_STABLE = '1.9.3'; +const APP_VERSION_STABLE = '1.9.4'; const APP_DATABASE_ATTRIBUTE_EMAIL = 'email'; const APP_DATABASE_ATTRIBUTE_ENUM = 'enum'; const APP_DATABASE_ATTRIBUTE_IP = 'ip'; diff --git a/src/Appwrite/Migration/Migration.php b/src/Appwrite/Migration/Migration.php index 359925e368..004e09cd23 100644 --- a/src/Appwrite/Migration/Migration.php +++ b/src/Appwrite/Migration/Migration.php @@ -96,6 +96,7 @@ abstract class Migration '1.9.1' => 'V24', '1.9.2' => 'V24', '1.9.3' => 'V24', + '1.9.4' => 'V25', ]; /** diff --git a/src/Appwrite/Migration/Version/V25.php b/src/Appwrite/Migration/Version/V25.php new file mode 100644 index 0000000000..6ba8afef0c --- /dev/null +++ b/src/Appwrite/Migration/Version/V25.php @@ -0,0 +1,116 @@ +migrateCollections(); + + Console::info('Migrating documents'); + $this->forEachDocument($this->migrateDocument(...)); + } + + /** + * @throws Throwable + */ + private function migrateCollections(): void + { + if ($this->project->getSequence() === 'console') { + return; + } + + $id = 'migrations'; + + $this->dbForProject->purgeCachedCollection($id); + $this->dbForProject->purgeCachedDocument(Database::METADATA, $id); + + $attributes = [ + 'resourceInternalId', + 'parentResourceId', + 'parentResourceInternalId', + 'parentResourceType', + ]; + try { + $this->createAttributesFromCollection($this->dbForProject, $id, $attributes); + } catch (Throwable $th) { + Console::warning('Failed to create attributes "' . \implode(', ', $attributes) . "\" in collection {$id}: {$th->getMessage()}"); + } + + try { + $this->dbForProject->deleteIndex($id, '_key_resource_id'); + } catch (Throwable $th) { + Console::warning("Failed to delete index \"_key_resource_id\" from {$id}: {$th->getMessage()}"); + } + + $indexes = [ + '_key_resourceId', + '_key_resourceType', + '_key_resourceInternalId', + '_key_parentResourceId', + '_key_parentResourceType', + '_key_parentResourceInternalId', + ]; + foreach ($indexes as $index) { + try { + $this->createIndexFromCollection($this->dbForProject, $id, $index); + } catch (Throwable $th) { + Console::warning("Failed to create index \"{$index}\" from {$id}: {$th->getMessage()}"); + } + } + + $this->dbForProject->purgeCachedCollection($id); + } + + private function migrateDocument(Document $document): Document + { + if ($document->getCollection() !== 'migrations') { + return $document; + } + + if (!empty($document->getAttribute('parentResourceId'))) { + return $document; + } + + $resourceId = $document->getAttribute('resourceId'); + if (empty($resourceId) || !\str_contains($resourceId, ':')) { + return $document; + } + + [$parentId, $childId] = \explode(':', $resourceId, 2); + $parentResourceType = $document->getAttribute('resourceType'); + + $document + ->setAttribute('resourceId', $childId) + ->setAttribute('resourceType', 'collection') + ->setAttribute('parentResourceId', $parentId) + ->setAttribute('parentResourceType', $parentResourceType); + + try { + $database = $this->dbForProject->getDocument('databases', $parentId); + if (!$database->isEmpty()) { + $document->setAttribute('parentResourceInternalId', (string) $database->getSequence()); + + $collection = $this->dbForProject->getDocument('database_' . $database->getSequence(), $childId); + if (!$collection->isEmpty()) { + $document->setAttribute('resourceInternalId', (string) $collection->getSequence()); + } + } + } catch (Throwable $th) { + Console::warning("Failed to backfill internal IDs for migration {$document->getId()}: {$th->getMessage()}"); + } + + return $document; + } +} diff --git a/src/Appwrite/Platform/Modules/Migrations/Http/Migrations/CSV/Exports/Create.php b/src/Appwrite/Platform/Modules/Migrations/Http/Migrations/CSV/Exports/Create.php index 0ab3cecf1a..bab990bee8 100644 --- a/src/Appwrite/Platform/Modules/Migrations/Http/Migrations/CSV/Exports/Create.php +++ b/src/Appwrite/Platform/Modules/Migrations/Http/Migrations/CSV/Exports/Create.php @@ -9,7 +9,6 @@ use Appwrite\Extend\Exception; use Appwrite\SDK\AuthType; use Appwrite\SDK\Method; use Appwrite\SDK\Response as SDKResponse; -use Appwrite\Utopia\Database\Validator\CompoundUID; use Appwrite\Utopia\Response; use Utopia\Database\Database; use Utopia\Database\Document; @@ -18,6 +17,7 @@ use Utopia\Database\Helpers\ID; use Utopia\Database\Query; use Utopia\Database\Validator\Authorization; use Utopia\Database\Validator\Queries\Documents; +use Utopia\Database\Validator\UID; use Utopia\Migration\Resource; use Utopia\Migration\Sources\Appwrite as AppwriteSource; use Utopia\Migration\Sources\CSV; @@ -60,7 +60,8 @@ class Create extends Action ) ] )) - ->param('resourceId', null, new CompoundUID(), 'Composite ID in the format {databaseId:collectionId}, identifying a collection within a database to export.') + ->param('databaseId', '', new UID(), 'Database ID containing the source collection.') + ->param('collectionId', '', new UID(), 'Collection ID to export documents from.') ->param('filename', '', new Text(255), 'The name of the file to be created for the export, excluding the .csv extension.') ->param('columns', [], new ArrayList(new Text(Database::LENGTH_KEY)), 'List of attributes to export. If empty, all attributes will be exported. You can use the `*` wildcard to export all attributes from the collection.', true) ->param('queries', [], new ArrayList(new Text(0)), 'Array of query strings generated using the Query class provided by the SDK to filter documents to export. [Learn more about queries](https://appwrite.io/docs/databases#querying-documents). Maximum of ' . APP_LIMIT_ARRAY_PARAMS_SIZE . ' queries are allowed, each ' . APP_LIMIT_ARRAY_ELEMENT_SIZE . ' characters long.', true) @@ -82,7 +83,8 @@ class Create extends Action } public function action( - string $resourceId, + string $databaseId, + string $collectionId, string $filename, array $columns, array $queries, @@ -112,14 +114,6 @@ class Create extends Action throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND); } - [$databaseId, $collectionId] = \explode(':', $resourceId, 2); - if (empty($databaseId)) { - throw new Exception(Exception::DATABASE_NOT_FOUND); - } - if (empty($collectionId)) { - throw new Exception(Exception::COLLECTION_NOT_FOUND); - } - $database = $authorization->skip(fn () => $dbForProject->getDocument('databases', $databaseId)); if ($database->isEmpty()) { throw new Exception(Exception::DATABASE_NOT_FOUND); @@ -150,7 +144,7 @@ class Create extends Action } $resources = Transfer::extractServices([self::transferGroupForDatabaseType($databaseType)]); - $resourceType = self::resourceTypeForDatabaseType($databaseType); + $parentResourceType = self::resourceTypeForDatabaseType($databaseType); $migration = $dbForProject->createDocument('migrations', new Document([ '$id' => ID::unique(), @@ -159,8 +153,12 @@ class Create extends Action 'source' => AppwriteSource::getName(), 'destination' => CSV::getName(), 'resources' => $resources, - 'resourceId' => $resourceId, - 'resourceType' => $resourceType, + 'resourceId' => $collection->getId(), + 'resourceInternalId' => $collection->getSequence(), + 'resourceType' => Resource::TYPE_COLLECTION, + 'parentResourceId' => $database->getId(), + 'parentResourceInternalId' => $database->getSequence(), + 'parentResourceType' => $parentResourceType, 'statusCounters' => '{}', 'resourceData' => '{}', 'errors' => [], diff --git a/src/Appwrite/Platform/Modules/Migrations/Http/Migrations/CSV/Imports/Create.php b/src/Appwrite/Platform/Modules/Migrations/Http/Migrations/CSV/Imports/Create.php index 5cc21241c3..b692dfe756 100644 --- a/src/Appwrite/Platform/Modules/Migrations/Http/Migrations/CSV/Imports/Create.php +++ b/src/Appwrite/Platform/Modules/Migrations/Http/Migrations/CSV/Imports/Create.php @@ -10,7 +10,6 @@ use Appwrite\OpenSSL\OpenSSL; use Appwrite\SDK\AuthType; use Appwrite\SDK\Method; use Appwrite\SDK\Response as SDKResponse; -use Appwrite\Utopia\Database\Validator\CompoundUID; use Appwrite\Utopia\Response; use Utopia\Compression\Algorithms\GZIP; use Utopia\Compression\Algorithms\Zstd; @@ -65,7 +64,8 @@ class Create extends Action )) ->param('bucketId', '', fn (Database $dbForProject) => new UID($dbForProject->getAdapter()->getMaxUIDLength()), 'Storage bucket unique ID. You can create a new storage bucket using the Storage service [server integration](https://appwrite.io/docs/server/storage#createBucket).', false, ['dbForProject']) ->param('fileId', '', fn (Database $dbForProject) => new UID($dbForProject->getAdapter()->getMaxUIDLength()), 'File ID.', false, ['dbForProject']) - ->param('resourceId', null, new CompoundUID(), 'Composite ID in the format {databaseId:collectionId}, identifying a collection within a database.') + ->param('databaseId', '', new UID(), 'Database ID containing the target collection.') + ->param('collectionId', '', new UID(), 'Collection ID to import documents into.') ->param('internalFile', false, new Boolean(), 'Is the file stored in an internal bucket?', true) ->inject('response') ->inject('dbForProject') @@ -83,7 +83,8 @@ class Create extends Action public function action( string $bucketId, string $fileId, - string $resourceId, + string $databaseId, + string $collectionId, bool $internalFile, Response $response, Database $dbForProject, @@ -158,15 +159,24 @@ class Create extends Action throw new \Exception('Unable to copy file'); } - [$databaseId] = \explode(':', $resourceId, 2); $database = $authorization->skip(fn () => $dbForProject->getDocument('databases', $databaseId)); + if ($database->isEmpty()) { + throw new Exception(Exception::DATABASE_NOT_FOUND); + } + $databaseType = $database->getAttribute('type'); if (!\in_array($databaseType, CSV_ALLOWED_DATABASE_TYPES)) { throw new Exception(Exception::MIGRATION_DATABASE_TYPE_UNSUPPORTED, 'Database type not supported for csv'); } + + $collection = $authorization->skip(fn () => $dbForProject->getDocument('database_' . $database->getSequence(), $collectionId)); + if ($collection->isEmpty()) { + throw new Exception(Exception::COLLECTION_NOT_FOUND); + } + $fileSize = $deviceForMigrations->getFileSize($newPath); $resources = Transfer::extractServices([self::transferGroupForDatabaseType($databaseType)]); - $resourceType = self::resourceTypeForDatabaseType($databaseType); + $parentResourceType = self::resourceTypeForDatabaseType($databaseType); $migration = $dbForProject->createDocument('migrations', new Document([ '$id' => $migrationId, @@ -175,8 +185,12 @@ class Create extends Action 'source' => CSV::getName(), 'destination' => AppwriteSource::getName(), 'resources' => $resources, - 'resourceId' => $resourceId, - 'resourceType' => $resourceType, + 'resourceId' => $collection->getId(), + 'resourceInternalId' => $collection->getSequence(), + 'resourceType' => Resource::TYPE_COLLECTION, + 'parentResourceId' => $database->getId(), + 'parentResourceInternalId' => $database->getSequence(), + 'parentResourceType' => $parentResourceType, 'statusCounters' => '{}', 'resourceData' => '{}', 'errors' => [], diff --git a/src/Appwrite/Platform/Modules/Migrations/Http/Migrations/JSON/Exports/Create.php b/src/Appwrite/Platform/Modules/Migrations/Http/Migrations/JSON/Exports/Create.php index d968bd91f6..2196128e5c 100644 --- a/src/Appwrite/Platform/Modules/Migrations/Http/Migrations/JSON/Exports/Create.php +++ b/src/Appwrite/Platform/Modules/Migrations/Http/Migrations/JSON/Exports/Create.php @@ -9,7 +9,6 @@ use Appwrite\Extend\Exception; use Appwrite\SDK\AuthType; use Appwrite\SDK\Method; use Appwrite\SDK\Response as SDKResponse; -use Appwrite\Utopia\Database\Validator\CompoundUID; use Appwrite\Utopia\Response; use Utopia\Database\Database; use Utopia\Database\Document; @@ -18,6 +17,7 @@ use Utopia\Database\Helpers\ID; use Utopia\Database\Query; use Utopia\Database\Validator\Authorization; use Utopia\Database\Validator\Queries\Documents; +use Utopia\Database\Validator\UID; use Utopia\Migration\Resource; use Utopia\Migration\Sources\Appwrite as AppwriteSource; use Utopia\Migration\Sources\JSON as JSONSource; @@ -60,7 +60,8 @@ class Create extends Action ) ] )) - ->param('resourceId', null, new CompoundUID(), 'Composite ID in the format {databaseId:collectionId}, identifying a collection within a database to export.') + ->param('databaseId', '', new UID(), 'Database ID containing the source collection.') + ->param('collectionId', '', new UID(), 'Collection ID to export documents from.') ->param('filename', '', new Text(255), 'The name of the file to be created for the export, excluding the .json extension.') ->param('columns', [], new ArrayList(new Text(Database::LENGTH_KEY)), 'List of attributes to export. If empty, all attributes will be exported. You can use the `*` wildcard to export all attributes from the collection.', true) ->param('queries', [], new ArrayList(new Text(0)), 'Array of query strings generated using the Query class provided by the SDK to filter documents to export. [Learn more about queries](https://appwrite.io/docs/databases#querying-documents). Maximum of ' . APP_LIMIT_ARRAY_PARAMS_SIZE . ' queries are allowed, each ' . APP_LIMIT_ARRAY_ELEMENT_SIZE . ' characters long.', true) @@ -78,7 +79,8 @@ class Create extends Action } public function action( - string $resourceId, + string $databaseId, + string $collectionId, string $filename, array $columns, array $queries, @@ -104,14 +106,6 @@ class Create extends Action throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND); } - [$databaseId, $collectionId] = \explode(':', $resourceId, 2); - if (empty($databaseId)) { - throw new Exception(Exception::DATABASE_NOT_FOUND); - } - if (empty($collectionId)) { - throw new Exception(Exception::COLLECTION_NOT_FOUND); - } - $database = $authorization->skip(fn () => $dbForProject->getDocument('databases', $databaseId)); if ($database->isEmpty()) { throw new Exception(Exception::DATABASE_NOT_FOUND); @@ -139,7 +133,7 @@ class Create extends Action } $resources = Transfer::extractServices([self::transferGroupForDatabaseType($databaseType)]); - $resourceType = self::resourceTypeForDatabaseType($databaseType); + $parentResourceType = self::resourceTypeForDatabaseType($databaseType); $migration = $dbForProject->createDocument('migrations', new Document([ '$id' => ID::unique(), @@ -148,8 +142,12 @@ class Create extends Action 'source' => AppwriteSource::getName(), 'destination' => JSONSource::getName(), 'resources' => $resources, - 'resourceId' => $resourceId, - 'resourceType' => $resourceType, + 'resourceId' => $collection->getId(), + 'resourceInternalId' => $collection->getSequence(), + 'resourceType' => Resource::TYPE_COLLECTION, + 'parentResourceId' => $database->getId(), + 'parentResourceInternalId' => $database->getSequence(), + 'parentResourceType' => $parentResourceType, 'statusCounters' => '{}', 'resourceData' => '{}', 'errors' => [], diff --git a/src/Appwrite/Platform/Modules/Migrations/Http/Migrations/JSON/Imports/Create.php b/src/Appwrite/Platform/Modules/Migrations/Http/Migrations/JSON/Imports/Create.php index 55081b2645..50a88d3fda 100644 --- a/src/Appwrite/Platform/Modules/Migrations/Http/Migrations/JSON/Imports/Create.php +++ b/src/Appwrite/Platform/Modules/Migrations/Http/Migrations/JSON/Imports/Create.php @@ -10,7 +10,6 @@ use Appwrite\OpenSSL\OpenSSL; use Appwrite\SDK\AuthType; use Appwrite\SDK\Method; use Appwrite\SDK\Response as SDKResponse; -use Appwrite\Utopia\Database\Validator\CompoundUID; use Appwrite\Utopia\Response; use Utopia\Compression\Algorithms\GZIP; use Utopia\Compression\Algorithms\Zstd; @@ -64,7 +63,8 @@ class Create extends Action )) ->param('bucketId', '', new UID(), 'Storage bucket unique ID. You can create a new storage bucket using the Storage service [server integration](https://appwrite.io/docs/server/storage#createBucket).') ->param('fileId', '', new UID(), 'File ID.') - ->param('resourceId', null, new CompoundUID(), 'Composite ID in the format {databaseId:collectionId}, identifying a collection within a database.') + ->param('databaseId', '', new UID(), 'Database ID containing the target collection.') + ->param('collectionId', '', new UID(), 'Collection ID to import documents into.') ->param('internalFile', false, new Boolean(), 'Is the file stored in an internal bucket?', true) ->inject('response') ->inject('dbForProject') @@ -82,7 +82,8 @@ class Create extends Action public function action( string $bucketId, string $fileId, - string $resourceId, + string $databaseId, + string $collectionId, bool $internalFile, Response $response, Database $dbForProject, @@ -159,14 +160,19 @@ class Create extends Action $fileSize = $deviceForMigrations->getFileSize($newPath); - [$databaseId] = \explode(':', $resourceId, 2); $database = $authorization->skip(fn () => $dbForProject->getDocument('databases', $databaseId)); if ($database->isEmpty()) { throw new Exception(Exception::DATABASE_NOT_FOUND); } + + $collection = $authorization->skip(fn () => $dbForProject->getDocument('database_' . $database->getSequence(), $collectionId)); + if ($collection->isEmpty()) { + throw new Exception(Exception::COLLECTION_NOT_FOUND); + } + $databaseType = $database->getAttribute('type'); $resources = Transfer::extractServices([self::transferGroupForDatabaseType($databaseType)]); - $resourceType = self::resourceTypeForDatabaseType($databaseType); + $parentResourceType = self::resourceTypeForDatabaseType($databaseType); $migration = $dbForProject->createDocument('migrations', new Document([ '$id' => $migrationId, @@ -175,8 +181,12 @@ class Create extends Action 'source' => JSONSource::getName(), 'destination' => AppwriteSource::getName(), 'resources' => $resources, - 'resourceId' => $resourceId, - 'resourceType' => $resourceType, + 'resourceId' => $collection->getId(), + 'resourceInternalId' => $collection->getSequence(), + 'resourceType' => Resource::TYPE_COLLECTION, + 'parentResourceId' => $database->getId(), + 'parentResourceInternalId' => $database->getSequence(), + 'parentResourceType' => $parentResourceType, 'statusCounters' => '{}', 'resourceData' => '{}', 'errors' => [], diff --git a/src/Appwrite/Platform/Workers/Migrations.php b/src/Appwrite/Platform/Workers/Migrations.php index 69f72b8e27..3bebc4dcc9 100644 --- a/src/Appwrite/Platform/Workers/Migrations.php +++ b/src/Appwrite/Platform/Workers/Migrations.php @@ -189,7 +189,7 @@ class Migrations extends Action { $source = $migration->getAttribute('source'); $destination = $migration->getAttribute('destination'); - $resourceId = $migration->getAttribute('resourceId'); + $resourceId = $this->getCompoundResourceId($migration); $credentials = $migration->getAttribute('credentials'); $migrationOptions = $migration->getAttribute('options'); /** @var Database|null $projectDB */ @@ -294,7 +294,7 @@ class Migrations extends Action ), DestinationCSV::getName() => new DestinationCSV( $this->deviceForFiles, - $migration->getAttribute('resourceId'), + $this->getCompoundResourceId($migration), $options['bucketId'], $options['filename'], $options['columns'], @@ -305,7 +305,7 @@ class Migrations extends Action ), DestinationJSON::getName() => new DestinationJSON( $this->deviceForFiles, - $migration->getAttribute('resourceId'), + $this->getCompoundResourceId($migration), $options['bucketId'] ?? 'default', $options['filename'], $options['columns'] ?? [], @@ -507,8 +507,8 @@ class Migrations extends Action } $this->updateMigrationDocument($migration, $project, $queueForRealtime); }, - $migration->getAttribute('resourceId'), - $migration->getAttribute('resourceType') + $this->getCompoundResourceId($migration), + $migration->getAttribute('parentResourceType') ); $destination->shutdown(); @@ -578,6 +578,7 @@ class Migrations extends Action $publisherForUsage, $migration->getAttribute('source'), $authorization, + $migration->getAttribute('parentResourceId'), $migration->getAttribute('resourceId') ); } @@ -608,6 +609,23 @@ class Migrations extends Action return ($this->getDatabasesDB)($database); } + /** + * Returns a "{parentResourceId}:{resourceId}" string when both are set, or just + * the available ID otherwise. The utopia-php/migration library expects this + * compound shape on its source/destination constructors. + */ + protected function getCompoundResourceId(Document $migration): ?string + { + $parentResourceId = $migration->getAttribute('parentResourceId'); + $resourceId = $migration->getAttribute('resourceId'); + + if (!empty($parentResourceId) && !empty($resourceId)) { + return $parentResourceId . ':' . $resourceId; + } + + return $resourceId ?? $parentResourceId; + } + /** * Handle actions to be performed when a CSV export migration is successfully completed * @@ -890,7 +908,7 @@ class Migrations extends Action return $errors; } - private function processMigrationResourceStats(array $resources, Context $usage, Document $projectDocument, UsagePublisher $publisherForUsage, string $source, Authorization $authorization, ?string $resourceId) + private function processMigrationResourceStats(array $resources, Context $usage, Document $projectDocument, UsagePublisher $publisherForUsage, string $source, Authorization $authorization, ?string $parentResourceId, ?string $resourceId) { $resourceName = $resources['name']; $count = $resources['count']; @@ -898,9 +916,8 @@ class Migrations extends Action $tableInternalId = $resources['tableId']; if ($source === CSV::getName()) { - [$databaseId, $tableId] = explode(':', $resourceId); - $database = $authorization->skip(fn () => $this->dbForProject->getDocument('databases', $databaseId)); - $table = $authorization->skip(fn () => $this->dbForProject->getDocument('database_' . $database->getSequence(), $tableId)); + $database = $authorization->skip(fn () => $this->dbForProject->getDocument('databases', (string) $parentResourceId)); + $table = $authorization->skip(fn () => $this->dbForProject->getDocument('database_' . $database->getSequence(), (string) $resourceId)); $databaseInternalId = (int) $database->getSequence(); $tableInternalId = (int) $table->getSequence(); } diff --git a/src/Appwrite/Utopia/Database/Validator/Queries/Migrations.php b/src/Appwrite/Utopia/Database/Validator/Queries/Migrations.php index c49788872e..93abdc2c60 100644 --- a/src/Appwrite/Utopia/Database/Validator/Queries/Migrations.php +++ b/src/Appwrite/Utopia/Database/Validator/Queries/Migrations.php @@ -11,10 +11,14 @@ class Migrations extends Base 'destination', 'resources', 'resourceId', + 'resourceInternalId', 'resourceType', + 'parentResourceId', + 'parentResourceInternalId', + 'parentResourceType', 'statusCounters', 'resourceData', - 'errors' + 'errors', ]; /** diff --git a/src/Appwrite/Utopia/Request/Filters/V25.php b/src/Appwrite/Utopia/Request/Filters/V25.php new file mode 100644 index 0000000000..39a682d9e6 --- /dev/null +++ b/src/Appwrite/Utopia/Request/Filters/V25.php @@ -0,0 +1,40 @@ +parseMigrationResource($content); + break; + } + + return $content; + } + + protected function parseMigrationResource(array $content): array + { + if (!isset($content['resourceId']) || !\is_string($content['resourceId'])) { + return $content; + } + + if (\str_contains($content['resourceId'], ':')) { + [$databaseId, $collectionId] = \explode(':', $content['resourceId'], 2); + $content['databaseId'] = $content['databaseId'] ?? $databaseId; + $content['collectionId'] = $content['collectionId'] ?? $collectionId; + } + + unset($content['resourceId']); + + return $content; + } +} diff --git a/src/Appwrite/Utopia/Response/Filters/V25.php b/src/Appwrite/Utopia/Response/Filters/V25.php new file mode 100644 index 0000000000..b53f895166 --- /dev/null +++ b/src/Appwrite/Utopia/Response/Filters/V25.php @@ -0,0 +1,38 @@ + $this->parseMigration($content), + Response::MODEL_MIGRATION_LIST => $this->handleList($content, 'migrations', fn ($item) => $this->parseMigration($item)), + default => $content, + }; + } + + protected function parseMigration(array $content): array + { + $parentResourceId = $content['parentResourceId'] ?? ''; + $resourceId = $content['resourceId'] ?? ''; + + if ($parentResourceId !== '' && $resourceId !== '') { + $content['resourceId'] = $parentResourceId . ':' . $resourceId; + } + + $content['resourceType'] = $content['parentResourceType'] ?? $content['resourceType'] ?? ''; + + unset($content['resourceInternalId']); + unset($content['parentResourceId']); + unset($content['parentResourceInternalId']); + unset($content['parentResourceType']); + + return $content; + } +} diff --git a/src/Appwrite/Utopia/Response/Model/Migration.php b/src/Appwrite/Utopia/Response/Model/Migration.php index df8b2d79ec..0fbf7362b7 100644 --- a/src/Appwrite/Utopia/Response/Model/Migration.php +++ b/src/Appwrite/Utopia/Response/Model/Migration.php @@ -62,9 +62,44 @@ class Migration extends Model ]) ->addRule('resourceId', [ 'type' => self::TYPE_STRING, - 'description' => 'Id of the resource to migrate.', + 'description' => 'ID of the resource being migrated.', 'default' => '', - 'example' => 'databaseId:collectionId', + 'example' => 'collectionId', + 'array' => false + ]) + ->addRule('resourceInternalId', [ + 'type' => self::TYPE_STRING, + 'description' => 'Internal ID of the resource being migrated.', + 'default' => '', + 'example' => '1', + 'array' => false + ]) + ->addRule('resourceType', [ + 'type' => self::TYPE_STRING, + 'description' => 'Type of the resource being migrated.', + 'default' => '', + 'example' => 'collection', + 'array' => false + ]) + ->addRule('parentResourceId', [ + 'type' => self::TYPE_STRING, + 'description' => 'ID of the parent resource that contains the migrated resource.', + 'default' => '', + 'example' => 'databaseId', + 'array' => false + ]) + ->addRule('parentResourceInternalId', [ + 'type' => self::TYPE_STRING, + 'description' => 'Internal ID of the parent resource that contains the migrated resource.', + 'default' => '', + 'example' => '1', + 'array' => false + ]) + ->addRule('parentResourceType', [ + 'type' => self::TYPE_STRING, + 'description' => 'Type of the parent resource that contains the migrated resource.', + 'default' => '', + 'example' => 'database', 'array' => false ]) ->addRule('statusCounters', [ diff --git a/tests/e2e/Services/Migrations/MigrationsBase.php b/tests/e2e/Services/Migrations/MigrationsBase.php index 4346e5a5fa..6ed0f9a738 100644 --- a/tests/e2e/Services/Migrations/MigrationsBase.php +++ b/tests/e2e/Services/Migrations/MigrationsBase.php @@ -1328,7 +1328,8 @@ trait MigrationsBase [ 'fileId' => $fileIds['missing-column'], 'bucketId' => $bucketIds['missing-column'], - 'resourceId' => $databaseId . ':' . $tableId, + 'databaseId' => $databaseId, + 'collectionId' => $tableId, ] ); @@ -1360,7 +1361,8 @@ trait MigrationsBase [ 'fileId' => $fileIds['missing-row'], 'bucketId' => $bucketIds['missing-row'], - 'resourceId' => $databaseId . ':' . $tableId, + 'databaseId' => $databaseId, + 'collectionId' => $tableId, ] ); @@ -1392,7 +1394,8 @@ trait MigrationsBase [ 'fileId' => $fileIds['irrelevant-column'], 'bucketId' => $bucketIds['irrelevant-column'], - 'resourceId' => $databaseId . ':' . $tableId, + 'databaseId' => $databaseId, + 'collectionId' => $tableId, ] ); @@ -1419,7 +1422,8 @@ trait MigrationsBase 'endpoint' => $this->webEndpoint, 'fileId' => $fileIds['default'], 'bucketId' => $bucketIds['default'], - 'resourceId' => $databaseId . ':' . $tableId, + 'databaseId' => $databaseId, + 'collectionId' => $tableId, ] ); @@ -1461,7 +1465,8 @@ trait MigrationsBase 'endpoint' => $this->webEndpoint, 'fileId' => $fileIds['documents-internals'], 'bucketId' => $bucketIds['documents-internals'], - 'resourceId' => $databaseId . ':' . $tableId, + 'databaseId' => $databaseId, + 'collectionId' => $tableId, ] ); @@ -1644,7 +1649,8 @@ trait MigrationsBase 'content-type' => 'application/json', 'x-appwrite-project' => $this->getProject()['$id'] ], $this->getHeaders()), [ - 'resourceId' => $databaseId . ':' . $collectionId, + 'databaseId' => $databaseId, + 'collectionId' => $collectionId, 'filename' => 'test-export', 'columns' => [], 'delimiter' => ',', @@ -2622,7 +2628,8 @@ trait MigrationsBase $migration = $this->performCsvMigration([ 'fileId' => $fileId, 'bucketId' => $bucketId, - 'resourceId' => $databaseId . ':' . $collectionId, + 'databaseId' => $databaseId, + 'collectionId' => $collectionId, ]); $this->assertEquals(202, $migration['headers']['status-code']); @@ -2750,7 +2757,8 @@ trait MigrationsBase 'content-type' => 'application/json', 'x-appwrite-project' => $this->getProject()['$id'], ], $this->getHeaders()), [ - 'resourceId' => $databaseId . ':' . $collectionId, + 'databaseId' => $databaseId, + 'collectionId' => $collectionId, 'filename' => $filename, 'columns' => [], 'queries' => [], @@ -3951,7 +3959,8 @@ trait MigrationsBase [ 'fileId' => $fileIds['missing-column'], 'bucketId' => $bucketIds['missing-column'], - 'resourceId' => $databaseId . ':' . $tableId, + 'databaseId' => $databaseId, + 'collectionId' => $tableId, ] ); @@ -3988,7 +3997,8 @@ trait MigrationsBase [ 'fileId' => $fileIds['irrelevant-column'], 'bucketId' => $bucketIds['irrelevant-column'], - 'resourceId' => $databaseId . ':' . $tableId, + 'databaseId' => $databaseId, + 'collectionId' => $tableId, ] ); @@ -4015,7 +4025,8 @@ trait MigrationsBase 'endpoint' => $this->endpoint, 'fileId' => $fileIds['default'], 'bucketId' => $bucketIds['default'], - 'resourceId' => $databaseId . ':' . $tableId, + 'databaseId' => $databaseId, + 'collectionId' => $tableId, ] ); @@ -4057,7 +4068,8 @@ trait MigrationsBase 'endpoint' => $this->endpoint, 'fileId' => $fileIds['documents-internals'], 'bucketId' => $bucketIds['documents-internals'], - 'resourceId' => $databaseId . ':' . $tableId, + 'databaseId' => $databaseId, + 'collectionId' => $tableId, ] ); @@ -4180,7 +4192,8 @@ trait MigrationsBase 'content-type' => 'application/json', 'x-appwrite-project' => $this->getProject()['$id'] ], $this->getHeaders()), [ - 'resourceId' => $databaseId . ':' . $collectionId, + 'databaseId' => $databaseId, + 'collectionId' => $collectionId, 'filename' => 'test-json-export', 'columns' => [], 'queries' => [], @@ -4291,7 +4304,8 @@ trait MigrationsBase // Trigger JSON export $migration = $this->client->call(Client::METHOD_POST, '/migrations/json/exports', $headers, [ - 'resourceId' => $databaseId . ':' . $collectionId, + 'databaseId' => $databaseId, + 'collectionId' => $collectionId, 'filename' => 'vectorsdb-export-test', 'columns' => [], 'queries' => [], @@ -4364,7 +4378,8 @@ trait MigrationsBase $migration = $this->performJsonMigration([ 'fileId' => $fileId, 'bucketId' => $bucketId, - 'resourceId' => $databaseId . ':' . $collectionId, + 'databaseId' => $databaseId, + 'collectionId' => $collectionId, ]); $this->assertEquals(202, $migration['headers']['status-code']); @@ -4435,7 +4450,8 @@ trait MigrationsBase // Trigger JSON export $migration = $this->client->call(Client::METHOD_POST, '/migrations/json/exports', $headers, [ - 'resourceId' => $databaseId . ':' . $collectionId, + 'databaseId' => $databaseId, + 'collectionId' => $collectionId, 'filename' => 'documentsdb-export-test', 'columns' => [], 'queries' => [], @@ -4507,7 +4523,8 @@ trait MigrationsBase $migration = $this->performJsonMigration([ 'fileId' => $fileId, 'bucketId' => $bucketId, - 'resourceId' => $databaseId . ':' . $collectionId, + 'databaseId' => $databaseId, + 'collectionId' => $collectionId, ]); $this->assertEquals(202, $migration['headers']['status-code']);