Appwrite/appwrite
1
0
Fork 0
mirror of https://github.com/appwrite/appwrite.git synced 2026-05-26 13:51:13 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor(migrations): split composite resourceId into parent/leaf

Browse Source 
The migrations collection used a composite "{databaseId}:{collectionId}"
in resourceId for CSV/JSON imports and exports, which mixed two
identifiers in one column and broke queryability. Split it into parent
(database) and leaf (collection) attributes following the convention
used by other collections.

Schema (collection bumped to V25):
- Adds resourceInternalId, parentResourceId, parentResourceInternalId,
  parentResourceType to the migrations collection
- Drops _key_resource_id; adds per-attribute indexes for every
  resource-related attribute so all of them are queryable
- V25 backfills existing migration documents by splitting the legacy
  composite resourceId, mapping the legacy resourceType (database type)
  to parentResourceType, and looking up sequence IDs

API:
- /v1/migrations/csv|json/imports|exports replace the CompoundUID
  resourceId param with separate UID databaseId + collectionId params
- Worker now reads parent/leaf fields directly and recomposes the
  legacy "id:id" shape only at the boundary to utopia-php/migration

Backwards compatibility:
- Request V25 filter splits old-SDK resourceId into databaseId/collectionId
- Response V25 filter recomposes resourceId for old-SDK consumers and
  strips the new fields
- Query validator now allows querying every resource-related attribute

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
Jake Barnby
2026-05-01 12:37:01 +12:00
co-authored by Claude Opus 4.7
parent 08ad7d7f71
commit fb4930741e
15 changed files with 448 additions and 73 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/config/collections/projects.php
+80 -1
View File
@@ -2483,6 +2483,17 @@ return [
'array' => false,
'filters' => [],
],
[
'$id' => ID::custom('resourceInternalId'),
'type' => Database::VAR_STRING,
'format' => '',
'size' => Database::LENGTH_KEY,
'signed' => true,
'required' => false,
'default' => null,
'array' => false,
'filters' => [],
],
[
'$id' => ID::custom('resourceType'),
'type' => Database::VAR_STRING,
@@ -2494,6 +2505,39 @@ return [
'array' => false,
'filters' => [],
],
[
'$id' => ID::custom('parentResourceId'),
'type' => Database::VAR_STRING,
'format' => '',
'size' => Database::LENGTH_KEY,
'signed' => true,
'required' => false,
'default' => null,
'array' => false,
'filters' => [],
],
[
'$id' => ID::custom('parentResourceInternalId'),
'type' => Database::VAR_STRING,
'format' => '',
'size' => Database::LENGTH_KEY,
'signed' => true,
'required' => false,
'default' => null,
'array' => false,
'filters' => [],
],
[
'$id' => ID::custom('parentResourceType'),
'type' => Database::VAR_STRING,
'format' => '',
'size' => Database::LENGTH_KEY,
'signed' => true,
'required' => false,
'default' => null,
'array' => false,
'filters' => [],
],
],
'indexes' => [
[
@@ -2518,12 +2562,47 @@ return [
'orders' => [Database::ORDER_ASC],
],
[
'$id' => '_key_resource_id',
'$id' => '_key_resourceId',
'type' => Database::INDEX_KEY,
'attributes' => ['resourceId'],
'lengths' => [Database::LENGTH_KEY],
'orders' => [Database::ORDER_DESC],
],
[
'$id' => '_key_resourceType',
'type' => Database::INDEX_KEY,
'attributes' => ['resourceType'],
'lengths' => [Database::LENGTH_KEY],
'orders' => [Database::ORDER_ASC],
],
[
'$id' => '_key_resourceInternalId',
'type' => Database::INDEX_KEY,
'attributes' => ['resourceInternalId'],
'lengths' => [Database::LENGTH_KEY],
'orders' => [Database::ORDER_ASC],
],
[
'$id' => '_key_parentResourceId',
'type' => Database::INDEX_KEY,
'attributes' => ['parentResourceId'],
'lengths' => [Database::LENGTH_KEY],
'orders' => [Database::ORDER_ASC],
],
[
'$id' => '_key_parentResourceType',
'type' => Database::INDEX_KEY,
'attributes' => ['parentResourceType'],
'lengths' => [Database::LENGTH_KEY],
'orders' => [Database::ORDER_ASC],
],
[
'$id' => '_key_parentResourceInternalId',
'type' => Database::INDEX_KEY,
'attributes' => ['parentResourceInternalId'],
'lengths' => [Database::LENGTH_KEY],
'orders' => [Database::ORDER_ASC],
],
[
'$id' => ID::custom('_fulltext_search'),
'type' => Database::INDEX_FULLTEXT,
app/controllers/general.php
+8
View File
@@ -28,6 +28,7 @@ use Appwrite\Utopia\Request\Filters\V21 as RequestV21;
use Appwrite\Utopia\Request\Filters\V22 as RequestV22;
use Appwrite\Utopia\Request\Filters\V23 as RequestV23;
use Appwrite\Utopia\Request\Filters\V24 as RequestV24;
use Appwrite\Utopia\Request\Filters\V25 as RequestV25;
use Appwrite\Utopia\Response;
use Appwrite\Utopia\Response\Filters\V16 as ResponseV16;
use Appwrite\Utopia\Response\Filters\V17 as ResponseV17;
@@ -38,6 +39,7 @@ use Appwrite\Utopia\Response\Filters\V21 as ResponseV21;
use Appwrite\Utopia\Response\Filters\V22 as ResponseV22;
use Appwrite\Utopia\Response\Filters\V23 as ResponseV23;
use Appwrite\Utopia\Response\Filters\V24 as ResponseV24;
use Appwrite\Utopia\Response\Filters\V25 as ResponseV25;
use Appwrite\Utopia\View;
use Executor\Executor;
use MaxMind\Db\Reader;
@@ -904,6 +906,9 @@ Http::init()
if (version_compare($requestFormat, '1.9.3', '<')) {
$request->addFilter(new RequestV24());
}
if (version_compare($requestFormat, '1.9.4', '<')) {
$request->addFilter(new RequestV25());
}
}
$localeParam = (string) $request->getParam('locale', $request->getHeader('x-appwrite-locale', ''));
@@ -928,6 +933,9 @@ Http::init()
*/
$responseFormat = $request->getHeader('x-appwrite-response-format', System::getEnv('_APP_SYSTEM_RESPONSE_FORMAT', ''));
if ($responseFormat) {
if (version_compare($responseFormat, '1.9.4', '<')) {
$response->addFilter(new ResponseV25());
}
if (version_compare($responseFormat, '1.9.3', '<')) {
$response->addFilter(new ResponseV24());
}
app/init/constants.php
+1 -1
View File
@@ -45,7 +45,7 @@ const APP_RESOURCE_TOKEN_ACCESS = 24 * 60 * 60; // 24 hours
const APP_FILE_ACCESS = 24 * 60 * 60; // 24 hours
const APP_CACHE_UPDATE = 24 * 60 * 60; // 24 hours
const APP_CACHE_BUSTER = 4324;
const APP_VERSION_STABLE = '1.9.3';
const APP_VERSION_STABLE = '1.9.4';
const APP_DATABASE_ATTRIBUTE_EMAIL = 'email';
const APP_DATABASE_ATTRIBUTE_ENUM = 'enum';
const APP_DATABASE_ATTRIBUTE_IP = 'ip';
src/Appwrite/Migration/Migration.php
+1
View File
@@ -96,6 +96,7 @@ abstract class Migration
'1.9.1' => 'V24',
'1.9.2' => 'V24',
'1.9.3' => 'V24',
'1.9.4' => 'V25',
];
/**
src/Appwrite/Migration/Version/V25.php
+116
View File
@@ -0,0 +1,116 @@
<?php
namespace Appwrite\Migration\Version;
use Appwrite\Migration\Migration;
use Throwable;
use Utopia\Console;
use Utopia\Database\Database;
use Utopia\Database\Document;
class V25 extends Migration
{
/**
* @throws Throwable
*/
public function execute(): void
{
Console::info('Migrating collections');
$this->migrateCollections();
Console::info('Migrating documents');
$this->forEachDocument($this->migrateDocument(...));
}
/**
* @throws Throwable
*/
private function migrateCollections(): void
{
if ($this->project->getSequence() === 'console') {
return;
}
$id = 'migrations';
$this->dbForProject->purgeCachedCollection($id);
$this->dbForProject->purgeCachedDocument(Database::METADATA, $id);
$attributes = [
'resourceInternalId',
'parentResourceId',
'parentResourceInternalId',
'parentResourceType',
];
try {
$this->createAttributesFromCollection($this->dbForProject, $id, $attributes);
} catch (Throwable $th) {
Console::warning('Failed to create attributes "' . \implode(', ', $attributes) . "\" in collection {$id}: {$th->getMessage()}");
}
try {
$this->dbForProject->deleteIndex($id, '_key_resource_id');
} catch (Throwable $th) {
Console::warning("Failed to delete index \"_key_resource_id\" from {$id}: {$th->getMessage()}");
}
$indexes = [
'_key_resourceId',
'_key_resourceType',
'_key_resourceInternalId',
'_key_parentResourceId',
'_key_parentResourceType',
'_key_parentResourceInternalId',
];
foreach ($indexes as $index) {
try {
$this->createIndexFromCollection($this->dbForProject, $id, $index);
} catch (Throwable $th) {
Console::warning("Failed to create index \"{$index}\" from {$id}: {$th->getMessage()}");
}
}
$this->dbForProject->purgeCachedCollection($id);
}
private function migrateDocument(Document $document): Document
{
if ($document->getCollection() !== 'migrations') {
return $document;
}
if (!empty($document->getAttribute('parentResourceId'))) {
return $document;
}
$resourceId = $document->getAttribute('resourceId');
if (empty($resourceId) || !\str_contains($resourceId, ':')) {
return $document;
}
[$parentId, $childId] = \explode(':', $resourceId, 2);
$parentResourceType = $document->getAttribute('resourceType');
$document
->setAttribute('resourceId', $childId)
->setAttribute('resourceType', 'collection')
->setAttribute('parentResourceId', $parentId)
->setAttribute('parentResourceType', $parentResourceType);
try {
$database = $this->dbForProject->getDocument('databases', $parentId);
if (!$database->isEmpty()) {
$document->setAttribute('parentResourceInternalId', (string) $database->getSequence());
$collection = $this->dbForProject->getDocument('database_' . $database->getSequence(), $childId);
if (!$collection->isEmpty()) {
$document->setAttribute('resourceInternalId', (string) $collection->getSequence());
}
}
} catch (Throwable $th) {
Console::warning("Failed to backfill internal IDs for migration {$document->getId()}: {$th->getMessage()}");
}
return $document;
}
}
src/Appwrite/Platform/Modules/Migrations/Http/Migrations/CSV/Exports/Create.php
+12 -14
View File
@@ -9,7 +9,6 @@ use Appwrite\Extend\Exception;
use Appwrite\SDK\AuthType;
use Appwrite\SDK\Method;
use Appwrite\SDK\Response as SDKResponse;
use Appwrite\Utopia\Database\Validator\CompoundUID;
use Appwrite\Utopia\Response;
use Utopia\Database\Database;
use Utopia\Database\Document;
@@ -18,6 +17,7 @@ use Utopia\Database\Helpers\ID;
use Utopia\Database\Query;
use Utopia\Database\Validator\Authorization;
use Utopia\Database\Validator\Queries\Documents;
use Utopia\Database\Validator\UID;
use Utopia\Migration\Resource;
use Utopia\Migration\Sources\Appwrite as AppwriteSource;
use Utopia\Migration\Sources\CSV;
@@ -60,7 +60,8 @@ class Create extends Action
)
]
))
->param('resourceId', null, new CompoundUID(), 'Composite ID in the format {databaseId:collectionId}, identifying a collection within a database to export.')
->param('databaseId', '', new UID(), 'Database ID containing the source collection.')
->param('collectionId', '', new UID(), 'Collection ID to export documents from.')
->param('filename', '', new Text(255), 'The name of the file to be created for the export, excluding the .csv extension.')
->param('columns', [], new ArrayList(new Text(Database::LENGTH_KEY)), 'List of attributes to export. If empty, all attributes will be exported. You can use the `*` wildcard to export all attributes from the collection.', true)
->param('queries', [], new ArrayList(new Text(0)), 'Array of query strings generated using the Query class provided by the SDK to filter documents to export. [Learn more about queries](https://appwrite.io/docs/databases#querying-documents). Maximum of ' . APP_LIMIT_ARRAY_PARAMS_SIZE . ' queries are allowed, each ' . APP_LIMIT_ARRAY_ELEMENT_SIZE . ' characters long.', true)
@@ -82,7 +83,8 @@ class Create extends Action
}
public function action(
string $resourceId,
string $databaseId,
string $collectionId,
string $filename,
array $columns,
array $queries,
@@ -112,14 +114,6 @@ class Create extends Action
throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND);
}
[$databaseId, $collectionId] = \explode(':', $resourceId, 2);
if (empty($databaseId)) {
throw new Exception(Exception::DATABASE_NOT_FOUND);
}
if (empty($collectionId)) {
throw new Exception(Exception::COLLECTION_NOT_FOUND);
}
$database = $authorization->skip(fn () => $dbForProject->getDocument('databases', $databaseId));
if ($database->isEmpty()) {
throw new Exception(Exception::DATABASE_NOT_FOUND);
@@ -150,7 +144,7 @@ class Create extends Action
}
$resources = Transfer::extractServices([self::transferGroupForDatabaseType($databaseType)]);
$resourceType = self::resourceTypeForDatabaseType($databaseType);
$parentResourceType = self::resourceTypeForDatabaseType($databaseType);
$migration = $dbForProject->createDocument('migrations', new Document([
'$id' => ID::unique(),
@@ -159,8 +153,12 @@ class Create extends Action
'source' => AppwriteSource::getName(),
'destination' => CSV::getName(),
'resources' => $resources,
'resourceId' => $resourceId,
'resourceType' => $resourceType,
'resourceId' => $collection->getId(),
'resourceInternalId' => $collection->getSequence(),
'resourceType' => Resource::TYPE_COLLECTION,
'parentResourceId' => $database->getId(),
'parentResourceInternalId' => $database->getSequence(),
'parentResourceType' => $parentResourceType,
'statusCounters' => '{}',
'resourceData' => '{}',
'errors' => [],
src/Appwrite/Platform/Modules/Migrations/Http/Migrations/CSV/Imports/Create.php
+21 -7
View File
@@ -10,7 +10,6 @@ use Appwrite\OpenSSL\OpenSSL;
use Appwrite\SDK\AuthType;
use Appwrite\SDK\Method;
use Appwrite\SDK\Response as SDKResponse;
use Appwrite\Utopia\Database\Validator\CompoundUID;
use Appwrite\Utopia\Response;
use Utopia\Compression\Algorithms\GZIP;
use Utopia\Compression\Algorithms\Zstd;
@@ -65,7 +64,8 @@ class Create extends Action
))
->param('bucketId', '', fn (Database $dbForProject) => new UID($dbForProject->getAdapter()->getMaxUIDLength()), 'Storage bucket unique ID. You can create a new storage bucket using the Storage service [server integration](https://appwrite.io/docs/server/storage#createBucket).', false, ['dbForProject'])
->param('fileId', '', fn (Database $dbForProject) => new UID($dbForProject->getAdapter()->getMaxUIDLength()), 'File ID.', false, ['dbForProject'])
->param('resourceId', null, new CompoundUID(), 'Composite ID in the format {databaseId:collectionId}, identifying a collection within a database.')
->param('databaseId', '', new UID(), 'Database ID containing the target collection.')
->param('collectionId', '', new UID(), 'Collection ID to import documents into.')
->param('internalFile', false, new Boolean(), 'Is the file stored in an internal bucket?', true)
->inject('response')
->inject('dbForProject')
@@ -83,7 +83,8 @@ class Create extends Action
public function action(
string $bucketId,
string $fileId,
string $resourceId,
string $databaseId,
string $collectionId,
bool $internalFile,
Response $response,
Database $dbForProject,
@@ -158,15 +159,24 @@ class Create extends Action
throw new \Exception('Unable to copy file');
}
[$databaseId] = \explode(':', $resourceId, 2);
$database = $authorization->skip(fn () => $dbForProject->getDocument('databases', $databaseId));
if ($database->isEmpty()) {
throw new Exception(Exception::DATABASE_NOT_FOUND);
}
$databaseType = $database->getAttribute('type');
if (!\in_array($databaseType, CSV_ALLOWED_DATABASE_TYPES)) {
throw new Exception(Exception::MIGRATION_DATABASE_TYPE_UNSUPPORTED, 'Database type not supported for csv');
}
$collection = $authorization->skip(fn () => $dbForProject->getDocument('database_' . $database->getSequence(), $collectionId));
if ($collection->isEmpty()) {
throw new Exception(Exception::COLLECTION_NOT_FOUND);
}
$fileSize = $deviceForMigrations->getFileSize($newPath);
$resources = Transfer::extractServices([self::transferGroupForDatabaseType($databaseType)]);
$resourceType = self::resourceTypeForDatabaseType($databaseType);
$parentResourceType = self::resourceTypeForDatabaseType($databaseType);
$migration = $dbForProject->createDocument('migrations', new Document([
'$id' => $migrationId,
@@ -175,8 +185,12 @@ class Create extends Action
'source' => CSV::getName(),
'destination' => AppwriteSource::getName(),
'resources' => $resources,
'resourceId' => $resourceId,
'resourceType' => $resourceType,
'resourceId' => $collection->getId(),
'resourceInternalId' => $collection->getSequence(),
'resourceType' => Resource::TYPE_COLLECTION,
'parentResourceId' => $database->getId(),
'parentResourceInternalId' => $database->getSequence(),
'parentResourceType' => $parentResourceType,
'statusCounters' => '{}',
'resourceData' => '{}',
'errors' => [],
src/Appwrite/Platform/Modules/Migrations/Http/Migrations/JSON/Exports/Create.php
+12 -14
View File
@@ -9,7 +9,6 @@ use Appwrite\Extend\Exception;
use Appwrite\SDK\AuthType;
use Appwrite\SDK\Method;
use Appwrite\SDK\Response as SDKResponse;
use Appwrite\Utopia\Database\Validator\CompoundUID;
use Appwrite\Utopia\Response;
use Utopia\Database\Database;
use Utopia\Database\Document;
@@ -18,6 +17,7 @@ use Utopia\Database\Helpers\ID;
use Utopia\Database\Query;
use Utopia\Database\Validator\Authorization;
use Utopia\Database\Validator\Queries\Documents;
use Utopia\Database\Validator\UID;
use Utopia\Migration\Resource;
use Utopia\Migration\Sources\Appwrite as AppwriteSource;
use Utopia\Migration\Sources\JSON as JSONSource;
@@ -60,7 +60,8 @@ class Create extends Action
)
]
))
->param('resourceId', null, new CompoundUID(), 'Composite ID in the format {databaseId:collectionId}, identifying a collection within a database to export.')
->param('databaseId', '', new UID(), 'Database ID containing the source collection.')
->param('collectionId', '', new UID(), 'Collection ID to export documents from.')
->param('filename', '', new Text(255), 'The name of the file to be created for the export, excluding the .json extension.')
->param('columns', [], new ArrayList(new Text(Database::LENGTH_KEY)), 'List of attributes to export. If empty, all attributes will be exported. You can use the `*` wildcard to export all attributes from the collection.', true)
->param('queries', [], new ArrayList(new Text(0)), 'Array of query strings generated using the Query class provided by the SDK to filter documents to export. [Learn more about queries](https://appwrite.io/docs/databases#querying-documents). Maximum of ' . APP_LIMIT_ARRAY_PARAMS_SIZE . ' queries are allowed, each ' . APP_LIMIT_ARRAY_ELEMENT_SIZE . ' characters long.', true)
@@ -78,7 +79,8 @@ class Create extends Action
}
public function action(
string $resourceId,
string $databaseId,
string $collectionId,
string $filename,
array $columns,
array $queries,
@@ -104,14 +106,6 @@ class Create extends Action
throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND);
}
[$databaseId, $collectionId] = \explode(':', $resourceId, 2);
if (empty($databaseId)) {
throw new Exception(Exception::DATABASE_NOT_FOUND);
}
if (empty($collectionId)) {
throw new Exception(Exception::COLLECTION_NOT_FOUND);
}
$database = $authorization->skip(fn () => $dbForProject->getDocument('databases', $databaseId));
if ($database->isEmpty()) {
throw new Exception(Exception::DATABASE_NOT_FOUND);
@@ -139,7 +133,7 @@ class Create extends Action
}
$resources = Transfer::extractServices([self::transferGroupForDatabaseType($databaseType)]);
$resourceType = self::resourceTypeForDatabaseType($databaseType);
$parentResourceType = self::resourceTypeForDatabaseType($databaseType);
$migration = $dbForProject->createDocument('migrations', new Document([
'$id' => ID::unique(),
@@ -148,8 +142,12 @@ class Create extends Action
'source' => AppwriteSource::getName(),
'destination' => JSONSource::getName(),
'resources' => $resources,
'resourceId' => $resourceId,
'resourceType' => $resourceType,
'resourceId' => $collection->getId(),
'resourceInternalId' => $collection->getSequence(),
'resourceType' => Resource::TYPE_COLLECTION,
'parentResourceId' => $database->getId(),
'parentResourceInternalId' => $database->getSequence(),
'parentResourceType' => $parentResourceType,
'statusCounters' => '{}',
'resourceData' => '{}',
'errors' => [],
src/Appwrite/Platform/Modules/Migrations/Http/Migrations/JSON/Imports/Create.php
+17 -7
View File
@@ -10,7 +10,6 @@ use Appwrite\OpenSSL\OpenSSL;
use Appwrite\SDK\AuthType;
use Appwrite\SDK\Method;
use Appwrite\SDK\Response as SDKResponse;
use Appwrite\Utopia\Database\Validator\CompoundUID;
use Appwrite\Utopia\Response;
use Utopia\Compression\Algorithms\GZIP;
use Utopia\Compression\Algorithms\Zstd;
@@ -64,7 +63,8 @@ class Create extends Action
))
->param('bucketId', '', new UID(), 'Storage bucket unique ID. You can create a new storage bucket using the Storage service [server integration](https://appwrite.io/docs/server/storage#createBucket).')
->param('fileId', '', new UID(), 'File ID.')
->param('resourceId', null, new CompoundUID(), 'Composite ID in the format {databaseId:collectionId}, identifying a collection within a database.')
->param('databaseId', '', new UID(), 'Database ID containing the target collection.')
->param('collectionId', '', new UID(), 'Collection ID to import documents into.')
->param('internalFile', false, new Boolean(), 'Is the file stored in an internal bucket?', true)
->inject('response')
->inject('dbForProject')
@@ -82,7 +82,8 @@ class Create extends Action
public function action(
string $bucketId,
string $fileId,
string $resourceId,
string $databaseId,
string $collectionId,
bool $internalFile,
Response $response,
Database $dbForProject,
@@ -159,14 +160,19 @@ class Create extends Action
$fileSize = $deviceForMigrations->getFileSize($newPath);
[$databaseId] = \explode(':', $resourceId, 2);
$database = $authorization->skip(fn () => $dbForProject->getDocument('databases', $databaseId));
if ($database->isEmpty()) {
throw new Exception(Exception::DATABASE_NOT_FOUND);
}
$collection = $authorization->skip(fn () => $dbForProject->getDocument('database_' . $database->getSequence(), $collectionId));
if ($collection->isEmpty()) {
throw new Exception(Exception::COLLECTION_NOT_FOUND);
}
$databaseType = $database->getAttribute('type');
$resources = Transfer::extractServices([self::transferGroupForDatabaseType($databaseType)]);
$resourceType = self::resourceTypeForDatabaseType($databaseType);
$parentResourceType = self::resourceTypeForDatabaseType($databaseType);
$migration = $dbForProject->createDocument('migrations', new Document([
'$id' => $migrationId,
@@ -175,8 +181,12 @@ class Create extends Action
'source' => JSONSource::getName(),
'destination' => AppwriteSource::getName(),
'resources' => $resources,
'resourceId' => $resourceId,
'resourceType' => $resourceType,
'resourceId' => $collection->getId(),
'resourceInternalId' => $collection->getSequence(),
'resourceType' => Resource::TYPE_COLLECTION,
'parentResourceId' => $database->getId(),
'parentResourceInternalId' => $database->getSequence(),
'parentResourceType' => $parentResourceType,
'statusCounters' => '{}',
'resourceData' => '{}',
'errors' => [],
src/Appwrite/Platform/Workers/Migrations.php
+26 -9
View File
@@ -189,7 +189,7 @@ class Migrations extends Action
{
$source = $migration->getAttribute('source');
$destination = $migration->getAttribute('destination');
$resourceId = $migration->getAttribute('resourceId');
$resourceId = $this->getCompoundResourceId($migration);
$credentials = $migration->getAttribute('credentials');
$migrationOptions = $migration->getAttribute('options');
/** @var Database|null $projectDB */
@@ -294,7 +294,7 @@ class Migrations extends Action
),
DestinationCSV::getName() => new DestinationCSV(
$this->deviceForFiles,
$migration->getAttribute('resourceId'),
$this->getCompoundResourceId($migration),
$options['bucketId'],
$options['filename'],
$options['columns'],
@@ -305,7 +305,7 @@ class Migrations extends Action
),
DestinationJSON::getName() => new DestinationJSON(
$this->deviceForFiles,
$migration->getAttribute('resourceId'),
$this->getCompoundResourceId($migration),
$options['bucketId'] ?? 'default',
$options['filename'],
$options['columns'] ?? [],
@@ -507,8 +507,8 @@ class Migrations extends Action
}
$this->updateMigrationDocument($migration, $project, $queueForRealtime);
},
$migration->getAttribute('resourceId'),
$migration->getAttribute('resourceType')
$this->getCompoundResourceId($migration),
$migration->getAttribute('parentResourceType')
);
$destination->shutdown();
@@ -578,6 +578,7 @@ class Migrations extends Action
$publisherForUsage,
$migration->getAttribute('source'),
$authorization,
$migration->getAttribute('parentResourceId'),
$migration->getAttribute('resourceId')
);
}
@@ -608,6 +609,23 @@ class Migrations extends Action
return ($this->getDatabasesDB)($database);
}
/**
* Returns a "{parentResourceId}:{resourceId}" string when both are set, or just
* the available ID otherwise. The utopia-php/migration library expects this
* compound shape on its source/destination constructors.
*/
protected function getCompoundResourceId(Document $migration): ?string
{
$parentResourceId = $migration->getAttribute('parentResourceId');
$resourceId = $migration->getAttribute('resourceId');
if (!empty($parentResourceId) && !empty($resourceId)) {
return $parentResourceId . ':' . $resourceId;
}
return $resourceId ?? $parentResourceId;
}
/**
* Handle actions to be performed when a CSV export migration is successfully completed
*
@@ -890,7 +908,7 @@ class Migrations extends Action
return $errors;
}
private function processMigrationResourceStats(array $resources, Context $usage, Document $projectDocument, UsagePublisher $publisherForUsage, string $source, Authorization $authorization, ?string $resourceId)
private function processMigrationResourceStats(array $resources, Context $usage, Document $projectDocument, UsagePublisher $publisherForUsage, string $source, Authorization $authorization, ?string $parentResourceId, ?string $resourceId)
{
$resourceName = $resources['name'];
$count = $resources['count'];
@@ -898,9 +916,8 @@ class Migrations extends Action
$tableInternalId = $resources['tableId'];
if ($source === CSV::getName()) {
[$databaseId, $tableId] = explode(':', $resourceId);
$database = $authorization->skip(fn () => $this->dbForProject->getDocument('databases', $databaseId));
$table = $authorization->skip(fn () => $this->dbForProject->getDocument('database_' . $database->getSequence(), $tableId));
$database = $authorization->skip(fn () => $this->dbForProject->getDocument('databases', (string) $parentResourceId));
$table = $authorization->skip(fn () => $this->dbForProject->getDocument('database_' . $database->getSequence(), (string) $resourceId));
$databaseInternalId = (int) $database->getSequence();
$tableInternalId = (int) $table->getSequence();
}
src/Appwrite/Utopia/Database/Validator/Queries/Migrations.php
+5 -1
View File
@@ -11,10 +11,14 @@ class Migrations extends Base
'destination',
'resources',
'resourceId',
'resourceInternalId',
'resourceType',
'parentResourceId',
'parentResourceInternalId',
'parentResourceType',
'statusCounters',
'resourceData',
'errors'
'errors',
];
/**
src/Appwrite/Utopia/Request/Filters/V25.php
+40
View File
@@ -0,0 +1,40 @@
<?php
namespace Appwrite\Utopia\Request\Filters;
use Appwrite\Utopia\Request\Filter;
class V25 extends Filter
{
// Convert 1.9.3 params to 1.9.4
public function parse(array $content, string $model): array
{
switch ($model) {
case 'migrations.createCSVImport':
case 'migrations.createCSVExport':
case 'migrations.createJSONImport':
case 'migrations.createJSONExport':
$content = $this->parseMigrationResource($content);
break;
}
return $content;
}
protected function parseMigrationResource(array $content): array
{
if (!isset($content['resourceId']) || !\is_string($content['resourceId'])) {
return $content;
}
if (\str_contains($content['resourceId'], ':')) {
[$databaseId, $collectionId] = \explode(':', $content['resourceId'], 2);
$content['databaseId'] = $content['databaseId'] ?? $databaseId;
$content['collectionId'] = $content['collectionId'] ?? $collectionId;
}
unset($content['resourceId']);
return $content;
}
}
src/Appwrite/Utopia/Response/Filters/V25.php
+38
View File
@@ -0,0 +1,38 @@
<?php
namespace Appwrite\Utopia\Response\Filters;
use Appwrite\Utopia\Response;
use Appwrite\Utopia\Response\Filter;
// Convert 1.9.4 Data format to 1.9.3 format
class V25 extends Filter
{
public function parse(array $content, string $model): array
{
return match ($model) {
Response::MODEL_MIGRATION => $this->parseMigration($content),
Response::MODEL_MIGRATION_LIST => $this->handleList($content, 'migrations', fn ($item) => $this->parseMigration($item)),
default => $content,
};
}
protected function parseMigration(array $content): array
{
$parentResourceId = $content['parentResourceId'] ?? '';
$resourceId = $content['resourceId'] ?? '';
if ($parentResourceId !== '' && $resourceId !== '') {
$content['resourceId'] = $parentResourceId . ':' . $resourceId;
}
$content['resourceType'] = $content['parentResourceType'] ?? $content['resourceType'] ?? '';
unset($content['resourceInternalId']);
unset($content['parentResourceId']);
unset($content['parentResourceInternalId']);
unset($content['parentResourceType']);
return $content;
}
}
src/Appwrite/Utopia/Response/Model/Migration.php
+37 -2
View File
@@ -62,9 +62,44 @@ class Migration extends Model
])
->addRule('resourceId', [
'type' => self::TYPE_STRING,
'description' => 'Id of the resource to migrate.',
'description' => 'ID of the resource being migrated.',
'default' => '',
'example' => 'databaseId:collectionId',
'example' => 'collectionId',
'array' => false
])
->addRule('resourceInternalId', [
'type' => self::TYPE_STRING,
'description' => 'Internal ID of the resource being migrated.',
'default' => '',
'example' => '1',
'array' => false
])
->addRule('resourceType', [
'type' => self::TYPE_STRING,
'description' => 'Type of the resource being migrated.',
'default' => '',
'example' => 'collection',
'array' => false
])
->addRule('parentResourceId', [
'type' => self::TYPE_STRING,
'description' => 'ID of the parent resource that contains the migrated resource.',
'default' => '',
'example' => 'databaseId',
'array' => false
])
->addRule('parentResourceInternalId', [
'type' => self::TYPE_STRING,
'description' => 'Internal ID of the parent resource that contains the migrated resource.',
'default' => '',
'example' => '1',
'array' => false
])
->addRule('parentResourceType', [
'type' => self::TYPE_STRING,
'description' => 'Type of the parent resource that contains the migrated resource.',
'default' => '',
'example' => 'database',
'array' => false
])
->addRule('statusCounters', [
tests/e2e/Services/Migrations/MigrationsBase.php
+34 -17
View File
@@ -1328,7 +1328,8 @@ trait MigrationsBase
[
'fileId' => $fileIds['missing-column'],
'bucketId' => $bucketIds['missing-column'],
'resourceId' => $databaseId . ':' . $tableId,
'databaseId' => $databaseId,
'collectionId' => $tableId,
]
);
@@ -1360,7 +1361,8 @@ trait MigrationsBase
[
'fileId' => $fileIds['missing-row'],
'bucketId' => $bucketIds['missing-row'],
'resourceId' => $databaseId . ':' . $tableId,
'databaseId' => $databaseId,
'collectionId' => $tableId,
]
);
@@ -1392,7 +1394,8 @@ trait MigrationsBase
[
'fileId' => $fileIds['irrelevant-column'],
'bucketId' => $bucketIds['irrelevant-column'],
'resourceId' => $databaseId . ':' . $tableId,
'databaseId' => $databaseId,
'collectionId' => $tableId,
]
);
@@ -1419,7 +1422,8 @@ trait MigrationsBase
'endpoint' => $this->webEndpoint,
'fileId' => $fileIds['default'],
'bucketId' => $bucketIds['default'],
'resourceId' => $databaseId . ':' . $tableId,
'databaseId' => $databaseId,
'collectionId' => $tableId,
]
);
@@ -1461,7 +1465,8 @@ trait MigrationsBase
'endpoint' => $this->webEndpoint,
'fileId' => $fileIds['documents-internals'],
'bucketId' => $bucketIds['documents-internals'],
'resourceId' => $databaseId . ':' . $tableId,
'databaseId' => $databaseId,
'collectionId' => $tableId,
]
);
@@ -1644,7 +1649,8 @@ trait MigrationsBase
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id']
], $this->getHeaders()), [
'resourceId' => $databaseId . ':' . $collectionId,
'databaseId' => $databaseId,
'collectionId' => $collectionId,
'filename' => 'test-export',
'columns' => [],
'delimiter' => ',',
@@ -2622,7 +2628,8 @@ trait MigrationsBase
$migration = $this->performCsvMigration([
'fileId' => $fileId,
'bucketId' => $bucketId,
'resourceId' => $databaseId . ':' . $collectionId,
'databaseId' => $databaseId,
'collectionId' => $collectionId,
]);
$this->assertEquals(202, $migration['headers']['status-code']);
@@ -2750,7 +2757,8 @@ trait MigrationsBase
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
], $this->getHeaders()), [
'resourceId' => $databaseId . ':' . $collectionId,
'databaseId' => $databaseId,
'collectionId' => $collectionId,
'filename' => $filename,
'columns' => [],
'queries' => [],
@@ -3951,7 +3959,8 @@ trait MigrationsBase
[
'fileId' => $fileIds['missing-column'],
'bucketId' => $bucketIds['missing-column'],
'resourceId' => $databaseId . ':' . $tableId,
'databaseId' => $databaseId,
'collectionId' => $tableId,
]
);
@@ -3988,7 +3997,8 @@ trait MigrationsBase
[
'fileId' => $fileIds['irrelevant-column'],
'bucketId' => $bucketIds['irrelevant-column'],
'resourceId' => $databaseId . ':' . $tableId,
'databaseId' => $databaseId,
'collectionId' => $tableId,
]
);
@@ -4015,7 +4025,8 @@ trait MigrationsBase
'endpoint' => $this->endpoint,
'fileId' => $fileIds['default'],
'bucketId' => $bucketIds['default'],
'resourceId' => $databaseId . ':' . $tableId,
'databaseId' => $databaseId,
'collectionId' => $tableId,
]
);
@@ -4057,7 +4068,8 @@ trait MigrationsBase
'endpoint' => $this->endpoint,
'fileId' => $fileIds['documents-internals'],
'bucketId' => $bucketIds['documents-internals'],
'resourceId' => $databaseId . ':' . $tableId,
'databaseId' => $databaseId,
'collectionId' => $tableId,
]
);
@@ -4180,7 +4192,8 @@ trait MigrationsBase
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id']
], $this->getHeaders()), [
'resourceId' => $databaseId . ':' . $collectionId,
'databaseId' => $databaseId,
'collectionId' => $collectionId,
'filename' => 'test-json-export',
'columns' => [],
'queries' => [],
@@ -4291,7 +4304,8 @@ trait MigrationsBase
// Trigger JSON export
$migration = $this->client->call(Client::METHOD_POST, '/migrations/json/exports', $headers, [
'resourceId' => $databaseId . ':' . $collectionId,
'databaseId' => $databaseId,
'collectionId' => $collectionId,
'filename' => 'vectorsdb-export-test',
'columns' => [],
'queries' => [],
@@ -4364,7 +4378,8 @@ trait MigrationsBase
$migration = $this->performJsonMigration([
'fileId' => $fileId,
'bucketId' => $bucketId,
'resourceId' => $databaseId . ':' . $collectionId,
'databaseId' => $databaseId,
'collectionId' => $collectionId,
]);
$this->assertEquals(202, $migration['headers']['status-code']);
@@ -4435,7 +4450,8 @@ trait MigrationsBase
// Trigger JSON export
$migration = $this->client->call(Client::METHOD_POST, '/migrations/json/exports', $headers, [
'resourceId' => $databaseId . ':' . $collectionId,
'databaseId' => $databaseId,
'collectionId' => $collectionId,
'filename' => 'documentsdb-export-test',
'columns' => [],
'queries' => [],
@@ -4507,7 +4523,8 @@ trait MigrationsBase
$migration = $this->performJsonMigration([
'fileId' => $fileId,
'bucketId' => $bucketId,
'resourceId' => $databaseId . ':' . $collectionId,
'databaseId' => $databaseId,
'collectionId' => $collectionId,
]);
$this->assertEquals(202, $migration['headers']['status-code']);