Merge pull request #12193 from appwrite/feat/migration-resource-attrs

feat(migrations): expose queryable resource links
This commit is contained in:
Jake Barnby authored and GitHub committed 2026-07-21 02:31:21 +12:00
commit 7dfadfe2dc
20 files changed
+1403 -121

No files matched your search

+133
View File
@@ -2267,6 +2267,17 @@ return [
'array' => false,
'filters' => [],
],
[
'$id' => ID::custom('resourceInternalId'),
'type' => Database::VAR_STRING,
'format' => '',
'size' => Database::LENGTH_KEY,
'signed' => true,
'required' => false,
'default' => null,
'array' => false,
'filters' => [],
],
[
'$id' => ID::custom('resourceType'),
'type' => Database::VAR_STRING,
@@ -2278,6 +2289,72 @@ return [
'array' => false,
'filters' => [],
],
[
'$id' => ID::custom('parentResourceId'),
'type' => Database::VAR_STRING,
'format' => '',
'size' => Database::LENGTH_KEY,
'signed' => true,
'required' => false,
'default' => null,
'array' => false,
'filters' => [],
],
[
'$id' => ID::custom('parentResourceInternalId'),
'type' => Database::VAR_STRING,
'format' => '',
'size' => Database::LENGTH_KEY,
'signed' => true,
'required' => false,
'default' => null,
'array' => false,
'filters' => [],
],
[
'$id' => ID::custom('parentResourceType'),
'type' => Database::VAR_STRING,
'format' => '',
'size' => Database::LENGTH_KEY,
'signed' => true,
'required' => false,
'default' => null,
'array' => false,
'filters' => [],
],
[
'$id' => ID::custom('destinationResourceId'),
'type' => Database::VAR_STRING,
'format' => '',
'size' => Database::LENGTH_KEY,
'signed' => true,
'required' => false,
'default' => null,
'array' => false,
'filters' => [],
],
[
'$id' => ID::custom('destinationResourceInternalId'),
'type' => Database::VAR_STRING,
'format' => '',
'size' => Database::LENGTH_KEY,
'signed' => true,
'required' => false,
'default' => null,
'array' => false,
'filters' => [],
],
[
'$id' => ID::custom('destinationResourceType'),
'type' => Database::VAR_STRING,
'format' => '',
'size' => Database::LENGTH_KEY,
'signed' => true,
'required' => false,
'default' => null,
'array' => false,
'filters' => [],
],
],
'indexes' => [
[
@@ -2308,6 +2385,62 @@ return [
'lengths' => [Database::LENGTH_KEY],
'orders' => [Database::ORDER_DESC],
],
[
'$id' => '_key_resourceType',
'type' => Database::INDEX_KEY,
'attributes' => ['resourceType'],
'lengths' => [Database::LENGTH_KEY],
'orders' => [Database::ORDER_ASC],
],
[
'$id' => '_key_resourceInternalId',
'type' => Database::INDEX_KEY,
'attributes' => ['resourceInternalId'],
'lengths' => [Database::LENGTH_KEY],
'orders' => [Database::ORDER_ASC],
],
[
'$id' => '_key_parentResourceId',
'type' => Database::INDEX_KEY,
'attributes' => ['parentResourceId'],
'lengths' => [Database::LENGTH_KEY],
'orders' => [Database::ORDER_ASC],
],
[
'$id' => '_key_parentResourceType',
'type' => Database::INDEX_KEY,
'attributes' => ['parentResourceType'],
'lengths' => [Database::LENGTH_KEY],
'orders' => [Database::ORDER_ASC],
],
[
'$id' => '_key_parentResourceInternalId',
'type' => Database::INDEX_KEY,
'attributes' => ['parentResourceInternalId'],
'lengths' => [Database::LENGTH_KEY],
'orders' => [Database::ORDER_ASC],
],
[
'$id' => '_key_destinationResourceId',
'type' => Database::INDEX_KEY,
'attributes' => ['destinationResourceId'],
'lengths' => [Database::LENGTH_KEY],
'orders' => [Database::ORDER_ASC],
],
[
'$id' => '_key_destinationResourceInternalId',
'type' => Database::INDEX_KEY,
'attributes' => ['destinationResourceInternalId'],
'lengths' => [Database::LENGTH_KEY],
'orders' => [Database::ORDER_ASC],
],
[
'$id' => '_key_destinationResourceType',
'type' => Database::INDEX_KEY,
'attributes' => ['destinationResourceType'],
'lengths' => [Database::LENGTH_KEY],
'orders' => [Database::ORDER_ASC],
],
[
'$id' => ID::custom('_fulltext_search'),
'type' => Database::INDEX_FULLTEXT,
+8
View File
@@ -34,6 +34,7 @@ use Appwrite\Utopia\Request\Filters\V23 as RequestV23;
use Appwrite\Utopia\Request\Filters\V24 as RequestV24;
use Appwrite\Utopia\Request\Filters\V25 as RequestV25;
use Appwrite\Utopia\Request\Filters\V26 as RequestV26;
use Appwrite\Utopia\Request\Filters\V27 as RequestV27;
use Appwrite\Utopia\Response;
use Appwrite\Utopia\Response\Filters\V16 as ResponseV16;
use Appwrite\Utopia\Response\Filters\V17 as ResponseV17;
@@ -46,6 +47,7 @@ use Appwrite\Utopia\Response\Filters\V23 as ResponseV23;
use Appwrite\Utopia\Response\Filters\V24 as ResponseV24;
use Appwrite\Utopia\Response\Filters\V25 as ResponseV25;
use Appwrite\Utopia\Response\Filters\V26 as ResponseV26;
use Appwrite\Utopia\Response\Filters\V27 as ResponseV27;
use Appwrite\Utopia\View;
use Executor\Exception\Timeout as ExecutorTimeout;
use Executor\Executor;
@@ -917,6 +919,9 @@ Http::init()
if (version_compare($requestFormat, '1.9.5', '<')) {
$request->addFilter(new RequestV26());
}
if (version_compare($requestFormat, '2.0.0', '<')) {
$request->addFilter(new RequestV27());
}
}
$localeParam = (string) $request->getParam('locale', $request->getHeaderLine('x-appwrite-locale', ''));
@@ -941,6 +946,9 @@ Http::init()
*/
$responseFormat = $request->getHeaderLine('x-appwrite-response-format', System::getEnv('_APP_SYSTEM_RESPONSE_FORMAT', ''));
if ($responseFormat) {
if (version_compare($responseFormat, '2.0.0', '<')) {
$response->addFilter(new ResponseV27());
}
if (version_compare($responseFormat, '1.9.5', '<')) {
$response->addFilter(new ResponseV26());
}
+1 -1
View File
@@ -83,7 +83,7 @@
"utopia-php/lock": "0.2.*",
"utopia-php/logger": "0.8.*",
"utopia-php/messaging": "^2.0",
"utopia-php/migration": "1.17.*",
"utopia-php/migration": "^1.18.0",
"utopia-php/platform": "^1.0@RC",
"utopia-php/pools": "1.*",
"utopia-php/span": "3.0.*",
Generated
+7 -7
View File
@@ -4,7 +4,7 @@
"Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies",
"This file is @generated automatically"
],
"content-hash": "185f7ef75a7b75691ca976b28c808281",
"content-hash": "f007708a2c4301c72ae22feadcad6c8f",
"packages": [
{
"name": "adhocore/jwt",
@@ -4432,16 +4432,16 @@
},
{
"name": "utopia-php/migration",
"version": "1.17.3",
"version": "1.18.0",
"source": {
"type": "git",
"url": "https://github.com/utopia-php/migration.git",
"reference": "daf1008800633416a186c18069e3a5784de88d1c"
"reference": "b531ea43293fde5ceb72c29d312489aa0ded0c83"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/utopia-php/migration/zipball/daf1008800633416a186c18069e3a5784de88d1c",
"reference": "daf1008800633416a186c18069e3a5784de88d1c",
"url": "https://api.github.com/repos/utopia-php/migration/zipball/b531ea43293fde5ceb72c29d312489aa0ded0c83",
"reference": "b531ea43293fde5ceb72c29d312489aa0ded0c83",
"shasum": ""
},
"require": {
@@ -4481,9 +4481,9 @@
],
"support": {
"issues": "https://github.com/utopia-php/migration/issues",
"source": "https://github.com/utopia-php/migration/tree/1.17.3"
"source": "https://github.com/utopia-php/migration/tree/1.18.0"
},
"time": "2026-07-14T14:04:00+00:00"
"time": "2026-07-20T10:00:12+00:00"
},
{
"name": "utopia-php/mongo",
+162
View File
@@ -9,6 +9,7 @@ use Utopia\Console;
use Utopia\Database\Database;
use Utopia\Database\Document;
use Utopia\Database\Query;
use Utopia\Migration\Resource;
class V25 extends Migration
{
@@ -19,6 +20,9 @@ class V25 extends Migration
{
Console::info('Migrating collections');
$this->migrateCollections();
Console::info('Migrating documents');
$this->forEachDocument($this->migrateDocument(...));
}
/**
@@ -104,7 +108,165 @@ class V25 extends Migration
}
}
break;
case 'migrations':
if ($collectionType === 'projects') {
$attributes = [
'resourceInternalId',
'parentResourceId',
'parentResourceInternalId',
'parentResourceType',
'destinationResourceId',
'destinationResourceInternalId',
'destinationResourceType',
];
try {
$this->createAttributesFromCollection($this->dbForProject, $id, $attributes);
} catch (Throwable $th) {
Console::warning('Failed to create attributes "' . \implode(', ', $attributes) . "\" in collection {$id}: {$th->getMessage()}");
}
$indexes = [
'_key_resourceType',
'_key_resourceInternalId',
'_key_parentResourceId',
'_key_parentResourceType',
'_key_parentResourceInternalId',
'_key_destinationResourceId',
'_key_destinationResourceInternalId',
'_key_destinationResourceType',
];
foreach ($indexes as $index) {
try {
$this->createIndexFromCollection($this->dbForProject, $id, $index);
} catch (Throwable $th) {
Console::warning("Failed to create index \"{$index}\" from {$id}: {$th->getMessage()}");
}
}
$this->dbForProject->purgeCachedCollection($id);
}
break;
}
}
}
protected function migrateDocument(Document $document): Document
{
if ($document->getCollection() !== 'migrations') {
return $document;
}
$resourceId = (string) $document->getAttribute('resourceId', '');
$parentResourceId = (string) $document->getAttribute('parentResourceId', '');
$parentResourceType = '';
$split = false;
if ($parentResourceId === '') {
if (!\str_contains($resourceId, ':')) {
return $document;
}
[$parentResourceId, $resourceId] = \explode(':', $resourceId, 2);
if ($parentResourceId === '' || $resourceId === '') {
return $document;
}
$parentResourceType = (string) $document->getAttribute('resourceType', '');
$split = true;
}
if ($resourceId === '') {
return $document;
}
$internalIds = $this->resolveInternalIds($parentResourceId, $resourceId, $document);
if (
$split
&& (!isset($internalIds['parentResourceInternalId']) || !isset($internalIds['resourceInternalId']))
) {
return $document;
}
if ($split) {
$document
->setAttribute('resourceId', $resourceId)
->setAttribute('resourceType', Resource::TYPE_COLLECTION)
->setAttribute('parentResourceId', $parentResourceId);
if ($parentResourceType !== '') {
$document->setAttribute('parentResourceType', $parentResourceType);
}
}
if (
(string) $document->getAttribute('parentResourceInternalId', '') === ''
&& isset($internalIds['parentResourceInternalId'])
) {
$document->setAttribute('parentResourceInternalId', $internalIds['parentResourceInternalId']);
}
if (
(string) $document->getAttribute('resourceInternalId', '') === ''
&& isset($internalIds['resourceInternalId'])
) {
$document->setAttribute('resourceInternalId', $internalIds['resourceInternalId']);
}
return $document;
}
/**
* @return array{parentResourceInternalId?: string, resourceInternalId?: string}
*/
protected function resolveInternalIds(string $parentResourceId, string $resourceId, Document $migration): array
{
try {
$database = $this->dbForProject->getDocument('databases', $parentResourceId);
if (
$database->isEmpty()
|| $database->getSequence() === ''
|| !$this->predatesMigration($database, $migration)
) {
return [];
}
} catch (Throwable $th) {
Console::warning("Failed to resolve parent internal ID for migration {$migration->getId()}: {$th->getMessage()}");
return [];
}
$internalIds = [
'parentResourceInternalId' => (string) $database->getSequence(),
];
try {
$resource = $this->dbForProject->getDocument('database_' . $database->getSequence(), $resourceId);
if (
!$resource->isEmpty()
&& $resource->getSequence() !== ''
&& $this->predatesMigration($resource, $migration)
) {
$internalIds['resourceInternalId'] = (string) $resource->getSequence();
}
} catch (Throwable $th) {
Console::warning("Failed to resolve resource internal ID for migration {$migration->getId()}: {$th->getMessage()}");
}
return $internalIds;
}
/**
* A resource created after the migration is a reused public ID, not the
* generation the historical migration operated on.
*/
protected function predatesMigration(Document $resource, Document $migration): bool
{
$resourceCreatedAt = \strtotime($resource->getCreatedAt());
$migrationCreatedAt = \strtotime($migration->getCreatedAt());
if ($resourceCreatedAt === false || $migrationCreatedAt === false) {
return false;
}
return $resourceCreatedAt <= $migrationCreatedAt;
}
}
@@ -9,7 +9,6 @@ use Appwrite\Extend\Exception;
use Appwrite\SDK\AuthType;
use Appwrite\SDK\Method;
use Appwrite\SDK\Response as SDKResponse;
use Appwrite\Utopia\Database\Validator\CompoundUID;
use Appwrite\Utopia\Response;
use Utopia\Database\Database;
use Utopia\Database\Document;
@@ -18,6 +17,7 @@ use Utopia\Database\Helpers\ID;
use Utopia\Database\Query;
use Utopia\Database\Validator\Authorization;
use Utopia\Database\Validator\Queries\Documents;
use Utopia\Database\Validator\UID;
use Utopia\Migration\Resource;
use Utopia\Migration\Sources\Appwrite as AppwriteSource;
use Utopia\Migration\Sources\CSV;
@@ -60,7 +60,8 @@ class Create extends Action
)
]
))
->param('resourceId', null, new CompoundUID(), 'Composite ID in the format {databaseId:collectionId}, identifying a collection within a database to export.')
->param('databaseId', '', new UID(), 'Database ID containing the source collection.')
->param('collectionId', '', new UID(), 'Collection ID to export documents from.')
->param('filename', '', new Text(255), 'The name of the file to be created for the export, excluding the .csv extension.')
->param('columns', [], new ArrayList(new Text(Database::LENGTH_KEY)), 'List of attributes to export. If empty, all attributes will be exported. You can use the `*` wildcard to export all attributes from the collection.', true)
->param('queries', [], new ArrayList(new Text(0)), 'Array of query strings generated using the Query class provided by the SDK to filter documents to export. [Learn more about queries](https://appwrite.io/docs/databases#querying-documents). Maximum of ' . APP_LIMIT_ARRAY_PARAMS_SIZE . ' queries are allowed, each ' . APP_LIMIT_ARRAY_ELEMENT_SIZE . ' characters long.', true)
@@ -82,7 +83,8 @@ class Create extends Action
}
public function action(
string $resourceId,
string $databaseId,
string $collectionId,
string $filename,
array $columns,
array $queries,
@@ -112,14 +114,6 @@ class Create extends Action
throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND);
}
[$databaseId, $collectionId] = \explode(':', $resourceId, 2);
if (empty($databaseId)) {
throw new Exception(Exception::DATABASE_NOT_FOUND);
}
if (empty($collectionId)) {
throw new Exception(Exception::COLLECTION_NOT_FOUND);
}
$database = $authorization->skip(fn () => $dbForProject->getDocument('databases', $databaseId));
if ($database->isEmpty()) {
throw new Exception(Exception::DATABASE_NOT_FOUND);
@@ -150,7 +144,7 @@ class Create extends Action
}
$resources = Transfer::extractServices([self::transferGroupForDatabaseType($databaseType)]);
$resourceType = self::resourceTypeForDatabaseType($databaseType);
$parentResourceType = self::resourceTypeForDatabaseType($databaseType);
$migration = $dbForProject->createDocument('migrations', new Document([
'$id' => ID::unique(),
@@ -159,8 +153,12 @@ class Create extends Action
'source' => AppwriteSource::getName(),
'destination' => CSV::getName(),
'resources' => $resources,
'resourceId' => $resourceId,
'resourceType' => $resourceType,
'resourceId' => $collection->getId(),
'resourceInternalId' => $collection->getSequence(),
'resourceType' => Resource::TYPE_COLLECTION,
'parentResourceId' => $database->getId(),
'parentResourceInternalId' => $database->getSequence(),
'parentResourceType' => $parentResourceType,
'statusCounters' => '{}',
'resourceData' => '{}',
'errors' => [],
@@ -10,7 +10,6 @@ use Appwrite\OpenSSL\OpenSSL;
use Appwrite\SDK\AuthType;
use Appwrite\SDK\Method;
use Appwrite\SDK\Response as SDKResponse;
use Appwrite\Utopia\Database\Validator\CompoundUID;
use Appwrite\Utopia\Response;
use Utopia\Compression\Algorithms\GZIP;
use Utopia\Compression\Algorithms\Zstd;
@@ -68,7 +67,8 @@ class Create extends Action
))
->param('bucketId', '', fn (Database $dbForProject) => new UID($dbForProject->getAdapter()->getMaxUIDLength()), 'Storage bucket unique ID. You can create a new storage bucket using the Storage service [server integration](https://appwrite.io/docs/server/storage#createBucket).', false, ['dbForProject'])
->param('fileId', '', fn (Database $dbForProject) => new UID($dbForProject->getAdapter()->getMaxUIDLength()), 'File ID.', false, ['dbForProject'])
->param('resourceId', null, new CompoundUID(), 'Composite ID in the format {databaseId:collectionId}, identifying a collection within a database.')
->param('databaseId', '', new UID(), 'Database ID containing the target collection.')
->param('collectionId', '', new UID(), 'Collection ID to import documents into.')
->param('internalFile', false, new Boolean(), 'Is the file stored in an internal bucket?', true)
->param('onDuplicate', OnDuplicate::Fail->value, new WhiteList(OnDuplicate::values()), 'Behavior when a row with an existing $id is encountered. "fail" (default): abort on first conflict. "skip": silently ignore. "overwrite": replace existing row.', true, enum: new Enum(name: 'OnDuplicate'))
->inject('response')
@@ -87,7 +87,8 @@ class Create extends Action
public function action(
string $bucketId,
string $fileId,
string $resourceId,
string $databaseId,
string $collectionId,
bool $internalFile,
string $onDuplicate,
Response $response,
@@ -163,15 +164,24 @@ class Create extends Action
throw new \Exception('Unable to copy file');
}
[$databaseId] = \explode(':', $resourceId, 2);
$database = $authorization->skip(fn () => $dbForProject->getDocument('databases', $databaseId));
if ($database->isEmpty()) {
throw new Exception(Exception::DATABASE_NOT_FOUND);
}
$databaseType = $database->getAttribute('type');
if (!\in_array($databaseType, CSV_ALLOWED_DATABASE_TYPES)) {
throw new Exception(Exception::MIGRATION_DATABASE_TYPE_UNSUPPORTED, 'Database type not supported for csv');
}
$collection = $authorization->skip(fn () => $dbForProject->getDocument('database_' . $database->getSequence(), $collectionId));
if ($collection->isEmpty()) {
throw new Exception(Exception::COLLECTION_NOT_FOUND);
}
$fileSize = $deviceForMigrations->getFileSize($newPath);
$resources = Transfer::extractServices([self::transferGroupForDatabaseType($databaseType)]);
$resourceType = self::resourceTypeForDatabaseType($databaseType);
$parentResourceType = self::resourceTypeForDatabaseType($databaseType);
$migration = $dbForProject->createDocument('migrations', new Document([
'$id' => $migrationId,
@@ -180,8 +190,15 @@ class Create extends Action
'source' => CSV::getName(),
'destination' => AppwriteSource::getName(),
'resources' => $resources,
'resourceId' => $resourceId,
'resourceType' => $resourceType,
'resourceId' => $collection->getId(),
'resourceInternalId' => $collection->getSequence(),
'resourceType' => Resource::TYPE_COLLECTION,
'parentResourceId' => $database->getId(),
'parentResourceInternalId' => $database->getSequence(),
'parentResourceType' => $parentResourceType,
'destinationResourceId' => $database->getId(),
'destinationResourceInternalId' => $database->getSequence(),
'destinationResourceType' => $parentResourceType,
'statusCounters' => '{}',
'resourceData' => '{}',
'errors' => [],
@@ -9,7 +9,6 @@ use Appwrite\Extend\Exception;
use Appwrite\SDK\AuthType;
use Appwrite\SDK\Method;
use Appwrite\SDK\Response as SDKResponse;
use Appwrite\Utopia\Database\Validator\CompoundUID;
use Appwrite\Utopia\Response;
use Utopia\Database\Database;
use Utopia\Database\Document;
@@ -18,6 +17,7 @@ use Utopia\Database\Helpers\ID;
use Utopia\Database\Query;
use Utopia\Database\Validator\Authorization;
use Utopia\Database\Validator\Queries\Documents;
use Utopia\Database\Validator\UID;
use Utopia\Migration\Resource;
use Utopia\Migration\Sources\Appwrite as AppwriteSource;
use Utopia\Migration\Sources\JSON as JSONSource;
@@ -60,7 +60,8 @@ class Create extends Action
)
]
))
->param('resourceId', null, new CompoundUID(), 'Composite ID in the format {databaseId:collectionId}, identifying a collection within a database to export.')
->param('databaseId', '', new UID(), 'Database ID containing the source collection.')
->param('collectionId', '', new UID(), 'Collection ID to export documents from.')
->param('filename', '', new Text(255), 'The name of the file to be created for the export, excluding the .json extension.')
->param('columns', [], new ArrayList(new Text(Database::LENGTH_KEY)), 'List of attributes to export. If empty, all attributes will be exported. You can use the `*` wildcard to export all attributes from the collection.', true)
->param('queries', [], new ArrayList(new Text(0)), 'Array of query strings generated using the Query class provided by the SDK to filter documents to export. [Learn more about queries](https://appwrite.io/docs/databases#querying-documents). Maximum of ' . APP_LIMIT_ARRAY_PARAMS_SIZE . ' queries are allowed, each ' . APP_LIMIT_ARRAY_ELEMENT_SIZE . ' characters long.', true)
@@ -78,7 +79,8 @@ class Create extends Action
}
public function action(
string $resourceId,
string $databaseId,
string $collectionId,
string $filename,
array $columns,
array $queries,
@@ -104,14 +106,6 @@ class Create extends Action
throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND);
}
[$databaseId, $collectionId] = \explode(':', $resourceId, 2);
if (empty($databaseId)) {
throw new Exception(Exception::DATABASE_NOT_FOUND);
}
if (empty($collectionId)) {
throw new Exception(Exception::COLLECTION_NOT_FOUND);
}
$database = $authorization->skip(fn () => $dbForProject->getDocument('databases', $databaseId));
if ($database->isEmpty()) {
throw new Exception(Exception::DATABASE_NOT_FOUND);
@@ -139,7 +133,7 @@ class Create extends Action
}
$resources = Transfer::extractServices([self::transferGroupForDatabaseType($databaseType)]);
$resourceType = self::resourceTypeForDatabaseType($databaseType);
$parentResourceType = self::resourceTypeForDatabaseType($databaseType);
$migration = $dbForProject->createDocument('migrations', new Document([
'$id' => ID::unique(),
@@ -148,8 +142,12 @@ class Create extends Action
'source' => AppwriteSource::getName(),
'destination' => JSONSource::getName(),
'resources' => $resources,
'resourceId' => $resourceId,
'resourceType' => $resourceType,
'resourceId' => $collection->getId(),
'resourceInternalId' => $collection->getSequence(),
'resourceType' => Resource::TYPE_COLLECTION,
'parentResourceId' => $database->getId(),
'parentResourceInternalId' => $database->getSequence(),
'parentResourceType' => $parentResourceType,
'statusCounters' => '{}',
'resourceData' => '{}',
'errors' => [],
@@ -10,7 +10,6 @@ use Appwrite\OpenSSL\OpenSSL;
use Appwrite\SDK\AuthType;
use Appwrite\SDK\Method;
use Appwrite\SDK\Response as SDKResponse;
use Appwrite\Utopia\Database\Validator\CompoundUID;
use Appwrite\Utopia\Response;
use Utopia\Compression\Algorithms\GZIP;
use Utopia\Compression\Algorithms\Zstd;
@@ -67,7 +66,8 @@ class Create extends Action
))
->param('bucketId', '', new UID(), 'Storage bucket unique ID. You can create a new storage bucket using the Storage service [server integration](https://appwrite.io/docs/server/storage#createBucket).')
->param('fileId', '', new UID(), 'File ID.')
->param('resourceId', null, new CompoundUID(), 'Composite ID in the format {databaseId:collectionId}, identifying a collection within a database.')
->param('databaseId', '', new UID(), 'Database ID containing the target collection.')
->param('collectionId', '', new UID(), 'Collection ID to import documents into.')
->param('internalFile', false, new Boolean(), 'Is the file stored in an internal bucket?', true)
->param('onDuplicate', OnDuplicate::Fail->value, new WhiteList(OnDuplicate::values()), 'Behavior when a row with an existing $id is encountered. "fail" (default): abort on first conflict. "skip": silently ignore. "overwrite": replace existing row.', true, enum: new Enum(name: 'OnDuplicate'))
->inject('response')
@@ -86,7 +86,8 @@ class Create extends Action
public function action(
string $bucketId,
string $fileId,
string $resourceId,
string $databaseId,
string $collectionId,
bool $internalFile,
string $onDuplicate,
Response $response,
@@ -164,14 +165,19 @@ class Create extends Action
$fileSize = $deviceForMigrations->getFileSize($newPath);
[$databaseId] = \explode(':', $resourceId, 2);
$database = $authorization->skip(fn () => $dbForProject->getDocument('databases', $databaseId));
if ($database->isEmpty()) {
throw new Exception(Exception::DATABASE_NOT_FOUND);
}
$collection = $authorization->skip(fn () => $dbForProject->getDocument('database_' . $database->getSequence(), $collectionId));
if ($collection->isEmpty()) {
throw new Exception(Exception::COLLECTION_NOT_FOUND);
}
$databaseType = $database->getAttribute('type');
$resources = Transfer::extractServices([self::transferGroupForDatabaseType($databaseType)]);
$resourceType = self::resourceTypeForDatabaseType($databaseType);
$parentResourceType = self::resourceTypeForDatabaseType($databaseType);
$migration = $dbForProject->createDocument('migrations', new Document([
'$id' => $migrationId,
@@ -180,8 +186,15 @@ class Create extends Action
'source' => JSONSource::getName(),
'destination' => AppwriteSource::getName(),
'resources' => $resources,
'resourceId' => $resourceId,
'resourceType' => $resourceType,
'resourceId' => $collection->getId(),
'resourceInternalId' => $collection->getSequence(),
'resourceType' => Resource::TYPE_COLLECTION,
'parentResourceId' => $database->getId(),
'parentResourceInternalId' => $database->getSequence(),
'parentResourceType' => $parentResourceType,
'destinationResourceId' => $database->getId(),
'destinationResourceInternalId' => $database->getSequence(),
'destinationResourceType' => $parentResourceType,
'statusCounters' => '{}',
'resourceData' => '{}',
'errors' => [],
+99 -38
View File
@@ -192,7 +192,7 @@ class Migrations extends Action
{
$source = $migration->getAttribute('source');
$destination = $migration->getAttribute('destination');
$resourceId = $migration->getAttribute('resourceId');
[$databaseId, $tableId] = $this->resolveResourceIds($migration);
$credentials = $migration->getAttribute('credentials');
$migrationOptions = $migration->getAttribute('options');
/** @var Database|null $projectDB */
@@ -290,18 +290,20 @@ class Migrations extends Action
$projectDB,
$queries
),
CSV::getName() => new CSV(
$resourceId,
$migrationOptions['path'],
$this->deviceForMigrations,
$this->dbForProject,
$getDatabasesDB
CSV::getName() => CSV::fromResourceIds(
databaseId: $databaseId,
tableId: $tableId,
filePath: $migrationOptions['path'],
device: $this->deviceForMigrations,
dbForProject: $this->dbForProject,
getDatabasesDB: $getDatabasesDB,
),
JSON::getName() => new JSON(
$resourceId,
$migrationOptions['path'],
$this->deviceForMigrations,
$this->dbForProject,
JSON::getName() => JSON::fromResourceIds(
databaseId: $databaseId,
tableId: $tableId,
filePath: $migrationOptions['path'],
device: $this->deviceForMigrations,
dbForProject: $this->dbForProject,
),
default => throw new Exception(Exception::MIGRATION_SOURCE_TYPE_INVALID),
};
@@ -320,6 +322,7 @@ class Migrations extends Action
$destination = $migration->getAttribute('destination');
$options = $migration->getAttribute('options', []);
$credentials = $migration->getAttribute('credentials');
[$databaseId, $tableId] = $this->resolveResourceIds($migration);
return match ($destination) {
DestinationAppwrite::getName() => new DestinationAppwrite(
@@ -334,23 +337,25 @@ class Migrations extends Action
OnDuplicate::tryFrom($options['onDuplicate'] ?? '') ?? OnDuplicate::Fail,
$this->resolveDestinationDatabaseDsn(...),
),
DestinationCSV::getName() => new DestinationCSV(
$this->deviceForFiles,
$migration->getAttribute('resourceId'),
$options['bucketId'],
$migration->getId(),
$options['columns'],
$options['delimiter'],
$options['enclosure'],
$options['escape'],
$options['header'],
DestinationCSV::getName() => DestinationCSV::fromResourceIds(
deviceForFiles: $this->deviceForFiles,
databaseId: $databaseId,
tableId: $tableId,
directory: $options['bucketId'],
filename: $migration->getId(),
allowedColumns: $options['columns'],
delimiter: $options['delimiter'],
enclosure: $options['enclosure'],
escape: $options['escape'],
includeHeaders: $options['header'],
),
DestinationJSON::getName() => new DestinationJSON(
$this->deviceForFiles,
$migration->getAttribute('resourceId'),
$options['bucketId'] ?? 'default',
$migration->getId(),
$options['columns'] ?? [],
DestinationJSON::getName() => DestinationJSON::fromResourceIds(
deviceForFiles: $this->deviceForFiles,
databaseId: $databaseId,
tableId: $tableId,
directory: $options['bucketId'] ?? 'default',
filename: $migration->getId(),
allowedColumns: $options['columns'] ?? [],
),
default => throw new Exception(Exception::MIGRATION_DESTINATION_TYPE_INVALID),
};
@@ -535,7 +540,8 @@ class Migrations extends Action
$migration->setAttribute('stage', 'migrating');
$this->updateMigrationDocument($migration, $project, $queueForRealtime);
$transfer->run(
$context = $this->resolveResourceContext($migration);
$transfer->runWithResourceSelector(
$migration->getAttribute('resources'),
function ($resources) use ($migration, $transfer, $project, $queueForRealtime, &$aggregatedResources) {
$migration->setAttribute('resourceData', json_encode($transfer->getCache()));
@@ -573,8 +579,12 @@ class Migrations extends Action
}
$this->updateMigrationDocument($migration, $project, $queueForRealtime);
},
$migration->getAttribute('resourceId'),
$migration->getAttribute('resourceType')
resourceId: $context['resourceId'],
resourceInternalId: $context['resourceInternalId'],
resourceType: $context['resourceType'],
parentResourceId: $context['parentResourceId'],
parentResourceInternalId: $context['parentResourceInternalId'],
parentResourceType: $context['parentResourceType'],
);
$destination->shutdown();
@@ -590,6 +600,9 @@ class Migrations extends Action
return;
}
$destination->success();
$source->success();
$destinationType = $migration->getAttribute('destination');
if ($destinationType === DestinationCSV::getName() || $destinationType === DestinationJSON::getName()) {
$this->handleDataExportComplete($project, $migration, $publisherForMails, $queueForRealtime, $platform, $authorization);
@@ -692,11 +705,9 @@ class Migrations extends Action
$publisherForUsage,
$migration->getAttribute('source'),
$authorization,
$migration->getAttribute('resourceId')
...$this->resolveResourceIds($migration),
);
}
$destination?->success();
$source?->success();
}
} finally {
$source?->cleanup();
@@ -718,6 +729,45 @@ class Migrations extends Action
return ($this->getDatabasesDB)($database);
}
/** @return array{0: string, 1: string} */
protected function resolveResourceIds(Document $migration): array
{
$context = $this->resolveResourceContext($migration);
if ($context['parentResourceId'] !== '') {
return [$context['parentResourceId'], $context['resourceId']];
}
return [$context['resourceId'], ''];
}
/**
* @return array{resourceId: string, resourceInternalId: string, resourceType: string, parentResourceId: string, parentResourceInternalId: string, parentResourceType: string}
*/
protected function resolveResourceContext(Document $migration): array
{
$context = [
'resourceId' => (string) $migration->getAttribute('resourceId', ''),
'resourceInternalId' => (string) $migration->getAttribute('resourceInternalId', ''),
'resourceType' => (string) $migration->getAttribute('resourceType', ''),
'parentResourceId' => (string) $migration->getAttribute('parentResourceId', ''),
'parentResourceInternalId' => (string) $migration->getAttribute('parentResourceInternalId', ''),
'parentResourceType' => (string) $migration->getAttribute('parentResourceType', ''),
];
if (
$context['parentResourceId'] === ''
&& \array_key_exists($context['resourceType'], Resource::DATABASE_TYPE_RESOURCE_MAP)
&& \str_contains($context['resourceId'], ':')
) {
[$context['parentResourceId'], $context['resourceId']] = \explode(':', $context['resourceId'], 2);
$context['parentResourceType'] = $context['resourceType'];
$context['resourceType'] = Resource::TYPE_COLLECTION;
}
return $context;
}
/**
* Handle actions to be performed when a CSV export migration is successfully completed
*
@@ -1074,7 +1124,7 @@ class Migrations extends Action
return $errors;
}
private function processMigrationResourceStats(array $resources, Context $usage, Document $projectDocument, UsagePublisher $publisherForUsage, string $source, Authorization $authorization, ?string $resourceId)
private function processMigrationResourceStats(array $resources, Context $usage, Document $projectDocument, UsagePublisher $publisherForUsage, string $source, Authorization $authorization, ?string $parentResourceId, ?string $resourceId)
{
$resourceName = $resources['name'];
$count = $resources['count'];
@@ -1082,9 +1132,20 @@ class Migrations extends Action
$tableInternalId = $resources['tableId'];
if ($source === CSV::getName()) {
[$databaseId, $tableId] = explode(':', $resourceId);
$database = $authorization->skip(fn () => $this->dbForProject->getDocument('databases', $databaseId));
$table = $authorization->skip(fn () => $this->dbForProject->getDocument('database_' . $database->getSequence(), $tableId));
if (empty($parentResourceId) || empty($resourceId)) {
Console::warning("Skipping CSV migration usage stats: missing parent/leaf resource ID (parent: '{$parentResourceId}', leaf: '{$resourceId}')");
return;
}
$database = $authorization->skip(fn () => $this->dbForProject->getDocument('databases', $parentResourceId));
if ($database->isEmpty()) {
Console::warning("Skipping CSV migration usage stats: database '{$parentResourceId}' not found");
return;
}
$table = $authorization->skip(fn () => $this->dbForProject->getDocument('database_' . $database->getSequence(), $resourceId));
if ($table->isEmpty()) {
Console::warning("Skipping CSV migration usage stats: collection '{$resourceId}' not found in database '{$parentResourceId}'");
return;
}
$databaseInternalId = (int) $database->getSequence();
$tableInternalId = (int) $table->getSequence();
}
@@ -11,10 +11,17 @@ class Migrations extends Base
'destination',
'resources',
'resourceId',
'resourceInternalId',
'resourceType',
'parentResourceId',
'parentResourceInternalId',
'parentResourceType',
'destinationResourceId',
'destinationResourceInternalId',
'destinationResourceType',
'statusCounters',
'resourceData',
'errors'
'errors',
];
/**
@@ -0,0 +1,38 @@
<?php
namespace Appwrite\Utopia\Request\Filters;
use Appwrite\Utopia\Request\Filter;
class V27 extends Filter
{
// Convert 1.9.5 params to 2.0.0
public function parse(array $content, string $model): array
{
return match ($model) {
'migrations.createCSVImport',
'migrations.createCSVExport',
'migrations.createJSONImport',
'migrations.createJSONExport' => $this->parseMigrationResource($content),
default => $content,
};
}
protected function parseMigrationResource(array $content): array
{
if (!isset($content['resourceId']) || !\is_string($content['resourceId'])) {
return $content;
}
if (!\str_contains($content['resourceId'], ':')) {
return $content;
}
[$databaseId, $collectionId] = \explode(':', $content['resourceId'], 2);
$content['databaseId'] = $content['databaseId'] ?? $databaseId;
$content['collectionId'] = $content['collectionId'] ?? $collectionId;
unset($content['resourceId']);
return $content;
}
}
@@ -0,0 +1,44 @@
<?php
namespace Appwrite\Utopia\Response\Filters;
use Appwrite\Utopia\Response;
use Appwrite\Utopia\Response\Filter;
// Convert 2.0.0 Data format to 1.9.5 format
class V27 extends Filter
{
public function parse(array $content, string $model): array
{
return match ($model) {
Response::MODEL_MIGRATION => $this->parseMigration($content),
Response::MODEL_MIGRATION_LIST => $this->handleList($content, 'migrations', fn ($item) => $this->parseMigration($item)),
default => $content,
};
}
protected function parseMigration(array $content): array
{
$parentResourceId = $content['parentResourceId'] ?? '';
$resourceId = $content['resourceId'] ?? '';
if ($parentResourceId !== '' && $resourceId !== '') {
$content['resourceId'] = $parentResourceId . ':' . $resourceId;
}
$parentResourceType = $content['parentResourceType'] ?? '';
if ($parentResourceType !== '') {
$content['resourceType'] = $parentResourceType;
}
unset($content['resourceInternalId']);
unset($content['parentResourceId']);
unset($content['parentResourceInternalId']);
unset($content['parentResourceType']);
unset($content['destinationResourceId']);
unset($content['destinationResourceInternalId']);
unset($content['destinationResourceType']);
return $content;
}
}
@@ -62,9 +62,65 @@ class Migration extends Model
])
->addRule('resourceId', [
'type' => self::TYPE_STRING,
'description' => 'Id of the resource to migrate.',
'description' => 'ID of the resource being migrated.',
'default' => '',
'example' => 'databaseId:collectionId',
'example' => 'collectionId',
'array' => false
])
->addRule('resourceInternalId', [
'type' => self::TYPE_STRING,
'description' => 'Internal ID of the resource being migrated.',
'default' => '',
'example' => '1',
'array' => false
])
->addRule('resourceType', [
'type' => self::TYPE_STRING,
'description' => 'Type of the resource being migrated.',
'default' => '',
'example' => 'collection',
'array' => false
])
->addRule('parentResourceId', [
'type' => self::TYPE_STRING,
'description' => 'ID of the parent resource that contains the migrated resource.',
'default' => '',
'example' => 'databaseId',
'array' => false
])
->addRule('parentResourceInternalId', [
'type' => self::TYPE_STRING,
'description' => 'Internal ID of the parent resource that contains the migrated resource.',
'default' => '',
'example' => '1',
'array' => false
])
->addRule('parentResourceType', [
'type' => self::TYPE_STRING,
'description' => 'Type of the parent resource that contains the migrated resource.',
'default' => '',
'example' => 'database',
'array' => false
])
->addRule('destinationResourceId', [
'type' => self::TYPE_STRING,
'description' => 'ID of the destination resource created or overwritten by the migration.',
'default' => '',
'example' => 'databaseId',
'array' => false
])
->addRule('destinationResourceInternalId', [
'type' => self::TYPE_STRING,
'description' => 'Internal ID of the destination resource created or overwritten by the migration.',
'default' => '',
'example' => '1',
'array' => false
])
->addRule('destinationResourceType', [
'type' => self::TYPE_STRING,
'description' => 'Type of the destination resource created or overwritten by the migration.',
'default' => '',
'example' => 'database',
'array' => false
])
->addRule('statusCounters', [
+133 -30
View File
@@ -3496,8 +3496,10 @@ trait MigrationsBase
/**
* Import documents from a CSV file.
*
* @return array{databaseId: string, tableId: string, migrationId: string}
*/
public function testCreateCSVImport(): void
public function testCreateCSVImport(): array
{
// Make a database
$response = $this->client->call(Client::METHOD_POST, '/databases', [
@@ -3641,7 +3643,8 @@ trait MigrationsBase
[
'fileId' => $fileIds['missing-column'],
'bucketId' => $bucketIds['missing-column'],
'resourceId' => $databaseId . ':' . $tableId,
'databaseId' => $databaseId,
'collectionId' => $tableId,
]
);
@@ -3673,7 +3676,8 @@ trait MigrationsBase
[
'fileId' => $fileIds['missing-row'],
'bucketId' => $bucketIds['missing-row'],
'resourceId' => $databaseId . ':' . $tableId,
'databaseId' => $databaseId,
'collectionId' => $tableId,
]
);
@@ -3705,7 +3709,8 @@ trait MigrationsBase
[
'fileId' => $fileIds['irrelevant-column'],
'bucketId' => $bucketIds['irrelevant-column'],
'resourceId' => $databaseId . ':' . $tableId,
'databaseId' => $databaseId,
'collectionId' => $tableId,
]
);
@@ -3732,7 +3737,8 @@ trait MigrationsBase
'endpoint' => $this->webEndpoint,
'fileId' => $fileIds['default'],
'bucketId' => $bucketIds['default'],
'resourceId' => $databaseId . ':' . $tableId,
'databaseId' => $databaseId,
'collectionId' => $tableId,
]
);
@@ -3774,7 +3780,8 @@ trait MigrationsBase
'endpoint' => $this->webEndpoint,
'fileId' => $fileIds['documents-internals'],
'bucketId' => $bucketIds['documents-internals'],
'resourceId' => $databaseId . ':' . $tableId,
'databaseId' => $databaseId,
'collectionId' => $tableId,
]
);
@@ -3794,6 +3801,78 @@ trait MigrationsBase
$this->assertArrayHasKey(Resource::TYPE_ROW, $migration['body']['statusCounters']);
$this->assertEquals(25, $migration['body']['statusCounters'][Resource::TYPE_ROW]['success']);
}, 10_000, 500);
return [
'databaseId' => $databaseId,
'tableId' => $tableId,
'migrationId' => $migration['body']['$id'],
];
}
/**
* @param array{databaseId: string, tableId: string, migrationId: string} $data
*/
#[Depends('testCreateCSVImport')]
public function testListMigrationsByDatabaseResource(array $data): void
{
$databaseId = $data['databaseId'];
$query = Query::or([
Query::and([
Query::equal('resourceId', [$databaseId]),
Query::equal('resourceType', [Resource::TYPE_DATABASE]),
]),
Query::and([
Query::equal('parentResourceId', [$databaseId]),
Query::equal('parentResourceType', [Resource::TYPE_DATABASE]),
]),
Query::and([
Query::equal('destinationResourceId', [$databaseId]),
Query::equal('destinationResourceType', [Resource::TYPE_DATABASE]),
]),
]);
$response = $this->client->call(Client::METHOD_GET, '/migrations', array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
], $this->getHeaders()), [
'queries' => [
$query->toString(),
Query::limit(100)->toString(),
],
]);
$this->assertSame(200, $response['headers']['status-code']);
$this->assertContains($data['migrationId'], array_column($response['body']['migrations'], '$id'));
$migration = null;
foreach ($response['body']['migrations'] as $candidate) {
if ($candidate['$id'] === $data['migrationId']) {
$migration = $candidate;
break;
}
}
$this->assertNotNull($migration);
$this->assertSame($data['tableId'], $migration['resourceId']);
$this->assertSame($databaseId, $migration['parentResourceId']);
$this->assertNotSame('', $migration['parentResourceInternalId']);
$this->assertSame(Resource::TYPE_DATABASE, $migration['parentResourceType']);
$this->assertSame($databaseId, $migration['destinationResourceId']);
$this->assertSame($migration['parentResourceInternalId'], $migration['destinationResourceInternalId']);
$this->assertSame(Resource::TYPE_DATABASE, $migration['destinationResourceType']);
$response = $this->client->call(Client::METHOD_GET, '/migrations', array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
], $this->getHeaders()), [
'queries' => [
Query::equal('destinationResourceInternalId', [$migration['destinationResourceInternalId']])->toString(),
Query::equal('destinationResourceType', [Resource::TYPE_DATABASE])->toString(),
],
]);
$this->assertSame(200, $response['headers']['status-code']);
$this->assertContains($data['migrationId'], array_column($response['body']['migrations'], '$id'));
}
/**
@@ -3888,7 +3967,8 @@ trait MigrationsBase
$first = $this->performCsvMigration([
'fileId' => $fileId,
'bucketId' => $bucketId,
'resourceId' => $databaseId . ':' . $tableId,
'databaseId' => $databaseId,
'collectionId' => $tableId,
]);
$this->assertEventually(function () use ($first) {
$migration = $this->client->call(Client::METHOD_GET, '/migrations/' . $first['body']['$id'], array_merge([
@@ -3914,7 +3994,8 @@ trait MigrationsBase
$second = $this->performCsvMigration([
'fileId' => $fileId,
'bucketId' => $bucketId,
'resourceId' => $databaseId . ':' . $tableId,
'databaseId' => $databaseId,
'collectionId' => $tableId,
'onDuplicate' => 'skip',
]);
$this->assertEventually(function () use ($second) {
@@ -3955,7 +4036,8 @@ trait MigrationsBase
$first = $this->performCsvMigration([
'fileId' => $fileId,
'bucketId' => $bucketId,
'resourceId' => $databaseId . ':' . $tableId,
'databaseId' => $databaseId,
'collectionId' => $tableId,
]);
$this->assertEventually(function () use ($first) {
$migration = $this->client->call(Client::METHOD_GET, '/migrations/' . $first['body']['$id'], array_merge([
@@ -3981,7 +4063,8 @@ trait MigrationsBase
$second = $this->performCsvMigration([
'fileId' => $fileId,
'bucketId' => $bucketId,
'resourceId' => $databaseId . ':' . $tableId,
'databaseId' => $databaseId,
'collectionId' => $tableId,
'onDuplicate' => 'overwrite',
]);
$this->assertEventually(function () use ($second) {
@@ -4023,7 +4106,8 @@ trait MigrationsBase
$first = $this->performCsvMigration([
'fileId' => $fileId,
'bucketId' => $bucketId,
'resourceId' => $databaseId . ':' . $tableId,
'databaseId' => $databaseId,
'collectionId' => $tableId,
]);
$this->assertEventually(function () use ($first) {
$migration = $this->client->call(Client::METHOD_GET, '/migrations/' . $first['body']['$id'], array_merge([
@@ -4037,7 +4121,8 @@ trait MigrationsBase
$second = $this->performCsvMigration([
'fileId' => $fileId,
'bucketId' => $bucketId,
'resourceId' => $databaseId . ':' . $tableId,
'databaseId' => $databaseId,
'collectionId' => $tableId,
]);
$this->assertEventually(function () use ($second) {
$migration = $this->client->call(Client::METHOD_GET, '/migrations/' . $second['body']['$id'], array_merge([
@@ -4148,7 +4233,8 @@ trait MigrationsBase
$first = $this->performJsonMigration([
'fileId' => $fileId,
'bucketId' => $bucketId,
'resourceId' => $databaseId . ':' . $tableId,
'databaseId' => $databaseId,
'collectionId' => $tableId,
]);
$this->assertEventually(function () use ($first) {
$migration = $this->client->call(Client::METHOD_GET, '/migrations/' . $first['body']['$id'], array_merge([
@@ -4173,7 +4259,8 @@ trait MigrationsBase
$second = $this->performJsonMigration([
'fileId' => $fileId,
'bucketId' => $bucketId,
'resourceId' => $databaseId . ':' . $tableId,
'databaseId' => $databaseId,
'collectionId' => $tableId,
'onDuplicate' => 'skip',
]);
$this->assertEventually(function () use ($second) {
@@ -4211,7 +4298,8 @@ trait MigrationsBase
$first = $this->performJsonMigration([
'fileId' => $fileId,
'bucketId' => $bucketId,
'resourceId' => $databaseId . ':' . $tableId,
'databaseId' => $databaseId,
'collectionId' => $tableId,
]);
$this->assertEventually(function () use ($first) {
$migration = $this->client->call(Client::METHOD_GET, '/migrations/' . $first['body']['$id'], array_merge([
@@ -4235,7 +4323,8 @@ trait MigrationsBase
$second = $this->performJsonMigration([
'fileId' => $fileId,
'bucketId' => $bucketId,
'resourceId' => $databaseId . ':' . $tableId,
'databaseId' => $databaseId,
'collectionId' => $tableId,
'onDuplicate' => 'overwrite',
]);
$this->assertEventually(function () use ($second) {
@@ -4273,7 +4362,8 @@ trait MigrationsBase
$first = $this->performJsonMigration([
'fileId' => $fileId,
'bucketId' => $bucketId,
'resourceId' => $databaseId . ':' . $tableId,
'databaseId' => $databaseId,
'collectionId' => $tableId,
]);
$this->assertEventually(function () use ($first) {
$migration = $this->client->call(Client::METHOD_GET, '/migrations/' . $first['body']['$id'], array_merge([
@@ -4286,7 +4376,8 @@ trait MigrationsBase
$second = $this->performJsonMigration([
'fileId' => $fileId,
'bucketId' => $bucketId,
'resourceId' => $databaseId . ':' . $tableId,
'databaseId' => $databaseId,
'collectionId' => $tableId,
]);
$this->assertEventually(function () use ($second) {
$migration = $this->client->call(Client::METHOD_GET, '/migrations/' . $second['body']['$id'], array_merge([
@@ -4465,7 +4556,8 @@ trait MigrationsBase
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id']
], $this->getHeaders()), [
'resourceId' => $databaseId . ':' . $collectionId,
'databaseId' => $databaseId,
'collectionId' => $collectionId,
'filename' => 'test-export',
'columns' => [],
'delimiter' => ',',
@@ -5445,7 +5537,8 @@ trait MigrationsBase
$migration = $this->performCsvMigration([
'fileId' => $fileId,
'bucketId' => $bucketId,
'resourceId' => $databaseId . ':' . $collectionId,
'databaseId' => $databaseId,
'collectionId' => $collectionId,
]);
$this->assertEquals(202, $migration['headers']['status-code']);
@@ -5573,7 +5666,8 @@ trait MigrationsBase
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
], $this->getHeaders()), [
'resourceId' => $databaseId . ':' . $collectionId,
'databaseId' => $databaseId,
'collectionId' => $collectionId,
'filename' => $filename,
'columns' => [],
'queries' => [],
@@ -6787,7 +6881,8 @@ trait MigrationsBase
[
'fileId' => $fileIds['missing-column'],
'bucketId' => $bucketIds['missing-column'],
'resourceId' => $databaseId . ':' . $tableId,
'databaseId' => $databaseId,
'collectionId' => $tableId,
]
);
@@ -6824,7 +6919,8 @@ trait MigrationsBase
[
'fileId' => $fileIds['irrelevant-column'],
'bucketId' => $bucketIds['irrelevant-column'],
'resourceId' => $databaseId . ':' . $tableId,
'databaseId' => $databaseId,
'collectionId' => $tableId,
]
);
@@ -6851,7 +6947,8 @@ trait MigrationsBase
'endpoint' => $this->endpoint,
'fileId' => $fileIds['default'],
'bucketId' => $bucketIds['default'],
'resourceId' => $databaseId . ':' . $tableId,
'databaseId' => $databaseId,
'collectionId' => $tableId,
]
);
@@ -6893,7 +6990,8 @@ trait MigrationsBase
'endpoint' => $this->endpoint,
'fileId' => $fileIds['documents-internals'],
'bucketId' => $bucketIds['documents-internals'],
'resourceId' => $databaseId . ':' . $tableId,
'databaseId' => $databaseId,
'collectionId' => $tableId,
]
);
@@ -7016,7 +7114,8 @@ trait MigrationsBase
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id']
], $this->getHeaders()), [
'resourceId' => $databaseId . ':' . $collectionId,
'databaseId' => $databaseId,
'collectionId' => $collectionId,
'filename' => 'test-json-export',
'columns' => [],
'queries' => [],
@@ -7127,7 +7226,8 @@ trait MigrationsBase
// Trigger JSON export
$migration = $this->client->call(Client::METHOD_POST, '/migrations/json/exports', $headers, [
'resourceId' => $databaseId . ':' . $collectionId,
'databaseId' => $databaseId,
'collectionId' => $collectionId,
'filename' => 'vectorsdb-export-test',
'columns' => [],
'queries' => [],
@@ -7200,7 +7300,8 @@ trait MigrationsBase
$migration = $this->performJsonMigration([
'fileId' => $fileId,
'bucketId' => $bucketId,
'resourceId' => $databaseId . ':' . $collectionId,
'databaseId' => $databaseId,
'collectionId' => $collectionId,
]);
$this->assertEquals(202, $migration['headers']['status-code']);
@@ -7271,7 +7372,8 @@ trait MigrationsBase
// Trigger JSON export
$migration = $this->client->call(Client::METHOD_POST, '/migrations/json/exports', $headers, [
'resourceId' => $databaseId . ':' . $collectionId,
'databaseId' => $databaseId,
'collectionId' => $collectionId,
'filename' => 'documentsdb-export-test',
'columns' => [],
'queries' => [],
@@ -7343,7 +7445,8 @@ trait MigrationsBase
$migration = $this->performJsonMigration([
'fileId' => $fileId,
'bucketId' => $bucketId,
'resourceId' => $databaseId . ':' . $collectionId,
'databaseId' => $databaseId,
'collectionId' => $collectionId,
]);
$this->assertEquals(202, $migration['headers']['status-code']);
+148
View File
@@ -0,0 +1,148 @@
<?php
declare(strict_types=1);
namespace Tests\Unit\Migration\Version;
use Appwrite\Migration\Version\V25;
use PHPUnit\Framework\TestCase;
use Utopia\Database\Document;
use Utopia\Migration\Resource;
final class V25Test extends TestCase
{
public function testPreservesLegacyResourceWhenInternalIdsCannotBeResolved(): void
{
$migration = new class () extends V25 {
public function __construct()
{
}
public function migrate(Document $document): Document
{
return $this->migrateDocument($document);
}
#[\Override]
protected function resolveInternalIds(string $parentResourceId, string $resourceId, Document $migration): array
{
return [];
}
};
$document = new Document([
'$id' => 'migration',
'$collection' => 'migrations',
'resourceId' => 'database:collection',
'resourceInternalId' => null,
'resourceType' => Resource::TYPE_DATABASE,
'parentResourceInternalId' => null,
]);
$result = $migration->migrate($document);
$this->assertSame('database:collection', $result->getAttribute('resourceId'));
$this->assertSame(Resource::TYPE_DATABASE, $result->getAttribute('resourceType'));
$this->assertNull($result->getAttribute('parentResourceId'));
$this->assertNull($result->getAttribute('parentResourceType'));
$this->assertNull($result->getAttribute('resourceInternalId'));
$this->assertNull($result->getAttribute('parentResourceInternalId'));
}
public function testRerunResolvesMissingInternalIdsIdempotently(): void
{
$migration = new class () extends V25 {
/**
* @var array{parentResourceInternalId?: string, resourceInternalId?: string}
*/
private array $internalIds = [];
public function __construct()
{
}
public function migrate(Document $document): Document
{
return $this->migrateDocument($document);
}
/**
* @param array{parentResourceInternalId?: string, resourceInternalId?: string} $internalIds
*/
public function setInternalIds(array $internalIds): void
{
$this->internalIds = $internalIds;
}
/**
* @return array{parentResourceInternalId?: string, resourceInternalId?: string}
*/
#[\Override]
protected function resolveInternalIds(string $parentResourceId, string $resourceId, Document $migration): array
{
return $this->internalIds;
}
};
$document = new Document([
'$id' => 'migration',
'$collection' => 'migrations',
'resourceId' => 'database:collection',
'resourceInternalId' => null,
'resourceType' => Resource::TYPE_DATABASE,
'parentResourceInternalId' => null,
]);
$migration->migrate($document);
$this->assertSame('database:collection', $document->getAttribute('resourceId'));
$this->assertSame(Resource::TYPE_DATABASE, $document->getAttribute('resourceType'));
$this->assertNull($document->getAttribute('parentResourceId'));
$migration->setInternalIds([
'parentResourceInternalId' => '10',
]);
$migration->migrate($document);
$this->assertSame('database:collection', $document->getAttribute('resourceId'));
$this->assertSame(Resource::TYPE_DATABASE, $document->getAttribute('resourceType'));
$this->assertNull($document->getAttribute('parentResourceId'));
$migration->setInternalIds([
'parentResourceInternalId' => '10',
'resourceInternalId' => '20',
]);
$migration->migrate($document);
$afterResolution = $document->getArrayCopy();
$migration->migrate($document);
$this->assertSame('10', $document->getAttribute('parentResourceInternalId'));
$this->assertSame('20', $document->getAttribute('resourceInternalId'));
$this->assertSame('collection', $document->getAttribute('resourceId'));
$this->assertSame(Resource::TYPE_COLLECTION, $document->getAttribute('resourceType'));
$this->assertSame('database', $document->getAttribute('parentResourceId'));
$this->assertSame(Resource::TYPE_DATABASE, $document->getAttribute('parentResourceType'));
$this->assertSame($afterResolution, $document->getArrayCopy());
}
public function testRejectsResourcesCreatedAfterMigration(): void
{
$migration = new class () extends V25 {
public function __construct()
{
}
public function isCandidate(Document $resource, Document $migration): bool
{
return $this->predatesMigration($resource, $migration);
}
};
$document = new Document([
'$createdAt' => '2026-01-02T00:00:00.000+00:00',
]);
$this->assertTrue($migration->isCandidate(new Document([
'$createdAt' => '2026-01-01T00:00:00.000+00:00',
]), $document));
$this->assertFalse($migration->isCandidate(new Document([
'$createdAt' => '2026-01-03T00:00:00.000+00:00',
]), $document));
$this->assertFalse($migration->isCandidate(new Document(), $document));
}
}
@@ -0,0 +1,326 @@
<?php
declare(strict_types=1);
namespace Tests\Unit\Platform\Workers;
use Appwrite\Event\Publisher\Mail as MailPublisher;
use Appwrite\Event\Publisher\Usage as UsagePublisher;
use Appwrite\Event\Realtime;
use Appwrite\Platform\Workers\Migrations;
use Appwrite\Usage\Context;
use PHPUnit\Framework\MockObject\MockObject;
use PHPUnit\Framework\TestCase;
use Utopia\Database\Document;
use Utopia\Database\Validator\Authorization;
use Utopia\Migration\Destination;
use Utopia\Migration\Resource;
use Utopia\Migration\Source;
use Utopia\Queue\Publisher;
use Utopia\Queue\Queue;
final class MigrationsTest extends TestCase
{
public function testSuccessHooksRunBeforeCompletedMigrationIsPersisted(): void
{
$events = [];
$source = $this->createSourceMock();
$destination = $this->createDestinationMock();
$destination
->expects($this->once())
->method('success')
->willReturnCallback(static function () use (&$events): void {
$events[] = 'destination:success';
});
$source
->expects($this->once())
->method('success')
->willReturnCallback(static function () use (&$events): void {
$events[] = 'source:success';
});
$source->expects($this->never())->method('error');
$destination->expects($this->never())->method('error');
$migration = $this->createMigration();
$processor = $this->createProcessor($source, $destination, $events);
$this->process($processor, $migration);
$this->assertSame('completed', $migration->getAttribute('status'));
$this->assertSame('finished', $migration->getAttribute('stage'));
$this->assertSame([
'persist:processing:processing',
'persist:processing:migrating',
'destination:success',
'source:success',
'persist:completed:finished',
], $events);
}
public function testThrowingSourceSuccessHookPersistsFailedMigrationWithoutRerunningHooks(): void
{
$events = [];
$source = $this->createSourceMock();
$destination = $this->createDestinationMock();
$destination
->expects($this->once())
->method('success')
->willReturnCallback(static function () use (&$events): void {
$events[] = 'destination:success';
});
$source
->expects($this->once())
->method('success')
->willReturnCallback(static function () use (&$events): void {
$events[] = 'source:success';
throw new \RuntimeException('Finalization failed');
});
$source
->expects($this->once())
->method('error')
->willReturnCallback(static function () use (&$events): void {
$events[] = 'source:error';
});
$destination
->expects($this->once())
->method('error')
->willReturnCallback(static function () use (&$events): void {
$events[] = 'destination:error';
});
$migration = $this->createMigration();
$processor = $this->createProcessor($source, $destination, $events);
$this->process($processor, $migration);
$this->assertSame('failed', $migration->getAttribute('status'));
$this->assertSame('finished', $migration->getAttribute('stage'));
$this->assertSame([
'persist:processing:processing',
'persist:processing:migrating',
'destination:success',
'source:success',
'persist:failed:finished',
'source:error',
'destination:error',
], $events);
$this->assertNotContains('persist:completed:finished', $events);
}
public function testNullResourceTypeUsesEmptyResourceSelector(): void
{
$events = [];
$source = $this->createSourceMock();
$destination = $this->createDestinationMock();
$destination
->expects($this->once())
->method('success')
->willReturnCallback(static function () use (&$events): void {
$events[] = 'destination:success';
});
$source
->expects($this->once())
->method('success')
->willReturnCallback(static function () use (&$events): void {
$events[] = 'source:success';
});
$source->expects($this->never())->method('error');
$destination->expects($this->never())->method('error');
$migration = $this->createMigration(null);
$processor = $this->createProcessor($source, $destination, $events);
$this->process($processor, $migration);
$this->assertSame('completed', $migration->getAttribute('status'));
$this->assertSame([
'persist:processing:processing',
'persist:processing:migrating',
'destination:success',
'source:success',
'persist:completed:finished',
], $events);
}
public function testResourceContextUsesCanonicalRelationFields(): void
{
$worker = new class () extends Migrations {
/**
* @return array{resourceId: string, resourceInternalId: string, resourceType: string, parentResourceId: string, parentResourceInternalId: string, parentResourceType: string}
*/
public function context(Document $migration): array
{
return $this->resolveResourceContext($migration);
}
};
$this->assertSame([
'resourceId' => 'table-a',
'resourceInternalId' => '201',
'resourceType' => Resource::TYPE_COLLECTION,
'parentResourceId' => 'database-a',
'parentResourceInternalId' => '101',
'parentResourceType' => Resource::TYPE_DATABASE,
], $worker->context(new Document([
'resourceId' => 'table-a',
'resourceInternalId' => '201',
'resourceType' => Resource::TYPE_COLLECTION,
'parentResourceId' => 'database-a',
'parentResourceInternalId' => '101',
'parentResourceType' => Resource::TYPE_DATABASE,
])));
$this->assertSame([
'resourceId' => 'table-a',
'resourceInternalId' => '',
'resourceType' => Resource::TYPE_COLLECTION,
'parentResourceId' => 'database-a',
'parentResourceInternalId' => '',
'parentResourceType' => Resource::TYPE_DATABASE,
], $worker->context(new Document([
'resourceId' => 'database-a:table-a',
'resourceType' => Resource::TYPE_DATABASE,
])));
}
private function createSourceMock(): Source&MockObject
{
$target = $this->createMock(Source::class);
$target->method('getErrors')->willReturn([]);
$target->expects($this->once())->method('shutdown');
$target->expects($this->once())->method('cleanUp');
return $target;
}
private function createDestinationMock(): Destination&MockObject
{
$target = $this->createMock(Destination::class);
$target->method('getErrors')->willReturn([]);
$target->expects($this->once())->method('shutdown');
$target->expects($this->once())->method('cleanUp');
return $target;
}
private function createMigration(?string $resourceType = ''): Document
{
return new Document([
'$id' => 'migration',
'$sequence' => 1,
'credentials' => [],
'destination' => 'TestDestination',
'options' => [],
'resourceId' => '',
'resourceType' => $resourceType,
'resources' => [],
'source' => 'TestSource',
'stage' => 'pending',
'status' => 'pending',
]);
}
/**
* @param array<string> $events
*/
private function createProcessor(Source $source, Destination $destination, array &$events): \Closure
{
$record = static function (string $event) use (&$events): void {
$events[] = $event;
};
$worker = new class ($source, $destination, $record) extends Migrations {
public function __construct(
private readonly Source $migrationSource,
private readonly Destination $migrationDestination,
private readonly \Closure $record,
) {
}
public function process(
Document $migration,
Document $project,
Realtime $queueForRealtime,
MailPublisher $publisherForMails,
Context $usage,
UsagePublisher $publisherForUsage,
Authorization $authorization,
): void {
$this->project = $project;
$this->logError = static function (): void {
};
$this->processMigration(
$migration,
$queueForRealtime,
$publisherForMails,
$usage,
$publisherForUsage,
[],
$authorization,
);
}
#[\Override]
protected function generateAPIKey(Document $project): string
{
return 'key';
}
#[\Override]
protected function processSource(Document $migration): Source
{
return $this->migrationSource;
}
#[\Override]
protected function processDestination(Document $migration): Destination
{
return $this->migrationDestination;
}
#[\Override]
protected function updateMigrationDocument(
Document $migration,
Document $project,
Realtime $queueForRealtime,
): Document {
($this->record)('persist:'
. $migration->getAttribute('status')
. ':'
. $migration->getAttribute('stage'));
return $migration;
}
};
return $worker->process(...);
}
private function process(\Closure $processor, Document $migration): void
{
$publisher = $this->createStub(Publisher::class);
$queue = new Queue('test');
$host = \getenv('_APP_MIGRATION_HOST');
\putenv('_APP_MIGRATION_HOST=localhost');
try {
$processor(
$migration,
new Document([
'$id' => 'project',
'$sequence' => 1,
]),
new Realtime(),
new MailPublisher($publisher, $queue),
new Context(),
new UsagePublisher($publisher, $queue),
new Authorization(),
);
} finally {
\putenv($host === false ? '_APP_MIGRATION_HOST' : '_APP_MIGRATION_HOST=' . $host);
}
}
}
@@ -0,0 +1,33 @@
<?php
declare(strict_types=1);
namespace Tests\Unit\Utopia\Database\Validator\Queries;
use Appwrite\Utopia\Database\Validator\Queries\Migrations;
use PHPUnit\Framework\TestCase;
use Utopia\Database\Query;
final class MigrationsTest extends TestCase
{
public function testCanonicalResourceFieldsCanBeQueried(): void
{
$validator = new Migrations();
foreach ([
'resourceId',
'resourceInternalId',
'resourceType',
'parentResourceId',
'parentResourceInternalId',
'parentResourceType',
'destinationResourceId',
'destinationResourceInternalId',
'destinationResourceType',
] as $attribute) {
$query = Query::equal($attribute, ['value']);
$this->assertTrue($validator->isValid([$query]), $validator->getDescription());
}
}
}
@@ -0,0 +1,49 @@
<?php
declare(strict_types=1);
namespace Tests\Unit\Utopia\Request\Filters;
use Appwrite\Utopia\Request\Filters\V27;
use PHPUnit\Framework\TestCase;
final class V27Test extends TestCase
{
public function testSplitsLegacyMigrationResource(): void
{
$filter = new V27();
$result = $filter->parse([
'resourceId' => 'database:collection',
], 'migrations.createCSVImport');
$this->assertSame([
'databaseId' => 'database',
'collectionId' => 'collection',
], $result);
}
public function testExplicitResourceIdsTakePrecedence(): void
{
$filter = new V27();
$result = $filter->parse([
'resourceId' => 'legacyDatabase:legacyCollection',
'databaseId' => 'database',
'collectionId' => 'collection',
], 'migrations.createJSONExport');
$this->assertSame([
'databaseId' => 'database',
'collectionId' => 'collection',
], $result);
}
public function testLeavesMalformedResourceForValidation(): void
{
$filter = new V27();
$content = ['resourceId' => 'database'];
$this->assertSame($content, $filter->parse($content, 'migrations.createCSVExport'));
}
}
@@ -0,0 +1,88 @@
<?php
declare(strict_types=1);
namespace Tests\Unit\Utopia\Response\Filters;
use Appwrite\Utopia\Response;
use Appwrite\Utopia\Response\Filters\V27;
use PHPUnit\Framework\TestCase;
final class V27Test extends TestCase
{
public function testRestoresLegacyMigrationResourceShape(): void
{
$filter = new V27();
$result = $filter->parse([
'$id' => 'migration',
'resourceId' => 'collection',
'resourceInternalId' => '20',
'resourceType' => 'collection',
'parentResourceId' => 'database',
'parentResourceInternalId' => '10',
'parentResourceType' => 'database',
'destinationResourceId' => 'destination',
'destinationResourceInternalId' => '30',
'destinationResourceType' => 'database',
], Response::MODEL_MIGRATION);
$this->assertSame([
'$id' => 'migration',
'resourceId' => 'database:collection',
'resourceType' => 'database',
], $result);
}
public function testPreservesSingleRootMigration(): void
{
$filter = new V27();
$result = $filter->parse([
'$id' => 'migration',
'resourceId' => 'database',
'resourceInternalId' => '10',
'resourceType' => 'database',
'parentResourceId' => '',
'parentResourceInternalId' => '',
'parentResourceType' => '',
'destinationResourceId' => '',
'destinationResourceInternalId' => '',
'destinationResourceType' => '',
], Response::MODEL_MIGRATION);
$this->assertSame([
'$id' => 'migration',
'resourceId' => 'database',
'resourceType' => 'database',
], $result);
}
public function testTransformsMigrationList(): void
{
$filter = new V27();
$result = $filter->parse([
'total' => 1,
'migrations' => [[
'$id' => 'migration',
'resourceId' => 'collection',
'resourceType' => 'collection',
'parentResourceId' => 'database',
'parentResourceType' => 'database',
'destinationResourceId' => 'destination',
'destinationResourceInternalId' => '30',
'destinationResourceType' => 'database',
]],
], Response::MODEL_MIGRATION_LIST);
$this->assertSame([
'total' => 1,
'migrations' => [[
'$id' => 'migration',
'resourceId' => 'database:collection',
'resourceType' => 'database',
]],
], $result);
}
}