mirror of
https://github.com/appwrite/appwrite.git
synced 2026-11-18 20:36:03 +00:00
Merge pull request #12193 from appwrite/feat/migration-resource-attrs
feat(migrations): expose queryable resource links
This commit is contained in:
20 files changed
+1403
-121
No files matched your search
@@ -2267,6 +2267,17 @@ return [
|
||||
'array' => false,
|
||||
'filters' => [],
|
||||
],
|
||||
[
|
||||
'$id' => ID::custom('resourceInternalId'),
|
||||
'type' => Database::VAR_STRING,
|
||||
'format' => '',
|
||||
'size' => Database::LENGTH_KEY,
|
||||
'signed' => true,
|
||||
'required' => false,
|
||||
'default' => null,
|
||||
'array' => false,
|
||||
'filters' => [],
|
||||
],
|
||||
[
|
||||
'$id' => ID::custom('resourceType'),
|
||||
'type' => Database::VAR_STRING,
|
||||
@@ -2278,6 +2289,72 @@ return [
|
||||
'array' => false,
|
||||
'filters' => [],
|
||||
],
|
||||
[
|
||||
'$id' => ID::custom('parentResourceId'),
|
||||
'type' => Database::VAR_STRING,
|
||||
'format' => '',
|
||||
'size' => Database::LENGTH_KEY,
|
||||
'signed' => true,
|
||||
'required' => false,
|
||||
'default' => null,
|
||||
'array' => false,
|
||||
'filters' => [],
|
||||
],
|
||||
[
|
||||
'$id' => ID::custom('parentResourceInternalId'),
|
||||
'type' => Database::VAR_STRING,
|
||||
'format' => '',
|
||||
'size' => Database::LENGTH_KEY,
|
||||
'signed' => true,
|
||||
'required' => false,
|
||||
'default' => null,
|
||||
'array' => false,
|
||||
'filters' => [],
|
||||
],
|
||||
[
|
||||
'$id' => ID::custom('parentResourceType'),
|
||||
'type' => Database::VAR_STRING,
|
||||
'format' => '',
|
||||
'size' => Database::LENGTH_KEY,
|
||||
'signed' => true,
|
||||
'required' => false,
|
||||
'default' => null,
|
||||
'array' => false,
|
||||
'filters' => [],
|
||||
],
|
||||
[
|
||||
'$id' => ID::custom('destinationResourceId'),
|
||||
'type' => Database::VAR_STRING,
|
||||
'format' => '',
|
||||
'size' => Database::LENGTH_KEY,
|
||||
'signed' => true,
|
||||
'required' => false,
|
||||
'default' => null,
|
||||
'array' => false,
|
||||
'filters' => [],
|
||||
],
|
||||
[
|
||||
'$id' => ID::custom('destinationResourceInternalId'),
|
||||
'type' => Database::VAR_STRING,
|
||||
'format' => '',
|
||||
'size' => Database::LENGTH_KEY,
|
||||
'signed' => true,
|
||||
'required' => false,
|
||||
'default' => null,
|
||||
'array' => false,
|
||||
'filters' => [],
|
||||
],
|
||||
[
|
||||
'$id' => ID::custom('destinationResourceType'),
|
||||
'type' => Database::VAR_STRING,
|
||||
'format' => '',
|
||||
'size' => Database::LENGTH_KEY,
|
||||
'signed' => true,
|
||||
'required' => false,
|
||||
'default' => null,
|
||||
'array' => false,
|
||||
'filters' => [],
|
||||
],
|
||||
],
|
||||
'indexes' => [
|
||||
[
|
||||
@@ -2308,6 +2385,62 @@ return [
|
||||
'lengths' => [Database::LENGTH_KEY],
|
||||
'orders' => [Database::ORDER_DESC],
|
||||
],
|
||||
[
|
||||
'$id' => '_key_resourceType',
|
||||
'type' => Database::INDEX_KEY,
|
||||
'attributes' => ['resourceType'],
|
||||
'lengths' => [Database::LENGTH_KEY],
|
||||
'orders' => [Database::ORDER_ASC],
|
||||
],
|
||||
[
|
||||
'$id' => '_key_resourceInternalId',
|
||||
'type' => Database::INDEX_KEY,
|
||||
'attributes' => ['resourceInternalId'],
|
||||
'lengths' => [Database::LENGTH_KEY],
|
||||
'orders' => [Database::ORDER_ASC],
|
||||
],
|
||||
[
|
||||
'$id' => '_key_parentResourceId',
|
||||
'type' => Database::INDEX_KEY,
|
||||
'attributes' => ['parentResourceId'],
|
||||
'lengths' => [Database::LENGTH_KEY],
|
||||
'orders' => [Database::ORDER_ASC],
|
||||
],
|
||||
[
|
||||
'$id' => '_key_parentResourceType',
|
||||
'type' => Database::INDEX_KEY,
|
||||
'attributes' => ['parentResourceType'],
|
||||
'lengths' => [Database::LENGTH_KEY],
|
||||
'orders' => [Database::ORDER_ASC],
|
||||
],
|
||||
[
|
||||
'$id' => '_key_parentResourceInternalId',
|
||||
'type' => Database::INDEX_KEY,
|
||||
'attributes' => ['parentResourceInternalId'],
|
||||
'lengths' => [Database::LENGTH_KEY],
|
||||
'orders' => [Database::ORDER_ASC],
|
||||
],
|
||||
[
|
||||
'$id' => '_key_destinationResourceId',
|
||||
'type' => Database::INDEX_KEY,
|
||||
'attributes' => ['destinationResourceId'],
|
||||
'lengths' => [Database::LENGTH_KEY],
|
||||
'orders' => [Database::ORDER_ASC],
|
||||
],
|
||||
[
|
||||
'$id' => '_key_destinationResourceInternalId',
|
||||
'type' => Database::INDEX_KEY,
|
||||
'attributes' => ['destinationResourceInternalId'],
|
||||
'lengths' => [Database::LENGTH_KEY],
|
||||
'orders' => [Database::ORDER_ASC],
|
||||
],
|
||||
[
|
||||
'$id' => '_key_destinationResourceType',
|
||||
'type' => Database::INDEX_KEY,
|
||||
'attributes' => ['destinationResourceType'],
|
||||
'lengths' => [Database::LENGTH_KEY],
|
||||
'orders' => [Database::ORDER_ASC],
|
||||
],
|
||||
[
|
||||
'$id' => ID::custom('_fulltext_search'),
|
||||
'type' => Database::INDEX_FULLTEXT,
|
||||
|
||||
@@ -34,6 +34,7 @@ use Appwrite\Utopia\Request\Filters\V23 as RequestV23;
|
||||
use Appwrite\Utopia\Request\Filters\V24 as RequestV24;
|
||||
use Appwrite\Utopia\Request\Filters\V25 as RequestV25;
|
||||
use Appwrite\Utopia\Request\Filters\V26 as RequestV26;
|
||||
use Appwrite\Utopia\Request\Filters\V27 as RequestV27;
|
||||
use Appwrite\Utopia\Response;
|
||||
use Appwrite\Utopia\Response\Filters\V16 as ResponseV16;
|
||||
use Appwrite\Utopia\Response\Filters\V17 as ResponseV17;
|
||||
@@ -46,6 +47,7 @@ use Appwrite\Utopia\Response\Filters\V23 as ResponseV23;
|
||||
use Appwrite\Utopia\Response\Filters\V24 as ResponseV24;
|
||||
use Appwrite\Utopia\Response\Filters\V25 as ResponseV25;
|
||||
use Appwrite\Utopia\Response\Filters\V26 as ResponseV26;
|
||||
use Appwrite\Utopia\Response\Filters\V27 as ResponseV27;
|
||||
use Appwrite\Utopia\View;
|
||||
use Executor\Exception\Timeout as ExecutorTimeout;
|
||||
use Executor\Executor;
|
||||
@@ -917,6 +919,9 @@ Http::init()
|
||||
if (version_compare($requestFormat, '1.9.5', '<')) {
|
||||
$request->addFilter(new RequestV26());
|
||||
}
|
||||
if (version_compare($requestFormat, '2.0.0', '<')) {
|
||||
$request->addFilter(new RequestV27());
|
||||
}
|
||||
}
|
||||
|
||||
$localeParam = (string) $request->getParam('locale', $request->getHeaderLine('x-appwrite-locale', ''));
|
||||
@@ -941,6 +946,9 @@ Http::init()
|
||||
*/
|
||||
$responseFormat = $request->getHeaderLine('x-appwrite-response-format', System::getEnv('_APP_SYSTEM_RESPONSE_FORMAT', ''));
|
||||
if ($responseFormat) {
|
||||
if (version_compare($responseFormat, '2.0.0', '<')) {
|
||||
$response->addFilter(new ResponseV27());
|
||||
}
|
||||
if (version_compare($responseFormat, '1.9.5', '<')) {
|
||||
$response->addFilter(new ResponseV26());
|
||||
}
|
||||
|
||||
+1
-1
@@ -83,7 +83,7 @@
|
||||
"utopia-php/lock": "0.2.*",
|
||||
"utopia-php/logger": "0.8.*",
|
||||
"utopia-php/messaging": "^2.0",
|
||||
"utopia-php/migration": "1.17.*",
|
||||
"utopia-php/migration": "^1.18.0",
|
||||
"utopia-php/platform": "^1.0@RC",
|
||||
"utopia-php/pools": "1.*",
|
||||
"utopia-php/span": "3.0.*",
|
||||
|
||||
Generated
+7
-7
@@ -4,7 +4,7 @@
|
||||
"Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies",
|
||||
"This file is @generated automatically"
|
||||
],
|
||||
"content-hash": "185f7ef75a7b75691ca976b28c808281",
|
||||
"content-hash": "f007708a2c4301c72ae22feadcad6c8f",
|
||||
"packages": [
|
||||
{
|
||||
"name": "adhocore/jwt",
|
||||
@@ -4432,16 +4432,16 @@
|
||||
},
|
||||
{
|
||||
"name": "utopia-php/migration",
|
||||
"version": "1.17.3",
|
||||
"version": "1.18.0",
|
||||
"source": {
|
||||
"type": "git",
|
||||
"url": "https://github.com/utopia-php/migration.git",
|
||||
"reference": "daf1008800633416a186c18069e3a5784de88d1c"
|
||||
"reference": "b531ea43293fde5ceb72c29d312489aa0ded0c83"
|
||||
},
|
||||
"dist": {
|
||||
"type": "zip",
|
||||
"url": "https://api.github.com/repos/utopia-php/migration/zipball/daf1008800633416a186c18069e3a5784de88d1c",
|
||||
"reference": "daf1008800633416a186c18069e3a5784de88d1c",
|
||||
"url": "https://api.github.com/repos/utopia-php/migration/zipball/b531ea43293fde5ceb72c29d312489aa0ded0c83",
|
||||
"reference": "b531ea43293fde5ceb72c29d312489aa0ded0c83",
|
||||
"shasum": ""
|
||||
},
|
||||
"require": {
|
||||
@@ -4481,9 +4481,9 @@
|
||||
],
|
||||
"support": {
|
||||
"issues": "https://github.com/utopia-php/migration/issues",
|
||||
"source": "https://github.com/utopia-php/migration/tree/1.17.3"
|
||||
"source": "https://github.com/utopia-php/migration/tree/1.18.0"
|
||||
},
|
||||
"time": "2026-07-14T14:04:00+00:00"
|
||||
"time": "2026-07-20T10:00:12+00:00"
|
||||
},
|
||||
{
|
||||
"name": "utopia-php/mongo",
|
||||
|
||||
@@ -9,6 +9,7 @@ use Utopia\Console;
|
||||
use Utopia\Database\Database;
|
||||
use Utopia\Database\Document;
|
||||
use Utopia\Database\Query;
|
||||
use Utopia\Migration\Resource;
|
||||
|
||||
class V25 extends Migration
|
||||
{
|
||||
@@ -19,6 +20,9 @@ class V25 extends Migration
|
||||
{
|
||||
Console::info('Migrating collections');
|
||||
$this->migrateCollections();
|
||||
|
||||
Console::info('Migrating documents');
|
||||
$this->forEachDocument($this->migrateDocument(...));
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -104,7 +108,165 @@ class V25 extends Migration
|
||||
}
|
||||
}
|
||||
break;
|
||||
|
||||
case 'migrations':
|
||||
if ($collectionType === 'projects') {
|
||||
$attributes = [
|
||||
'resourceInternalId',
|
||||
'parentResourceId',
|
||||
'parentResourceInternalId',
|
||||
'parentResourceType',
|
||||
'destinationResourceId',
|
||||
'destinationResourceInternalId',
|
||||
'destinationResourceType',
|
||||
];
|
||||
try {
|
||||
$this->createAttributesFromCollection($this->dbForProject, $id, $attributes);
|
||||
} catch (Throwable $th) {
|
||||
Console::warning('Failed to create attributes "' . \implode(', ', $attributes) . "\" in collection {$id}: {$th->getMessage()}");
|
||||
}
|
||||
|
||||
$indexes = [
|
||||
'_key_resourceType',
|
||||
'_key_resourceInternalId',
|
||||
'_key_parentResourceId',
|
||||
'_key_parentResourceType',
|
||||
'_key_parentResourceInternalId',
|
||||
'_key_destinationResourceId',
|
||||
'_key_destinationResourceInternalId',
|
||||
'_key_destinationResourceType',
|
||||
];
|
||||
foreach ($indexes as $index) {
|
||||
try {
|
||||
$this->createIndexFromCollection($this->dbForProject, $id, $index);
|
||||
} catch (Throwable $th) {
|
||||
Console::warning("Failed to create index \"{$index}\" from {$id}: {$th->getMessage()}");
|
||||
}
|
||||
}
|
||||
|
||||
$this->dbForProject->purgeCachedCollection($id);
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
protected function migrateDocument(Document $document): Document
|
||||
{
|
||||
if ($document->getCollection() !== 'migrations') {
|
||||
return $document;
|
||||
}
|
||||
|
||||
$resourceId = (string) $document->getAttribute('resourceId', '');
|
||||
$parentResourceId = (string) $document->getAttribute('parentResourceId', '');
|
||||
$parentResourceType = '';
|
||||
$split = false;
|
||||
|
||||
if ($parentResourceId === '') {
|
||||
if (!\str_contains($resourceId, ':')) {
|
||||
return $document;
|
||||
}
|
||||
|
||||
[$parentResourceId, $resourceId] = \explode(':', $resourceId, 2);
|
||||
if ($parentResourceId === '' || $resourceId === '') {
|
||||
return $document;
|
||||
}
|
||||
|
||||
$parentResourceType = (string) $document->getAttribute('resourceType', '');
|
||||
$split = true;
|
||||
}
|
||||
|
||||
if ($resourceId === '') {
|
||||
return $document;
|
||||
}
|
||||
|
||||
$internalIds = $this->resolveInternalIds($parentResourceId, $resourceId, $document);
|
||||
if (
|
||||
$split
|
||||
&& (!isset($internalIds['parentResourceInternalId']) || !isset($internalIds['resourceInternalId']))
|
||||
) {
|
||||
return $document;
|
||||
}
|
||||
|
||||
if ($split) {
|
||||
$document
|
||||
->setAttribute('resourceId', $resourceId)
|
||||
->setAttribute('resourceType', Resource::TYPE_COLLECTION)
|
||||
->setAttribute('parentResourceId', $parentResourceId);
|
||||
|
||||
if ($parentResourceType !== '') {
|
||||
$document->setAttribute('parentResourceType', $parentResourceType);
|
||||
}
|
||||
}
|
||||
|
||||
if (
|
||||
(string) $document->getAttribute('parentResourceInternalId', '') === ''
|
||||
&& isset($internalIds['parentResourceInternalId'])
|
||||
) {
|
||||
$document->setAttribute('parentResourceInternalId', $internalIds['parentResourceInternalId']);
|
||||
}
|
||||
if (
|
||||
(string) $document->getAttribute('resourceInternalId', '') === ''
|
||||
&& isset($internalIds['resourceInternalId'])
|
||||
) {
|
||||
$document->setAttribute('resourceInternalId', $internalIds['resourceInternalId']);
|
||||
}
|
||||
|
||||
return $document;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return array{parentResourceInternalId?: string, resourceInternalId?: string}
|
||||
*/
|
||||
protected function resolveInternalIds(string $parentResourceId, string $resourceId, Document $migration): array
|
||||
{
|
||||
try {
|
||||
$database = $this->dbForProject->getDocument('databases', $parentResourceId);
|
||||
if (
|
||||
$database->isEmpty()
|
||||
|| $database->getSequence() === ''
|
||||
|| !$this->predatesMigration($database, $migration)
|
||||
) {
|
||||
return [];
|
||||
}
|
||||
} catch (Throwable $th) {
|
||||
Console::warning("Failed to resolve parent internal ID for migration {$migration->getId()}: {$th->getMessage()}");
|
||||
return [];
|
||||
}
|
||||
|
||||
$internalIds = [
|
||||
'parentResourceInternalId' => (string) $database->getSequence(),
|
||||
];
|
||||
|
||||
try {
|
||||
$resource = $this->dbForProject->getDocument('database_' . $database->getSequence(), $resourceId);
|
||||
if (
|
||||
!$resource->isEmpty()
|
||||
&& $resource->getSequence() !== ''
|
||||
&& $this->predatesMigration($resource, $migration)
|
||||
) {
|
||||
$internalIds['resourceInternalId'] = (string) $resource->getSequence();
|
||||
}
|
||||
} catch (Throwable $th) {
|
||||
Console::warning("Failed to resolve resource internal ID for migration {$migration->getId()}: {$th->getMessage()}");
|
||||
}
|
||||
|
||||
return $internalIds;
|
||||
}
|
||||
|
||||
/**
|
||||
* A resource created after the migration is a reused public ID, not the
|
||||
* generation the historical migration operated on.
|
||||
*/
|
||||
protected function predatesMigration(Document $resource, Document $migration): bool
|
||||
{
|
||||
$resourceCreatedAt = \strtotime($resource->getCreatedAt());
|
||||
$migrationCreatedAt = \strtotime($migration->getCreatedAt());
|
||||
|
||||
if ($resourceCreatedAt === false || $migrationCreatedAt === false) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return $resourceCreatedAt <= $migrationCreatedAt;
|
||||
}
|
||||
}
|
||||
@@ -9,7 +9,6 @@ use Appwrite\Extend\Exception;
|
||||
use Appwrite\SDK\AuthType;
|
||||
use Appwrite\SDK\Method;
|
||||
use Appwrite\SDK\Response as SDKResponse;
|
||||
use Appwrite\Utopia\Database\Validator\CompoundUID;
|
||||
use Appwrite\Utopia\Response;
|
||||
use Utopia\Database\Database;
|
||||
use Utopia\Database\Document;
|
||||
@@ -18,6 +17,7 @@ use Utopia\Database\Helpers\ID;
|
||||
use Utopia\Database\Query;
|
||||
use Utopia\Database\Validator\Authorization;
|
||||
use Utopia\Database\Validator\Queries\Documents;
|
||||
use Utopia\Database\Validator\UID;
|
||||
use Utopia\Migration\Resource;
|
||||
use Utopia\Migration\Sources\Appwrite as AppwriteSource;
|
||||
use Utopia\Migration\Sources\CSV;
|
||||
@@ -60,7 +60,8 @@ class Create extends Action
|
||||
)
|
||||
]
|
||||
))
|
||||
->param('resourceId', null, new CompoundUID(), 'Composite ID in the format {databaseId:collectionId}, identifying a collection within a database to export.')
|
||||
->param('databaseId', '', new UID(), 'Database ID containing the source collection.')
|
||||
->param('collectionId', '', new UID(), 'Collection ID to export documents from.')
|
||||
->param('filename', '', new Text(255), 'The name of the file to be created for the export, excluding the .csv extension.')
|
||||
->param('columns', [], new ArrayList(new Text(Database::LENGTH_KEY)), 'List of attributes to export. If empty, all attributes will be exported. You can use the `*` wildcard to export all attributes from the collection.', true)
|
||||
->param('queries', [], new ArrayList(new Text(0)), 'Array of query strings generated using the Query class provided by the SDK to filter documents to export. [Learn more about queries](https://appwrite.io/docs/databases#querying-documents). Maximum of ' . APP_LIMIT_ARRAY_PARAMS_SIZE . ' queries are allowed, each ' . APP_LIMIT_ARRAY_ELEMENT_SIZE . ' characters long.', true)
|
||||
@@ -82,7 +83,8 @@ class Create extends Action
|
||||
}
|
||||
|
||||
public function action(
|
||||
string $resourceId,
|
||||
string $databaseId,
|
||||
string $collectionId,
|
||||
string $filename,
|
||||
array $columns,
|
||||
array $queries,
|
||||
@@ -112,14 +114,6 @@ class Create extends Action
|
||||
throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND);
|
||||
}
|
||||
|
||||
[$databaseId, $collectionId] = \explode(':', $resourceId, 2);
|
||||
if (empty($databaseId)) {
|
||||
throw new Exception(Exception::DATABASE_NOT_FOUND);
|
||||
}
|
||||
if (empty($collectionId)) {
|
||||
throw new Exception(Exception::COLLECTION_NOT_FOUND);
|
||||
}
|
||||
|
||||
$database = $authorization->skip(fn () => $dbForProject->getDocument('databases', $databaseId));
|
||||
if ($database->isEmpty()) {
|
||||
throw new Exception(Exception::DATABASE_NOT_FOUND);
|
||||
@@ -150,7 +144,7 @@ class Create extends Action
|
||||
}
|
||||
|
||||
$resources = Transfer::extractServices([self::transferGroupForDatabaseType($databaseType)]);
|
||||
$resourceType = self::resourceTypeForDatabaseType($databaseType);
|
||||
$parentResourceType = self::resourceTypeForDatabaseType($databaseType);
|
||||
|
||||
$migration = $dbForProject->createDocument('migrations', new Document([
|
||||
'$id' => ID::unique(),
|
||||
@@ -159,8 +153,12 @@ class Create extends Action
|
||||
'source' => AppwriteSource::getName(),
|
||||
'destination' => CSV::getName(),
|
||||
'resources' => $resources,
|
||||
'resourceId' => $resourceId,
|
||||
'resourceType' => $resourceType,
|
||||
'resourceId' => $collection->getId(),
|
||||
'resourceInternalId' => $collection->getSequence(),
|
||||
'resourceType' => Resource::TYPE_COLLECTION,
|
||||
'parentResourceId' => $database->getId(),
|
||||
'parentResourceInternalId' => $database->getSequence(),
|
||||
'parentResourceType' => $parentResourceType,
|
||||
'statusCounters' => '{}',
|
||||
'resourceData' => '{}',
|
||||
'errors' => [],
|
||||
|
||||
@@ -10,7 +10,6 @@ use Appwrite\OpenSSL\OpenSSL;
|
||||
use Appwrite\SDK\AuthType;
|
||||
use Appwrite\SDK\Method;
|
||||
use Appwrite\SDK\Response as SDKResponse;
|
||||
use Appwrite\Utopia\Database\Validator\CompoundUID;
|
||||
use Appwrite\Utopia\Response;
|
||||
use Utopia\Compression\Algorithms\GZIP;
|
||||
use Utopia\Compression\Algorithms\Zstd;
|
||||
@@ -68,7 +67,8 @@ class Create extends Action
|
||||
))
|
||||
->param('bucketId', '', fn (Database $dbForProject) => new UID($dbForProject->getAdapter()->getMaxUIDLength()), 'Storage bucket unique ID. You can create a new storage bucket using the Storage service [server integration](https://appwrite.io/docs/server/storage#createBucket).', false, ['dbForProject'])
|
||||
->param('fileId', '', fn (Database $dbForProject) => new UID($dbForProject->getAdapter()->getMaxUIDLength()), 'File ID.', false, ['dbForProject'])
|
||||
->param('resourceId', null, new CompoundUID(), 'Composite ID in the format {databaseId:collectionId}, identifying a collection within a database.')
|
||||
->param('databaseId', '', new UID(), 'Database ID containing the target collection.')
|
||||
->param('collectionId', '', new UID(), 'Collection ID to import documents into.')
|
||||
->param('internalFile', false, new Boolean(), 'Is the file stored in an internal bucket?', true)
|
||||
->param('onDuplicate', OnDuplicate::Fail->value, new WhiteList(OnDuplicate::values()), 'Behavior when a row with an existing $id is encountered. "fail" (default): abort on first conflict. "skip": silently ignore. "overwrite": replace existing row.', true, enum: new Enum(name: 'OnDuplicate'))
|
||||
->inject('response')
|
||||
@@ -87,7 +87,8 @@ class Create extends Action
|
||||
public function action(
|
||||
string $bucketId,
|
||||
string $fileId,
|
||||
string $resourceId,
|
||||
string $databaseId,
|
||||
string $collectionId,
|
||||
bool $internalFile,
|
||||
string $onDuplicate,
|
||||
Response $response,
|
||||
@@ -163,15 +164,24 @@ class Create extends Action
|
||||
throw new \Exception('Unable to copy file');
|
||||
}
|
||||
|
||||
[$databaseId] = \explode(':', $resourceId, 2);
|
||||
$database = $authorization->skip(fn () => $dbForProject->getDocument('databases', $databaseId));
|
||||
if ($database->isEmpty()) {
|
||||
throw new Exception(Exception::DATABASE_NOT_FOUND);
|
||||
}
|
||||
|
||||
$databaseType = $database->getAttribute('type');
|
||||
if (!\in_array($databaseType, CSV_ALLOWED_DATABASE_TYPES)) {
|
||||
throw new Exception(Exception::MIGRATION_DATABASE_TYPE_UNSUPPORTED, 'Database type not supported for csv');
|
||||
}
|
||||
|
||||
$collection = $authorization->skip(fn () => $dbForProject->getDocument('database_' . $database->getSequence(), $collectionId));
|
||||
if ($collection->isEmpty()) {
|
||||
throw new Exception(Exception::COLLECTION_NOT_FOUND);
|
||||
}
|
||||
|
||||
$fileSize = $deviceForMigrations->getFileSize($newPath);
|
||||
$resources = Transfer::extractServices([self::transferGroupForDatabaseType($databaseType)]);
|
||||
$resourceType = self::resourceTypeForDatabaseType($databaseType);
|
||||
$parentResourceType = self::resourceTypeForDatabaseType($databaseType);
|
||||
|
||||
$migration = $dbForProject->createDocument('migrations', new Document([
|
||||
'$id' => $migrationId,
|
||||
@@ -180,8 +190,15 @@ class Create extends Action
|
||||
'source' => CSV::getName(),
|
||||
'destination' => AppwriteSource::getName(),
|
||||
'resources' => $resources,
|
||||
'resourceId' => $resourceId,
|
||||
'resourceType' => $resourceType,
|
||||
'resourceId' => $collection->getId(),
|
||||
'resourceInternalId' => $collection->getSequence(),
|
||||
'resourceType' => Resource::TYPE_COLLECTION,
|
||||
'parentResourceId' => $database->getId(),
|
||||
'parentResourceInternalId' => $database->getSequence(),
|
||||
'parentResourceType' => $parentResourceType,
|
||||
'destinationResourceId' => $database->getId(),
|
||||
'destinationResourceInternalId' => $database->getSequence(),
|
||||
'destinationResourceType' => $parentResourceType,
|
||||
'statusCounters' => '{}',
|
||||
'resourceData' => '{}',
|
||||
'errors' => [],
|
||||
|
||||
@@ -9,7 +9,6 @@ use Appwrite\Extend\Exception;
|
||||
use Appwrite\SDK\AuthType;
|
||||
use Appwrite\SDK\Method;
|
||||
use Appwrite\SDK\Response as SDKResponse;
|
||||
use Appwrite\Utopia\Database\Validator\CompoundUID;
|
||||
use Appwrite\Utopia\Response;
|
||||
use Utopia\Database\Database;
|
||||
use Utopia\Database\Document;
|
||||
@@ -18,6 +17,7 @@ use Utopia\Database\Helpers\ID;
|
||||
use Utopia\Database\Query;
|
||||
use Utopia\Database\Validator\Authorization;
|
||||
use Utopia\Database\Validator\Queries\Documents;
|
||||
use Utopia\Database\Validator\UID;
|
||||
use Utopia\Migration\Resource;
|
||||
use Utopia\Migration\Sources\Appwrite as AppwriteSource;
|
||||
use Utopia\Migration\Sources\JSON as JSONSource;
|
||||
@@ -60,7 +60,8 @@ class Create extends Action
|
||||
)
|
||||
]
|
||||
))
|
||||
->param('resourceId', null, new CompoundUID(), 'Composite ID in the format {databaseId:collectionId}, identifying a collection within a database to export.')
|
||||
->param('databaseId', '', new UID(), 'Database ID containing the source collection.')
|
||||
->param('collectionId', '', new UID(), 'Collection ID to export documents from.')
|
||||
->param('filename', '', new Text(255), 'The name of the file to be created for the export, excluding the .json extension.')
|
||||
->param('columns', [], new ArrayList(new Text(Database::LENGTH_KEY)), 'List of attributes to export. If empty, all attributes will be exported. You can use the `*` wildcard to export all attributes from the collection.', true)
|
||||
->param('queries', [], new ArrayList(new Text(0)), 'Array of query strings generated using the Query class provided by the SDK to filter documents to export. [Learn more about queries](https://appwrite.io/docs/databases#querying-documents). Maximum of ' . APP_LIMIT_ARRAY_PARAMS_SIZE . ' queries are allowed, each ' . APP_LIMIT_ARRAY_ELEMENT_SIZE . ' characters long.', true)
|
||||
@@ -78,7 +79,8 @@ class Create extends Action
|
||||
}
|
||||
|
||||
public function action(
|
||||
string $resourceId,
|
||||
string $databaseId,
|
||||
string $collectionId,
|
||||
string $filename,
|
||||
array $columns,
|
||||
array $queries,
|
||||
@@ -104,14 +106,6 @@ class Create extends Action
|
||||
throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND);
|
||||
}
|
||||
|
||||
[$databaseId, $collectionId] = \explode(':', $resourceId, 2);
|
||||
if (empty($databaseId)) {
|
||||
throw new Exception(Exception::DATABASE_NOT_FOUND);
|
||||
}
|
||||
if (empty($collectionId)) {
|
||||
throw new Exception(Exception::COLLECTION_NOT_FOUND);
|
||||
}
|
||||
|
||||
$database = $authorization->skip(fn () => $dbForProject->getDocument('databases', $databaseId));
|
||||
if ($database->isEmpty()) {
|
||||
throw new Exception(Exception::DATABASE_NOT_FOUND);
|
||||
@@ -139,7 +133,7 @@ class Create extends Action
|
||||
}
|
||||
|
||||
$resources = Transfer::extractServices([self::transferGroupForDatabaseType($databaseType)]);
|
||||
$resourceType = self::resourceTypeForDatabaseType($databaseType);
|
||||
$parentResourceType = self::resourceTypeForDatabaseType($databaseType);
|
||||
|
||||
$migration = $dbForProject->createDocument('migrations', new Document([
|
||||
'$id' => ID::unique(),
|
||||
@@ -148,8 +142,12 @@ class Create extends Action
|
||||
'source' => AppwriteSource::getName(),
|
||||
'destination' => JSONSource::getName(),
|
||||
'resources' => $resources,
|
||||
'resourceId' => $resourceId,
|
||||
'resourceType' => $resourceType,
|
||||
'resourceId' => $collection->getId(),
|
||||
'resourceInternalId' => $collection->getSequence(),
|
||||
'resourceType' => Resource::TYPE_COLLECTION,
|
||||
'parentResourceId' => $database->getId(),
|
||||
'parentResourceInternalId' => $database->getSequence(),
|
||||
'parentResourceType' => $parentResourceType,
|
||||
'statusCounters' => '{}',
|
||||
'resourceData' => '{}',
|
||||
'errors' => [],
|
||||
|
||||
@@ -10,7 +10,6 @@ use Appwrite\OpenSSL\OpenSSL;
|
||||
use Appwrite\SDK\AuthType;
|
||||
use Appwrite\SDK\Method;
|
||||
use Appwrite\SDK\Response as SDKResponse;
|
||||
use Appwrite\Utopia\Database\Validator\CompoundUID;
|
||||
use Appwrite\Utopia\Response;
|
||||
use Utopia\Compression\Algorithms\GZIP;
|
||||
use Utopia\Compression\Algorithms\Zstd;
|
||||
@@ -67,7 +66,8 @@ class Create extends Action
|
||||
))
|
||||
->param('bucketId', '', new UID(), 'Storage bucket unique ID. You can create a new storage bucket using the Storage service [server integration](https://appwrite.io/docs/server/storage#createBucket).')
|
||||
->param('fileId', '', new UID(), 'File ID.')
|
||||
->param('resourceId', null, new CompoundUID(), 'Composite ID in the format {databaseId:collectionId}, identifying a collection within a database.')
|
||||
->param('databaseId', '', new UID(), 'Database ID containing the target collection.')
|
||||
->param('collectionId', '', new UID(), 'Collection ID to import documents into.')
|
||||
->param('internalFile', false, new Boolean(), 'Is the file stored in an internal bucket?', true)
|
||||
->param('onDuplicate', OnDuplicate::Fail->value, new WhiteList(OnDuplicate::values()), 'Behavior when a row with an existing $id is encountered. "fail" (default): abort on first conflict. "skip": silently ignore. "overwrite": replace existing row.', true, enum: new Enum(name: 'OnDuplicate'))
|
||||
->inject('response')
|
||||
@@ -86,7 +86,8 @@ class Create extends Action
|
||||
public function action(
|
||||
string $bucketId,
|
||||
string $fileId,
|
||||
string $resourceId,
|
||||
string $databaseId,
|
||||
string $collectionId,
|
||||
bool $internalFile,
|
||||
string $onDuplicate,
|
||||
Response $response,
|
||||
@@ -164,14 +165,19 @@ class Create extends Action
|
||||
|
||||
$fileSize = $deviceForMigrations->getFileSize($newPath);
|
||||
|
||||
[$databaseId] = \explode(':', $resourceId, 2);
|
||||
$database = $authorization->skip(fn () => $dbForProject->getDocument('databases', $databaseId));
|
||||
if ($database->isEmpty()) {
|
||||
throw new Exception(Exception::DATABASE_NOT_FOUND);
|
||||
}
|
||||
|
||||
$collection = $authorization->skip(fn () => $dbForProject->getDocument('database_' . $database->getSequence(), $collectionId));
|
||||
if ($collection->isEmpty()) {
|
||||
throw new Exception(Exception::COLLECTION_NOT_FOUND);
|
||||
}
|
||||
|
||||
$databaseType = $database->getAttribute('type');
|
||||
$resources = Transfer::extractServices([self::transferGroupForDatabaseType($databaseType)]);
|
||||
$resourceType = self::resourceTypeForDatabaseType($databaseType);
|
||||
$parentResourceType = self::resourceTypeForDatabaseType($databaseType);
|
||||
|
||||
$migration = $dbForProject->createDocument('migrations', new Document([
|
||||
'$id' => $migrationId,
|
||||
@@ -180,8 +186,15 @@ class Create extends Action
|
||||
'source' => JSONSource::getName(),
|
||||
'destination' => AppwriteSource::getName(),
|
||||
'resources' => $resources,
|
||||
'resourceId' => $resourceId,
|
||||
'resourceType' => $resourceType,
|
||||
'resourceId' => $collection->getId(),
|
||||
'resourceInternalId' => $collection->getSequence(),
|
||||
'resourceType' => Resource::TYPE_COLLECTION,
|
||||
'parentResourceId' => $database->getId(),
|
||||
'parentResourceInternalId' => $database->getSequence(),
|
||||
'parentResourceType' => $parentResourceType,
|
||||
'destinationResourceId' => $database->getId(),
|
||||
'destinationResourceInternalId' => $database->getSequence(),
|
||||
'destinationResourceType' => $parentResourceType,
|
||||
'statusCounters' => '{}',
|
||||
'resourceData' => '{}',
|
||||
'errors' => [],
|
||||
|
||||
@@ -192,7 +192,7 @@ class Migrations extends Action
|
||||
{
|
||||
$source = $migration->getAttribute('source');
|
||||
$destination = $migration->getAttribute('destination');
|
||||
$resourceId = $migration->getAttribute('resourceId');
|
||||
[$databaseId, $tableId] = $this->resolveResourceIds($migration);
|
||||
$credentials = $migration->getAttribute('credentials');
|
||||
$migrationOptions = $migration->getAttribute('options');
|
||||
/** @var Database|null $projectDB */
|
||||
@@ -290,18 +290,20 @@ class Migrations extends Action
|
||||
$projectDB,
|
||||
$queries
|
||||
),
|
||||
CSV::getName() => new CSV(
|
||||
$resourceId,
|
||||
$migrationOptions['path'],
|
||||
$this->deviceForMigrations,
|
||||
$this->dbForProject,
|
||||
$getDatabasesDB
|
||||
CSV::getName() => CSV::fromResourceIds(
|
||||
databaseId: $databaseId,
|
||||
tableId: $tableId,
|
||||
filePath: $migrationOptions['path'],
|
||||
device: $this->deviceForMigrations,
|
||||
dbForProject: $this->dbForProject,
|
||||
getDatabasesDB: $getDatabasesDB,
|
||||
),
|
||||
JSON::getName() => new JSON(
|
||||
$resourceId,
|
||||
$migrationOptions['path'],
|
||||
$this->deviceForMigrations,
|
||||
$this->dbForProject,
|
||||
JSON::getName() => JSON::fromResourceIds(
|
||||
databaseId: $databaseId,
|
||||
tableId: $tableId,
|
||||
filePath: $migrationOptions['path'],
|
||||
device: $this->deviceForMigrations,
|
||||
dbForProject: $this->dbForProject,
|
||||
),
|
||||
default => throw new Exception(Exception::MIGRATION_SOURCE_TYPE_INVALID),
|
||||
};
|
||||
@@ -320,6 +322,7 @@ class Migrations extends Action
|
||||
$destination = $migration->getAttribute('destination');
|
||||
$options = $migration->getAttribute('options', []);
|
||||
$credentials = $migration->getAttribute('credentials');
|
||||
[$databaseId, $tableId] = $this->resolveResourceIds($migration);
|
||||
|
||||
return match ($destination) {
|
||||
DestinationAppwrite::getName() => new DestinationAppwrite(
|
||||
@@ -334,23 +337,25 @@ class Migrations extends Action
|
||||
OnDuplicate::tryFrom($options['onDuplicate'] ?? '') ?? OnDuplicate::Fail,
|
||||
$this->resolveDestinationDatabaseDsn(...),
|
||||
),
|
||||
DestinationCSV::getName() => new DestinationCSV(
|
||||
$this->deviceForFiles,
|
||||
$migration->getAttribute('resourceId'),
|
||||
$options['bucketId'],
|
||||
$migration->getId(),
|
||||
$options['columns'],
|
||||
$options['delimiter'],
|
||||
$options['enclosure'],
|
||||
$options['escape'],
|
||||
$options['header'],
|
||||
DestinationCSV::getName() => DestinationCSV::fromResourceIds(
|
||||
deviceForFiles: $this->deviceForFiles,
|
||||
databaseId: $databaseId,
|
||||
tableId: $tableId,
|
||||
directory: $options['bucketId'],
|
||||
filename: $migration->getId(),
|
||||
allowedColumns: $options['columns'],
|
||||
delimiter: $options['delimiter'],
|
||||
enclosure: $options['enclosure'],
|
||||
escape: $options['escape'],
|
||||
includeHeaders: $options['header'],
|
||||
),
|
||||
DestinationJSON::getName() => new DestinationJSON(
|
||||
$this->deviceForFiles,
|
||||
$migration->getAttribute('resourceId'),
|
||||
$options['bucketId'] ?? 'default',
|
||||
$migration->getId(),
|
||||
$options['columns'] ?? [],
|
||||
DestinationJSON::getName() => DestinationJSON::fromResourceIds(
|
||||
deviceForFiles: $this->deviceForFiles,
|
||||
databaseId: $databaseId,
|
||||
tableId: $tableId,
|
||||
directory: $options['bucketId'] ?? 'default',
|
||||
filename: $migration->getId(),
|
||||
allowedColumns: $options['columns'] ?? [],
|
||||
),
|
||||
default => throw new Exception(Exception::MIGRATION_DESTINATION_TYPE_INVALID),
|
||||
};
|
||||
@@ -535,7 +540,8 @@ class Migrations extends Action
|
||||
$migration->setAttribute('stage', 'migrating');
|
||||
$this->updateMigrationDocument($migration, $project, $queueForRealtime);
|
||||
|
||||
$transfer->run(
|
||||
$context = $this->resolveResourceContext($migration);
|
||||
$transfer->runWithResourceSelector(
|
||||
$migration->getAttribute('resources'),
|
||||
function ($resources) use ($migration, $transfer, $project, $queueForRealtime, &$aggregatedResources) {
|
||||
$migration->setAttribute('resourceData', json_encode($transfer->getCache()));
|
||||
@@ -573,8 +579,12 @@ class Migrations extends Action
|
||||
}
|
||||
$this->updateMigrationDocument($migration, $project, $queueForRealtime);
|
||||
},
|
||||
$migration->getAttribute('resourceId'),
|
||||
$migration->getAttribute('resourceType')
|
||||
resourceId: $context['resourceId'],
|
||||
resourceInternalId: $context['resourceInternalId'],
|
||||
resourceType: $context['resourceType'],
|
||||
parentResourceId: $context['parentResourceId'],
|
||||
parentResourceInternalId: $context['parentResourceInternalId'],
|
||||
parentResourceType: $context['parentResourceType'],
|
||||
);
|
||||
|
||||
$destination->shutdown();
|
||||
@@ -590,6 +600,9 @@ class Migrations extends Action
|
||||
return;
|
||||
}
|
||||
|
||||
$destination->success();
|
||||
$source->success();
|
||||
|
||||
$destinationType = $migration->getAttribute('destination');
|
||||
if ($destinationType === DestinationCSV::getName() || $destinationType === DestinationJSON::getName()) {
|
||||
$this->handleDataExportComplete($project, $migration, $publisherForMails, $queueForRealtime, $platform, $authorization);
|
||||
@@ -692,11 +705,9 @@ class Migrations extends Action
|
||||
$publisherForUsage,
|
||||
$migration->getAttribute('source'),
|
||||
$authorization,
|
||||
$migration->getAttribute('resourceId')
|
||||
...$this->resolveResourceIds($migration),
|
||||
);
|
||||
}
|
||||
$destination?->success();
|
||||
$source?->success();
|
||||
}
|
||||
} finally {
|
||||
$source?->cleanup();
|
||||
@@ -718,6 +729,45 @@ class Migrations extends Action
|
||||
return ($this->getDatabasesDB)($database);
|
||||
}
|
||||
|
||||
/** @return array{0: string, 1: string} */
|
||||
protected function resolveResourceIds(Document $migration): array
|
||||
{
|
||||
$context = $this->resolveResourceContext($migration);
|
||||
|
||||
if ($context['parentResourceId'] !== '') {
|
||||
return [$context['parentResourceId'], $context['resourceId']];
|
||||
}
|
||||
|
||||
return [$context['resourceId'], ''];
|
||||
}
|
||||
|
||||
/**
|
||||
* @return array{resourceId: string, resourceInternalId: string, resourceType: string, parentResourceId: string, parentResourceInternalId: string, parentResourceType: string}
|
||||
*/
|
||||
protected function resolveResourceContext(Document $migration): array
|
||||
{
|
||||
$context = [
|
||||
'resourceId' => (string) $migration->getAttribute('resourceId', ''),
|
||||
'resourceInternalId' => (string) $migration->getAttribute('resourceInternalId', ''),
|
||||
'resourceType' => (string) $migration->getAttribute('resourceType', ''),
|
||||
'parentResourceId' => (string) $migration->getAttribute('parentResourceId', ''),
|
||||
'parentResourceInternalId' => (string) $migration->getAttribute('parentResourceInternalId', ''),
|
||||
'parentResourceType' => (string) $migration->getAttribute('parentResourceType', ''),
|
||||
];
|
||||
|
||||
if (
|
||||
$context['parentResourceId'] === ''
|
||||
&& \array_key_exists($context['resourceType'], Resource::DATABASE_TYPE_RESOURCE_MAP)
|
||||
&& \str_contains($context['resourceId'], ':')
|
||||
) {
|
||||
[$context['parentResourceId'], $context['resourceId']] = \explode(':', $context['resourceId'], 2);
|
||||
$context['parentResourceType'] = $context['resourceType'];
|
||||
$context['resourceType'] = Resource::TYPE_COLLECTION;
|
||||
}
|
||||
|
||||
return $context;
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle actions to be performed when a CSV export migration is successfully completed
|
||||
*
|
||||
@@ -1074,7 +1124,7 @@ class Migrations extends Action
|
||||
return $errors;
|
||||
}
|
||||
|
||||
private function processMigrationResourceStats(array $resources, Context $usage, Document $projectDocument, UsagePublisher $publisherForUsage, string $source, Authorization $authorization, ?string $resourceId)
|
||||
private function processMigrationResourceStats(array $resources, Context $usage, Document $projectDocument, UsagePublisher $publisherForUsage, string $source, Authorization $authorization, ?string $parentResourceId, ?string $resourceId)
|
||||
{
|
||||
$resourceName = $resources['name'];
|
||||
$count = $resources['count'];
|
||||
@@ -1082,9 +1132,20 @@ class Migrations extends Action
|
||||
$tableInternalId = $resources['tableId'];
|
||||
|
||||
if ($source === CSV::getName()) {
|
||||
[$databaseId, $tableId] = explode(':', $resourceId);
|
||||
$database = $authorization->skip(fn () => $this->dbForProject->getDocument('databases', $databaseId));
|
||||
$table = $authorization->skip(fn () => $this->dbForProject->getDocument('database_' . $database->getSequence(), $tableId));
|
||||
if (empty($parentResourceId) || empty($resourceId)) {
|
||||
Console::warning("Skipping CSV migration usage stats: missing parent/leaf resource ID (parent: '{$parentResourceId}', leaf: '{$resourceId}')");
|
||||
return;
|
||||
}
|
||||
$database = $authorization->skip(fn () => $this->dbForProject->getDocument('databases', $parentResourceId));
|
||||
if ($database->isEmpty()) {
|
||||
Console::warning("Skipping CSV migration usage stats: database '{$parentResourceId}' not found");
|
||||
return;
|
||||
}
|
||||
$table = $authorization->skip(fn () => $this->dbForProject->getDocument('database_' . $database->getSequence(), $resourceId));
|
||||
if ($table->isEmpty()) {
|
||||
Console::warning("Skipping CSV migration usage stats: collection '{$resourceId}' not found in database '{$parentResourceId}'");
|
||||
return;
|
||||
}
|
||||
$databaseInternalId = (int) $database->getSequence();
|
||||
$tableInternalId = (int) $table->getSequence();
|
||||
}
|
||||
|
||||
@@ -11,10 +11,17 @@ class Migrations extends Base
|
||||
'destination',
|
||||
'resources',
|
||||
'resourceId',
|
||||
'resourceInternalId',
|
||||
'resourceType',
|
||||
'parentResourceId',
|
||||
'parentResourceInternalId',
|
||||
'parentResourceType',
|
||||
'destinationResourceId',
|
||||
'destinationResourceInternalId',
|
||||
'destinationResourceType',
|
||||
'statusCounters',
|
||||
'resourceData',
|
||||
'errors'
|
||||
'errors',
|
||||
];
|
||||
|
||||
/**
|
||||
|
||||
@@ -0,0 +1,38 @@
|
||||
<?php
|
||||
|
||||
namespace Appwrite\Utopia\Request\Filters;
|
||||
|
||||
use Appwrite\Utopia\Request\Filter;
|
||||
|
||||
class V27 extends Filter
|
||||
{
|
||||
// Convert 1.9.5 params to 2.0.0
|
||||
public function parse(array $content, string $model): array
|
||||
{
|
||||
return match ($model) {
|
||||
'migrations.createCSVImport',
|
||||
'migrations.createCSVExport',
|
||||
'migrations.createJSONImport',
|
||||
'migrations.createJSONExport' => $this->parseMigrationResource($content),
|
||||
default => $content,
|
||||
};
|
||||
}
|
||||
|
||||
protected function parseMigrationResource(array $content): array
|
||||
{
|
||||
if (!isset($content['resourceId']) || !\is_string($content['resourceId'])) {
|
||||
return $content;
|
||||
}
|
||||
|
||||
if (!\str_contains($content['resourceId'], ':')) {
|
||||
return $content;
|
||||
}
|
||||
|
||||
[$databaseId, $collectionId] = \explode(':', $content['resourceId'], 2);
|
||||
$content['databaseId'] = $content['databaseId'] ?? $databaseId;
|
||||
$content['collectionId'] = $content['collectionId'] ?? $collectionId;
|
||||
unset($content['resourceId']);
|
||||
|
||||
return $content;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
<?php
|
||||
|
||||
namespace Appwrite\Utopia\Response\Filters;
|
||||
|
||||
use Appwrite\Utopia\Response;
|
||||
use Appwrite\Utopia\Response\Filter;
|
||||
|
||||
// Convert 2.0.0 Data format to 1.9.5 format
|
||||
class V27 extends Filter
|
||||
{
|
||||
public function parse(array $content, string $model): array
|
||||
{
|
||||
return match ($model) {
|
||||
Response::MODEL_MIGRATION => $this->parseMigration($content),
|
||||
Response::MODEL_MIGRATION_LIST => $this->handleList($content, 'migrations', fn ($item) => $this->parseMigration($item)),
|
||||
default => $content,
|
||||
};
|
||||
}
|
||||
|
||||
protected function parseMigration(array $content): array
|
||||
{
|
||||
$parentResourceId = $content['parentResourceId'] ?? '';
|
||||
$resourceId = $content['resourceId'] ?? '';
|
||||
|
||||
if ($parentResourceId !== '' && $resourceId !== '') {
|
||||
$content['resourceId'] = $parentResourceId . ':' . $resourceId;
|
||||
}
|
||||
|
||||
$parentResourceType = $content['parentResourceType'] ?? '';
|
||||
if ($parentResourceType !== '') {
|
||||
$content['resourceType'] = $parentResourceType;
|
||||
}
|
||||
|
||||
unset($content['resourceInternalId']);
|
||||
unset($content['parentResourceId']);
|
||||
unset($content['parentResourceInternalId']);
|
||||
unset($content['parentResourceType']);
|
||||
unset($content['destinationResourceId']);
|
||||
unset($content['destinationResourceInternalId']);
|
||||
unset($content['destinationResourceType']);
|
||||
|
||||
return $content;
|
||||
}
|
||||
}
|
||||
@@ -62,9 +62,65 @@ class Migration extends Model
|
||||
])
|
||||
->addRule('resourceId', [
|
||||
'type' => self::TYPE_STRING,
|
||||
'description' => 'Id of the resource to migrate.',
|
||||
'description' => 'ID of the resource being migrated.',
|
||||
'default' => '',
|
||||
'example' => 'databaseId:collectionId',
|
||||
'example' => 'collectionId',
|
||||
'array' => false
|
||||
])
|
||||
->addRule('resourceInternalId', [
|
||||
'type' => self::TYPE_STRING,
|
||||
'description' => 'Internal ID of the resource being migrated.',
|
||||
'default' => '',
|
||||
'example' => '1',
|
||||
'array' => false
|
||||
])
|
||||
->addRule('resourceType', [
|
||||
'type' => self::TYPE_STRING,
|
||||
'description' => 'Type of the resource being migrated.',
|
||||
'default' => '',
|
||||
'example' => 'collection',
|
||||
'array' => false
|
||||
])
|
||||
->addRule('parentResourceId', [
|
||||
'type' => self::TYPE_STRING,
|
||||
'description' => 'ID of the parent resource that contains the migrated resource.',
|
||||
'default' => '',
|
||||
'example' => 'databaseId',
|
||||
'array' => false
|
||||
])
|
||||
->addRule('parentResourceInternalId', [
|
||||
'type' => self::TYPE_STRING,
|
||||
'description' => 'Internal ID of the parent resource that contains the migrated resource.',
|
||||
'default' => '',
|
||||
'example' => '1',
|
||||
'array' => false
|
||||
])
|
||||
->addRule('parentResourceType', [
|
||||
'type' => self::TYPE_STRING,
|
||||
'description' => 'Type of the parent resource that contains the migrated resource.',
|
||||
'default' => '',
|
||||
'example' => 'database',
|
||||
'array' => false
|
||||
])
|
||||
->addRule('destinationResourceId', [
|
||||
'type' => self::TYPE_STRING,
|
||||
'description' => 'ID of the destination resource created or overwritten by the migration.',
|
||||
'default' => '',
|
||||
'example' => 'databaseId',
|
||||
'array' => false
|
||||
])
|
||||
->addRule('destinationResourceInternalId', [
|
||||
'type' => self::TYPE_STRING,
|
||||
'description' => 'Internal ID of the destination resource created or overwritten by the migration.',
|
||||
'default' => '',
|
||||
'example' => '1',
|
||||
'array' => false
|
||||
])
|
||||
->addRule('destinationResourceType', [
|
||||
'type' => self::TYPE_STRING,
|
||||
'description' => 'Type of the destination resource created or overwritten by the migration.',
|
||||
'default' => '',
|
||||
'example' => 'database',
|
||||
'array' => false
|
||||
])
|
||||
->addRule('statusCounters', [
|
||||
|
||||
@@ -3496,8 +3496,10 @@ trait MigrationsBase
|
||||
|
||||
/**
|
||||
* Import documents from a CSV file.
|
||||
*
|
||||
* @return array{databaseId: string, tableId: string, migrationId: string}
|
||||
*/
|
||||
public function testCreateCSVImport(): void
|
||||
public function testCreateCSVImport(): array
|
||||
{
|
||||
// Make a database
|
||||
$response = $this->client->call(Client::METHOD_POST, '/databases', [
|
||||
@@ -3641,7 +3643,8 @@ trait MigrationsBase
|
||||
[
|
||||
'fileId' => $fileIds['missing-column'],
|
||||
'bucketId' => $bucketIds['missing-column'],
|
||||
'resourceId' => $databaseId . ':' . $tableId,
|
||||
'databaseId' => $databaseId,
|
||||
'collectionId' => $tableId,
|
||||
]
|
||||
);
|
||||
|
||||
@@ -3673,7 +3676,8 @@ trait MigrationsBase
|
||||
[
|
||||
'fileId' => $fileIds['missing-row'],
|
||||
'bucketId' => $bucketIds['missing-row'],
|
||||
'resourceId' => $databaseId . ':' . $tableId,
|
||||
'databaseId' => $databaseId,
|
||||
'collectionId' => $tableId,
|
||||
]
|
||||
);
|
||||
|
||||
@@ -3705,7 +3709,8 @@ trait MigrationsBase
|
||||
[
|
||||
'fileId' => $fileIds['irrelevant-column'],
|
||||
'bucketId' => $bucketIds['irrelevant-column'],
|
||||
'resourceId' => $databaseId . ':' . $tableId,
|
||||
'databaseId' => $databaseId,
|
||||
'collectionId' => $tableId,
|
||||
]
|
||||
);
|
||||
|
||||
@@ -3732,7 +3737,8 @@ trait MigrationsBase
|
||||
'endpoint' => $this->webEndpoint,
|
||||
'fileId' => $fileIds['default'],
|
||||
'bucketId' => $bucketIds['default'],
|
||||
'resourceId' => $databaseId . ':' . $tableId,
|
||||
'databaseId' => $databaseId,
|
||||
'collectionId' => $tableId,
|
||||
]
|
||||
);
|
||||
|
||||
@@ -3774,7 +3780,8 @@ trait MigrationsBase
|
||||
'endpoint' => $this->webEndpoint,
|
||||
'fileId' => $fileIds['documents-internals'],
|
||||
'bucketId' => $bucketIds['documents-internals'],
|
||||
'resourceId' => $databaseId . ':' . $tableId,
|
||||
'databaseId' => $databaseId,
|
||||
'collectionId' => $tableId,
|
||||
]
|
||||
);
|
||||
|
||||
@@ -3794,6 +3801,78 @@ trait MigrationsBase
|
||||
$this->assertArrayHasKey(Resource::TYPE_ROW, $migration['body']['statusCounters']);
|
||||
$this->assertEquals(25, $migration['body']['statusCounters'][Resource::TYPE_ROW]['success']);
|
||||
}, 10_000, 500);
|
||||
|
||||
return [
|
||||
'databaseId' => $databaseId,
|
||||
'tableId' => $tableId,
|
||||
'migrationId' => $migration['body']['$id'],
|
||||
];
|
||||
}
|
||||
|
||||
/**
|
||||
* @param array{databaseId: string, tableId: string, migrationId: string} $data
|
||||
*/
|
||||
#[Depends('testCreateCSVImport')]
|
||||
public function testListMigrationsByDatabaseResource(array $data): void
|
||||
{
|
||||
$databaseId = $data['databaseId'];
|
||||
$query = Query::or([
|
||||
Query::and([
|
||||
Query::equal('resourceId', [$databaseId]),
|
||||
Query::equal('resourceType', [Resource::TYPE_DATABASE]),
|
||||
]),
|
||||
Query::and([
|
||||
Query::equal('parentResourceId', [$databaseId]),
|
||||
Query::equal('parentResourceType', [Resource::TYPE_DATABASE]),
|
||||
]),
|
||||
Query::and([
|
||||
Query::equal('destinationResourceId', [$databaseId]),
|
||||
Query::equal('destinationResourceType', [Resource::TYPE_DATABASE]),
|
||||
]),
|
||||
]);
|
||||
|
||||
$response = $this->client->call(Client::METHOD_GET, '/migrations', array_merge([
|
||||
'content-type' => 'application/json',
|
||||
'x-appwrite-project' => $this->getProject()['$id'],
|
||||
], $this->getHeaders()), [
|
||||
'queries' => [
|
||||
$query->toString(),
|
||||
Query::limit(100)->toString(),
|
||||
],
|
||||
]);
|
||||
|
||||
$this->assertSame(200, $response['headers']['status-code']);
|
||||
$this->assertContains($data['migrationId'], array_column($response['body']['migrations'], '$id'));
|
||||
|
||||
$migration = null;
|
||||
foreach ($response['body']['migrations'] as $candidate) {
|
||||
if ($candidate['$id'] === $data['migrationId']) {
|
||||
$migration = $candidate;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
$this->assertNotNull($migration);
|
||||
$this->assertSame($data['tableId'], $migration['resourceId']);
|
||||
$this->assertSame($databaseId, $migration['parentResourceId']);
|
||||
$this->assertNotSame('', $migration['parentResourceInternalId']);
|
||||
$this->assertSame(Resource::TYPE_DATABASE, $migration['parentResourceType']);
|
||||
$this->assertSame($databaseId, $migration['destinationResourceId']);
|
||||
$this->assertSame($migration['parentResourceInternalId'], $migration['destinationResourceInternalId']);
|
||||
$this->assertSame(Resource::TYPE_DATABASE, $migration['destinationResourceType']);
|
||||
|
||||
$response = $this->client->call(Client::METHOD_GET, '/migrations', array_merge([
|
||||
'content-type' => 'application/json',
|
||||
'x-appwrite-project' => $this->getProject()['$id'],
|
||||
], $this->getHeaders()), [
|
||||
'queries' => [
|
||||
Query::equal('destinationResourceInternalId', [$migration['destinationResourceInternalId']])->toString(),
|
||||
Query::equal('destinationResourceType', [Resource::TYPE_DATABASE])->toString(),
|
||||
],
|
||||
]);
|
||||
|
||||
$this->assertSame(200, $response['headers']['status-code']);
|
||||
$this->assertContains($data['migrationId'], array_column($response['body']['migrations'], '$id'));
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -3888,7 +3967,8 @@ trait MigrationsBase
|
||||
$first = $this->performCsvMigration([
|
||||
'fileId' => $fileId,
|
||||
'bucketId' => $bucketId,
|
||||
'resourceId' => $databaseId . ':' . $tableId,
|
||||
'databaseId' => $databaseId,
|
||||
'collectionId' => $tableId,
|
||||
]);
|
||||
$this->assertEventually(function () use ($first) {
|
||||
$migration = $this->client->call(Client::METHOD_GET, '/migrations/' . $first['body']['$id'], array_merge([
|
||||
@@ -3914,7 +3994,8 @@ trait MigrationsBase
|
||||
$second = $this->performCsvMigration([
|
||||
'fileId' => $fileId,
|
||||
'bucketId' => $bucketId,
|
||||
'resourceId' => $databaseId . ':' . $tableId,
|
||||
'databaseId' => $databaseId,
|
||||
'collectionId' => $tableId,
|
||||
'onDuplicate' => 'skip',
|
||||
]);
|
||||
$this->assertEventually(function () use ($second) {
|
||||
@@ -3955,7 +4036,8 @@ trait MigrationsBase
|
||||
$first = $this->performCsvMigration([
|
||||
'fileId' => $fileId,
|
||||
'bucketId' => $bucketId,
|
||||
'resourceId' => $databaseId . ':' . $tableId,
|
||||
'databaseId' => $databaseId,
|
||||
'collectionId' => $tableId,
|
||||
]);
|
||||
$this->assertEventually(function () use ($first) {
|
||||
$migration = $this->client->call(Client::METHOD_GET, '/migrations/' . $first['body']['$id'], array_merge([
|
||||
@@ -3981,7 +4063,8 @@ trait MigrationsBase
|
||||
$second = $this->performCsvMigration([
|
||||
'fileId' => $fileId,
|
||||
'bucketId' => $bucketId,
|
||||
'resourceId' => $databaseId . ':' . $tableId,
|
||||
'databaseId' => $databaseId,
|
||||
'collectionId' => $tableId,
|
||||
'onDuplicate' => 'overwrite',
|
||||
]);
|
||||
$this->assertEventually(function () use ($second) {
|
||||
@@ -4023,7 +4106,8 @@ trait MigrationsBase
|
||||
$first = $this->performCsvMigration([
|
||||
'fileId' => $fileId,
|
||||
'bucketId' => $bucketId,
|
||||
'resourceId' => $databaseId . ':' . $tableId,
|
||||
'databaseId' => $databaseId,
|
||||
'collectionId' => $tableId,
|
||||
]);
|
||||
$this->assertEventually(function () use ($first) {
|
||||
$migration = $this->client->call(Client::METHOD_GET, '/migrations/' . $first['body']['$id'], array_merge([
|
||||
@@ -4037,7 +4121,8 @@ trait MigrationsBase
|
||||
$second = $this->performCsvMigration([
|
||||
'fileId' => $fileId,
|
||||
'bucketId' => $bucketId,
|
||||
'resourceId' => $databaseId . ':' . $tableId,
|
||||
'databaseId' => $databaseId,
|
||||
'collectionId' => $tableId,
|
||||
]);
|
||||
$this->assertEventually(function () use ($second) {
|
||||
$migration = $this->client->call(Client::METHOD_GET, '/migrations/' . $second['body']['$id'], array_merge([
|
||||
@@ -4148,7 +4233,8 @@ trait MigrationsBase
|
||||
$first = $this->performJsonMigration([
|
||||
'fileId' => $fileId,
|
||||
'bucketId' => $bucketId,
|
||||
'resourceId' => $databaseId . ':' . $tableId,
|
||||
'databaseId' => $databaseId,
|
||||
'collectionId' => $tableId,
|
||||
]);
|
||||
$this->assertEventually(function () use ($first) {
|
||||
$migration = $this->client->call(Client::METHOD_GET, '/migrations/' . $first['body']['$id'], array_merge([
|
||||
@@ -4173,7 +4259,8 @@ trait MigrationsBase
|
||||
$second = $this->performJsonMigration([
|
||||
'fileId' => $fileId,
|
||||
'bucketId' => $bucketId,
|
||||
'resourceId' => $databaseId . ':' . $tableId,
|
||||
'databaseId' => $databaseId,
|
||||
'collectionId' => $tableId,
|
||||
'onDuplicate' => 'skip',
|
||||
]);
|
||||
$this->assertEventually(function () use ($second) {
|
||||
@@ -4211,7 +4298,8 @@ trait MigrationsBase
|
||||
$first = $this->performJsonMigration([
|
||||
'fileId' => $fileId,
|
||||
'bucketId' => $bucketId,
|
||||
'resourceId' => $databaseId . ':' . $tableId,
|
||||
'databaseId' => $databaseId,
|
||||
'collectionId' => $tableId,
|
||||
]);
|
||||
$this->assertEventually(function () use ($first) {
|
||||
$migration = $this->client->call(Client::METHOD_GET, '/migrations/' . $first['body']['$id'], array_merge([
|
||||
@@ -4235,7 +4323,8 @@ trait MigrationsBase
|
||||
$second = $this->performJsonMigration([
|
||||
'fileId' => $fileId,
|
||||
'bucketId' => $bucketId,
|
||||
'resourceId' => $databaseId . ':' . $tableId,
|
||||
'databaseId' => $databaseId,
|
||||
'collectionId' => $tableId,
|
||||
'onDuplicate' => 'overwrite',
|
||||
]);
|
||||
$this->assertEventually(function () use ($second) {
|
||||
@@ -4273,7 +4362,8 @@ trait MigrationsBase
|
||||
$first = $this->performJsonMigration([
|
||||
'fileId' => $fileId,
|
||||
'bucketId' => $bucketId,
|
||||
'resourceId' => $databaseId . ':' . $tableId,
|
||||
'databaseId' => $databaseId,
|
||||
'collectionId' => $tableId,
|
||||
]);
|
||||
$this->assertEventually(function () use ($first) {
|
||||
$migration = $this->client->call(Client::METHOD_GET, '/migrations/' . $first['body']['$id'], array_merge([
|
||||
@@ -4286,7 +4376,8 @@ trait MigrationsBase
|
||||
$second = $this->performJsonMigration([
|
||||
'fileId' => $fileId,
|
||||
'bucketId' => $bucketId,
|
||||
'resourceId' => $databaseId . ':' . $tableId,
|
||||
'databaseId' => $databaseId,
|
||||
'collectionId' => $tableId,
|
||||
]);
|
||||
$this->assertEventually(function () use ($second) {
|
||||
$migration = $this->client->call(Client::METHOD_GET, '/migrations/' . $second['body']['$id'], array_merge([
|
||||
@@ -4465,7 +4556,8 @@ trait MigrationsBase
|
||||
'content-type' => 'application/json',
|
||||
'x-appwrite-project' => $this->getProject()['$id']
|
||||
], $this->getHeaders()), [
|
||||
'resourceId' => $databaseId . ':' . $collectionId,
|
||||
'databaseId' => $databaseId,
|
||||
'collectionId' => $collectionId,
|
||||
'filename' => 'test-export',
|
||||
'columns' => [],
|
||||
'delimiter' => ',',
|
||||
@@ -5445,7 +5537,8 @@ trait MigrationsBase
|
||||
$migration = $this->performCsvMigration([
|
||||
'fileId' => $fileId,
|
||||
'bucketId' => $bucketId,
|
||||
'resourceId' => $databaseId . ':' . $collectionId,
|
||||
'databaseId' => $databaseId,
|
||||
'collectionId' => $collectionId,
|
||||
]);
|
||||
|
||||
$this->assertEquals(202, $migration['headers']['status-code']);
|
||||
@@ -5573,7 +5666,8 @@ trait MigrationsBase
|
||||
'content-type' => 'application/json',
|
||||
'x-appwrite-project' => $this->getProject()['$id'],
|
||||
], $this->getHeaders()), [
|
||||
'resourceId' => $databaseId . ':' . $collectionId,
|
||||
'databaseId' => $databaseId,
|
||||
'collectionId' => $collectionId,
|
||||
'filename' => $filename,
|
||||
'columns' => [],
|
||||
'queries' => [],
|
||||
@@ -6787,7 +6881,8 @@ trait MigrationsBase
|
||||
[
|
||||
'fileId' => $fileIds['missing-column'],
|
||||
'bucketId' => $bucketIds['missing-column'],
|
||||
'resourceId' => $databaseId . ':' . $tableId,
|
||||
'databaseId' => $databaseId,
|
||||
'collectionId' => $tableId,
|
||||
]
|
||||
);
|
||||
|
||||
@@ -6824,7 +6919,8 @@ trait MigrationsBase
|
||||
[
|
||||
'fileId' => $fileIds['irrelevant-column'],
|
||||
'bucketId' => $bucketIds['irrelevant-column'],
|
||||
'resourceId' => $databaseId . ':' . $tableId,
|
||||
'databaseId' => $databaseId,
|
||||
'collectionId' => $tableId,
|
||||
]
|
||||
);
|
||||
|
||||
@@ -6851,7 +6947,8 @@ trait MigrationsBase
|
||||
'endpoint' => $this->endpoint,
|
||||
'fileId' => $fileIds['default'],
|
||||
'bucketId' => $bucketIds['default'],
|
||||
'resourceId' => $databaseId . ':' . $tableId,
|
||||
'databaseId' => $databaseId,
|
||||
'collectionId' => $tableId,
|
||||
]
|
||||
);
|
||||
|
||||
@@ -6893,7 +6990,8 @@ trait MigrationsBase
|
||||
'endpoint' => $this->endpoint,
|
||||
'fileId' => $fileIds['documents-internals'],
|
||||
'bucketId' => $bucketIds['documents-internals'],
|
||||
'resourceId' => $databaseId . ':' . $tableId,
|
||||
'databaseId' => $databaseId,
|
||||
'collectionId' => $tableId,
|
||||
]
|
||||
);
|
||||
|
||||
@@ -7016,7 +7114,8 @@ trait MigrationsBase
|
||||
'content-type' => 'application/json',
|
||||
'x-appwrite-project' => $this->getProject()['$id']
|
||||
], $this->getHeaders()), [
|
||||
'resourceId' => $databaseId . ':' . $collectionId,
|
||||
'databaseId' => $databaseId,
|
||||
'collectionId' => $collectionId,
|
||||
'filename' => 'test-json-export',
|
||||
'columns' => [],
|
||||
'queries' => [],
|
||||
@@ -7127,7 +7226,8 @@ trait MigrationsBase
|
||||
|
||||
// Trigger JSON export
|
||||
$migration = $this->client->call(Client::METHOD_POST, '/migrations/json/exports', $headers, [
|
||||
'resourceId' => $databaseId . ':' . $collectionId,
|
||||
'databaseId' => $databaseId,
|
||||
'collectionId' => $collectionId,
|
||||
'filename' => 'vectorsdb-export-test',
|
||||
'columns' => [],
|
||||
'queries' => [],
|
||||
@@ -7200,7 +7300,8 @@ trait MigrationsBase
|
||||
$migration = $this->performJsonMigration([
|
||||
'fileId' => $fileId,
|
||||
'bucketId' => $bucketId,
|
||||
'resourceId' => $databaseId . ':' . $collectionId,
|
||||
'databaseId' => $databaseId,
|
||||
'collectionId' => $collectionId,
|
||||
]);
|
||||
$this->assertEquals(202, $migration['headers']['status-code']);
|
||||
|
||||
@@ -7271,7 +7372,8 @@ trait MigrationsBase
|
||||
|
||||
// Trigger JSON export
|
||||
$migration = $this->client->call(Client::METHOD_POST, '/migrations/json/exports', $headers, [
|
||||
'resourceId' => $databaseId . ':' . $collectionId,
|
||||
'databaseId' => $databaseId,
|
||||
'collectionId' => $collectionId,
|
||||
'filename' => 'documentsdb-export-test',
|
||||
'columns' => [],
|
||||
'queries' => [],
|
||||
@@ -7343,7 +7445,8 @@ trait MigrationsBase
|
||||
$migration = $this->performJsonMigration([
|
||||
'fileId' => $fileId,
|
||||
'bucketId' => $bucketId,
|
||||
'resourceId' => $databaseId . ':' . $collectionId,
|
||||
'databaseId' => $databaseId,
|
||||
'collectionId' => $collectionId,
|
||||
]);
|
||||
$this->assertEquals(202, $migration['headers']['status-code']);
|
||||
|
||||
|
||||
@@ -0,0 +1,148 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace Tests\Unit\Migration\Version;
|
||||
|
||||
use Appwrite\Migration\Version\V25;
|
||||
use PHPUnit\Framework\TestCase;
|
||||
use Utopia\Database\Document;
|
||||
use Utopia\Migration\Resource;
|
||||
|
||||
final class V25Test extends TestCase
|
||||
{
|
||||
public function testPreservesLegacyResourceWhenInternalIdsCannotBeResolved(): void
|
||||
{
|
||||
$migration = new class () extends V25 {
|
||||
public function __construct()
|
||||
{
|
||||
}
|
||||
|
||||
public function migrate(Document $document): Document
|
||||
{
|
||||
return $this->migrateDocument($document);
|
||||
}
|
||||
|
||||
#[\Override]
|
||||
protected function resolveInternalIds(string $parentResourceId, string $resourceId, Document $migration): array
|
||||
{
|
||||
return [];
|
||||
}
|
||||
};
|
||||
$document = new Document([
|
||||
'$id' => 'migration',
|
||||
'$collection' => 'migrations',
|
||||
'resourceId' => 'database:collection',
|
||||
'resourceInternalId' => null,
|
||||
'resourceType' => Resource::TYPE_DATABASE,
|
||||
'parentResourceInternalId' => null,
|
||||
]);
|
||||
|
||||
$result = $migration->migrate($document);
|
||||
|
||||
$this->assertSame('database:collection', $result->getAttribute('resourceId'));
|
||||
$this->assertSame(Resource::TYPE_DATABASE, $result->getAttribute('resourceType'));
|
||||
$this->assertNull($result->getAttribute('parentResourceId'));
|
||||
$this->assertNull($result->getAttribute('parentResourceType'));
|
||||
$this->assertNull($result->getAttribute('resourceInternalId'));
|
||||
$this->assertNull($result->getAttribute('parentResourceInternalId'));
|
||||
}
|
||||
|
||||
public function testRerunResolvesMissingInternalIdsIdempotently(): void
|
||||
{
|
||||
$migration = new class () extends V25 {
|
||||
/**
|
||||
* @var array{parentResourceInternalId?: string, resourceInternalId?: string}
|
||||
*/
|
||||
private array $internalIds = [];
|
||||
|
||||
public function __construct()
|
||||
{
|
||||
}
|
||||
|
||||
public function migrate(Document $document): Document
|
||||
{
|
||||
return $this->migrateDocument($document);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param array{parentResourceInternalId?: string, resourceInternalId?: string} $internalIds
|
||||
*/
|
||||
public function setInternalIds(array $internalIds): void
|
||||
{
|
||||
$this->internalIds = $internalIds;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return array{parentResourceInternalId?: string, resourceInternalId?: string}
|
||||
*/
|
||||
#[\Override]
|
||||
protected function resolveInternalIds(string $parentResourceId, string $resourceId, Document $migration): array
|
||||
{
|
||||
return $this->internalIds;
|
||||
}
|
||||
};
|
||||
$document = new Document([
|
||||
'$id' => 'migration',
|
||||
'$collection' => 'migrations',
|
||||
'resourceId' => 'database:collection',
|
||||
'resourceInternalId' => null,
|
||||
'resourceType' => Resource::TYPE_DATABASE,
|
||||
'parentResourceInternalId' => null,
|
||||
]);
|
||||
|
||||
$migration->migrate($document);
|
||||
$this->assertSame('database:collection', $document->getAttribute('resourceId'));
|
||||
$this->assertSame(Resource::TYPE_DATABASE, $document->getAttribute('resourceType'));
|
||||
$this->assertNull($document->getAttribute('parentResourceId'));
|
||||
|
||||
$migration->setInternalIds([
|
||||
'parentResourceInternalId' => '10',
|
||||
]);
|
||||
$migration->migrate($document);
|
||||
$this->assertSame('database:collection', $document->getAttribute('resourceId'));
|
||||
$this->assertSame(Resource::TYPE_DATABASE, $document->getAttribute('resourceType'));
|
||||
$this->assertNull($document->getAttribute('parentResourceId'));
|
||||
|
||||
$migration->setInternalIds([
|
||||
'parentResourceInternalId' => '10',
|
||||
'resourceInternalId' => '20',
|
||||
]);
|
||||
$migration->migrate($document);
|
||||
$afterResolution = $document->getArrayCopy();
|
||||
$migration->migrate($document);
|
||||
|
||||
$this->assertSame('10', $document->getAttribute('parentResourceInternalId'));
|
||||
$this->assertSame('20', $document->getAttribute('resourceInternalId'));
|
||||
$this->assertSame('collection', $document->getAttribute('resourceId'));
|
||||
$this->assertSame(Resource::TYPE_COLLECTION, $document->getAttribute('resourceType'));
|
||||
$this->assertSame('database', $document->getAttribute('parentResourceId'));
|
||||
$this->assertSame(Resource::TYPE_DATABASE, $document->getAttribute('parentResourceType'));
|
||||
$this->assertSame($afterResolution, $document->getArrayCopy());
|
||||
}
|
||||
|
||||
public function testRejectsResourcesCreatedAfterMigration(): void
|
||||
{
|
||||
$migration = new class () extends V25 {
|
||||
public function __construct()
|
||||
{
|
||||
}
|
||||
|
||||
public function isCandidate(Document $resource, Document $migration): bool
|
||||
{
|
||||
return $this->predatesMigration($resource, $migration);
|
||||
}
|
||||
};
|
||||
$document = new Document([
|
||||
'$createdAt' => '2026-01-02T00:00:00.000+00:00',
|
||||
]);
|
||||
|
||||
$this->assertTrue($migration->isCandidate(new Document([
|
||||
'$createdAt' => '2026-01-01T00:00:00.000+00:00',
|
||||
]), $document));
|
||||
$this->assertFalse($migration->isCandidate(new Document([
|
||||
'$createdAt' => '2026-01-03T00:00:00.000+00:00',
|
||||
]), $document));
|
||||
$this->assertFalse($migration->isCandidate(new Document(), $document));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,326 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace Tests\Unit\Platform\Workers;
|
||||
|
||||
use Appwrite\Event\Publisher\Mail as MailPublisher;
|
||||
use Appwrite\Event\Publisher\Usage as UsagePublisher;
|
||||
use Appwrite\Event\Realtime;
|
||||
use Appwrite\Platform\Workers\Migrations;
|
||||
use Appwrite\Usage\Context;
|
||||
use PHPUnit\Framework\MockObject\MockObject;
|
||||
use PHPUnit\Framework\TestCase;
|
||||
use Utopia\Database\Document;
|
||||
use Utopia\Database\Validator\Authorization;
|
||||
use Utopia\Migration\Destination;
|
||||
use Utopia\Migration\Resource;
|
||||
use Utopia\Migration\Source;
|
||||
use Utopia\Queue\Publisher;
|
||||
use Utopia\Queue\Queue;
|
||||
|
||||
final class MigrationsTest extends TestCase
|
||||
{
|
||||
public function testSuccessHooksRunBeforeCompletedMigrationIsPersisted(): void
|
||||
{
|
||||
$events = [];
|
||||
$source = $this->createSourceMock();
|
||||
$destination = $this->createDestinationMock();
|
||||
|
||||
$destination
|
||||
->expects($this->once())
|
||||
->method('success')
|
||||
->willReturnCallback(static function () use (&$events): void {
|
||||
$events[] = 'destination:success';
|
||||
});
|
||||
$source
|
||||
->expects($this->once())
|
||||
->method('success')
|
||||
->willReturnCallback(static function () use (&$events): void {
|
||||
$events[] = 'source:success';
|
||||
});
|
||||
$source->expects($this->never())->method('error');
|
||||
$destination->expects($this->never())->method('error');
|
||||
|
||||
$migration = $this->createMigration();
|
||||
$processor = $this->createProcessor($source, $destination, $events);
|
||||
|
||||
$this->process($processor, $migration);
|
||||
|
||||
$this->assertSame('completed', $migration->getAttribute('status'));
|
||||
$this->assertSame('finished', $migration->getAttribute('stage'));
|
||||
$this->assertSame([
|
||||
'persist:processing:processing',
|
||||
'persist:processing:migrating',
|
||||
'destination:success',
|
||||
'source:success',
|
||||
'persist:completed:finished',
|
||||
], $events);
|
||||
}
|
||||
|
||||
public function testThrowingSourceSuccessHookPersistsFailedMigrationWithoutRerunningHooks(): void
|
||||
{
|
||||
$events = [];
|
||||
$source = $this->createSourceMock();
|
||||
$destination = $this->createDestinationMock();
|
||||
|
||||
$destination
|
||||
->expects($this->once())
|
||||
->method('success')
|
||||
->willReturnCallback(static function () use (&$events): void {
|
||||
$events[] = 'destination:success';
|
||||
});
|
||||
$source
|
||||
->expects($this->once())
|
||||
->method('success')
|
||||
->willReturnCallback(static function () use (&$events): void {
|
||||
$events[] = 'source:success';
|
||||
throw new \RuntimeException('Finalization failed');
|
||||
});
|
||||
$source
|
||||
->expects($this->once())
|
||||
->method('error')
|
||||
->willReturnCallback(static function () use (&$events): void {
|
||||
$events[] = 'source:error';
|
||||
});
|
||||
$destination
|
||||
->expects($this->once())
|
||||
->method('error')
|
||||
->willReturnCallback(static function () use (&$events): void {
|
||||
$events[] = 'destination:error';
|
||||
});
|
||||
|
||||
$migration = $this->createMigration();
|
||||
$processor = $this->createProcessor($source, $destination, $events);
|
||||
|
||||
$this->process($processor, $migration);
|
||||
|
||||
$this->assertSame('failed', $migration->getAttribute('status'));
|
||||
$this->assertSame('finished', $migration->getAttribute('stage'));
|
||||
$this->assertSame([
|
||||
'persist:processing:processing',
|
||||
'persist:processing:migrating',
|
||||
'destination:success',
|
||||
'source:success',
|
||||
'persist:failed:finished',
|
||||
'source:error',
|
||||
'destination:error',
|
||||
], $events);
|
||||
$this->assertNotContains('persist:completed:finished', $events);
|
||||
}
|
||||
|
||||
public function testNullResourceTypeUsesEmptyResourceSelector(): void
|
||||
{
|
||||
$events = [];
|
||||
$source = $this->createSourceMock();
|
||||
$destination = $this->createDestinationMock();
|
||||
|
||||
$destination
|
||||
->expects($this->once())
|
||||
->method('success')
|
||||
->willReturnCallback(static function () use (&$events): void {
|
||||
$events[] = 'destination:success';
|
||||
});
|
||||
$source
|
||||
->expects($this->once())
|
||||
->method('success')
|
||||
->willReturnCallback(static function () use (&$events): void {
|
||||
$events[] = 'source:success';
|
||||
});
|
||||
$source->expects($this->never())->method('error');
|
||||
$destination->expects($this->never())->method('error');
|
||||
|
||||
$migration = $this->createMigration(null);
|
||||
$processor = $this->createProcessor($source, $destination, $events);
|
||||
|
||||
$this->process($processor, $migration);
|
||||
|
||||
$this->assertSame('completed', $migration->getAttribute('status'));
|
||||
$this->assertSame([
|
||||
'persist:processing:processing',
|
||||
'persist:processing:migrating',
|
||||
'destination:success',
|
||||
'source:success',
|
||||
'persist:completed:finished',
|
||||
], $events);
|
||||
}
|
||||
|
||||
public function testResourceContextUsesCanonicalRelationFields(): void
|
||||
{
|
||||
$worker = new class () extends Migrations {
|
||||
/**
|
||||
* @return array{resourceId: string, resourceInternalId: string, resourceType: string, parentResourceId: string, parentResourceInternalId: string, parentResourceType: string}
|
||||
*/
|
||||
public function context(Document $migration): array
|
||||
{
|
||||
return $this->resolveResourceContext($migration);
|
||||
}
|
||||
};
|
||||
|
||||
$this->assertSame([
|
||||
'resourceId' => 'table-a',
|
||||
'resourceInternalId' => '201',
|
||||
'resourceType' => Resource::TYPE_COLLECTION,
|
||||
'parentResourceId' => 'database-a',
|
||||
'parentResourceInternalId' => '101',
|
||||
'parentResourceType' => Resource::TYPE_DATABASE,
|
||||
], $worker->context(new Document([
|
||||
'resourceId' => 'table-a',
|
||||
'resourceInternalId' => '201',
|
||||
'resourceType' => Resource::TYPE_COLLECTION,
|
||||
'parentResourceId' => 'database-a',
|
||||
'parentResourceInternalId' => '101',
|
||||
'parentResourceType' => Resource::TYPE_DATABASE,
|
||||
])));
|
||||
|
||||
$this->assertSame([
|
||||
'resourceId' => 'table-a',
|
||||
'resourceInternalId' => '',
|
||||
'resourceType' => Resource::TYPE_COLLECTION,
|
||||
'parentResourceId' => 'database-a',
|
||||
'parentResourceInternalId' => '',
|
||||
'parentResourceType' => Resource::TYPE_DATABASE,
|
||||
], $worker->context(new Document([
|
||||
'resourceId' => 'database-a:table-a',
|
||||
'resourceType' => Resource::TYPE_DATABASE,
|
||||
])));
|
||||
}
|
||||
|
||||
private function createSourceMock(): Source&MockObject
|
||||
{
|
||||
$target = $this->createMock(Source::class);
|
||||
$target->method('getErrors')->willReturn([]);
|
||||
$target->expects($this->once())->method('shutdown');
|
||||
$target->expects($this->once())->method('cleanUp');
|
||||
|
||||
return $target;
|
||||
}
|
||||
|
||||
private function createDestinationMock(): Destination&MockObject
|
||||
{
|
||||
$target = $this->createMock(Destination::class);
|
||||
$target->method('getErrors')->willReturn([]);
|
||||
$target->expects($this->once())->method('shutdown');
|
||||
$target->expects($this->once())->method('cleanUp');
|
||||
|
||||
return $target;
|
||||
}
|
||||
|
||||
private function createMigration(?string $resourceType = ''): Document
|
||||
{
|
||||
return new Document([
|
||||
'$id' => 'migration',
|
||||
'$sequence' => 1,
|
||||
'credentials' => [],
|
||||
'destination' => 'TestDestination',
|
||||
'options' => [],
|
||||
'resourceId' => '',
|
||||
'resourceType' => $resourceType,
|
||||
'resources' => [],
|
||||
'source' => 'TestSource',
|
||||
'stage' => 'pending',
|
||||
'status' => 'pending',
|
||||
]);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param array<string> $events
|
||||
*/
|
||||
private function createProcessor(Source $source, Destination $destination, array &$events): \Closure
|
||||
{
|
||||
$record = static function (string $event) use (&$events): void {
|
||||
$events[] = $event;
|
||||
};
|
||||
$worker = new class ($source, $destination, $record) extends Migrations {
|
||||
public function __construct(
|
||||
private readonly Source $migrationSource,
|
||||
private readonly Destination $migrationDestination,
|
||||
private readonly \Closure $record,
|
||||
) {
|
||||
}
|
||||
|
||||
public function process(
|
||||
Document $migration,
|
||||
Document $project,
|
||||
Realtime $queueForRealtime,
|
||||
MailPublisher $publisherForMails,
|
||||
Context $usage,
|
||||
UsagePublisher $publisherForUsage,
|
||||
Authorization $authorization,
|
||||
): void {
|
||||
$this->project = $project;
|
||||
$this->logError = static function (): void {
|
||||
};
|
||||
|
||||
$this->processMigration(
|
||||
$migration,
|
||||
$queueForRealtime,
|
||||
$publisherForMails,
|
||||
$usage,
|
||||
$publisherForUsage,
|
||||
[],
|
||||
$authorization,
|
||||
);
|
||||
}
|
||||
|
||||
#[\Override]
|
||||
protected function generateAPIKey(Document $project): string
|
||||
{
|
||||
return 'key';
|
||||
}
|
||||
|
||||
#[\Override]
|
||||
protected function processSource(Document $migration): Source
|
||||
{
|
||||
return $this->migrationSource;
|
||||
}
|
||||
|
||||
#[\Override]
|
||||
protected function processDestination(Document $migration): Destination
|
||||
{
|
||||
return $this->migrationDestination;
|
||||
}
|
||||
|
||||
#[\Override]
|
||||
protected function updateMigrationDocument(
|
||||
Document $migration,
|
||||
Document $project,
|
||||
Realtime $queueForRealtime,
|
||||
): Document {
|
||||
($this->record)('persist:'
|
||||
. $migration->getAttribute('status')
|
||||
. ':'
|
||||
. $migration->getAttribute('stage'));
|
||||
|
||||
return $migration;
|
||||
}
|
||||
};
|
||||
|
||||
return $worker->process(...);
|
||||
}
|
||||
|
||||
private function process(\Closure $processor, Document $migration): void
|
||||
{
|
||||
$publisher = $this->createStub(Publisher::class);
|
||||
$queue = new Queue('test');
|
||||
$host = \getenv('_APP_MIGRATION_HOST');
|
||||
\putenv('_APP_MIGRATION_HOST=localhost');
|
||||
|
||||
try {
|
||||
$processor(
|
||||
$migration,
|
||||
new Document([
|
||||
'$id' => 'project',
|
||||
'$sequence' => 1,
|
||||
]),
|
||||
new Realtime(),
|
||||
new MailPublisher($publisher, $queue),
|
||||
new Context(),
|
||||
new UsagePublisher($publisher, $queue),
|
||||
new Authorization(),
|
||||
);
|
||||
} finally {
|
||||
\putenv($host === false ? '_APP_MIGRATION_HOST' : '_APP_MIGRATION_HOST=' . $host);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace Tests\Unit\Utopia\Database\Validator\Queries;
|
||||
|
||||
use Appwrite\Utopia\Database\Validator\Queries\Migrations;
|
||||
use PHPUnit\Framework\TestCase;
|
||||
use Utopia\Database\Query;
|
||||
|
||||
final class MigrationsTest extends TestCase
|
||||
{
|
||||
public function testCanonicalResourceFieldsCanBeQueried(): void
|
||||
{
|
||||
$validator = new Migrations();
|
||||
|
||||
foreach ([
|
||||
'resourceId',
|
||||
'resourceInternalId',
|
||||
'resourceType',
|
||||
'parentResourceId',
|
||||
'parentResourceInternalId',
|
||||
'parentResourceType',
|
||||
'destinationResourceId',
|
||||
'destinationResourceInternalId',
|
||||
'destinationResourceType',
|
||||
] as $attribute) {
|
||||
$query = Query::equal($attribute, ['value']);
|
||||
|
||||
$this->assertTrue($validator->isValid([$query]), $validator->getDescription());
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace Tests\Unit\Utopia\Request\Filters;
|
||||
|
||||
use Appwrite\Utopia\Request\Filters\V27;
|
||||
use PHPUnit\Framework\TestCase;
|
||||
|
||||
final class V27Test extends TestCase
|
||||
{
|
||||
public function testSplitsLegacyMigrationResource(): void
|
||||
{
|
||||
$filter = new V27();
|
||||
|
||||
$result = $filter->parse([
|
||||
'resourceId' => 'database:collection',
|
||||
], 'migrations.createCSVImport');
|
||||
|
||||
$this->assertSame([
|
||||
'databaseId' => 'database',
|
||||
'collectionId' => 'collection',
|
||||
], $result);
|
||||
}
|
||||
|
||||
public function testExplicitResourceIdsTakePrecedence(): void
|
||||
{
|
||||
$filter = new V27();
|
||||
|
||||
$result = $filter->parse([
|
||||
'resourceId' => 'legacyDatabase:legacyCollection',
|
||||
'databaseId' => 'database',
|
||||
'collectionId' => 'collection',
|
||||
], 'migrations.createJSONExport');
|
||||
|
||||
$this->assertSame([
|
||||
'databaseId' => 'database',
|
||||
'collectionId' => 'collection',
|
||||
], $result);
|
||||
}
|
||||
|
||||
public function testLeavesMalformedResourceForValidation(): void
|
||||
{
|
||||
$filter = new V27();
|
||||
$content = ['resourceId' => 'database'];
|
||||
|
||||
$this->assertSame($content, $filter->parse($content, 'migrations.createCSVExport'));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,88 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace Tests\Unit\Utopia\Response\Filters;
|
||||
|
||||
use Appwrite\Utopia\Response;
|
||||
use Appwrite\Utopia\Response\Filters\V27;
|
||||
use PHPUnit\Framework\TestCase;
|
||||
|
||||
final class V27Test extends TestCase
|
||||
{
|
||||
public function testRestoresLegacyMigrationResourceShape(): void
|
||||
{
|
||||
$filter = new V27();
|
||||
|
||||
$result = $filter->parse([
|
||||
'$id' => 'migration',
|
||||
'resourceId' => 'collection',
|
||||
'resourceInternalId' => '20',
|
||||
'resourceType' => 'collection',
|
||||
'parentResourceId' => 'database',
|
||||
'parentResourceInternalId' => '10',
|
||||
'parentResourceType' => 'database',
|
||||
'destinationResourceId' => 'destination',
|
||||
'destinationResourceInternalId' => '30',
|
||||
'destinationResourceType' => 'database',
|
||||
], Response::MODEL_MIGRATION);
|
||||
|
||||
$this->assertSame([
|
||||
'$id' => 'migration',
|
||||
'resourceId' => 'database:collection',
|
||||
'resourceType' => 'database',
|
||||
], $result);
|
||||
}
|
||||
|
||||
public function testPreservesSingleRootMigration(): void
|
||||
{
|
||||
$filter = new V27();
|
||||
|
||||
$result = $filter->parse([
|
||||
'$id' => 'migration',
|
||||
'resourceId' => 'database',
|
||||
'resourceInternalId' => '10',
|
||||
'resourceType' => 'database',
|
||||
'parentResourceId' => '',
|
||||
'parentResourceInternalId' => '',
|
||||
'parentResourceType' => '',
|
||||
'destinationResourceId' => '',
|
||||
'destinationResourceInternalId' => '',
|
||||
'destinationResourceType' => '',
|
||||
], Response::MODEL_MIGRATION);
|
||||
|
||||
$this->assertSame([
|
||||
'$id' => 'migration',
|
||||
'resourceId' => 'database',
|
||||
'resourceType' => 'database',
|
||||
], $result);
|
||||
}
|
||||
|
||||
public function testTransformsMigrationList(): void
|
||||
{
|
||||
$filter = new V27();
|
||||
|
||||
$result = $filter->parse([
|
||||
'total' => 1,
|
||||
'migrations' => [[
|
||||
'$id' => 'migration',
|
||||
'resourceId' => 'collection',
|
||||
'resourceType' => 'collection',
|
||||
'parentResourceId' => 'database',
|
||||
'parentResourceType' => 'database',
|
||||
'destinationResourceId' => 'destination',
|
||||
'destinationResourceInternalId' => '30',
|
||||
'destinationResourceType' => 'database',
|
||||
]],
|
||||
], Response::MODEL_MIGRATION_LIST);
|
||||
|
||||
$this->assertSame([
|
||||
'total' => 1,
|
||||
'migrations' => [[
|
||||
'$id' => 'migration',
|
||||
'resourceId' => 'database:collection',
|
||||
'resourceType' => 'database',
|
||||
]],
|
||||
], $result);
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user