Merge pull request #139 from utopia-php/multitype-db

This commit is contained in:
Jake Barnby authored and GitHub committed 2026-03-11 01:50:35 +13:00
commit b5626b9b05
19 files changed
+1568 -587

No files matched your search

+1
View File
@@ -46,6 +46,7 @@ class Cache
case Resource::TYPE_TABLE:
case Resource::TYPE_COLLECTION:
/** @var Table $resource */
$keys[] = $resource->getDatabase()->getType();
$keys[] = $resource->getDatabase()->getSequence();
break;
+226 -176
View File
@@ -41,6 +41,7 @@ use Utopia\Migration\Resources\Auth\Hash;
use Utopia\Migration\Resources\Auth\Membership;
use Utopia\Migration\Resources\Auth\Team;
use Utopia\Migration\Resources\Auth\User;
use Utopia\Migration\Resources\Database\Attribute;
use Utopia\Migration\Resources\Database\Column;
use Utopia\Migration\Resources\Database\Database;
use Utopia\Migration\Resources\Database\Index;
@@ -74,6 +75,11 @@ class Appwrite extends Destination
private Teams $teams;
private Users $users;
/**
* @var callable(UtopiaDocument $database): UtopiaDatabase
*/
protected $getDatabasesDB;
/**
* @var array<UtopiaDocument>
*/
@@ -83,14 +89,16 @@ class Appwrite extends Destination
* @param string $project
* @param string $endpoint
* @param string $key
* @param UtopiaDatabase $database
* @param UtopiaDatabase $dbForProject
* @param callable(UtopiaDocument $database):UtopiaDatabase $getDatabasesDB
* @param array<array<string, mixed>> $collectionStructure
*/
public function __construct(
string $project,
string $endpoint,
string $key,
protected UtopiaDatabase $database,
protected UtopiaDatabase $dbForProject,
callable $getDatabasesDB,
protected array $collectionStructure
) {
$this->project = $project;
@@ -108,6 +116,8 @@ class Appwrite extends Destination
$this->storage = new Storage($this->client);
$this->teams = new Teams($this->client);
$this->users = new Users($this->client);
$this->getDatabasesDB = $getDatabasesDB;
}
public static function getName(): string
@@ -128,6 +138,8 @@ class Appwrite extends Destination
// Database
Resource::TYPE_DATABASE,
Resource::TYPE_DATABASE_DOCUMENTSDB,
Resource::TYPE_DATABASE_VECTORSDB,
Resource::TYPE_TABLE,
Resource::TYPE_COLUMN,
Resource::TYPE_INDEX,
@@ -304,7 +316,7 @@ class Appwrite extends Destination
$isLast = $index === $total - 1;
try {
$this->database->setPreserveDates(true);
$this->dbForProject->setPreserveDates(true);
$responseResource = match ($resource->getGroup()) {
Transfer::GROUP_DATABASES => $this->importDatabaseResource($resource, $isLast),
@@ -329,7 +341,7 @@ class Appwrite extends Destination
$responseResource = $resource;
} finally {
$this->database->setPreserveDates(false);
$this->dbForProject->setPreserveDates(false);
}
$this->cache->update($responseResource);
@@ -347,18 +359,20 @@ class Appwrite extends Destination
{
switch ($resource->getName()) {
case Resource::TYPE_DATABASE:
case Resource::TYPE_DATABASE_DOCUMENTSDB:
case Resource::TYPE_DATABASE_VECTORSDB:
/** @var Database $resource */
$success = $this->createDatabase($resource);
break;
case Resource::TYPE_TABLE:
case Resource::TYPE_COLLECTION:
/** @var Table $resource */
$success = $this->createTable($resource);
$success = $this->createEntity($resource);
break;
case Resource::TYPE_COLUMN:
case Resource::TYPE_ATTRIBUTE:
/** @var Column $resource */
$success = $this->createColumn($resource);
$success = $this->createField($resource);
break;
case Resource::TYPE_INDEX:
/** @var Index $resource */
@@ -367,7 +381,7 @@ class Appwrite extends Destination
case Resource::TYPE_ROW:
case Resource::TYPE_DOCUMENT:
/** @var Row $resource */
$success = $this->createRow($resource, $isLast);
$success = $this->createRecord($resource, $isLast);
break;
default:
$success = false;
@@ -406,7 +420,7 @@ class Appwrite extends Destination
$createdAt = $this->normalizeDateTime($resource->getCreatedAt());
$updatedAt = $this->normalizeDateTime($resource->getUpdatedAt(), $createdAt);
$database = $this->database->createDocument('databases', new UtopiaDocument([
$database = $this->dbForProject->createDocument('databases', new UtopiaDocument([
'$id' => $resource->getId(),
'name' => $resource->getDatabaseName(),
'enabled' => $resource->getEnabled(),
@@ -415,6 +429,8 @@ class Appwrite extends Destination
'$updatedAt' => $updatedAt,
'originalId' => empty($resource->getOriginalId()) ? null : $resource->getOriginalId(),
'type' => empty($resource->getType()) ? 'legacy' : $resource->getType(),
// source and destination can be in different location
'database' => $resource->getDatabase()
]));
$resource->setSequence($database->getSequence());
@@ -429,7 +445,7 @@ class Appwrite extends Destination
$this->collectionStructure['indexes']
);
$this->database->createCollection(
$this->dbForProject->createCollection(
'database_' . $database->getSequence(),
$columns,
$indexes
@@ -444,7 +460,7 @@ class Appwrite extends Destination
* @throws StructureException
* @throws Exception
*/
protected function createTable(Table $resource): bool
protected function createEntity(Table $resource): bool
{
if ($resource->getId() == 'unique()') {
$resource->setId(ID::unique());
@@ -461,7 +477,7 @@ class Appwrite extends Destination
);
}
$database = $this->database->getDocument(
$database = $this->dbForProject->getDocument(
'databases',
$resource->getDatabase()->getId()
);
@@ -478,7 +494,14 @@ class Appwrite extends Destination
$createdAt = $this->normalizeDateTime($resource->getCreatedAt());
$updatedAt = $this->normalizeDateTime($resource->getUpdatedAt(), $createdAt);
$table = $this->database->createDocument('database_' . $database->getSequence(), new UtopiaDocument([
$dbForDatabases = ($this->getDatabasesDB)($database);
// passing null in creates only creates the metadata collection
if (!$dbForDatabases->exists(null, UtopiaDatabase::METADATA)) {
$dbForDatabases->create();
}
$table = $this->dbForProject->createDocument('database_' . $database->getSequence(), new UtopiaDocument([
'$id' => $resource->getId(),
'databaseInternalId' => $database->getSequence(),
'databaseId' => $resource->getDatabase()->getId(),
@@ -493,7 +516,7 @@ class Appwrite extends Destination
$resource->setSequence($table->getSequence());
$this->database->createCollection(
$dbForDatabases->createCollection(
'database_' . $database->getSequence() . '_collection_' . $resource->getSequence(),
permissions: $resource->getPermissions(),
documentSecurity: $resource->getRowSecurity()
@@ -507,19 +530,27 @@ class Appwrite extends Destination
* @throws \Exception
* @throws \Throwable
*/
protected function createColumn(Column $resource): bool
protected function createField(Column|Attribute $resource): bool
{
if ($resource->getTable()->getDatabase()->getType() === Resource::TYPE_DATABASE_DOCUMENTSDB) {
$resource->setStatus(Resource::STATUS_SKIPPED, 'Columns not supported for DocumentsDB');
return false;
}
// column will be matching attribute as well
// column type will be matching attribute type as well
$type = match ($resource->getType()) {
Column::TYPE_DATETIME => UtopiaDatabase::VAR_DATETIME,
Column::TYPE_BOOLEAN => UtopiaDatabase::VAR_BOOLEAN,
Column::TYPE_INTEGER => UtopiaDatabase::VAR_INTEGER,
Column::TYPE_FLOAT => UtopiaDatabase::VAR_FLOAT,
Column::TYPE_RELATIONSHIP => UtopiaDatabase::VAR_RELATIONSHIP,
Column::TYPE_STRING,
Column::TYPE_IP,
Column::TYPE_EMAIL,
Column::TYPE_URL,
Column::TYPE_ENUM => UtopiaDatabase::VAR_STRING,
Column::TYPE_POINT => UtopiaDatabase::VAR_POINT,
Column::TYPE_LINE => UtopiaDatabase::VAR_LINESTRING,
Column::TYPE_POLYGON => UtopiaDatabase::VAR_POLYGON,
@@ -527,10 +558,13 @@ class Appwrite extends Destination
Column::TYPE_VARCHAR => UtopiaDatabase::VAR_VARCHAR,
Column::TYPE_MEDIUMTEXT => UtopiaDatabase::VAR_MEDIUMTEXT,
Column::TYPE_LONGTEXT => UtopiaDatabase::VAR_LONGTEXT,
default => throw new \Exception('Invalid resource type '.$resource->getType(), Exception::CODE_VALIDATION),
Column::TYPE_OBJECT => UtopiaDatabase::VAR_OBJECT,
Column::TYPE_VECTOR => UtopiaDatabase::VAR_VECTOR,
default => throw new \Exception('Invalid resource type ' . $resource->getType(), Exception::CODE_VALIDATION),
};
$database = $this->database->getDocument(
$database = $this->dbForProject->getDocument(
'databases',
$resource->getTable()->getDatabase()->getId(),
);
@@ -544,7 +578,7 @@ class Appwrite extends Destination
);
}
$table = $this->database->getDocument(
$table = $this->dbForProject->getDocument(
'database_' . $database->getSequence(),
$resource->getTable()->getId(),
);
@@ -589,7 +623,7 @@ class Appwrite extends Destination
if ($type === UtopiaDatabase::VAR_RELATIONSHIP) {
$resource->getOptions()['side'] = UtopiaDatabase::RELATION_SIDE_PARENT;
$relatedTable = $this->database->getDocument(
$relatedTable = $this->dbForProject->getDocument(
'database_' . $database->getSequence(),
$resource->getOptions()['relatedCollection']
);
@@ -605,7 +639,7 @@ class Appwrite extends Destination
$createdAt = $this->normalizeDateTime($resource->getCreatedAt());
$updatedAt = $this->normalizeDateTime($resource->getUpdatedAt(), $createdAt);
$dbForDatabases = ($this->getDatabasesDB)($database);
try {
$column = new UtopiaDocument([
'$id' => ID::custom($database->getSequence() . '_' . $table->getSequence() . '_' . $resource->getKey()),
@@ -629,9 +663,9 @@ class Appwrite extends Destination
'$updatedAt' => $updatedAt,
]);
$this->database->checkAttribute($table, $column);
$this->dbForProject->checkAttribute($table, $column);
$column = $this->database->createDocument('attributes', $column);
$column = $this->dbForProject->createDocument('attributes', $column);
} catch (DuplicateException) {
throw new Exception(
resourceName: $resource->getName(),
@@ -647,13 +681,13 @@ class Appwrite extends Destination
message: 'Attribute limit exceeded',
);
} catch (\Throwable $e) {
$this->database->purgeCachedDocument('database_' . $database->getSequence(), $table->getId());
$this->database->purgeCachedCollection('database_' . $database->getSequence() . '_collection_' . $table->getSequence());
$this->dbForProject->purgeCachedDocument('database_' . $database->getSequence(), $table->getId());
$dbForDatabases->purgeCachedCollection('database_' . $database->getSequence() . '_collection_' . $table->getSequence());
throw $e;
}
$this->database->purgeCachedDocument('database_' . $database->getSequence(), $table->getId());
$this->database->purgeCachedCollection('database_' . $database->getSequence() . '_collection_' . $table->getSequence());
$this->dbForProject->purgeCachedDocument('database_' . $database->getSequence(), $table->getId());
$dbForDatabases->purgeCachedCollection('database_' . $database->getSequence() . '_collection_' . $table->getSequence());
$options = $resource->getOptions();
$twoWayKey = null;
@@ -687,9 +721,9 @@ class Appwrite extends Destination
'$updatedAt' => $updatedAt,
]);
$this->database->createDocument('attributes', $twoWayAttribute);
$this->dbForProject->createDocument('attributes', $twoWayAttribute);
} catch (DuplicateException) {
$this->database->deleteDocument('attributes', $column->getId());
$this->dbForProject->deleteDocument('attributes', $column->getId());
throw new Exception(
resourceName: $resource->getName(),
@@ -698,7 +732,7 @@ class Appwrite extends Destination
message: 'Attribute already exists',
);
} catch (LimitException) {
$this->database->deleteDocument('attributes', $column->getId());
$this->dbForProject->deleteDocument('attributes', $column->getId());
throw new Exception(
resourceName: $resource->getName(),
@@ -707,8 +741,8 @@ class Appwrite extends Destination
message: 'Column limit exceeded',
);
} catch (\Throwable $e) {
$this->database->purgeCachedDocument('database_' . $database->getSequence(), $relatedTable->getId());
$this->database->purgeCachedCollection('database_' . $database->getSequence() . '_collection_' . $relatedTable->getSequence());
$this->dbForProject->purgeCachedDocument('database_' . $database->getSequence(), $relatedTable->getId());
$dbForDatabases->purgeCachedCollection('database_' . $database->getSequence() . '_collection_' . $relatedTable->getSequence());
throw $e;
}
}
@@ -716,7 +750,7 @@ class Appwrite extends Destination
try {
switch ($type) {
case UtopiaDatabase::VAR_RELATIONSHIP:
if (!$this->database->createRelationship(
if (!$dbForDatabases->createRelationship(
collection: 'database_' . $database->getSequence() . '_collection_' . $table->getSequence(),
relatedCollection: 'database_' . $database->getSequence() . '_collection_' . $relatedTable->getSequence(),
type: $options['relationType'],
@@ -734,7 +768,7 @@ class Appwrite extends Destination
}
break;
default:
if (!$this->database->createAttribute(
if (!$dbForDatabases->createAttribute(
'database_' . $database->getSequence() . '_collection_' . $table->getSequence(),
$resource->getKey(),
$type,
@@ -751,10 +785,10 @@ class Appwrite extends Destination
}
}
} catch (\Throwable) {
$this->database->deleteDocument('attributes', $column->getId());
$this->dbForProject->deleteDocument('attributes', $column->getId());
if (isset($twoWayAttribute)) {
$this->database->deleteDocument('attributes', $twoWayAttribute->getId());
$this->dbForProject->deleteDocument('attributes', $twoWayAttribute->getId());
}
throw new Exception(
@@ -766,11 +800,11 @@ class Appwrite extends Destination
}
if ($type === UtopiaDatabase::VAR_RELATIONSHIP && $options['twoWay']) {
$this->database->purgeCachedDocument('database_' . $database->getSequence(), $relatedTable->getId());
$this->dbForProject->purgeCachedDocument('database_' . $database->getSequence(), $relatedTable->getId());
}
$this->database->purgeCachedDocument('database_' . $database->getSequence(), $table->getId());
$this->database->purgeCachedCollection('database_' . $database->getSequence() . '_collection_' . $table->getSequence());
$this->dbForProject->purgeCachedDocument('database_' . $database->getSequence(), $table->getId());
$dbForDatabases->purgeCachedCollection('database_' . $database->getSequence() . '_collection_' . $table->getSequence());
return true;
}
@@ -781,7 +815,7 @@ class Appwrite extends Destination
*/
protected function createIndex(Index $resource): bool
{
$database = $this->database->getDocument(
$database = $this->dbForProject->getDocument(
'databases',
$resource->getTable()->getDatabase()->getId(),
);
@@ -794,7 +828,7 @@ class Appwrite extends Destination
);
}
$table = $this->database->getDocument(
$table = $this->dbForProject->getDocument(
'database_' . $database->getSequence(),
$resource->getTable()->getId(),
);
@@ -806,13 +840,14 @@ class Appwrite extends Destination
message: 'Table not found',
);
}
$dbForDatabases = ($this->getDatabasesDB)($database);
$count = $this->database->count('indexes', [
$count = $this->dbForProject->count('indexes', [
Query::equal('collectionInternalId', [$table->getSequence()]),
Query::equal('databaseInternalId', [$database->getSequence()])
], $this->database->getLimitForIndexes());
], $dbForDatabases->getLimitForIndexes());
if ($count >= $this->database->getLimitForIndexes()) {
if ($count >= $dbForDatabases->getLimitForIndexes()) {
throw new Exception(
resourceName: $resource->getName(),
resourceGroup: $resource->getGroup(),
@@ -821,101 +856,11 @@ class Appwrite extends Destination
);
}
/**
* @var array<UtopiaDocument> $tableColumns
*/
$tableColumns = $table->getAttribute('attributes', []);
/**
* @var array<UtopiaDocument> $tableIndexes
*/
$tableIndexes = $table->getAttribute('indexes', []);
$oldColumns = \array_map(
fn ($attr) => $attr->getArrayCopy(),
$tableColumns
);
$oldColumns[] = [
'key' => '$id',
'type' => UtopiaDatabase::VAR_STRING,
'status' => 'available',
'required' => true,
'array' => false,
'default' => null,
'size' => UtopiaDatabase::LENGTH_KEY
];
$oldColumns[] = [
'key' => '$createdAt',
'type' => UtopiaDatabase::VAR_DATETIME,
'status' => 'available',
'signed' => false,
'required' => false,
'array' => false,
'default' => null,
'size' => 0
];
$oldColumns[] = [
'key' => '$updatedAt',
'type' => UtopiaDatabase::VAR_DATETIME,
'status' => 'available',
'signed' => false,
'required' => false,
'array' => false,
'default' => null,
'size' => 0
];
// Lengths hidden by default
$lengths = [];
foreach ($resource->getColumns() as $i => $column) {
// find attribute metadata in collection document
$columnIndex = \array_search(
$column,
\array_column($oldColumns, 'key')
);
if ($columnIndex === false) {
throw new Exception(
resourceName: $resource->getName(),
resourceGroup: $resource->getGroup(),
resourceId: $resource->getId(),
message: 'Column not found in table: ' . $column,
);
}
$columnStatus = $oldColumns[$columnIndex]['status'];
$columnType = $oldColumns[$columnIndex]['type'];
$columnSize = $oldColumns[$columnIndex]['size'];
$columnArray = $oldColumns[$columnIndex]['array'] ?? false;
if ($columnType === UtopiaDatabase::VAR_RELATIONSHIP) {
throw new Exception(
resourceName: $resource->getName(),
resourceGroup: $resource->getGroup(),
resourceId: $resource->getId(),
message: 'Relationship columns are not supported in indexes',
);
}
// Ensure attribute is available
if ($columnStatus !== 'available') {
throw new Exception(
resourceName: $resource->getName(),
resourceGroup: $resource->getGroup(),
resourceId: $resource->getId(),
message: 'Column not available: ' . $column,
);
}
$lengths[$i] = null;
if ($columnArray === true) {
$lengths[$i] = UtopiaDatabase::MAX_ARRAY_INDEX_LENGTH;
}
if ($dbForDatabases->getAdapter()->getSupportForAttributes()) {
$this->validateFieldsForIndexes($resource, $table, $lengths);
}
$createdAt = $this->normalizeDateTime($resource->getCreatedAt());
@@ -937,20 +882,33 @@ class Appwrite extends Destination
'$updatedAt' => $updatedAt,
]);
/**
* @var array<UtopiaDocument> $tableColumns
*/
$tableColumns = $table->getAttribute('attributes', []);
$tableIndexes = $table->getAttribute('indexes', []);
$validator = new IndexValidator(
$tableColumns,
$tableIndexes,
$this->database->getAdapter()->getMaxIndexLength(),
$this->database->getAdapter()->getInternalIndexesKeys(),
$this->database->getAdapter()->getSupportForIndexArray(),
$this->database->getAdapter()->getSupportForSpatialIndexNull(),
$this->database->getAdapter()->getSupportForSpatialIndexOrder(),
$this->database->getAdapter()->getSupportForVectors(),
$this->database->getAdapter()->getSupportForAttributes(),
$this->database->getAdapter()->getSupportForMultipleFulltextIndexes(),
$this->database->getAdapter()->getSupportForIdenticalIndexes(),
$dbForDatabases->getAdapter()->getMaxIndexLength(),
$dbForDatabases->getAdapter()->getInternalIndexesKeys(),
$dbForDatabases->getAdapter()->getSupportForIndexArray(),
$dbForDatabases->getAdapter()->getSupportForSpatialIndexNull(),
$dbForDatabases->getAdapter()->getSupportForSpatialIndexOrder(),
$dbForDatabases->getAdapter()->getSupportForVectors(),
$dbForDatabases->getAdapter()->getSupportForAttributes(),
$dbForDatabases->getAdapter()->getSupportForMultipleFulltextIndexes(),
$dbForDatabases->getAdapter()->getSupportForIdenticalIndexes(),
$dbForDatabases->getAdapter()->getSupportForObjectIndexes(),
$dbForDatabases->getAdapter()->getSupportForTrigramIndex(),
$dbForDatabases->getAdapter()->getSupportForSpatialAttributes(),
$dbForDatabases->getAdapter()->getSupportForIndex(),
$dbForDatabases->getAdapter()->getSupportForUniqueIndex(),
$dbForDatabases->getAdapter()->getSupportForFulltextIndex()
);
if (!$validator->isValid($index)) {
throw new Exception(
resourceName: $resource->getName(),
@@ -960,10 +918,10 @@ class Appwrite extends Destination
);
}
$index = $this->database->createDocument('indexes', $index);
$index = $this->dbForProject->createDocument('indexes', $index);
try {
$result = $this->database->createIndex(
$result = $dbForDatabases->createIndex(
'database_' . $database->getSequence() . '_collection_' . $table->getSequence(),
$resource->getKey(),
$resource->getType(),
@@ -981,7 +939,7 @@ class Appwrite extends Destination
);
}
} catch (\Throwable $th) {
$this->database->deleteDocument('indexes', $index->getId());
$this->dbForProject->deleteDocument('indexes', $index->getId());
throw new Exception(
resourceName: $resource->getName(),
@@ -991,7 +949,7 @@ class Appwrite extends Destination
);
}
$this->database->purgeCachedDocument(
$this->dbForProject->purgeCachedDocument(
'database_' . $database->getSequence(),
$table->getId()
);
@@ -1005,7 +963,7 @@ class Appwrite extends Destination
* @throws StructureException
* @throws Exception
*/
protected function createRow(Row $resource, bool $isLast): bool
protected function createRecord(Row $resource, bool $isLast): bool
{
if ($resource->getId() == 'unique()') {
$resource->setId(ID::unique());
@@ -1025,7 +983,7 @@ class Appwrite extends Destination
// Check if document has already been created
$exists = \array_key_exists(
$resource->getId(),
$this->cache->get(Resource::TYPE_ROW)
$this->cache->get($resource->getName())
);
if ($exists) {
@@ -1070,45 +1028,46 @@ class Appwrite extends Destination
if ($isLast) {
try {
$database = $this->database->getDocument(
$database = $this->dbForProject->getDocument(
'databases',
$resource->getTable()->getDatabase()->getId(),
);
$table = $this->database->getDocument(
$table = $this->dbForProject->getDocument(
'database_' . $database->getSequence(),
$resource->getTable()->getId(),
);
$databaseInternalId = $database->getSequence();
$tableInternalId = $table->getSequence();
$dbForDatabases = ($this->getDatabasesDB)($database);
/**
* This is in case an attribute was deleted from Appwrite attributes collection but was not deleted from the table
* When creating an archive we select * which will include orphan attribute from the schema
*/
foreach ($this->rowBuffer as $row) {
foreach ($row as $key => $value) {
if (\str_starts_with($key, '$')) {
continue;
}
/** @var \Utopia\Database\Document $attribute */
$found = false;
foreach ($table->getAttribute('attributes', []) as $attribute) {
if ($attribute->getAttribute('key') == $key) {
$found = true;
break;
if ($dbForDatabases->getAdapter()->getSupportForAttributes()) {
foreach ($this->rowBuffer as $row) {
foreach ($row as $key => $value) {
if (\str_starts_with($key, '$')) {
continue;
}
}
if (! $found) {
$row->removeAttribute($key);
/** @var \Utopia\Database\Document $attribute */
$found = false;
foreach ($table->getAttribute('attributes', []) as $attribute) {
if ($attribute->getAttribute('key') == $key) {
$found = true;
break;
}
}
if (! $found) {
$row->removeAttribute($key);
}
}
}
}
$this->database->skipRelationshipsExistCheck(fn () => $this->database->createDocuments(
$dbForDatabases->skipRelationshipsExistCheck(fn () => $dbForDatabases->createDocuments(
'database_' . $databaseInternalId . '_collection_' . $tableInternalId,
$this->rowBuffer
));
@@ -1908,8 +1867,8 @@ class Appwrite extends Destination
protected function createSubscriber(Subscriber $resource): void
{
$target = match ($resource->getProviderType()) {
'push' => $this->database->getDocument('targets', $resource->getTargetId()),
'email', 'sms' => $this->database->findOne('targets', [
'push' => $this->dbForProject->getDocument('targets', $resource->getTargetId()),
'email', 'sms' => $this->dbForProject->findOne('targets', [
Query::equal('userId', [$resource->getUserId()]),
Query::equal('providerType', [$resource->getProviderType()]),
]),
@@ -1920,12 +1879,12 @@ class Appwrite extends Destination
throw new \Exception('Target not found for subscriber: ' . $resource->getId());
}
$topic = $this->database->getDocument('topics', $resource->getTopicId());
$topic = $this->dbForProject->getDocument('topics', $resource->getTopicId());
if ($topic->isEmpty()) {
throw new \Exception('Topic not found: ' . $resource->getTopicId());
}
$user = $this->database->getDocument('users', $resource->getUserId());
$user = $this->dbForProject->getDocument('users', $resource->getUserId());
if ($user->isEmpty()) {
throw new \Exception('User not found: ' . $resource->getUserId());
}
@@ -1933,7 +1892,7 @@ class Appwrite extends Destination
$createdAt = $this->normalizeDateTime($resource->getCreatedAt());
$updatedAt = $this->normalizeDateTime($resource->getUpdatedAt(), $createdAt);
$this->database->createDocument('subscribers', new UtopiaDocument([
$this->dbForProject->createDocument('subscribers', new UtopiaDocument([
'$id' => $resource->getId(),
'$createdAt' => $createdAt,
'$updatedAt' => $updatedAt,
@@ -1963,7 +1922,7 @@ class Appwrite extends Destination
default => throw new \Exception('Unknown provider type: ' . $resource->getProviderType()),
};
$this->database->increaseDocumentAttribute('topics', $resource->getTopicId(), $totalAttribute);
$this->dbForProject->increaseDocumentAttribute('topics', $resource->getTopicId(), $totalAttribute);
}
/**
@@ -2002,7 +1961,7 @@ class Appwrite extends Destination
default => '',
};
$this->database->createDocument('messages', new UtopiaDocument([
$this->dbForProject->createDocument('messages', new UtopiaDocument([
'$id' => $resource->getId(),
'$createdAt' => $createdAt,
'$updatedAt' => $updatedAt,
@@ -2157,4 +2116,95 @@ class Appwrite extends Destination
return $deployment;
}
private function validateFieldsForIndexes(Index $resource, UtopiaDocument $table, array &$lengths)
{
/**
* @var array<UtopiaDocument> $tableColumns
*/
$tableColumns = $table->getAttribute('attributes', []);
$oldColumns = \array_map(
fn ($attr) => $attr->getArrayCopy(),
$tableColumns
);
$oldColumns[] = [
'key' => '$id',
'type' => UtopiaDatabase::VAR_STRING,
'status' => 'available',
'required' => true,
'array' => false,
'default' => null,
'size' => UtopiaDatabase::LENGTH_KEY
];
$oldColumns[] = [
'key' => '$createdAt',
'type' => UtopiaDatabase::VAR_DATETIME,
'status' => 'available',
'signed' => false,
'required' => false,
'array' => false,
'default' => null,
'size' => 0
];
$oldColumns[] = [
'key' => '$updatedAt',
'type' => UtopiaDatabase::VAR_DATETIME,
'status' => 'available',
'signed' => false,
'required' => false,
'array' => false,
'default' => null,
'size' => 0
];
foreach ($resource->getColumns() as $i => $column) {
// find attribute metadata in collection document
$columnIndex = \array_search(
$column,
\array_column($oldColumns, 'key')
);
if ($columnIndex === false) {
throw new Exception(
resourceName: $resource->getName(),
resourceGroup: $resource->getGroup(),
resourceId: $resource->getId(),
message: 'Column not found in table: ' . $column,
);
}
$columnStatus = $oldColumns[$columnIndex]['status'];
$columnType = $oldColumns[$columnIndex]['type'];
$columnArray = $oldColumns[$columnIndex]['array'] ?? false;
if ($columnType === UtopiaDatabase::VAR_RELATIONSHIP) {
throw new Exception(
resourceName: $resource->getName(),
resourceGroup: $resource->getGroup(),
resourceId: $resource->getId(),
message: 'Relationship columns are not supported in indexes',
);
}
// Ensure attribute is available
if ($columnStatus !== 'available') {
throw new Exception(
resourceName: $resource->getName(),
resourceGroup: $resource->getGroup(),
resourceId: $resource->getId(),
message: 'Column not available: ' . $column,
);
}
$lengths[$i] = null;
if ($columnArray === true) {
$lengths[$i] = UtopiaDatabase::MAX_ARRAY_INDEX_LENGTH;
}
}
}
}
+42
View File
@@ -30,6 +30,13 @@ abstract class Resource implements \JsonSerializable
public const TYPE_DATABASE = 'database';
public const TYPE_DATABASE_LEGACY = 'legacy';
public const TYPE_DATABASE_TABLESDB = 'tablesdb';
public const TYPE_DATABASE_DOCUMENTSDB = 'documentsdb';
public const TYPE_DATABASE_VECTORSDB = 'vectorsdb';
public const TYPE_ROW = 'row';
public const TYPE_FILE = 'file';
@@ -84,6 +91,8 @@ abstract class Resource implements \JsonSerializable
self::TYPE_BUCKET,
self::TYPE_TABLE,
self::TYPE_DATABASE,
self::TYPE_DATABASE_VECTORSDB,
self::TYPE_DATABASE_DOCUMENTSDB,
self::TYPE_ROW,
self::TYPE_FILE,
self::TYPE_FUNCTION,
@@ -108,6 +117,39 @@ abstract class Resource implements \JsonSerializable
self::TYPE_COLLECTION,
];
// index terminology is same for all
public const DATABASE_TYPE_RESOURCE_MAP = [
self::TYPE_DATABASE => [
'entity' => self::TYPE_TABLE,
'field' => self::TYPE_COLUMN,
'record' => self::TYPE_ROW,
],
self::TYPE_DATABASE_DOCUMENTSDB => [
'entity' => self::TYPE_COLLECTION,
// HACK: not required in documentsdb but adding it for consistency in the db reader(not gonna impact)
'field' => self::TYPE_ATTRIBUTE,
'record' => self::TYPE_DOCUMENT,
],
self::TYPE_DATABASE_VECTORSDB => [
'entity' => self::TYPE_COLLECTION,
'field' => self::TYPE_ATTRIBUTE,
'record' => self::TYPE_DOCUMENT,
]
];
public const ENTITY_TYPE_RESOURCE_MAP = [
self::TYPE_TABLE => [
'field' => self::TYPE_COLUMN,
'record' => self::TYPE_ROW,
'index' => self::TYPE_INDEX
],
self::TYPE_COLLECTION => [
'field' => self::TYPE_ATTRIBUTE,
'record' => self::TYPE_DOCUMENT,
'index' => self::TYPE_INDEX
],
];
protected string $id = '';
protected string $originalId = '';
@@ -0,0 +1,189 @@
<?php
namespace Utopia\Migration\Resources\Database;
use Utopia\Migration\Resource;
use Utopia\Migration\Transfer;
class Attribute extends Resource
{
public const TYPE_STRING = 'string';
public const TYPE_INTEGER = 'integer';
public const TYPE_FLOAT = 'double';
public const TYPE_BOOLEAN = 'boolean';
public const TYPE_DATETIME = 'datetime';
public const TYPE_EMAIL = 'email';
public const TYPE_ENUM = 'enum';
public const TYPE_IP = 'ip';
public const TYPE_URL = 'url';
public const TYPE_RELATIONSHIP = 'relationship';
public const TYPE_POINT = 'point';
public const TYPE_LINE = 'linestring';
public const TYPE_POLYGON = 'polygon';
public const TYPE_OBJECT = 'object';
public const TYPE_VECTOR = 'vector';
/**
* @param string $key
* @param Table $table
* @param string $fieldType The actual field type (e.g., 'string', 'integer', 'email')
* @param int $size
* @param bool $required
* @param mixed|null $default
* @param bool $array
* @param bool $signed
* @param string $format
* @param array<string, mixed> $formatOptions
* @param array<string> $filters
* @param array<string, mixed> $options
* @param string $createdAt
* @param string $updatedAt
*/
public function __construct(
protected readonly string $key,
protected readonly Table $table,
protected readonly string $fieldType = '',
protected readonly int $size = 0,
protected readonly bool $required = false,
protected readonly mixed $default = null,
protected readonly bool $array = false,
protected readonly bool $signed = false,
protected readonly string $format = '',
protected readonly array $formatOptions = [],
protected readonly array $filters = [],
protected array $options = [],
protected string $createdAt = '',
protected string $updatedAt = '',
) {
}
/**
* @return array<string, mixed>
*/
public function jsonSerialize(): array
{
return [
'key' => $this->key,
'table' => $this->table,
'type' => $this->getType(),
'size' => $this->size,
'required' => $this->required,
'default' => $this->default,
'array' => $this->array,
'signed' => $this->signed,
'format' => $this->format,
'formatOptions' => $this->formatOptions,
'filters' => $this->filters,
'options' => $this->options,
'createdAt' => $this->createdAt,
'updatedAt' => $this->updatedAt,
];
}
public static function getName(): string
{
return Resource::TYPE_ATTRIBUTE;
}
public function getType(): string
{
return $this->fieldType;
}
/**
* Convert a Column resource to an Attribute resource.
*
* @param Column $column
* @return self
*/
public static function fromColumn(Column $column): self
{
return new self(
$column->getKey(),
$column->getTable(),
$column->getType(),
$column->getSize(),
$column->isRequired(),
$column->getDefault(),
$column->isArray(),
$column->isSigned(),
$column->getFormat(),
$column->getFormatOptions(),
$column->getFilters(),
$column->getOptions(),
$column->getCreatedAt(),
$column->getUpdatedAt()
);
}
public function getGroup(): string
{
return Transfer::GROUP_DATABASES;
}
public function getKey(): string
{
return $this->key;
}
public function getTable(): Table
{
return $this->table;
}
public function getSize(): int
{
return $this->size;
}
public function isRequired(): bool
{
return $this->required;
}
public function getDefault(): mixed
{
return $this->default;
}
public function isArray(): bool
{
return $this->array;
}
public function isSigned(): bool
{
return $this->signed;
}
public function getFormat(): string
{
return $this->format;
}
/**
* @return array<string, mixed>
*/
public function getFormatOptions(): array
{
return $this->formatOptions;
}
/**
* @return array<string>
*/
public function getFilters(): array
{
return $this->filters;
}
/**
* @return array<string, mixed>
*/
public function &getOptions(): array
{
return $this->options;
}
}
@@ -0,0 +1,50 @@
<?php
namespace Utopia\Migration\Resources\Database;
use Utopia\Migration\Resource;
class Collection extends Table
{
public static function getName(): string
{
return Resource::TYPE_COLLECTION;
}
/**
* @param array{
* database: array{
* id: string,
* name: string,
* type: string
* },
* name: string,
* id: string,
* documentSecurity?: bool,
* rowSecurity?: bool,
* permissions: ?array<string>,
* createdAt: string,
* updatedAt: string,
* enabled: bool
* } $array
*/
public static function fromArray(array $array): self
{
$database = match ($array['database']['type']) {
Resource::TYPE_DATABASE_DOCUMENTSDB => DocumentsDB::fromArray($array['database']),
Resource::TYPE_DATABASE_VECTORSDB => VectorsDB::fromArray($array['database']),
default => Database::fromArray($array['database'])
};
return new self(
$database,
name: $array['name'],
id: $array['id'],
rowSecurity: $array['rowSecurity'] ?? $array['documentSecurity'],
permissions: $array['permissions'] ?? [],
createdAt: $array['createdAt'] ?? '',
updatedAt: $array['updatedAt'] ?? '',
enabled: $array['enabled'] ?? true,
);
}
}
@@ -27,6 +27,9 @@ abstract class Column extends Resource
public const TYPE_LINE = 'linestring';
public const TYPE_POLYGON = 'polygon';
public const TYPE_OBJECT = 'object';
public const TYPE_VECTOR = 'vector';
/**
* @param string $key
* @param Table $table
@@ -157,4 +160,16 @@ abstract class Column extends Resource
{
return $this->options;
}
/**
* Convert this Column resource to an Attribute resource.
* This provides a deterministic way to derive attributes from columns,
* eliminating the need to maintain duplicate per-type Attribute implementations.
*
* @return Attribute
*/
public function getAttribute(): Attribute
{
return Attribute::fromColumn($this);
}
}
@@ -0,0 +1,74 @@
<?php
namespace Utopia\Migration\Resources\Database\Columns;
use Utopia\Migration\Resources\Database\Column;
use Utopia\Migration\Resources\Database\Table;
class ObjectType extends Column
{
public function __construct(
string $key,
Table $table,
bool $required = false,
?array $default = null,
string $createdAt = '',
string $updatedAt = ''
) {
parent::__construct(
$key,
$table,
required: $required,
default: $default,
createdAt: $createdAt,
updatedAt: $updatedAt
);
}
/**
* @param array{
* key: string,
* collection?: array{
* database: array{
* id: string,
* name: string,
* },
* name: string,
* id: string,
* documentSecurity: bool,
* permissions: ?array<string>
* },
* table?: array{
* database: array{
* id: string,
* name: string,
* },
* name: string,
* id: string,
* rowSecurity: bool,
* permissions: ?array<string>
* },
* required: bool,
* default: ?array<mixed>,
* createdAt: string,
* updatedAt: string,
* } $array
* @return self
*/
public static function fromArray(array $array): self
{
return new self(
$array['key'],
Table::fromArray($array['table'] ?? $array['collection']),
required: $array['required'],
default: $array['default'],
createdAt: $array['createdAt'] ?? '',
updatedAt: $array['updatedAt'] ?? '',
);
}
public function getType(): string
{
return Column::TYPE_OBJECT;
}
}
@@ -0,0 +1,78 @@
<?php
namespace Utopia\Migration\Resources\Database\Columns;
use Utopia\Migration\Resources\Database\Column;
use Utopia\Migration\Resources\Database\Table;
class Vector extends Column
{
public function __construct(
string $key,
Table $table,
bool $required = false,
int $size = 0,
?array $default = null,
string $createdAt = '',
string $updatedAt = ''
) {
parent::__construct(
$key,
$table,
size: $size,
required: $required,
default: $default,
createdAt: $createdAt,
updatedAt: $updatedAt
);
}
/**
* @param array{
* key: string,
* collection?: array{
* database: array{
* id: string,
* name: string,
* },
* name: string,
* id: string,
* documentSecurity: bool,
* permissions: ?array<string>
* },
* table?: array{
* database: array{
* id: string,
* name: string,
* },
* name: string,
* id: string,
* rowSecurity: bool,
* permissions: ?array<string>
* },
* size: int,
* required: bool,
* default: ?array<mixed>,
* createdAt: string,
* updatedAt: string,
* } $array
* @return self
*/
public static function fromArray(array $array): self
{
return new self(
$array['key'],
Table::fromArray($array['table'] ?? $array['collection']),
required: $array['required'],
size:$array['size'],
default: $array['default'],
createdAt: $array['createdAt'] ?? '',
updatedAt: $array['updatedAt'] ?? '',
);
}
public function getType(): string
{
return Column::TYPE_VECTOR;
}
}
@@ -26,6 +26,7 @@ class Database extends Resource
protected bool $enabled = true,
protected string $originalId = '',
protected string $type = '',
protected string $database = ''
) {
$this->id = $id;
}
@@ -38,6 +39,7 @@ class Database extends Resource
* updatedAt: string,
* enabled: bool,
* originalId: string|null,
* database: string
* } $array
*/
public static function fromArray(array $array): self
@@ -50,6 +52,7 @@ class Database extends Resource
enabled: $array['enabled'] ?? true,
originalId: $array['originalId'] ?? '',
type: $array['type'] ?? 'legacy',
database: $array['database']
);
}
@@ -65,6 +68,7 @@ class Database extends Resource
'updatedAt' => $this->updatedAt,
'enabled' => $this->enabled,
'type' => $this->type,
'database' => $this->database
];
}
@@ -92,4 +96,9 @@ class Database extends Resource
{
return $this->type;
}
public function getDatabase(): string
{
return $this->database;
}
}
@@ -0,0 +1,51 @@
<?php
namespace Utopia\Migration\Resources\Database;
use Utopia\Migration\Resource;
class Document extends Row
{
public static function getName(): string
{
return Resource::TYPE_DOCUMENT;
}
/**
* @param array{
* id: string,
* collection?: array{
* database: array{
* id: string,
* name: string,
* },
* name: string,
* id: string,
* documentSecurity: bool,
* permissions: ?array<string>
* },
* table?: array{
* database: array{
* id: string,
* name: string,
* },
* name: string,
* id: string,
* rowSecurity: bool,
* permissions: ?array<string>
* },
* data: array<string, mixed>,
* permissions: ?array<string>
* } $array
*/
public static function fromArray(array $array): self
{
// keeping table and collection to have backward compat
return new self(
$array['id'],
Collection::fromArray($array['table'] ?? $array['collection']),
$array['data'],
$array['permissions'] ?? []
);
}
}
@@ -0,0 +1,38 @@
<?php
namespace Utopia\Migration\Resources\Database;
use Utopia\Migration\Resource;
class DocumentsDB extends Database
{
public static function getName(): string
{
return Resource::TYPE_DATABASE_DOCUMENTSDB;
}
/**
* @param array{
* id: string,
* name: string,
* createdAt: string,
* updatedAt: string,
* enabled: bool,
* originalId: string|null,
* database: string
* } $array
*/
public static function fromArray(array $array): self
{
return new self(
$array['id'],
$array['name'],
createdAt: $array['createdAt'] ?? '',
updatedAt: $array['updatedAt'] ?? '',
enabled: $array['enabled'] ?? true,
originalId: $array['originalId'] ?? '',
type: $array['type'] ?? Resource::TYPE_DATABASE_DOCUMENTSDB,
database: $array['database']
);
}
}
@@ -0,0 +1,39 @@
<?php
namespace Utopia\Migration\Resources\Database;
use Utopia\Migration\Resource;
class VectorsDB extends Database
{
public static function getName(): string
{
return Resource::TYPE_DATABASE_VECTORSDB;
}
/**
* @param array{
* id: string,
* name: string,
* createdAt: string,
* updatedAt: string,
* enabled: bool,
* originalId: string|null,
* database: string,
* type: string
* } $array
*/
public static function fromArray(array $array): self
{
return new self(
$array['id'],
$array['name'],
createdAt: $array['createdAt'] ?? '',
updatedAt: $array['updatedAt'] ?? '',
enabled: $array['enabled'] ?? true,
originalId: $array['originalId'] ?? '',
type: $array['type'] ?? Resource::TYPE_DATABASE_VECTORSDB,
database: $array['database']
);
}
}
+8
View File
@@ -74,6 +74,14 @@ abstract class Source extends Target
if (!in_array($resource->getName(), $resources)) {
$resource->setStatus(Resource::STATUS_SKIPPED);
} else {
/**
* Rows and documents are counted in cache in an aggregated way (per status count)
* to prevent memory blowups. Destination updates already account for these, and
* adding them here causes double counting (source + destination).
*/
if (\in_array($resource->getName(), [Resource::TYPE_ROW, Resource::TYPE_DOCUMENT], true)) {
continue;
}
$prunedResources[] = $resource;
}
}
File diff suppressed because it is too large. Load diff
@@ -107,4 +107,6 @@ interface Reader
* @return QueryType|string
*/
public function queryLimit(int $limit): mixed;
public function getSupportForAttributes(): bool;
}
@@ -247,4 +247,9 @@ class API implements Reader
{
return Query::limit($limit);
}
public function getSupportForAttributes(): bool
{
return true;
}
}
@@ -8,8 +8,10 @@ use Utopia\Database\Exception as DatabaseException;
use Utopia\Database\Query;
use Utopia\Migration\Exception;
use Utopia\Migration\Resource;
use Utopia\Migration\Resources\Database\Collection as CollectionResource;
use Utopia\Migration\Resources\Database\Column as ColumnResource;
use Utopia\Migration\Resources\Database\Database as DatabaseResource;
use Utopia\Migration\Resources\Database\Document as DocumentResource;
use Utopia\Migration\Resources\Database\Index as IndexResource;
use Utopia\Migration\Resources\Database\Row as RowResource;
use Utopia\Migration\Resources\Database\Table as TableResource;
@@ -20,18 +22,35 @@ use Utopia\Migration\Sources\Appwrite\Reader;
*/
class Database implements Reader
{
public function __construct(private readonly UtopiaDatabase $dbForProject)
{
/**
* @var callable(UtopiaDocument|null): UtopiaDatabase
*/
private mixed $getDatabasesDB;
public function __construct(
private readonly UtopiaDatabase $dbForProject,
?callable $getDatabasesDB = null,
private readonly ?string $projectId = null
) {
$this->getDatabasesDB = $getDatabasesDB;
}
public function report(array $resources, array &$report, array $resourceIds = []): mixed
{
$relevantResources = [
// tablesdb
Resource::TYPE_DATABASE,
Resource::TYPE_TABLE,
Resource::TYPE_ROW,
Resource::TYPE_COLUMN,
Resource::TYPE_INDEX,
// vectorsdb
Resource::TYPE_DATABASE_VECTORSDB,
// Documentsdb
Resource::TYPE_DATABASE_DOCUMENTSDB,
Resource::TYPE_COLLECTION,
Resource::TYPE_DOCUMENT,
Resource::TYPE_ATTRIBUTE,
];
if (!Resource::isSupported($relevantResources, $resources)) {
@@ -43,38 +62,53 @@ class Database implements Reader
$report[$resourceType] = 0;
}
}
$databaseResources = array_keys(Resource::DATABASE_TYPE_RESOURCE_MAP);
$databaseQueries = [];
if (!empty($resourceIds[Resource::TYPE_DATABASE])) {
$databaseIds = (array) $resourceIds[Resource::TYPE_DATABASE];
foreach ($databaseResources as $databaseResourceType) {
if (!empty($resourceIds[$databaseResourceType])) {
$databaseIds = (array) $resourceIds[$databaseResourceType];
$databaseQueries[] = Query::equal('$id', $databaseIds);
$databaseQueries[] = Query::equal('$id', $databaseIds);
}
}
if (in_array(Resource::TYPE_DATABASE, $resources)) {
$report[Resource::TYPE_DATABASE] = $this->countResources('databases', $databaseQueries);
}
if (count(array_intersect($resources, $relevantResources)) === 1 &&
in_array(Resource::TYPE_DATABASE, $resources)) {
if (
count(array_intersect($resources, $relevantResources)) === 1 &&
Resource::isSupported(array_keys(Resource::DATABASE_TYPE_RESOURCE_MAP), $resources)
) {
return null;
}
$dbResources = [];
$databases = $this->listDatabases($databaseQueries);
foreach ($databases as $database) {
$databaseType = $database->getAttribute('type');
if (in_array($databaseType, [Resource::TYPE_DATABASE_LEGACY,Resource::TYPE_DATABASE_TABLESDB])) {
$databaseType = Resource::TYPE_DATABASE;
}
if (!isset(Resource::DATABASE_TYPE_RESOURCE_MAP[$databaseType])) {
$databaseType = Resource::TYPE_DATABASE;
}
if (Resource::isSupported($databaseType, $resources)) {
$report[$databaseType] += 1;
}
}
// Process each database
foreach ($databases as $database) {
$databaseType = $database->getAttribute('type', Resource::TYPE_DATABASE);
if (\in_array($databaseType, [Resource::TYPE_DATABASE_LEGACY, Resource::TYPE_DATABASE_TABLESDB], true)) {
$databaseType = Resource::TYPE_DATABASE;
}
if (!isset(Resource::DATABASE_TYPE_RESOURCE_MAP[$databaseType])) {
$databaseType = Resource::TYPE_DATABASE;
}
$databaseSpecificResources = Resource::DATABASE_TYPE_RESOURCE_MAP[$databaseType];
$databaseSequence = $database->getSequence();
$tableId = "database_{$databaseSequence}";
if (Resource::isSupported(Resource::TYPE_TABLE, $resources)) {
$report[Resource::TYPE_TABLE] += $this->countResources($tableId);
}
if (!Resource::isSupported([Resource::TYPE_ROW, Resource::TYPE_COLUMN, Resource::TYPE_INDEX], $resources)) {
continue;
}
if (!isset($dbResources[$database->getId()])) {
$dbResources[$database->getId()] = new DatabaseResource(
@@ -82,19 +116,29 @@ class Database implements Reader
$database->getAttribute('name'),
$database->getCreatedAt(),
$database->getUpdatedAt(),
$database->getAttribute('enabled', true),
$database->getAttribute('originalId', ''),
$database->getAttribute('type', ''),
$database->getAttribute('database', '')
);
}
$dbResource = $dbResources[$database->getId()];
$tables = $this->listTables($dbResource);
$count = count($tables);
if (Resource::isSupported($databaseSpecificResources['entity'], $resources)) {
$report[$databaseSpecificResources['entity']] += $count;
}
foreach ($tables as $table) {
$tableSequence = $table->getSequence();
if (Resource::isSupported(Resource::TYPE_ROW, $resources)) {
if (Resource::isSupported($databaseSpecificResources['record'], $resources)) {
$rowTableId = "database_{$databaseSequence}_collection_{$tableSequence}";
$report[Resource::TYPE_ROW] += $this->countResources($rowTableId);
$count = $this->countResources($rowTableId, [], $dbResource);
$report[$databaseSpecificResources['record']] += $count;
}
$commonQueries = [
@@ -102,8 +146,12 @@ class Database implements Reader
Query::equal('collectionInternalId', [$tableSequence]),
];
if (Resource::isSupported(Resource::TYPE_COLUMN, $resources)) {
$report[Resource::TYPE_COLUMN] += $this->countResources('attributes', $commonQueries);
if (
isset($databaseSpecificResources['field']) &&
Resource::isSupported($databaseSpecificResources['field'], $resources)
) {
$count = $this->countResources('attributes', $commonQueries);
$report[$databaseSpecificResources['field']] += $count;
}
if (in_array(Resource::TYPE_INDEX, $resources)) {
@@ -302,8 +350,11 @@ class Database implements Reader
$tableId = "database_{$database->getSequence()}_collection_{$table->getSequence()}";
// Use the appropriate database instance for this specific database
$dbInstance = $this->getDatabase($resource->getDatabase()->getDatabase());
try {
$rows = $this->dbForProject->find($tableId, $queries);
$rows = $dbInstance->find($tableId, $queries);
} catch (DatabaseException $e) {
throw new Exception(
resourceName: $resource->getName(),
@@ -352,7 +403,10 @@ class Database implements Reader
$tableId = "database_{$database->getSequence()}_collection_{$table->getSequence()}";
return $this->dbForProject->getDocument(
// Use the appropriate database instance for this specific database
$dbInstance = $this->getDatabase($resource->getDatabase()->getDatabase());
return $dbInstance->getDocument(
$tableId,
$rowId,
$queries
@@ -392,19 +446,32 @@ class Database implements Reader
switch ($resource::class) {
case DatabaseResource::class:
/** @var DatabaseResource $resource */
// Databases are always in dbForProject metadata
$document = $this->dbForProject->getDocument('databases', $resource->getId());
break;
case TableResource::class:
case CollectionResource::class:
/** @var TableResource|CollectionResource $resource */
// Tables/Collections metadata is in dbForProject
$database = $this->dbForProject->getDocument('databases', $resource->getDatabase()->getId());
$document = $this->dbForProject->getDocument('database_' . $database->getSequence(), $resource->getId());
break;
case ColumnResource::class:
/** @var ColumnResource $resource */
// Columns (attributes) are in dbForProject metadata
$document = $this->dbForProject->getDocument('attributes', $resource->getId());
break;
case IndexResource::class:
/** @var IndexResource $resource */
// Indexes are in dbForProject metadata
$document = $this->dbForProject->getDocument('indexes', $resource->getId());
break;
case RowResource::class:
case DocumentResource::class:
/** @var RowResource|DocumentResource $resource */
// Rows/Documents are in the specific database instance
// getRow() already uses getDatabase() internally
$document = $this->getRow($resource->getTable(), $resource->getId());
$document = new UtopiaDocument($document);
break;
@@ -420,13 +487,38 @@ class Database implements Reader
return Query::limit($limit);
}
public function getSupportForAttributes(): bool
{
return $this->dbForProject->getAdapter()->getSupportForAttributes();
}
/**
* @param string $table
* @param array $queries
* @param DatabaseResource|null $databaseResource
* @return int
*/
private function countResources(string $table, array $queries = []): int
private function countResources(string $table, array $queries = [], ?DatabaseResource $databaseResource = null): int
{
// Use the appropriate database instance for row data
if ($databaseResource !== null) {
$dbInstance = $this->getDatabase($databaseResource->getDatabase());
return $dbInstance->count($table, $queries);
}
// Use dbForProject for metadata tables
return $this->dbForProject->count($table, $queries);
}
/**
* Get the appropriate database instance for the given database DSN
*/
private function getDatabase(?string $databaseDSN = null): UtopiaDatabase
{
if ($this->getDatabasesDB !== null && $databaseDSN !== null) {
return ($this->getDatabasesDB)(new UtopiaDocument(['database' => $databaseDSN]));
}
return $this->dbForProject;
}
}
+57 -16
View File
@@ -4,11 +4,9 @@ namespace Utopia\Migration\Sources;
use Utopia\Database\Database as UtopiaDatabase;
use Utopia\Migration\Exception;
use Utopia\Migration\Resource;
use Utopia\Migration\Resource as UtopiaResource;
use Utopia\Migration\Resources\Database\Column;
use Utopia\Migration\Resources\Database\Database;
use Utopia\Migration\Resources\Database\Row;
use Utopia\Migration\Resources\Database\Table;
use Utopia\Migration\Resources\Storage\File;
use Utopia\Migration\Source;
use Utopia\Migration\Sources\Appwrite\Reader;
@@ -44,12 +42,13 @@ class CSV extends Source
string $resourceId,
string $filePath,
Device $device,
?UtopiaDatabase $dbForProject
?UtopiaDatabase $dbForProject,
?callable $getDatabasesDB = null,
) {
$this->device = $device;
$this->filePath = $filePath;
$this->resourceId = $resourceId;
$this->database = new DatabaseReader($dbForProject);
$this->database = new DatabaseReader($dbForProject, $getDatabasesDB);
}
public static function getName(): string
@@ -61,6 +60,7 @@ class CSV extends Source
{
return [
UtopiaResource::TYPE_ROW,
UtopiaResource::TYPE_DOCUMENT,
];
}
@@ -104,7 +104,7 @@ class CSV extends Source
protected function exportGroupDatabases(int $batchSize, array $resources): void
{
try {
if (UtopiaResource::isSupported(UtopiaResource::TYPE_ROW, $resources)) {
if (UtopiaResource::isSupported($this->getSupportedResources(), $resources)) {
$this->exportRows($batchSize);
}
} catch (\Throwable $e) {
@@ -132,8 +132,47 @@ class CSV extends Source
$lastColumn = null;
[$databaseId, $tableId] = \explode(':', $this->resourceId);
$database = new Database($databaseId, '');
$table = new Table($database, '', $tableId);
$databases = $this->database->listDatabases([
$this->database->queryEqual('$id', [$databaseId]),
$this->database->queryLimit(1),
]);
if (empty($databases)) {
throw new \Exception('Database not found');
}
$databaseDocument = $databases[0];
$databaseType = $databaseDocument->getAttribute('type', UtopiaResource::TYPE_DATABASE);
if (\in_array($databaseType, [UtopiaResource::TYPE_DATABASE_LEGACY, UtopiaResource::TYPE_DATABASE_TABLESDB], true)) {
$databaseType = UtopiaResource::TYPE_DATABASE;
}
$databasePayload = [
'id' => $databaseDocument->getId(),
'name' => $databaseDocument->getAttribute('name', $databaseDocument->getId()),
'originalId' => $databaseDocument->getAttribute('originalId', ''),
'type' => $databaseType,
'database' => $databaseDocument->getAttribute('database', ''),
];
$tablePayload = [
'id' => $tableId,
'name' => $tableId,
'documentSecurity' => false,
'rowSecurity' => false,
'permissions' => [],
'createdAt' => '',
'updatedAt' => '',
'database' => [
'id' => $databasePayload['id'],
'name' => $databasePayload['name'],
'type' => $databasePayload['type'],
'database' => $databasePayload['database'],
],
];
$table = Appwrite::getEntity($databaseType, $tablePayload);
while (true) {
$queries = [$this->database->queryLimit($batchSize)];
@@ -195,7 +234,7 @@ class CSV extends Source
}
}
$this->withCsvStream(function ($stream, $delimiter) use ($columnTypes, $requiredColumns, $manyToManyKeys, $arrayKeys, $table, $batchSize) {
$this->withCsvStream(function ($stream, $delimiter) use ($columnTypes, $databaseType, $requiredColumns, $manyToManyKeys, $arrayKeys, $tablePayload, $batchSize) {
$headers = \fgetcsv($stream, 0, $delimiter, '"', '"');
if (!\is_array($headers) || \count($headers) === 0) {
@@ -300,7 +339,9 @@ class CSV extends Source
),
Column::TYPE_POINT,
Column::TYPE_LINE,
Column::TYPE_POLYGON => \is_string($parsedValue) ? json_decode($parsedValue) : null,
Column::TYPE_POLYGON,
Column::TYPE_VECTOR,
Column::TYPE_OBJECT => \is_string($parsedValue) ? json_decode($parsedValue, true) : null,
default => $parsedValue,
},
};
@@ -311,12 +352,12 @@ class CSV extends Source
unset($parsedData['$id'], $parsedData['$permissions']);
$row = new Row(
$rowId,
$table,
$parsedData,
$permissions,
);
$row = Appwrite::getRecord($databaseType, [
'id' => $rowId,
'table' => $tablePayload,
'data' => $parsedData,
'permissions' => $permissions
]);
$buffer[] = $row;
+42 -3
View File
@@ -14,7 +14,13 @@ class Transfer
public const GROUP_SITES = 'sites';
// separating databases and tablesdb out for easier separation in extract services
// migration can use group_databases for mentioning all resources but when mentioning specific resources go with specific type databases
public const GROUP_DATABASES = 'databases';
public const GROUP_DATABASES_TABLES_DB = 'tablesdb';
public const GROUP_DATABASES_DOCUMENTS_DB = 'documentsdb';
public const GROUP_DATABASES_VECTOR_DB = 'vectorsdb';
public const GROUP_SETTINGS = 'settings';
@@ -44,7 +50,7 @@ class Transfer
Resource::TYPE_SITE_DEPLOYMENT
];
public const GROUP_DATABASES_RESOURCES = [
public const GROUP_TABLESDB_RESOURCES = [
Resource::TYPE_DATABASE,
Resource::TYPE_TABLE,
Resource::TYPE_INDEX,
@@ -52,6 +58,34 @@ class Transfer
Resource::TYPE_ROW,
];
public const GROUP_DOCUMENTSDB_RESOURCES = [
Resource::TYPE_DATABASE_DOCUMENTSDB,
Resource::TYPE_COLLECTION,
Resource::TYPE_INDEX,
Resource::TYPE_DOCUMENT
];
public const GROUP_VECTORSDB_RESOURCES = [
Resource::TYPE_DATABASE_VECTORSDB,
Resource::TYPE_COLLECTION,
Resource::TYPE_ATTRIBUTE,
Resource::TYPE_INDEX,
Resource::TYPE_DOCUMENT
];
public const GROUP_DATABASES_RESOURCES = [
Resource::TYPE_DATABASE,
Resource::TYPE_DATABASE_DOCUMENTSDB,
Resource::TYPE_DATABASE_VECTORSDB,
Resource::TYPE_TABLE,
Resource::TYPE_INDEX,
Resource::TYPE_COLUMN,
Resource::TYPE_ROW,
Resource::TYPE_DOCUMENT,
Resource::TYPE_COLLECTION,
Resource::TYPE_ATTRIBUTE
];
public const GROUP_SETTINGS_RESOURCES = [];
public const GROUP_MESSAGING_RESOURCES = [
@@ -92,6 +126,8 @@ class Transfer
public const ROOT_RESOURCES = [
Resource::TYPE_BUCKET,
Resource::TYPE_DATABASE,
Resource::TYPE_DATABASE_DOCUMENTSDB,
Resource::TYPE_DATABASE_VECTORSDB,
Resource::TYPE_FUNCTION,
Resource::TYPE_SITE,
Resource::TYPE_USER,
@@ -171,7 +207,7 @@ class Transfer
$status[$resourceType][$k] = $resource;
if ($status[$resourceType]['pending'] > 0) {
$status[$resourceType]['pending'] -= $resource;
$status[$resourceType]['pending'] -= \min($status[$resourceType]['pending'], $resource);
}
continue;
@@ -256,7 +292,7 @@ class Transfer
}
if (!in_array($rootResourceType, self::ROOT_RESOURCES)) {
throw new \Exception('Resource type must be one of ' . implode(', ', self::ROOT_RESOURCES));
throw new \Exception('Got '.$rootResourceType.' Resource type must be one of ' . implode(', ', self::ROOT_RESOURCES));
}
$rootResources = \array_intersect($computedResources, self::ROOT_RESOURCES);
@@ -358,6 +394,9 @@ class Transfer
self::GROUP_GENERAL => array_merge($resources, []),
self::GROUP_AUTH => array_merge($resources, self::GROUP_AUTH_RESOURCES),
self::GROUP_DATABASES => array_merge($resources, self::GROUP_DATABASES_RESOURCES),
self::GROUP_DATABASES_TABLES_DB => array_merge($resources, self::GROUP_TABLESDB_RESOURCES),
self::GROUP_DATABASES_DOCUMENTS_DB => array_merge($resources, self::GROUP_DOCUMENTSDB_RESOURCES),
self::GROUP_DATABASES_VECTOR_DB => array_merge($resources, self::GROUP_VECTORSDB_RESOURCES),
self::GROUP_SETTINGS => array_merge($resources, self::GROUP_SETTINGS_RESOURCES),
self::GROUP_MESSAGING => array_merge($resources, self::GROUP_MESSAGING_RESOURCES),
default => throw new \Exception('No service group found'),