mirror of
https://github.com/appwrite/appwrite.git
synced 2026-11-18 20:36:03 +00:00
Merge pull request #139 from utopia-php/multitype-db
This commit is contained in:
19 files changed
+1568
-587
No files matched your search
@@ -46,6 +46,7 @@ class Cache
|
||||
case Resource::TYPE_TABLE:
|
||||
case Resource::TYPE_COLLECTION:
|
||||
/** @var Table $resource */
|
||||
$keys[] = $resource->getDatabase()->getType();
|
||||
$keys[] = $resource->getDatabase()->getSequence();
|
||||
break;
|
||||
|
||||
|
||||
@@ -41,6 +41,7 @@ use Utopia\Migration\Resources\Auth\Hash;
|
||||
use Utopia\Migration\Resources\Auth\Membership;
|
||||
use Utopia\Migration\Resources\Auth\Team;
|
||||
use Utopia\Migration\Resources\Auth\User;
|
||||
use Utopia\Migration\Resources\Database\Attribute;
|
||||
use Utopia\Migration\Resources\Database\Column;
|
||||
use Utopia\Migration\Resources\Database\Database;
|
||||
use Utopia\Migration\Resources\Database\Index;
|
||||
@@ -74,6 +75,11 @@ class Appwrite extends Destination
|
||||
private Teams $teams;
|
||||
private Users $users;
|
||||
|
||||
/**
|
||||
* @var callable(UtopiaDocument $database): UtopiaDatabase
|
||||
*/
|
||||
protected $getDatabasesDB;
|
||||
|
||||
/**
|
||||
* @var array<UtopiaDocument>
|
||||
*/
|
||||
@@ -83,14 +89,16 @@ class Appwrite extends Destination
|
||||
* @param string $project
|
||||
* @param string $endpoint
|
||||
* @param string $key
|
||||
* @param UtopiaDatabase $database
|
||||
* @param UtopiaDatabase $dbForProject
|
||||
* @param callable(UtopiaDocument $database):UtopiaDatabase $getDatabasesDB
|
||||
* @param array<array<string, mixed>> $collectionStructure
|
||||
*/
|
||||
public function __construct(
|
||||
string $project,
|
||||
string $endpoint,
|
||||
string $key,
|
||||
protected UtopiaDatabase $database,
|
||||
protected UtopiaDatabase $dbForProject,
|
||||
callable $getDatabasesDB,
|
||||
protected array $collectionStructure
|
||||
) {
|
||||
$this->project = $project;
|
||||
@@ -108,6 +116,8 @@ class Appwrite extends Destination
|
||||
$this->storage = new Storage($this->client);
|
||||
$this->teams = new Teams($this->client);
|
||||
$this->users = new Users($this->client);
|
||||
|
||||
$this->getDatabasesDB = $getDatabasesDB;
|
||||
}
|
||||
|
||||
public static function getName(): string
|
||||
@@ -128,6 +138,8 @@ class Appwrite extends Destination
|
||||
|
||||
// Database
|
||||
Resource::TYPE_DATABASE,
|
||||
Resource::TYPE_DATABASE_DOCUMENTSDB,
|
||||
Resource::TYPE_DATABASE_VECTORSDB,
|
||||
Resource::TYPE_TABLE,
|
||||
Resource::TYPE_COLUMN,
|
||||
Resource::TYPE_INDEX,
|
||||
@@ -304,7 +316,7 @@ class Appwrite extends Destination
|
||||
$isLast = $index === $total - 1;
|
||||
|
||||
try {
|
||||
$this->database->setPreserveDates(true);
|
||||
$this->dbForProject->setPreserveDates(true);
|
||||
|
||||
$responseResource = match ($resource->getGroup()) {
|
||||
Transfer::GROUP_DATABASES => $this->importDatabaseResource($resource, $isLast),
|
||||
@@ -329,7 +341,7 @@ class Appwrite extends Destination
|
||||
|
||||
$responseResource = $resource;
|
||||
} finally {
|
||||
$this->database->setPreserveDates(false);
|
||||
$this->dbForProject->setPreserveDates(false);
|
||||
}
|
||||
|
||||
$this->cache->update($responseResource);
|
||||
@@ -347,18 +359,20 @@ class Appwrite extends Destination
|
||||
{
|
||||
switch ($resource->getName()) {
|
||||
case Resource::TYPE_DATABASE:
|
||||
case Resource::TYPE_DATABASE_DOCUMENTSDB:
|
||||
case Resource::TYPE_DATABASE_VECTORSDB:
|
||||
/** @var Database $resource */
|
||||
$success = $this->createDatabase($resource);
|
||||
break;
|
||||
case Resource::TYPE_TABLE:
|
||||
case Resource::TYPE_COLLECTION:
|
||||
/** @var Table $resource */
|
||||
$success = $this->createTable($resource);
|
||||
$success = $this->createEntity($resource);
|
||||
break;
|
||||
case Resource::TYPE_COLUMN:
|
||||
case Resource::TYPE_ATTRIBUTE:
|
||||
/** @var Column $resource */
|
||||
$success = $this->createColumn($resource);
|
||||
$success = $this->createField($resource);
|
||||
break;
|
||||
case Resource::TYPE_INDEX:
|
||||
/** @var Index $resource */
|
||||
@@ -367,7 +381,7 @@ class Appwrite extends Destination
|
||||
case Resource::TYPE_ROW:
|
||||
case Resource::TYPE_DOCUMENT:
|
||||
/** @var Row $resource */
|
||||
$success = $this->createRow($resource, $isLast);
|
||||
$success = $this->createRecord($resource, $isLast);
|
||||
break;
|
||||
default:
|
||||
$success = false;
|
||||
@@ -406,7 +420,7 @@ class Appwrite extends Destination
|
||||
$createdAt = $this->normalizeDateTime($resource->getCreatedAt());
|
||||
$updatedAt = $this->normalizeDateTime($resource->getUpdatedAt(), $createdAt);
|
||||
|
||||
$database = $this->database->createDocument('databases', new UtopiaDocument([
|
||||
$database = $this->dbForProject->createDocument('databases', new UtopiaDocument([
|
||||
'$id' => $resource->getId(),
|
||||
'name' => $resource->getDatabaseName(),
|
||||
'enabled' => $resource->getEnabled(),
|
||||
@@ -415,6 +429,8 @@ class Appwrite extends Destination
|
||||
'$updatedAt' => $updatedAt,
|
||||
'originalId' => empty($resource->getOriginalId()) ? null : $resource->getOriginalId(),
|
||||
'type' => empty($resource->getType()) ? 'legacy' : $resource->getType(),
|
||||
// source and destination can be in different location
|
||||
'database' => $resource->getDatabase()
|
||||
]));
|
||||
|
||||
$resource->setSequence($database->getSequence());
|
||||
@@ -429,7 +445,7 @@ class Appwrite extends Destination
|
||||
$this->collectionStructure['indexes']
|
||||
);
|
||||
|
||||
$this->database->createCollection(
|
||||
$this->dbForProject->createCollection(
|
||||
'database_' . $database->getSequence(),
|
||||
$columns,
|
||||
$indexes
|
||||
@@ -444,7 +460,7 @@ class Appwrite extends Destination
|
||||
* @throws StructureException
|
||||
* @throws Exception
|
||||
*/
|
||||
protected function createTable(Table $resource): bool
|
||||
protected function createEntity(Table $resource): bool
|
||||
{
|
||||
if ($resource->getId() == 'unique()') {
|
||||
$resource->setId(ID::unique());
|
||||
@@ -461,7 +477,7 @@ class Appwrite extends Destination
|
||||
);
|
||||
}
|
||||
|
||||
$database = $this->database->getDocument(
|
||||
$database = $this->dbForProject->getDocument(
|
||||
'databases',
|
||||
$resource->getDatabase()->getId()
|
||||
);
|
||||
@@ -478,7 +494,14 @@ class Appwrite extends Destination
|
||||
$createdAt = $this->normalizeDateTime($resource->getCreatedAt());
|
||||
$updatedAt = $this->normalizeDateTime($resource->getUpdatedAt(), $createdAt);
|
||||
|
||||
$table = $this->database->createDocument('database_' . $database->getSequence(), new UtopiaDocument([
|
||||
$dbForDatabases = ($this->getDatabasesDB)($database);
|
||||
|
||||
// passing null in creates only creates the metadata collection
|
||||
if (!$dbForDatabases->exists(null, UtopiaDatabase::METADATA)) {
|
||||
$dbForDatabases->create();
|
||||
}
|
||||
|
||||
$table = $this->dbForProject->createDocument('database_' . $database->getSequence(), new UtopiaDocument([
|
||||
'$id' => $resource->getId(),
|
||||
'databaseInternalId' => $database->getSequence(),
|
||||
'databaseId' => $resource->getDatabase()->getId(),
|
||||
@@ -493,7 +516,7 @@ class Appwrite extends Destination
|
||||
|
||||
$resource->setSequence($table->getSequence());
|
||||
|
||||
$this->database->createCollection(
|
||||
$dbForDatabases->createCollection(
|
||||
'database_' . $database->getSequence() . '_collection_' . $resource->getSequence(),
|
||||
permissions: $resource->getPermissions(),
|
||||
documentSecurity: $resource->getRowSecurity()
|
||||
@@ -507,19 +530,27 @@ class Appwrite extends Destination
|
||||
* @throws \Exception
|
||||
* @throws \Throwable
|
||||
*/
|
||||
protected function createColumn(Column $resource): bool
|
||||
protected function createField(Column|Attribute $resource): bool
|
||||
{
|
||||
if ($resource->getTable()->getDatabase()->getType() === Resource::TYPE_DATABASE_DOCUMENTSDB) {
|
||||
$resource->setStatus(Resource::STATUS_SKIPPED, 'Columns not supported for DocumentsDB');
|
||||
return false;
|
||||
}
|
||||
// column will be matching attribute as well
|
||||
// column type will be matching attribute type as well
|
||||
$type = match ($resource->getType()) {
|
||||
Column::TYPE_DATETIME => UtopiaDatabase::VAR_DATETIME,
|
||||
Column::TYPE_BOOLEAN => UtopiaDatabase::VAR_BOOLEAN,
|
||||
Column::TYPE_INTEGER => UtopiaDatabase::VAR_INTEGER,
|
||||
Column::TYPE_FLOAT => UtopiaDatabase::VAR_FLOAT,
|
||||
Column::TYPE_RELATIONSHIP => UtopiaDatabase::VAR_RELATIONSHIP,
|
||||
|
||||
Column::TYPE_STRING,
|
||||
Column::TYPE_IP,
|
||||
Column::TYPE_EMAIL,
|
||||
Column::TYPE_URL,
|
||||
Column::TYPE_ENUM => UtopiaDatabase::VAR_STRING,
|
||||
|
||||
Column::TYPE_POINT => UtopiaDatabase::VAR_POINT,
|
||||
Column::TYPE_LINE => UtopiaDatabase::VAR_LINESTRING,
|
||||
Column::TYPE_POLYGON => UtopiaDatabase::VAR_POLYGON,
|
||||
@@ -527,10 +558,13 @@ class Appwrite extends Destination
|
||||
Column::TYPE_VARCHAR => UtopiaDatabase::VAR_VARCHAR,
|
||||
Column::TYPE_MEDIUMTEXT => UtopiaDatabase::VAR_MEDIUMTEXT,
|
||||
Column::TYPE_LONGTEXT => UtopiaDatabase::VAR_LONGTEXT,
|
||||
default => throw new \Exception('Invalid resource type '.$resource->getType(), Exception::CODE_VALIDATION),
|
||||
Column::TYPE_OBJECT => UtopiaDatabase::VAR_OBJECT,
|
||||
Column::TYPE_VECTOR => UtopiaDatabase::VAR_VECTOR,
|
||||
|
||||
default => throw new \Exception('Invalid resource type ' . $resource->getType(), Exception::CODE_VALIDATION),
|
||||
};
|
||||
|
||||
$database = $this->database->getDocument(
|
||||
$database = $this->dbForProject->getDocument(
|
||||
'databases',
|
||||
$resource->getTable()->getDatabase()->getId(),
|
||||
);
|
||||
@@ -544,7 +578,7 @@ class Appwrite extends Destination
|
||||
);
|
||||
}
|
||||
|
||||
$table = $this->database->getDocument(
|
||||
$table = $this->dbForProject->getDocument(
|
||||
'database_' . $database->getSequence(),
|
||||
$resource->getTable()->getId(),
|
||||
);
|
||||
@@ -589,7 +623,7 @@ class Appwrite extends Destination
|
||||
|
||||
if ($type === UtopiaDatabase::VAR_RELATIONSHIP) {
|
||||
$resource->getOptions()['side'] = UtopiaDatabase::RELATION_SIDE_PARENT;
|
||||
$relatedTable = $this->database->getDocument(
|
||||
$relatedTable = $this->dbForProject->getDocument(
|
||||
'database_' . $database->getSequence(),
|
||||
$resource->getOptions()['relatedCollection']
|
||||
);
|
||||
@@ -605,7 +639,7 @@ class Appwrite extends Destination
|
||||
|
||||
$createdAt = $this->normalizeDateTime($resource->getCreatedAt());
|
||||
$updatedAt = $this->normalizeDateTime($resource->getUpdatedAt(), $createdAt);
|
||||
|
||||
$dbForDatabases = ($this->getDatabasesDB)($database);
|
||||
try {
|
||||
$column = new UtopiaDocument([
|
||||
'$id' => ID::custom($database->getSequence() . '_' . $table->getSequence() . '_' . $resource->getKey()),
|
||||
@@ -629,9 +663,9 @@ class Appwrite extends Destination
|
||||
'$updatedAt' => $updatedAt,
|
||||
]);
|
||||
|
||||
$this->database->checkAttribute($table, $column);
|
||||
$this->dbForProject->checkAttribute($table, $column);
|
||||
|
||||
$column = $this->database->createDocument('attributes', $column);
|
||||
$column = $this->dbForProject->createDocument('attributes', $column);
|
||||
} catch (DuplicateException) {
|
||||
throw new Exception(
|
||||
resourceName: $resource->getName(),
|
||||
@@ -647,13 +681,13 @@ class Appwrite extends Destination
|
||||
message: 'Attribute limit exceeded',
|
||||
);
|
||||
} catch (\Throwable $e) {
|
||||
$this->database->purgeCachedDocument('database_' . $database->getSequence(), $table->getId());
|
||||
$this->database->purgeCachedCollection('database_' . $database->getSequence() . '_collection_' . $table->getSequence());
|
||||
$this->dbForProject->purgeCachedDocument('database_' . $database->getSequence(), $table->getId());
|
||||
$dbForDatabases->purgeCachedCollection('database_' . $database->getSequence() . '_collection_' . $table->getSequence());
|
||||
throw $e;
|
||||
}
|
||||
|
||||
$this->database->purgeCachedDocument('database_' . $database->getSequence(), $table->getId());
|
||||
$this->database->purgeCachedCollection('database_' . $database->getSequence() . '_collection_' . $table->getSequence());
|
||||
$this->dbForProject->purgeCachedDocument('database_' . $database->getSequence(), $table->getId());
|
||||
$dbForDatabases->purgeCachedCollection('database_' . $database->getSequence() . '_collection_' . $table->getSequence());
|
||||
$options = $resource->getOptions();
|
||||
|
||||
$twoWayKey = null;
|
||||
@@ -687,9 +721,9 @@ class Appwrite extends Destination
|
||||
'$updatedAt' => $updatedAt,
|
||||
]);
|
||||
|
||||
$this->database->createDocument('attributes', $twoWayAttribute);
|
||||
$this->dbForProject->createDocument('attributes', $twoWayAttribute);
|
||||
} catch (DuplicateException) {
|
||||
$this->database->deleteDocument('attributes', $column->getId());
|
||||
$this->dbForProject->deleteDocument('attributes', $column->getId());
|
||||
|
||||
throw new Exception(
|
||||
resourceName: $resource->getName(),
|
||||
@@ -698,7 +732,7 @@ class Appwrite extends Destination
|
||||
message: 'Attribute already exists',
|
||||
);
|
||||
} catch (LimitException) {
|
||||
$this->database->deleteDocument('attributes', $column->getId());
|
||||
$this->dbForProject->deleteDocument('attributes', $column->getId());
|
||||
|
||||
throw new Exception(
|
||||
resourceName: $resource->getName(),
|
||||
@@ -707,8 +741,8 @@ class Appwrite extends Destination
|
||||
message: 'Column limit exceeded',
|
||||
);
|
||||
} catch (\Throwable $e) {
|
||||
$this->database->purgeCachedDocument('database_' . $database->getSequence(), $relatedTable->getId());
|
||||
$this->database->purgeCachedCollection('database_' . $database->getSequence() . '_collection_' . $relatedTable->getSequence());
|
||||
$this->dbForProject->purgeCachedDocument('database_' . $database->getSequence(), $relatedTable->getId());
|
||||
$dbForDatabases->purgeCachedCollection('database_' . $database->getSequence() . '_collection_' . $relatedTable->getSequence());
|
||||
throw $e;
|
||||
}
|
||||
}
|
||||
@@ -716,7 +750,7 @@ class Appwrite extends Destination
|
||||
try {
|
||||
switch ($type) {
|
||||
case UtopiaDatabase::VAR_RELATIONSHIP:
|
||||
if (!$this->database->createRelationship(
|
||||
if (!$dbForDatabases->createRelationship(
|
||||
collection: 'database_' . $database->getSequence() . '_collection_' . $table->getSequence(),
|
||||
relatedCollection: 'database_' . $database->getSequence() . '_collection_' . $relatedTable->getSequence(),
|
||||
type: $options['relationType'],
|
||||
@@ -734,7 +768,7 @@ class Appwrite extends Destination
|
||||
}
|
||||
break;
|
||||
default:
|
||||
if (!$this->database->createAttribute(
|
||||
if (!$dbForDatabases->createAttribute(
|
||||
'database_' . $database->getSequence() . '_collection_' . $table->getSequence(),
|
||||
$resource->getKey(),
|
||||
$type,
|
||||
@@ -751,10 +785,10 @@ class Appwrite extends Destination
|
||||
}
|
||||
}
|
||||
} catch (\Throwable) {
|
||||
$this->database->deleteDocument('attributes', $column->getId());
|
||||
$this->dbForProject->deleteDocument('attributes', $column->getId());
|
||||
|
||||
if (isset($twoWayAttribute)) {
|
||||
$this->database->deleteDocument('attributes', $twoWayAttribute->getId());
|
||||
$this->dbForProject->deleteDocument('attributes', $twoWayAttribute->getId());
|
||||
}
|
||||
|
||||
throw new Exception(
|
||||
@@ -766,11 +800,11 @@ class Appwrite extends Destination
|
||||
}
|
||||
|
||||
if ($type === UtopiaDatabase::VAR_RELATIONSHIP && $options['twoWay']) {
|
||||
$this->database->purgeCachedDocument('database_' . $database->getSequence(), $relatedTable->getId());
|
||||
$this->dbForProject->purgeCachedDocument('database_' . $database->getSequence(), $relatedTable->getId());
|
||||
}
|
||||
|
||||
$this->database->purgeCachedDocument('database_' . $database->getSequence(), $table->getId());
|
||||
$this->database->purgeCachedCollection('database_' . $database->getSequence() . '_collection_' . $table->getSequence());
|
||||
$this->dbForProject->purgeCachedDocument('database_' . $database->getSequence(), $table->getId());
|
||||
$dbForDatabases->purgeCachedCollection('database_' . $database->getSequence() . '_collection_' . $table->getSequence());
|
||||
|
||||
return true;
|
||||
}
|
||||
@@ -781,7 +815,7 @@ class Appwrite extends Destination
|
||||
*/
|
||||
protected function createIndex(Index $resource): bool
|
||||
{
|
||||
$database = $this->database->getDocument(
|
||||
$database = $this->dbForProject->getDocument(
|
||||
'databases',
|
||||
$resource->getTable()->getDatabase()->getId(),
|
||||
);
|
||||
@@ -794,7 +828,7 @@ class Appwrite extends Destination
|
||||
);
|
||||
}
|
||||
|
||||
$table = $this->database->getDocument(
|
||||
$table = $this->dbForProject->getDocument(
|
||||
'database_' . $database->getSequence(),
|
||||
$resource->getTable()->getId(),
|
||||
);
|
||||
@@ -806,13 +840,14 @@ class Appwrite extends Destination
|
||||
message: 'Table not found',
|
||||
);
|
||||
}
|
||||
$dbForDatabases = ($this->getDatabasesDB)($database);
|
||||
|
||||
$count = $this->database->count('indexes', [
|
||||
$count = $this->dbForProject->count('indexes', [
|
||||
Query::equal('collectionInternalId', [$table->getSequence()]),
|
||||
Query::equal('databaseInternalId', [$database->getSequence()])
|
||||
], $this->database->getLimitForIndexes());
|
||||
], $dbForDatabases->getLimitForIndexes());
|
||||
|
||||
if ($count >= $this->database->getLimitForIndexes()) {
|
||||
if ($count >= $dbForDatabases->getLimitForIndexes()) {
|
||||
throw new Exception(
|
||||
resourceName: $resource->getName(),
|
||||
resourceGroup: $resource->getGroup(),
|
||||
@@ -821,101 +856,11 @@ class Appwrite extends Destination
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* @var array<UtopiaDocument> $tableColumns
|
||||
*/
|
||||
$tableColumns = $table->getAttribute('attributes', []);
|
||||
|
||||
/**
|
||||
* @var array<UtopiaDocument> $tableIndexes
|
||||
*/
|
||||
$tableIndexes = $table->getAttribute('indexes', []);
|
||||
|
||||
$oldColumns = \array_map(
|
||||
fn ($attr) => $attr->getArrayCopy(),
|
||||
$tableColumns
|
||||
);
|
||||
|
||||
$oldColumns[] = [
|
||||
'key' => '$id',
|
||||
'type' => UtopiaDatabase::VAR_STRING,
|
||||
'status' => 'available',
|
||||
'required' => true,
|
||||
'array' => false,
|
||||
'default' => null,
|
||||
'size' => UtopiaDatabase::LENGTH_KEY
|
||||
];
|
||||
|
||||
$oldColumns[] = [
|
||||
'key' => '$createdAt',
|
||||
'type' => UtopiaDatabase::VAR_DATETIME,
|
||||
'status' => 'available',
|
||||
'signed' => false,
|
||||
'required' => false,
|
||||
'array' => false,
|
||||
'default' => null,
|
||||
'size' => 0
|
||||
];
|
||||
|
||||
$oldColumns[] = [
|
||||
'key' => '$updatedAt',
|
||||
'type' => UtopiaDatabase::VAR_DATETIME,
|
||||
'status' => 'available',
|
||||
'signed' => false,
|
||||
'required' => false,
|
||||
'array' => false,
|
||||
'default' => null,
|
||||
'size' => 0
|
||||
];
|
||||
|
||||
// Lengths hidden by default
|
||||
$lengths = [];
|
||||
|
||||
foreach ($resource->getColumns() as $i => $column) {
|
||||
// find attribute metadata in collection document
|
||||
$columnIndex = \array_search(
|
||||
$column,
|
||||
\array_column($oldColumns, 'key')
|
||||
);
|
||||
|
||||
if ($columnIndex === false) {
|
||||
throw new Exception(
|
||||
resourceName: $resource->getName(),
|
||||
resourceGroup: $resource->getGroup(),
|
||||
resourceId: $resource->getId(),
|
||||
message: 'Column not found in table: ' . $column,
|
||||
);
|
||||
}
|
||||
|
||||
$columnStatus = $oldColumns[$columnIndex]['status'];
|
||||
$columnType = $oldColumns[$columnIndex]['type'];
|
||||
$columnSize = $oldColumns[$columnIndex]['size'];
|
||||
$columnArray = $oldColumns[$columnIndex]['array'] ?? false;
|
||||
|
||||
if ($columnType === UtopiaDatabase::VAR_RELATIONSHIP) {
|
||||
throw new Exception(
|
||||
resourceName: $resource->getName(),
|
||||
resourceGroup: $resource->getGroup(),
|
||||
resourceId: $resource->getId(),
|
||||
message: 'Relationship columns are not supported in indexes',
|
||||
);
|
||||
}
|
||||
|
||||
// Ensure attribute is available
|
||||
if ($columnStatus !== 'available') {
|
||||
throw new Exception(
|
||||
resourceName: $resource->getName(),
|
||||
resourceGroup: $resource->getGroup(),
|
||||
resourceId: $resource->getId(),
|
||||
message: 'Column not available: ' . $column,
|
||||
);
|
||||
}
|
||||
|
||||
$lengths[$i] = null;
|
||||
|
||||
if ($columnArray === true) {
|
||||
$lengths[$i] = UtopiaDatabase::MAX_ARRAY_INDEX_LENGTH;
|
||||
}
|
||||
if ($dbForDatabases->getAdapter()->getSupportForAttributes()) {
|
||||
$this->validateFieldsForIndexes($resource, $table, $lengths);
|
||||
}
|
||||
|
||||
$createdAt = $this->normalizeDateTime($resource->getCreatedAt());
|
||||
@@ -937,20 +882,33 @@ class Appwrite extends Destination
|
||||
'$updatedAt' => $updatedAt,
|
||||
]);
|
||||
|
||||
/**
|
||||
* @var array<UtopiaDocument> $tableColumns
|
||||
*/
|
||||
$tableColumns = $table->getAttribute('attributes', []);
|
||||
$tableIndexes = $table->getAttribute('indexes', []);
|
||||
|
||||
$validator = new IndexValidator(
|
||||
$tableColumns,
|
||||
$tableIndexes,
|
||||
$this->database->getAdapter()->getMaxIndexLength(),
|
||||
$this->database->getAdapter()->getInternalIndexesKeys(),
|
||||
$this->database->getAdapter()->getSupportForIndexArray(),
|
||||
$this->database->getAdapter()->getSupportForSpatialIndexNull(),
|
||||
$this->database->getAdapter()->getSupportForSpatialIndexOrder(),
|
||||
$this->database->getAdapter()->getSupportForVectors(),
|
||||
$this->database->getAdapter()->getSupportForAttributes(),
|
||||
$this->database->getAdapter()->getSupportForMultipleFulltextIndexes(),
|
||||
$this->database->getAdapter()->getSupportForIdenticalIndexes(),
|
||||
$dbForDatabases->getAdapter()->getMaxIndexLength(),
|
||||
$dbForDatabases->getAdapter()->getInternalIndexesKeys(),
|
||||
$dbForDatabases->getAdapter()->getSupportForIndexArray(),
|
||||
$dbForDatabases->getAdapter()->getSupportForSpatialIndexNull(),
|
||||
$dbForDatabases->getAdapter()->getSupportForSpatialIndexOrder(),
|
||||
$dbForDatabases->getAdapter()->getSupportForVectors(),
|
||||
$dbForDatabases->getAdapter()->getSupportForAttributes(),
|
||||
$dbForDatabases->getAdapter()->getSupportForMultipleFulltextIndexes(),
|
||||
$dbForDatabases->getAdapter()->getSupportForIdenticalIndexes(),
|
||||
$dbForDatabases->getAdapter()->getSupportForObjectIndexes(),
|
||||
$dbForDatabases->getAdapter()->getSupportForTrigramIndex(),
|
||||
$dbForDatabases->getAdapter()->getSupportForSpatialAttributes(),
|
||||
$dbForDatabases->getAdapter()->getSupportForIndex(),
|
||||
$dbForDatabases->getAdapter()->getSupportForUniqueIndex(),
|
||||
$dbForDatabases->getAdapter()->getSupportForFulltextIndex()
|
||||
);
|
||||
|
||||
|
||||
if (!$validator->isValid($index)) {
|
||||
throw new Exception(
|
||||
resourceName: $resource->getName(),
|
||||
@@ -960,10 +918,10 @@ class Appwrite extends Destination
|
||||
);
|
||||
}
|
||||
|
||||
$index = $this->database->createDocument('indexes', $index);
|
||||
$index = $this->dbForProject->createDocument('indexes', $index);
|
||||
|
||||
try {
|
||||
$result = $this->database->createIndex(
|
||||
$result = $dbForDatabases->createIndex(
|
||||
'database_' . $database->getSequence() . '_collection_' . $table->getSequence(),
|
||||
$resource->getKey(),
|
||||
$resource->getType(),
|
||||
@@ -981,7 +939,7 @@ class Appwrite extends Destination
|
||||
);
|
||||
}
|
||||
} catch (\Throwable $th) {
|
||||
$this->database->deleteDocument('indexes', $index->getId());
|
||||
$this->dbForProject->deleteDocument('indexes', $index->getId());
|
||||
|
||||
throw new Exception(
|
||||
resourceName: $resource->getName(),
|
||||
@@ -991,7 +949,7 @@ class Appwrite extends Destination
|
||||
);
|
||||
}
|
||||
|
||||
$this->database->purgeCachedDocument(
|
||||
$this->dbForProject->purgeCachedDocument(
|
||||
'database_' . $database->getSequence(),
|
||||
$table->getId()
|
||||
);
|
||||
@@ -1005,7 +963,7 @@ class Appwrite extends Destination
|
||||
* @throws StructureException
|
||||
* @throws Exception
|
||||
*/
|
||||
protected function createRow(Row $resource, bool $isLast): bool
|
||||
protected function createRecord(Row $resource, bool $isLast): bool
|
||||
{
|
||||
if ($resource->getId() == 'unique()') {
|
||||
$resource->setId(ID::unique());
|
||||
@@ -1025,7 +983,7 @@ class Appwrite extends Destination
|
||||
// Check if document has already been created
|
||||
$exists = \array_key_exists(
|
||||
$resource->getId(),
|
||||
$this->cache->get(Resource::TYPE_ROW)
|
||||
$this->cache->get($resource->getName())
|
||||
);
|
||||
|
||||
if ($exists) {
|
||||
@@ -1070,45 +1028,46 @@ class Appwrite extends Destination
|
||||
|
||||
if ($isLast) {
|
||||
try {
|
||||
$database = $this->database->getDocument(
|
||||
$database = $this->dbForProject->getDocument(
|
||||
'databases',
|
||||
$resource->getTable()->getDatabase()->getId(),
|
||||
);
|
||||
|
||||
$table = $this->database->getDocument(
|
||||
$table = $this->dbForProject->getDocument(
|
||||
'database_' . $database->getSequence(),
|
||||
$resource->getTable()->getId(),
|
||||
);
|
||||
|
||||
$databaseInternalId = $database->getSequence();
|
||||
$tableInternalId = $table->getSequence();
|
||||
|
||||
$dbForDatabases = ($this->getDatabasesDB)($database);
|
||||
/**
|
||||
* This is in case an attribute was deleted from Appwrite attributes collection but was not deleted from the table
|
||||
* When creating an archive we select * which will include orphan attribute from the schema
|
||||
*/
|
||||
foreach ($this->rowBuffer as $row) {
|
||||
foreach ($row as $key => $value) {
|
||||
if (\str_starts_with($key, '$')) {
|
||||
continue;
|
||||
}
|
||||
|
||||
/** @var \Utopia\Database\Document $attribute */
|
||||
$found = false;
|
||||
foreach ($table->getAttribute('attributes', []) as $attribute) {
|
||||
if ($attribute->getAttribute('key') == $key) {
|
||||
$found = true;
|
||||
break;
|
||||
if ($dbForDatabases->getAdapter()->getSupportForAttributes()) {
|
||||
foreach ($this->rowBuffer as $row) {
|
||||
foreach ($row as $key => $value) {
|
||||
if (\str_starts_with($key, '$')) {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
if (! $found) {
|
||||
$row->removeAttribute($key);
|
||||
/** @var \Utopia\Database\Document $attribute */
|
||||
$found = false;
|
||||
foreach ($table->getAttribute('attributes', []) as $attribute) {
|
||||
if ($attribute->getAttribute('key') == $key) {
|
||||
$found = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (! $found) {
|
||||
$row->removeAttribute($key);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
$this->database->skipRelationshipsExistCheck(fn () => $this->database->createDocuments(
|
||||
$dbForDatabases->skipRelationshipsExistCheck(fn () => $dbForDatabases->createDocuments(
|
||||
'database_' . $databaseInternalId . '_collection_' . $tableInternalId,
|
||||
$this->rowBuffer
|
||||
));
|
||||
@@ -1908,8 +1867,8 @@ class Appwrite extends Destination
|
||||
protected function createSubscriber(Subscriber $resource): void
|
||||
{
|
||||
$target = match ($resource->getProviderType()) {
|
||||
'push' => $this->database->getDocument('targets', $resource->getTargetId()),
|
||||
'email', 'sms' => $this->database->findOne('targets', [
|
||||
'push' => $this->dbForProject->getDocument('targets', $resource->getTargetId()),
|
||||
'email', 'sms' => $this->dbForProject->findOne('targets', [
|
||||
Query::equal('userId', [$resource->getUserId()]),
|
||||
Query::equal('providerType', [$resource->getProviderType()]),
|
||||
]),
|
||||
@@ -1920,12 +1879,12 @@ class Appwrite extends Destination
|
||||
throw new \Exception('Target not found for subscriber: ' . $resource->getId());
|
||||
}
|
||||
|
||||
$topic = $this->database->getDocument('topics', $resource->getTopicId());
|
||||
$topic = $this->dbForProject->getDocument('topics', $resource->getTopicId());
|
||||
if ($topic->isEmpty()) {
|
||||
throw new \Exception('Topic not found: ' . $resource->getTopicId());
|
||||
}
|
||||
|
||||
$user = $this->database->getDocument('users', $resource->getUserId());
|
||||
$user = $this->dbForProject->getDocument('users', $resource->getUserId());
|
||||
if ($user->isEmpty()) {
|
||||
throw new \Exception('User not found: ' . $resource->getUserId());
|
||||
}
|
||||
@@ -1933,7 +1892,7 @@ class Appwrite extends Destination
|
||||
$createdAt = $this->normalizeDateTime($resource->getCreatedAt());
|
||||
$updatedAt = $this->normalizeDateTime($resource->getUpdatedAt(), $createdAt);
|
||||
|
||||
$this->database->createDocument('subscribers', new UtopiaDocument([
|
||||
$this->dbForProject->createDocument('subscribers', new UtopiaDocument([
|
||||
'$id' => $resource->getId(),
|
||||
'$createdAt' => $createdAt,
|
||||
'$updatedAt' => $updatedAt,
|
||||
@@ -1963,7 +1922,7 @@ class Appwrite extends Destination
|
||||
default => throw new \Exception('Unknown provider type: ' . $resource->getProviderType()),
|
||||
};
|
||||
|
||||
$this->database->increaseDocumentAttribute('topics', $resource->getTopicId(), $totalAttribute);
|
||||
$this->dbForProject->increaseDocumentAttribute('topics', $resource->getTopicId(), $totalAttribute);
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -2002,7 +1961,7 @@ class Appwrite extends Destination
|
||||
default => '',
|
||||
};
|
||||
|
||||
$this->database->createDocument('messages', new UtopiaDocument([
|
||||
$this->dbForProject->createDocument('messages', new UtopiaDocument([
|
||||
'$id' => $resource->getId(),
|
||||
'$createdAt' => $createdAt,
|
||||
'$updatedAt' => $updatedAt,
|
||||
@@ -2157,4 +2116,95 @@ class Appwrite extends Destination
|
||||
|
||||
return $deployment;
|
||||
}
|
||||
|
||||
private function validateFieldsForIndexes(Index $resource, UtopiaDocument $table, array &$lengths)
|
||||
{
|
||||
/**
|
||||
* @var array<UtopiaDocument> $tableColumns
|
||||
*/
|
||||
$tableColumns = $table->getAttribute('attributes', []);
|
||||
|
||||
$oldColumns = \array_map(
|
||||
fn ($attr) => $attr->getArrayCopy(),
|
||||
$tableColumns
|
||||
);
|
||||
|
||||
$oldColumns[] = [
|
||||
'key' => '$id',
|
||||
'type' => UtopiaDatabase::VAR_STRING,
|
||||
'status' => 'available',
|
||||
'required' => true,
|
||||
'array' => false,
|
||||
'default' => null,
|
||||
'size' => UtopiaDatabase::LENGTH_KEY
|
||||
];
|
||||
|
||||
$oldColumns[] = [
|
||||
'key' => '$createdAt',
|
||||
'type' => UtopiaDatabase::VAR_DATETIME,
|
||||
'status' => 'available',
|
||||
'signed' => false,
|
||||
'required' => false,
|
||||
'array' => false,
|
||||
'default' => null,
|
||||
'size' => 0
|
||||
];
|
||||
|
||||
$oldColumns[] = [
|
||||
'key' => '$updatedAt',
|
||||
'type' => UtopiaDatabase::VAR_DATETIME,
|
||||
'status' => 'available',
|
||||
'signed' => false,
|
||||
'required' => false,
|
||||
'array' => false,
|
||||
'default' => null,
|
||||
'size' => 0
|
||||
];
|
||||
|
||||
foreach ($resource->getColumns() as $i => $column) {
|
||||
// find attribute metadata in collection document
|
||||
$columnIndex = \array_search(
|
||||
$column,
|
||||
\array_column($oldColumns, 'key')
|
||||
);
|
||||
|
||||
if ($columnIndex === false) {
|
||||
throw new Exception(
|
||||
resourceName: $resource->getName(),
|
||||
resourceGroup: $resource->getGroup(),
|
||||
resourceId: $resource->getId(),
|
||||
message: 'Column not found in table: ' . $column,
|
||||
);
|
||||
}
|
||||
|
||||
$columnStatus = $oldColumns[$columnIndex]['status'];
|
||||
$columnType = $oldColumns[$columnIndex]['type'];
|
||||
$columnArray = $oldColumns[$columnIndex]['array'] ?? false;
|
||||
|
||||
if ($columnType === UtopiaDatabase::VAR_RELATIONSHIP) {
|
||||
throw new Exception(
|
||||
resourceName: $resource->getName(),
|
||||
resourceGroup: $resource->getGroup(),
|
||||
resourceId: $resource->getId(),
|
||||
message: 'Relationship columns are not supported in indexes',
|
||||
);
|
||||
}
|
||||
|
||||
// Ensure attribute is available
|
||||
if ($columnStatus !== 'available') {
|
||||
throw new Exception(
|
||||
resourceName: $resource->getName(),
|
||||
resourceGroup: $resource->getGroup(),
|
||||
resourceId: $resource->getId(),
|
||||
message: 'Column not available: ' . $column,
|
||||
);
|
||||
}
|
||||
|
||||
$lengths[$i] = null;
|
||||
|
||||
if ($columnArray === true) {
|
||||
$lengths[$i] = UtopiaDatabase::MAX_ARRAY_INDEX_LENGTH;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -30,6 +30,13 @@ abstract class Resource implements \JsonSerializable
|
||||
|
||||
public const TYPE_DATABASE = 'database';
|
||||
|
||||
public const TYPE_DATABASE_LEGACY = 'legacy';
|
||||
|
||||
public const TYPE_DATABASE_TABLESDB = 'tablesdb';
|
||||
|
||||
public const TYPE_DATABASE_DOCUMENTSDB = 'documentsdb';
|
||||
public const TYPE_DATABASE_VECTORSDB = 'vectorsdb';
|
||||
|
||||
public const TYPE_ROW = 'row';
|
||||
|
||||
public const TYPE_FILE = 'file';
|
||||
@@ -84,6 +91,8 @@ abstract class Resource implements \JsonSerializable
|
||||
self::TYPE_BUCKET,
|
||||
self::TYPE_TABLE,
|
||||
self::TYPE_DATABASE,
|
||||
self::TYPE_DATABASE_VECTORSDB,
|
||||
self::TYPE_DATABASE_DOCUMENTSDB,
|
||||
self::TYPE_ROW,
|
||||
self::TYPE_FILE,
|
||||
self::TYPE_FUNCTION,
|
||||
@@ -108,6 +117,39 @@ abstract class Resource implements \JsonSerializable
|
||||
self::TYPE_COLLECTION,
|
||||
];
|
||||
|
||||
// index terminology is same for all
|
||||
public const DATABASE_TYPE_RESOURCE_MAP = [
|
||||
self::TYPE_DATABASE => [
|
||||
'entity' => self::TYPE_TABLE,
|
||||
'field' => self::TYPE_COLUMN,
|
||||
'record' => self::TYPE_ROW,
|
||||
],
|
||||
self::TYPE_DATABASE_DOCUMENTSDB => [
|
||||
'entity' => self::TYPE_COLLECTION,
|
||||
// HACK: not required in documentsdb but adding it for consistency in the db reader(not gonna impact)
|
||||
'field' => self::TYPE_ATTRIBUTE,
|
||||
'record' => self::TYPE_DOCUMENT,
|
||||
],
|
||||
self::TYPE_DATABASE_VECTORSDB => [
|
||||
'entity' => self::TYPE_COLLECTION,
|
||||
'field' => self::TYPE_ATTRIBUTE,
|
||||
'record' => self::TYPE_DOCUMENT,
|
||||
]
|
||||
];
|
||||
|
||||
public const ENTITY_TYPE_RESOURCE_MAP = [
|
||||
self::TYPE_TABLE => [
|
||||
'field' => self::TYPE_COLUMN,
|
||||
'record' => self::TYPE_ROW,
|
||||
'index' => self::TYPE_INDEX
|
||||
],
|
||||
self::TYPE_COLLECTION => [
|
||||
'field' => self::TYPE_ATTRIBUTE,
|
||||
'record' => self::TYPE_DOCUMENT,
|
||||
'index' => self::TYPE_INDEX
|
||||
],
|
||||
];
|
||||
|
||||
protected string $id = '';
|
||||
|
||||
protected string $originalId = '';
|
||||
|
||||
@@ -0,0 +1,189 @@
|
||||
<?php
|
||||
|
||||
namespace Utopia\Migration\Resources\Database;
|
||||
|
||||
use Utopia\Migration\Resource;
|
||||
use Utopia\Migration\Transfer;
|
||||
|
||||
class Attribute extends Resource
|
||||
{
|
||||
public const TYPE_STRING = 'string';
|
||||
public const TYPE_INTEGER = 'integer';
|
||||
public const TYPE_FLOAT = 'double';
|
||||
public const TYPE_BOOLEAN = 'boolean';
|
||||
public const TYPE_DATETIME = 'datetime';
|
||||
public const TYPE_EMAIL = 'email';
|
||||
public const TYPE_ENUM = 'enum';
|
||||
public const TYPE_IP = 'ip';
|
||||
public const TYPE_URL = 'url';
|
||||
public const TYPE_RELATIONSHIP = 'relationship';
|
||||
|
||||
public const TYPE_POINT = 'point';
|
||||
public const TYPE_LINE = 'linestring';
|
||||
public const TYPE_POLYGON = 'polygon';
|
||||
|
||||
public const TYPE_OBJECT = 'object';
|
||||
public const TYPE_VECTOR = 'vector';
|
||||
|
||||
/**
|
||||
* @param string $key
|
||||
* @param Table $table
|
||||
* @param string $fieldType The actual field type (e.g., 'string', 'integer', 'email')
|
||||
* @param int $size
|
||||
* @param bool $required
|
||||
* @param mixed|null $default
|
||||
* @param bool $array
|
||||
* @param bool $signed
|
||||
* @param string $format
|
||||
* @param array<string, mixed> $formatOptions
|
||||
* @param array<string> $filters
|
||||
* @param array<string, mixed> $options
|
||||
* @param string $createdAt
|
||||
* @param string $updatedAt
|
||||
*/
|
||||
public function __construct(
|
||||
protected readonly string $key,
|
||||
protected readonly Table $table,
|
||||
protected readonly string $fieldType = '',
|
||||
protected readonly int $size = 0,
|
||||
protected readonly bool $required = false,
|
||||
protected readonly mixed $default = null,
|
||||
protected readonly bool $array = false,
|
||||
protected readonly bool $signed = false,
|
||||
protected readonly string $format = '',
|
||||
protected readonly array $formatOptions = [],
|
||||
protected readonly array $filters = [],
|
||||
protected array $options = [],
|
||||
protected string $createdAt = '',
|
||||
protected string $updatedAt = '',
|
||||
) {
|
||||
}
|
||||
|
||||
/**
|
||||
* @return array<string, mixed>
|
||||
*/
|
||||
public function jsonSerialize(): array
|
||||
{
|
||||
return [
|
||||
'key' => $this->key,
|
||||
'table' => $this->table,
|
||||
'type' => $this->getType(),
|
||||
'size' => $this->size,
|
||||
'required' => $this->required,
|
||||
'default' => $this->default,
|
||||
'array' => $this->array,
|
||||
'signed' => $this->signed,
|
||||
'format' => $this->format,
|
||||
'formatOptions' => $this->formatOptions,
|
||||
'filters' => $this->filters,
|
||||
'options' => $this->options,
|
||||
'createdAt' => $this->createdAt,
|
||||
'updatedAt' => $this->updatedAt,
|
||||
];
|
||||
}
|
||||
|
||||
public static function getName(): string
|
||||
{
|
||||
return Resource::TYPE_ATTRIBUTE;
|
||||
}
|
||||
|
||||
public function getType(): string
|
||||
{
|
||||
return $this->fieldType;
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert a Column resource to an Attribute resource.
|
||||
*
|
||||
* @param Column $column
|
||||
* @return self
|
||||
*/
|
||||
public static function fromColumn(Column $column): self
|
||||
{
|
||||
return new self(
|
||||
$column->getKey(),
|
||||
$column->getTable(),
|
||||
$column->getType(),
|
||||
$column->getSize(),
|
||||
$column->isRequired(),
|
||||
$column->getDefault(),
|
||||
$column->isArray(),
|
||||
$column->isSigned(),
|
||||
$column->getFormat(),
|
||||
$column->getFormatOptions(),
|
||||
$column->getFilters(),
|
||||
$column->getOptions(),
|
||||
$column->getCreatedAt(),
|
||||
$column->getUpdatedAt()
|
||||
);
|
||||
}
|
||||
|
||||
public function getGroup(): string
|
||||
{
|
||||
return Transfer::GROUP_DATABASES;
|
||||
}
|
||||
|
||||
public function getKey(): string
|
||||
{
|
||||
return $this->key;
|
||||
}
|
||||
|
||||
public function getTable(): Table
|
||||
{
|
||||
return $this->table;
|
||||
}
|
||||
|
||||
public function getSize(): int
|
||||
{
|
||||
return $this->size;
|
||||
}
|
||||
|
||||
public function isRequired(): bool
|
||||
{
|
||||
return $this->required;
|
||||
}
|
||||
|
||||
public function getDefault(): mixed
|
||||
{
|
||||
return $this->default;
|
||||
}
|
||||
|
||||
public function isArray(): bool
|
||||
{
|
||||
return $this->array;
|
||||
}
|
||||
|
||||
public function isSigned(): bool
|
||||
{
|
||||
return $this->signed;
|
||||
}
|
||||
|
||||
public function getFormat(): string
|
||||
{
|
||||
return $this->format;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return array<string, mixed>
|
||||
*/
|
||||
public function getFormatOptions(): array
|
||||
{
|
||||
return $this->formatOptions;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return array<string>
|
||||
*/
|
||||
public function getFilters(): array
|
||||
{
|
||||
return $this->filters;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return array<string, mixed>
|
||||
*/
|
||||
public function &getOptions(): array
|
||||
{
|
||||
return $this->options;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,50 @@
|
||||
<?php
|
||||
|
||||
namespace Utopia\Migration\Resources\Database;
|
||||
|
||||
use Utopia\Migration\Resource;
|
||||
|
||||
class Collection extends Table
|
||||
{
|
||||
public static function getName(): string
|
||||
{
|
||||
return Resource::TYPE_COLLECTION;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param array{
|
||||
* database: array{
|
||||
* id: string,
|
||||
* name: string,
|
||||
* type: string
|
||||
* },
|
||||
* name: string,
|
||||
* id: string,
|
||||
* documentSecurity?: bool,
|
||||
* rowSecurity?: bool,
|
||||
* permissions: ?array<string>,
|
||||
* createdAt: string,
|
||||
* updatedAt: string,
|
||||
* enabled: bool
|
||||
* } $array
|
||||
*/
|
||||
public static function fromArray(array $array): self
|
||||
{
|
||||
$database = match ($array['database']['type']) {
|
||||
Resource::TYPE_DATABASE_DOCUMENTSDB => DocumentsDB::fromArray($array['database']),
|
||||
Resource::TYPE_DATABASE_VECTORSDB => VectorsDB::fromArray($array['database']),
|
||||
default => Database::fromArray($array['database'])
|
||||
};
|
||||
|
||||
return new self(
|
||||
$database,
|
||||
name: $array['name'],
|
||||
id: $array['id'],
|
||||
rowSecurity: $array['rowSecurity'] ?? $array['documentSecurity'],
|
||||
permissions: $array['permissions'] ?? [],
|
||||
createdAt: $array['createdAt'] ?? '',
|
||||
updatedAt: $array['updatedAt'] ?? '',
|
||||
enabled: $array['enabled'] ?? true,
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -27,6 +27,9 @@ abstract class Column extends Resource
|
||||
public const TYPE_LINE = 'linestring';
|
||||
public const TYPE_POLYGON = 'polygon';
|
||||
|
||||
public const TYPE_OBJECT = 'object';
|
||||
public const TYPE_VECTOR = 'vector';
|
||||
|
||||
/**
|
||||
* @param string $key
|
||||
* @param Table $table
|
||||
@@ -157,4 +160,16 @@ abstract class Column extends Resource
|
||||
{
|
||||
return $this->options;
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert this Column resource to an Attribute resource.
|
||||
* This provides a deterministic way to derive attributes from columns,
|
||||
* eliminating the need to maintain duplicate per-type Attribute implementations.
|
||||
*
|
||||
* @return Attribute
|
||||
*/
|
||||
public function getAttribute(): Attribute
|
||||
{
|
||||
return Attribute::fromColumn($this);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,74 @@
|
||||
<?php
|
||||
|
||||
namespace Utopia\Migration\Resources\Database\Columns;
|
||||
|
||||
use Utopia\Migration\Resources\Database\Column;
|
||||
use Utopia\Migration\Resources\Database\Table;
|
||||
|
||||
class ObjectType extends Column
|
||||
{
|
||||
public function __construct(
|
||||
string $key,
|
||||
Table $table,
|
||||
bool $required = false,
|
||||
?array $default = null,
|
||||
string $createdAt = '',
|
||||
string $updatedAt = ''
|
||||
) {
|
||||
parent::__construct(
|
||||
$key,
|
||||
$table,
|
||||
required: $required,
|
||||
default: $default,
|
||||
createdAt: $createdAt,
|
||||
updatedAt: $updatedAt
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param array{
|
||||
* key: string,
|
||||
* collection?: array{
|
||||
* database: array{
|
||||
* id: string,
|
||||
* name: string,
|
||||
* },
|
||||
* name: string,
|
||||
* id: string,
|
||||
* documentSecurity: bool,
|
||||
* permissions: ?array<string>
|
||||
* },
|
||||
* table?: array{
|
||||
* database: array{
|
||||
* id: string,
|
||||
* name: string,
|
||||
* },
|
||||
* name: string,
|
||||
* id: string,
|
||||
* rowSecurity: bool,
|
||||
* permissions: ?array<string>
|
||||
* },
|
||||
* required: bool,
|
||||
* default: ?array<mixed>,
|
||||
* createdAt: string,
|
||||
* updatedAt: string,
|
||||
* } $array
|
||||
* @return self
|
||||
*/
|
||||
public static function fromArray(array $array): self
|
||||
{
|
||||
return new self(
|
||||
$array['key'],
|
||||
Table::fromArray($array['table'] ?? $array['collection']),
|
||||
required: $array['required'],
|
||||
default: $array['default'],
|
||||
createdAt: $array['createdAt'] ?? '',
|
||||
updatedAt: $array['updatedAt'] ?? '',
|
||||
);
|
||||
}
|
||||
|
||||
public function getType(): string
|
||||
{
|
||||
return Column::TYPE_OBJECT;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,78 @@
|
||||
<?php
|
||||
|
||||
namespace Utopia\Migration\Resources\Database\Columns;
|
||||
|
||||
use Utopia\Migration\Resources\Database\Column;
|
||||
use Utopia\Migration\Resources\Database\Table;
|
||||
|
||||
class Vector extends Column
|
||||
{
|
||||
public function __construct(
|
||||
string $key,
|
||||
Table $table,
|
||||
bool $required = false,
|
||||
int $size = 0,
|
||||
?array $default = null,
|
||||
string $createdAt = '',
|
||||
string $updatedAt = ''
|
||||
) {
|
||||
parent::__construct(
|
||||
$key,
|
||||
$table,
|
||||
size: $size,
|
||||
required: $required,
|
||||
default: $default,
|
||||
createdAt: $createdAt,
|
||||
updatedAt: $updatedAt
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param array{
|
||||
* key: string,
|
||||
* collection?: array{
|
||||
* database: array{
|
||||
* id: string,
|
||||
* name: string,
|
||||
* },
|
||||
* name: string,
|
||||
* id: string,
|
||||
* documentSecurity: bool,
|
||||
* permissions: ?array<string>
|
||||
* },
|
||||
* table?: array{
|
||||
* database: array{
|
||||
* id: string,
|
||||
* name: string,
|
||||
* },
|
||||
* name: string,
|
||||
* id: string,
|
||||
* rowSecurity: bool,
|
||||
* permissions: ?array<string>
|
||||
* },
|
||||
* size: int,
|
||||
* required: bool,
|
||||
* default: ?array<mixed>,
|
||||
* createdAt: string,
|
||||
* updatedAt: string,
|
||||
* } $array
|
||||
* @return self
|
||||
*/
|
||||
public static function fromArray(array $array): self
|
||||
{
|
||||
return new self(
|
||||
$array['key'],
|
||||
Table::fromArray($array['table'] ?? $array['collection']),
|
||||
required: $array['required'],
|
||||
size:$array['size'],
|
||||
default: $array['default'],
|
||||
createdAt: $array['createdAt'] ?? '',
|
||||
updatedAt: $array['updatedAt'] ?? '',
|
||||
);
|
||||
}
|
||||
|
||||
public function getType(): string
|
||||
{
|
||||
return Column::TYPE_VECTOR;
|
||||
}
|
||||
}
|
||||
@@ -26,6 +26,7 @@ class Database extends Resource
|
||||
protected bool $enabled = true,
|
||||
protected string $originalId = '',
|
||||
protected string $type = '',
|
||||
protected string $database = ''
|
||||
) {
|
||||
$this->id = $id;
|
||||
}
|
||||
@@ -38,6 +39,7 @@ class Database extends Resource
|
||||
* updatedAt: string,
|
||||
* enabled: bool,
|
||||
* originalId: string|null,
|
||||
* database: string
|
||||
* } $array
|
||||
*/
|
||||
public static function fromArray(array $array): self
|
||||
@@ -50,6 +52,7 @@ class Database extends Resource
|
||||
enabled: $array['enabled'] ?? true,
|
||||
originalId: $array['originalId'] ?? '',
|
||||
type: $array['type'] ?? 'legacy',
|
||||
database: $array['database']
|
||||
);
|
||||
}
|
||||
|
||||
@@ -65,6 +68,7 @@ class Database extends Resource
|
||||
'updatedAt' => $this->updatedAt,
|
||||
'enabled' => $this->enabled,
|
||||
'type' => $this->type,
|
||||
'database' => $this->database
|
||||
];
|
||||
}
|
||||
|
||||
@@ -92,4 +96,9 @@ class Database extends Resource
|
||||
{
|
||||
return $this->type;
|
||||
}
|
||||
|
||||
public function getDatabase(): string
|
||||
{
|
||||
return $this->database;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,51 @@
|
||||
<?php
|
||||
|
||||
namespace Utopia\Migration\Resources\Database;
|
||||
|
||||
use Utopia\Migration\Resource;
|
||||
|
||||
class Document extends Row
|
||||
{
|
||||
public static function getName(): string
|
||||
{
|
||||
return Resource::TYPE_DOCUMENT;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param array{
|
||||
* id: string,
|
||||
* collection?: array{
|
||||
* database: array{
|
||||
* id: string,
|
||||
* name: string,
|
||||
* },
|
||||
* name: string,
|
||||
* id: string,
|
||||
* documentSecurity: bool,
|
||||
* permissions: ?array<string>
|
||||
* },
|
||||
* table?: array{
|
||||
* database: array{
|
||||
* id: string,
|
||||
* name: string,
|
||||
* },
|
||||
* name: string,
|
||||
* id: string,
|
||||
* rowSecurity: bool,
|
||||
* permissions: ?array<string>
|
||||
* },
|
||||
* data: array<string, mixed>,
|
||||
* permissions: ?array<string>
|
||||
* } $array
|
||||
*/
|
||||
public static function fromArray(array $array): self
|
||||
{
|
||||
// keeping table and collection to have backward compat
|
||||
return new self(
|
||||
$array['id'],
|
||||
Collection::fromArray($array['table'] ?? $array['collection']),
|
||||
$array['data'],
|
||||
$array['permissions'] ?? []
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
<?php
|
||||
|
||||
namespace Utopia\Migration\Resources\Database;
|
||||
|
||||
use Utopia\Migration\Resource;
|
||||
|
||||
class DocumentsDB extends Database
|
||||
{
|
||||
public static function getName(): string
|
||||
{
|
||||
return Resource::TYPE_DATABASE_DOCUMENTSDB;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param array{
|
||||
* id: string,
|
||||
* name: string,
|
||||
* createdAt: string,
|
||||
* updatedAt: string,
|
||||
* enabled: bool,
|
||||
* originalId: string|null,
|
||||
* database: string
|
||||
* } $array
|
||||
*/
|
||||
public static function fromArray(array $array): self
|
||||
{
|
||||
return new self(
|
||||
$array['id'],
|
||||
$array['name'],
|
||||
createdAt: $array['createdAt'] ?? '',
|
||||
updatedAt: $array['updatedAt'] ?? '',
|
||||
enabled: $array['enabled'] ?? true,
|
||||
originalId: $array['originalId'] ?? '',
|
||||
type: $array['type'] ?? Resource::TYPE_DATABASE_DOCUMENTSDB,
|
||||
database: $array['database']
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,39 @@
|
||||
<?php
|
||||
|
||||
namespace Utopia\Migration\Resources\Database;
|
||||
|
||||
use Utopia\Migration\Resource;
|
||||
|
||||
class VectorsDB extends Database
|
||||
{
|
||||
public static function getName(): string
|
||||
{
|
||||
return Resource::TYPE_DATABASE_VECTORSDB;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param array{
|
||||
* id: string,
|
||||
* name: string,
|
||||
* createdAt: string,
|
||||
* updatedAt: string,
|
||||
* enabled: bool,
|
||||
* originalId: string|null,
|
||||
* database: string,
|
||||
* type: string
|
||||
* } $array
|
||||
*/
|
||||
public static function fromArray(array $array): self
|
||||
{
|
||||
return new self(
|
||||
$array['id'],
|
||||
$array['name'],
|
||||
createdAt: $array['createdAt'] ?? '',
|
||||
updatedAt: $array['updatedAt'] ?? '',
|
||||
enabled: $array['enabled'] ?? true,
|
||||
originalId: $array['originalId'] ?? '',
|
||||
type: $array['type'] ?? Resource::TYPE_DATABASE_VECTORSDB,
|
||||
database: $array['database']
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -74,6 +74,14 @@ abstract class Source extends Target
|
||||
if (!in_array($resource->getName(), $resources)) {
|
||||
$resource->setStatus(Resource::STATUS_SKIPPED);
|
||||
} else {
|
||||
/**
|
||||
* Rows and documents are counted in cache in an aggregated way (per status count)
|
||||
* to prevent memory blowups. Destination updates already account for these, and
|
||||
* adding them here causes double counting (source + destination).
|
||||
*/
|
||||
if (\in_array($resource->getName(), [Resource::TYPE_ROW, Resource::TYPE_DOCUMENT], true)) {
|
||||
continue;
|
||||
}
|
||||
$prunedResources[] = $resource;
|
||||
}
|
||||
}
|
||||
|
||||
+522
-364
File diff suppressed because it is too large.
Load diff
@@ -107,4 +107,6 @@ interface Reader
|
||||
* @return QueryType|string
|
||||
*/
|
||||
public function queryLimit(int $limit): mixed;
|
||||
public function getSupportForAttributes(): bool;
|
||||
|
||||
}
|
||||
@@ -247,4 +247,9 @@ class API implements Reader
|
||||
{
|
||||
return Query::limit($limit);
|
||||
}
|
||||
|
||||
public function getSupportForAttributes(): bool
|
||||
{
|
||||
return true;
|
||||
}
|
||||
}
|
||||
@@ -8,8 +8,10 @@ use Utopia\Database\Exception as DatabaseException;
|
||||
use Utopia\Database\Query;
|
||||
use Utopia\Migration\Exception;
|
||||
use Utopia\Migration\Resource;
|
||||
use Utopia\Migration\Resources\Database\Collection as CollectionResource;
|
||||
use Utopia\Migration\Resources\Database\Column as ColumnResource;
|
||||
use Utopia\Migration\Resources\Database\Database as DatabaseResource;
|
||||
use Utopia\Migration\Resources\Database\Document as DocumentResource;
|
||||
use Utopia\Migration\Resources\Database\Index as IndexResource;
|
||||
use Utopia\Migration\Resources\Database\Row as RowResource;
|
||||
use Utopia\Migration\Resources\Database\Table as TableResource;
|
||||
@@ -20,18 +22,35 @@ use Utopia\Migration\Sources\Appwrite\Reader;
|
||||
*/
|
||||
class Database implements Reader
|
||||
{
|
||||
public function __construct(private readonly UtopiaDatabase $dbForProject)
|
||||
{
|
||||
/**
|
||||
* @var callable(UtopiaDocument|null): UtopiaDatabase
|
||||
*/
|
||||
private mixed $getDatabasesDB;
|
||||
|
||||
public function __construct(
|
||||
private readonly UtopiaDatabase $dbForProject,
|
||||
?callable $getDatabasesDB = null,
|
||||
private readonly ?string $projectId = null
|
||||
) {
|
||||
$this->getDatabasesDB = $getDatabasesDB;
|
||||
}
|
||||
|
||||
public function report(array $resources, array &$report, array $resourceIds = []): mixed
|
||||
{
|
||||
$relevantResources = [
|
||||
// tablesdb
|
||||
Resource::TYPE_DATABASE,
|
||||
Resource::TYPE_TABLE,
|
||||
Resource::TYPE_ROW,
|
||||
Resource::TYPE_COLUMN,
|
||||
Resource::TYPE_INDEX,
|
||||
// vectorsdb
|
||||
Resource::TYPE_DATABASE_VECTORSDB,
|
||||
// Documentsdb
|
||||
Resource::TYPE_DATABASE_DOCUMENTSDB,
|
||||
Resource::TYPE_COLLECTION,
|
||||
Resource::TYPE_DOCUMENT,
|
||||
Resource::TYPE_ATTRIBUTE,
|
||||
];
|
||||
|
||||
if (!Resource::isSupported($relevantResources, $resources)) {
|
||||
@@ -43,38 +62,53 @@ class Database implements Reader
|
||||
$report[$resourceType] = 0;
|
||||
}
|
||||
}
|
||||
|
||||
$databaseResources = array_keys(Resource::DATABASE_TYPE_RESOURCE_MAP);
|
||||
$databaseQueries = [];
|
||||
if (!empty($resourceIds[Resource::TYPE_DATABASE])) {
|
||||
$databaseIds = (array) $resourceIds[Resource::TYPE_DATABASE];
|
||||
foreach ($databaseResources as $databaseResourceType) {
|
||||
if (!empty($resourceIds[$databaseResourceType])) {
|
||||
$databaseIds = (array) $resourceIds[$databaseResourceType];
|
||||
|
||||
$databaseQueries[] = Query::equal('$id', $databaseIds);
|
||||
$databaseQueries[] = Query::equal('$id', $databaseIds);
|
||||
}
|
||||
}
|
||||
|
||||
if (in_array(Resource::TYPE_DATABASE, $resources)) {
|
||||
$report[Resource::TYPE_DATABASE] = $this->countResources('databases', $databaseQueries);
|
||||
}
|
||||
|
||||
if (count(array_intersect($resources, $relevantResources)) === 1 &&
|
||||
in_array(Resource::TYPE_DATABASE, $resources)) {
|
||||
if (
|
||||
count(array_intersect($resources, $relevantResources)) === 1 &&
|
||||
Resource::isSupported(array_keys(Resource::DATABASE_TYPE_RESOURCE_MAP), $resources)
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
|
||||
$dbResources = [];
|
||||
$databases = $this->listDatabases($databaseQueries);
|
||||
|
||||
foreach ($databases as $database) {
|
||||
$databaseType = $database->getAttribute('type');
|
||||
if (in_array($databaseType, [Resource::TYPE_DATABASE_LEGACY,Resource::TYPE_DATABASE_TABLESDB])) {
|
||||
$databaseType = Resource::TYPE_DATABASE;
|
||||
}
|
||||
if (!isset(Resource::DATABASE_TYPE_RESOURCE_MAP[$databaseType])) {
|
||||
$databaseType = Resource::TYPE_DATABASE;
|
||||
}
|
||||
if (Resource::isSupported($databaseType, $resources)) {
|
||||
$report[$databaseType] += 1;
|
||||
}
|
||||
}
|
||||
|
||||
// Process each database
|
||||
foreach ($databases as $database) {
|
||||
$databaseType = $database->getAttribute('type', Resource::TYPE_DATABASE);
|
||||
if (\in_array($databaseType, [Resource::TYPE_DATABASE_LEGACY, Resource::TYPE_DATABASE_TABLESDB], true)) {
|
||||
$databaseType = Resource::TYPE_DATABASE;
|
||||
}
|
||||
|
||||
if (!isset(Resource::DATABASE_TYPE_RESOURCE_MAP[$databaseType])) {
|
||||
$databaseType = Resource::TYPE_DATABASE;
|
||||
}
|
||||
|
||||
$databaseSpecificResources = Resource::DATABASE_TYPE_RESOURCE_MAP[$databaseType];
|
||||
|
||||
$databaseSequence = $database->getSequence();
|
||||
$tableId = "database_{$databaseSequence}";
|
||||
|
||||
if (Resource::isSupported(Resource::TYPE_TABLE, $resources)) {
|
||||
$report[Resource::TYPE_TABLE] += $this->countResources($tableId);
|
||||
}
|
||||
|
||||
if (!Resource::isSupported([Resource::TYPE_ROW, Resource::TYPE_COLUMN, Resource::TYPE_INDEX], $resources)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!isset($dbResources[$database->getId()])) {
|
||||
$dbResources[$database->getId()] = new DatabaseResource(
|
||||
@@ -82,19 +116,29 @@ class Database implements Reader
|
||||
$database->getAttribute('name'),
|
||||
$database->getCreatedAt(),
|
||||
$database->getUpdatedAt(),
|
||||
$database->getAttribute('enabled', true),
|
||||
$database->getAttribute('originalId', ''),
|
||||
$database->getAttribute('type', ''),
|
||||
$database->getAttribute('database', '')
|
||||
);
|
||||
}
|
||||
|
||||
$dbResource = $dbResources[$database->getId()];
|
||||
|
||||
$tables = $this->listTables($dbResource);
|
||||
$count = count($tables);
|
||||
|
||||
if (Resource::isSupported($databaseSpecificResources['entity'], $resources)) {
|
||||
$report[$databaseSpecificResources['entity']] += $count;
|
||||
}
|
||||
|
||||
foreach ($tables as $table) {
|
||||
$tableSequence = $table->getSequence();
|
||||
|
||||
if (Resource::isSupported(Resource::TYPE_ROW, $resources)) {
|
||||
if (Resource::isSupported($databaseSpecificResources['record'], $resources)) {
|
||||
$rowTableId = "database_{$databaseSequence}_collection_{$tableSequence}";
|
||||
$report[Resource::TYPE_ROW] += $this->countResources($rowTableId);
|
||||
$count = $this->countResources($rowTableId, [], $dbResource);
|
||||
$report[$databaseSpecificResources['record']] += $count;
|
||||
}
|
||||
|
||||
$commonQueries = [
|
||||
@@ -102,8 +146,12 @@ class Database implements Reader
|
||||
Query::equal('collectionInternalId', [$tableSequence]),
|
||||
];
|
||||
|
||||
if (Resource::isSupported(Resource::TYPE_COLUMN, $resources)) {
|
||||
$report[Resource::TYPE_COLUMN] += $this->countResources('attributes', $commonQueries);
|
||||
if (
|
||||
isset($databaseSpecificResources['field']) &&
|
||||
Resource::isSupported($databaseSpecificResources['field'], $resources)
|
||||
) {
|
||||
$count = $this->countResources('attributes', $commonQueries);
|
||||
$report[$databaseSpecificResources['field']] += $count;
|
||||
}
|
||||
|
||||
if (in_array(Resource::TYPE_INDEX, $resources)) {
|
||||
@@ -302,8 +350,11 @@ class Database implements Reader
|
||||
|
||||
$tableId = "database_{$database->getSequence()}_collection_{$table->getSequence()}";
|
||||
|
||||
// Use the appropriate database instance for this specific database
|
||||
$dbInstance = $this->getDatabase($resource->getDatabase()->getDatabase());
|
||||
|
||||
try {
|
||||
$rows = $this->dbForProject->find($tableId, $queries);
|
||||
$rows = $dbInstance->find($tableId, $queries);
|
||||
} catch (DatabaseException $e) {
|
||||
throw new Exception(
|
||||
resourceName: $resource->getName(),
|
||||
@@ -352,7 +403,10 @@ class Database implements Reader
|
||||
|
||||
$tableId = "database_{$database->getSequence()}_collection_{$table->getSequence()}";
|
||||
|
||||
return $this->dbForProject->getDocument(
|
||||
// Use the appropriate database instance for this specific database
|
||||
$dbInstance = $this->getDatabase($resource->getDatabase()->getDatabase());
|
||||
|
||||
return $dbInstance->getDocument(
|
||||
$tableId,
|
||||
$rowId,
|
||||
$queries
|
||||
@@ -392,19 +446,32 @@ class Database implements Reader
|
||||
|
||||
switch ($resource::class) {
|
||||
case DatabaseResource::class:
|
||||
/** @var DatabaseResource $resource */
|
||||
// Databases are always in dbForProject metadata
|
||||
$document = $this->dbForProject->getDocument('databases', $resource->getId());
|
||||
break;
|
||||
case TableResource::class:
|
||||
case CollectionResource::class:
|
||||
/** @var TableResource|CollectionResource $resource */
|
||||
// Tables/Collections metadata is in dbForProject
|
||||
$database = $this->dbForProject->getDocument('databases', $resource->getDatabase()->getId());
|
||||
$document = $this->dbForProject->getDocument('database_' . $database->getSequence(), $resource->getId());
|
||||
break;
|
||||
case ColumnResource::class:
|
||||
/** @var ColumnResource $resource */
|
||||
// Columns (attributes) are in dbForProject metadata
|
||||
$document = $this->dbForProject->getDocument('attributes', $resource->getId());
|
||||
break;
|
||||
case IndexResource::class:
|
||||
/** @var IndexResource $resource */
|
||||
// Indexes are in dbForProject metadata
|
||||
$document = $this->dbForProject->getDocument('indexes', $resource->getId());
|
||||
break;
|
||||
case RowResource::class:
|
||||
case DocumentResource::class:
|
||||
/** @var RowResource|DocumentResource $resource */
|
||||
// Rows/Documents are in the specific database instance
|
||||
// getRow() already uses getDatabase() internally
|
||||
$document = $this->getRow($resource->getTable(), $resource->getId());
|
||||
$document = new UtopiaDocument($document);
|
||||
break;
|
||||
@@ -420,13 +487,38 @@ class Database implements Reader
|
||||
return Query::limit($limit);
|
||||
}
|
||||
|
||||
public function getSupportForAttributes(): bool
|
||||
{
|
||||
return $this->dbForProject->getAdapter()->getSupportForAttributes();
|
||||
}
|
||||
|
||||
/**
|
||||
* @param string $table
|
||||
* @param array $queries
|
||||
* @param DatabaseResource|null $databaseResource
|
||||
* @return int
|
||||
*/
|
||||
private function countResources(string $table, array $queries = []): int
|
||||
private function countResources(string $table, array $queries = [], ?DatabaseResource $databaseResource = null): int
|
||||
{
|
||||
// Use the appropriate database instance for row data
|
||||
if ($databaseResource !== null) {
|
||||
$dbInstance = $this->getDatabase($databaseResource->getDatabase());
|
||||
return $dbInstance->count($table, $queries);
|
||||
}
|
||||
|
||||
// Use dbForProject for metadata tables
|
||||
return $this->dbForProject->count($table, $queries);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the appropriate database instance for the given database DSN
|
||||
*/
|
||||
private function getDatabase(?string $databaseDSN = null): UtopiaDatabase
|
||||
{
|
||||
if ($this->getDatabasesDB !== null && $databaseDSN !== null) {
|
||||
return ($this->getDatabasesDB)(new UtopiaDocument(['database' => $databaseDSN]));
|
||||
}
|
||||
|
||||
return $this->dbForProject;
|
||||
}
|
||||
}
|
||||
@@ -4,11 +4,9 @@ namespace Utopia\Migration\Sources;
|
||||
|
||||
use Utopia\Database\Database as UtopiaDatabase;
|
||||
use Utopia\Migration\Exception;
|
||||
use Utopia\Migration\Resource;
|
||||
use Utopia\Migration\Resource as UtopiaResource;
|
||||
use Utopia\Migration\Resources\Database\Column;
|
||||
use Utopia\Migration\Resources\Database\Database;
|
||||
use Utopia\Migration\Resources\Database\Row;
|
||||
use Utopia\Migration\Resources\Database\Table;
|
||||
use Utopia\Migration\Resources\Storage\File;
|
||||
use Utopia\Migration\Source;
|
||||
use Utopia\Migration\Sources\Appwrite\Reader;
|
||||
@@ -44,12 +42,13 @@ class CSV extends Source
|
||||
string $resourceId,
|
||||
string $filePath,
|
||||
Device $device,
|
||||
?UtopiaDatabase $dbForProject
|
||||
?UtopiaDatabase $dbForProject,
|
||||
?callable $getDatabasesDB = null,
|
||||
) {
|
||||
$this->device = $device;
|
||||
$this->filePath = $filePath;
|
||||
$this->resourceId = $resourceId;
|
||||
$this->database = new DatabaseReader($dbForProject);
|
||||
$this->database = new DatabaseReader($dbForProject, $getDatabasesDB);
|
||||
}
|
||||
|
||||
public static function getName(): string
|
||||
@@ -61,6 +60,7 @@ class CSV extends Source
|
||||
{
|
||||
return [
|
||||
UtopiaResource::TYPE_ROW,
|
||||
UtopiaResource::TYPE_DOCUMENT,
|
||||
];
|
||||
}
|
||||
|
||||
@@ -104,7 +104,7 @@ class CSV extends Source
|
||||
protected function exportGroupDatabases(int $batchSize, array $resources): void
|
||||
{
|
||||
try {
|
||||
if (UtopiaResource::isSupported(UtopiaResource::TYPE_ROW, $resources)) {
|
||||
if (UtopiaResource::isSupported($this->getSupportedResources(), $resources)) {
|
||||
$this->exportRows($batchSize);
|
||||
}
|
||||
} catch (\Throwable $e) {
|
||||
@@ -132,8 +132,47 @@ class CSV extends Source
|
||||
$lastColumn = null;
|
||||
|
||||
[$databaseId, $tableId] = \explode(':', $this->resourceId);
|
||||
$database = new Database($databaseId, '');
|
||||
$table = new Table($database, '', $tableId);
|
||||
|
||||
$databases = $this->database->listDatabases([
|
||||
$this->database->queryEqual('$id', [$databaseId]),
|
||||
$this->database->queryLimit(1),
|
||||
]);
|
||||
|
||||
if (empty($databases)) {
|
||||
throw new \Exception('Database not found');
|
||||
}
|
||||
|
||||
$databaseDocument = $databases[0];
|
||||
$databaseType = $databaseDocument->getAttribute('type', UtopiaResource::TYPE_DATABASE);
|
||||
if (\in_array($databaseType, [UtopiaResource::TYPE_DATABASE_LEGACY, UtopiaResource::TYPE_DATABASE_TABLESDB], true)) {
|
||||
$databaseType = UtopiaResource::TYPE_DATABASE;
|
||||
}
|
||||
|
||||
$databasePayload = [
|
||||
'id' => $databaseDocument->getId(),
|
||||
'name' => $databaseDocument->getAttribute('name', $databaseDocument->getId()),
|
||||
'originalId' => $databaseDocument->getAttribute('originalId', ''),
|
||||
'type' => $databaseType,
|
||||
'database' => $databaseDocument->getAttribute('database', ''),
|
||||
];
|
||||
|
||||
$tablePayload = [
|
||||
'id' => $tableId,
|
||||
'name' => $tableId,
|
||||
'documentSecurity' => false,
|
||||
'rowSecurity' => false,
|
||||
'permissions' => [],
|
||||
'createdAt' => '',
|
||||
'updatedAt' => '',
|
||||
'database' => [
|
||||
'id' => $databasePayload['id'],
|
||||
'name' => $databasePayload['name'],
|
||||
'type' => $databasePayload['type'],
|
||||
'database' => $databasePayload['database'],
|
||||
],
|
||||
];
|
||||
|
||||
$table = Appwrite::getEntity($databaseType, $tablePayload);
|
||||
|
||||
while (true) {
|
||||
$queries = [$this->database->queryLimit($batchSize)];
|
||||
@@ -195,7 +234,7 @@ class CSV extends Source
|
||||
}
|
||||
}
|
||||
|
||||
$this->withCsvStream(function ($stream, $delimiter) use ($columnTypes, $requiredColumns, $manyToManyKeys, $arrayKeys, $table, $batchSize) {
|
||||
$this->withCsvStream(function ($stream, $delimiter) use ($columnTypes, $databaseType, $requiredColumns, $manyToManyKeys, $arrayKeys, $tablePayload, $batchSize) {
|
||||
$headers = \fgetcsv($stream, 0, $delimiter, '"', '"');
|
||||
|
||||
if (!\is_array($headers) || \count($headers) === 0) {
|
||||
@@ -300,7 +339,9 @@ class CSV extends Source
|
||||
),
|
||||
Column::TYPE_POINT,
|
||||
Column::TYPE_LINE,
|
||||
Column::TYPE_POLYGON => \is_string($parsedValue) ? json_decode($parsedValue) : null,
|
||||
Column::TYPE_POLYGON,
|
||||
Column::TYPE_VECTOR,
|
||||
Column::TYPE_OBJECT => \is_string($parsedValue) ? json_decode($parsedValue, true) : null,
|
||||
default => $parsedValue,
|
||||
},
|
||||
};
|
||||
@@ -311,12 +352,12 @@ class CSV extends Source
|
||||
|
||||
unset($parsedData['$id'], $parsedData['$permissions']);
|
||||
|
||||
$row = new Row(
|
||||
$rowId,
|
||||
$table,
|
||||
$parsedData,
|
||||
$permissions,
|
||||
);
|
||||
$row = Appwrite::getRecord($databaseType, [
|
||||
'id' => $rowId,
|
||||
'table' => $tablePayload,
|
||||
'data' => $parsedData,
|
||||
'permissions' => $permissions
|
||||
]);
|
||||
|
||||
$buffer[] = $row;
|
||||
|
||||
|
||||
@@ -14,7 +14,13 @@ class Transfer
|
||||
|
||||
public const GROUP_SITES = 'sites';
|
||||
|
||||
// separating databases and tablesdb out for easier separation in extract services
|
||||
// migration can use group_databases for mentioning all resources but when mentioning specific resources go with specific type databases
|
||||
public const GROUP_DATABASES = 'databases';
|
||||
public const GROUP_DATABASES_TABLES_DB = 'tablesdb';
|
||||
public const GROUP_DATABASES_DOCUMENTS_DB = 'documentsdb';
|
||||
|
||||
public const GROUP_DATABASES_VECTOR_DB = 'vectorsdb';
|
||||
|
||||
public const GROUP_SETTINGS = 'settings';
|
||||
|
||||
@@ -44,7 +50,7 @@ class Transfer
|
||||
Resource::TYPE_SITE_DEPLOYMENT
|
||||
];
|
||||
|
||||
public const GROUP_DATABASES_RESOURCES = [
|
||||
public const GROUP_TABLESDB_RESOURCES = [
|
||||
Resource::TYPE_DATABASE,
|
||||
Resource::TYPE_TABLE,
|
||||
Resource::TYPE_INDEX,
|
||||
@@ -52,6 +58,34 @@ class Transfer
|
||||
Resource::TYPE_ROW,
|
||||
];
|
||||
|
||||
public const GROUP_DOCUMENTSDB_RESOURCES = [
|
||||
Resource::TYPE_DATABASE_DOCUMENTSDB,
|
||||
Resource::TYPE_COLLECTION,
|
||||
Resource::TYPE_INDEX,
|
||||
Resource::TYPE_DOCUMENT
|
||||
];
|
||||
|
||||
public const GROUP_VECTORSDB_RESOURCES = [
|
||||
Resource::TYPE_DATABASE_VECTORSDB,
|
||||
Resource::TYPE_COLLECTION,
|
||||
Resource::TYPE_ATTRIBUTE,
|
||||
Resource::TYPE_INDEX,
|
||||
Resource::TYPE_DOCUMENT
|
||||
];
|
||||
|
||||
public const GROUP_DATABASES_RESOURCES = [
|
||||
Resource::TYPE_DATABASE,
|
||||
Resource::TYPE_DATABASE_DOCUMENTSDB,
|
||||
Resource::TYPE_DATABASE_VECTORSDB,
|
||||
Resource::TYPE_TABLE,
|
||||
Resource::TYPE_INDEX,
|
||||
Resource::TYPE_COLUMN,
|
||||
Resource::TYPE_ROW,
|
||||
Resource::TYPE_DOCUMENT,
|
||||
Resource::TYPE_COLLECTION,
|
||||
Resource::TYPE_ATTRIBUTE
|
||||
];
|
||||
|
||||
public const GROUP_SETTINGS_RESOURCES = [];
|
||||
|
||||
public const GROUP_MESSAGING_RESOURCES = [
|
||||
@@ -92,6 +126,8 @@ class Transfer
|
||||
public const ROOT_RESOURCES = [
|
||||
Resource::TYPE_BUCKET,
|
||||
Resource::TYPE_DATABASE,
|
||||
Resource::TYPE_DATABASE_DOCUMENTSDB,
|
||||
Resource::TYPE_DATABASE_VECTORSDB,
|
||||
Resource::TYPE_FUNCTION,
|
||||
Resource::TYPE_SITE,
|
||||
Resource::TYPE_USER,
|
||||
@@ -171,7 +207,7 @@ class Transfer
|
||||
$status[$resourceType][$k] = $resource;
|
||||
|
||||
if ($status[$resourceType]['pending'] > 0) {
|
||||
$status[$resourceType]['pending'] -= $resource;
|
||||
$status[$resourceType]['pending'] -= \min($status[$resourceType]['pending'], $resource);
|
||||
}
|
||||
|
||||
continue;
|
||||
@@ -256,7 +292,7 @@ class Transfer
|
||||
}
|
||||
|
||||
if (!in_array($rootResourceType, self::ROOT_RESOURCES)) {
|
||||
throw new \Exception('Resource type must be one of ' . implode(', ', self::ROOT_RESOURCES));
|
||||
throw new \Exception('Got '.$rootResourceType.' Resource type must be one of ' . implode(', ', self::ROOT_RESOURCES));
|
||||
}
|
||||
|
||||
$rootResources = \array_intersect($computedResources, self::ROOT_RESOURCES);
|
||||
@@ -358,6 +394,9 @@ class Transfer
|
||||
self::GROUP_GENERAL => array_merge($resources, []),
|
||||
self::GROUP_AUTH => array_merge($resources, self::GROUP_AUTH_RESOURCES),
|
||||
self::GROUP_DATABASES => array_merge($resources, self::GROUP_DATABASES_RESOURCES),
|
||||
self::GROUP_DATABASES_TABLES_DB => array_merge($resources, self::GROUP_TABLESDB_RESOURCES),
|
||||
self::GROUP_DATABASES_DOCUMENTS_DB => array_merge($resources, self::GROUP_DOCUMENTSDB_RESOURCES),
|
||||
self::GROUP_DATABASES_VECTOR_DB => array_merge($resources, self::GROUP_VECTORSDB_RESOURCES),
|
||||
self::GROUP_SETTINGS => array_merge($resources, self::GROUP_SETTINGS_RESOURCES),
|
||||
self::GROUP_MESSAGING => array_merge($resources, self::GROUP_MESSAGING_RESOURCES),
|
||||
default => throw new \Exception('No service group found'),
|
||||
|
||||
Reference in new issue
Block a user