Appwrite/appwrite
1
0
Fork 0
mirror of https://github.com/appwrite/appwrite.git synced 2026-05-26 13:51:13 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix: dirty migration

Browse Source
This commit is contained in:
Torsten Dittmann
2022-03-01 16:51:45 +01:00
parent b800c5d224
commit 24b29b86f8
3 changed files with 269 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/init.php
+1 -1
View File
@@ -71,7 +71,7 @@ const APP_LIMIT_ENCRYPTION = 20000000; //20MB
const APP_LIMIT_COMPRESSION = 20000000; //20MB
const APP_LIMIT_PREVIEW = 10000000; //10MB file size limit for preview endpoint
const APP_CACHE_BUSTER = 201;
const APP_VERSION_STABLE = '0.12.3';
const APP_VERSION_STABLE = '0.13.0';
const APP_DATABASE_ATTRIBUTE_EMAIL = 'email';
const APP_DATABASE_ATTRIBUTE_ENUM = 'enum';
const APP_DATABASE_ATTRIBUTE_IP = 'ip';
composer.lock
Generated
+21 -21
View File
@@ -4,7 +4,7 @@
"Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies",
"This file is @generated automatically"
],
"content-hash": "318c53aaac3cdfbdb728acc5e59b8057",
"content-hash": "b7919cd5b669605d051fadd3818523a7",
"packages": [
{
"name": "adhocore/jwt",
@@ -2129,16 +2129,16 @@
},
{
"name": "utopia-php/database",
"version": "0.15.1",
"version": "0.15.2",
"source": {
"type": "git",
"url": "https://github.com/utopia-php/database.git",
"reference": "eb4f61ec40d697acdfd574638ecd075e4f44b864"
"reference": "4ae9f1162d6640ccfe28afa0bbc60b907a7ad1c0"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/utopia-php/database/zipball/eb4f61ec40d697acdfd574638ecd075e4f44b864",
"reference": "eb4f61ec40d697acdfd574638ecd075e4f44b864",
"url": "https://api.github.com/repos/utopia-php/database/zipball/4ae9f1162d6640ccfe28afa0bbc60b907a7ad1c0",
"reference": "4ae9f1162d6640ccfe28afa0bbc60b907a7ad1c0",
"shasum": ""
},
"require": {
@@ -2186,9 +2186,9 @@
],
"support": {
"issues": "https://github.com/utopia-php/database/issues",
"source": "https://github.com/utopia-php/database/tree/0.15.1"
"source": "https://github.com/utopia-php/database/tree/0.15.2"
},
"time": "2022-02-22T09:33:37+00:00"
"time": "2022-02-28T15:53:37+00:00"
},
{
"name": "utopia-php/domains",
@@ -3075,7 +3075,7 @@
"source": {
"type": "git",
"url": "https://github.com/appwrite/sdk-generator",
"reference": "4a43aa70c2f0b243edfc689f618a85f7d0817287"
"reference": "3af54e71d0088b72b3223efc511b77e254fb7a1d"
},
"require": {
"ext-curl": "*",
@@ -3110,7 +3110,7 @@
}
],
"description": "Appwrite PHP library for generating API SDKs for multiple programming languages and platforms",
"time": "2022-02-22T10:51:55+00:00"
"time": "2022-03-01T09:53:31+00:00"
},
{
"name": "composer/pcre",
@@ -4226,16 +4226,16 @@
},
{
"name": "phpunit/php-code-coverage",
"version": "9.2.13",
"version": "9.2.14",
"source": {
"type": "git",
"url": "https://github.com/sebastianbergmann/php-code-coverage.git",
"reference": "deac8540cb7bd40b2b8cfa679b76202834fd04e8"
"reference": "9f4d60b6afe5546421462b76cd4e633ebc364ab4"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/sebastianbergmann/php-code-coverage/zipball/deac8540cb7bd40b2b8cfa679b76202834fd04e8",
"reference": "deac8540cb7bd40b2b8cfa679b76202834fd04e8",
"url": "https://api.github.com/repos/sebastianbergmann/php-code-coverage/zipball/9f4d60b6afe5546421462b76cd4e633ebc364ab4",
"reference": "9f4d60b6afe5546421462b76cd4e633ebc364ab4",
"shasum": ""
},
"require": {
@@ -4291,7 +4291,7 @@
],
"support": {
"issues": "https://github.com/sebastianbergmann/php-code-coverage/issues",
"source": "https://github.com/sebastianbergmann/php-code-coverage/tree/9.2.13"
"source": "https://github.com/sebastianbergmann/php-code-coverage/tree/9.2.14"
},
"funding": [
{
@@ -4299,7 +4299,7 @@
"type": "github"
}
],
"time": "2022-02-23T17:02:38+00:00"
"time": "2022-02-28T12:38:02+00:00"
},
{
"name": "phpunit/php-file-iterator",
@@ -5706,16 +5706,16 @@
},
{
"name": "symfony/console",
"version": "v6.0.3",
"version": "v6.0.5",
"source": {
"type": "git",
"url": "https://github.com/symfony/console.git",
"reference": "22e8efd019c3270c4f79376234a3f8752cd25490"
"reference": "3bebf4108b9e07492a2a4057d207aa5a77d146b1"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/symfony/console/zipball/22e8efd019c3270c4f79376234a3f8752cd25490",
"reference": "22e8efd019c3270c4f79376234a3f8752cd25490",
"url": "https://api.github.com/repos/symfony/console/zipball/3bebf4108b9e07492a2a4057d207aa5a77d146b1",
"reference": "3bebf4108b9e07492a2a4057d207aa5a77d146b1",
"shasum": ""
},
"require": {
@@ -5781,7 +5781,7 @@
"terminal"
],
"support": {
"source": "https://github.com/symfony/console/tree/v6.0.3"
"source": "https://github.com/symfony/console/tree/v6.0.5"
},
"funding": [
{
@@ -5797,7 +5797,7 @@
"type": "tidelift"
}
],
"time": "2022-01-26T17:23:29+00:00"
"time": "2022-02-25T10:48:52+00:00"
},
{
"name": "symfony/polyfill-intl-grapheme",
src/Appwrite/Migration/Version/V12.php
+247 -1
View File
@@ -3,9 +3,11 @@
namespace Appwrite\Migration\Version;
use Appwrite\Migration\Migration;
use Utopia\App;
use Utopia\CLI\Console;
use Utopia\Database\Database;
use Utopia\Database\Document;
use Utopia\Database\Query;
class V12 extends Migration
{
@@ -35,6 +37,7 @@ class V12 extends Migration
Console::info('Migrating Permissions');
$this->fixPermissions();
Console::info('Migrating Collections');
$this->migrateCustomCollections();
$this->fixCollections();
Console::info('Migrating Documents');
$this->forEachDocument([$this, 'fixDocument']);
@@ -54,7 +57,7 @@ class V12 extends Migration
* Remove empty generated Console Project.
*/
if ($this->consoleDB->getNamespace() === '_project_console' && $projectId === 'console') {
$all = [];
$all = ['_console_bucket_1', '_console_bucket_1_perms'];
foreach ($this->collections as $collection) {
$all[] = "_{$projectId}_{$collection['$id']}";
$all[] = "_{$projectId}_{$collection['$id']}_perms";
@@ -69,6 +72,9 @@ class V12 extends Migration
*/
foreach ($this->collections as $collection) {
$id = $collection['$id'];
if (in_array($id, ['buckets', 'deployments', 'builds'])) {
continue;
}
$this->pdo->prepare("ALTER TABLE IF EXISTS _project_{$projectId}_{$id} RENAME TO _{$projectId}_{$id}")->execute();
$this->pdo->prepare("CREATE TABLE IF NOT EXISTS _{$projectId}_{$id}_perms (
@@ -92,6 +98,10 @@ class V12 extends Migration
{
foreach ($this->collections as $collection) {
$id = $collection['$id'];
if (in_array($id, ['buckets', 'deployments', 'builds'])) {
continue;
}
Console::log("- {$id}");
switch ($id) {
case 'sessions':
@@ -132,11 +142,219 @@ class V12 extends Migration
Console::warning("'search' from {$id}: {$th->getMessage()}");
}
break;
case 'files':
/**
* Create bucket table if not exists.
*/
$this->createCollection('buckets');
if (!$this->projectDB->findOne('buckets', [new Query('$id', Query::TYPE_EQUAL, ['default'])])) {
$this->projectDB->createDocument('buckets', new Document([
'$id' => 'default',
'$collection' => 'buckets',
'dateCreated' => \time(),
'dateUpdated' => \time(),
'name' => 'Default',
'permission' => 'file',
'maximumFileSize' => (int) App::getEnv('_APP_STORAGE_LIMIT', 0), // 10MB
'allowedFileExtensions' => [],
'enabled' => true,
'encryption' => true,
'antivirus' => true,
'$read' => ['role:all'],
'$write' => ['role:all'],
'search' => 'buckets Default',
]));
$this->createCollection('files', 'bucket_1');
$nextDocument = null;
$path = "/storage/uploads/app-{$this->project->getId()}";
/**
* Rename folder on volumes.
*/
if (is_dir("{$path}/")) {
mkdir("/storage/uploads/app-{$this->project->getId()}/default");
foreach (new \DirectoryIterator($path) as $fileinfo) {
if ($fileinfo->isDir() && !$fileinfo->isDot() && $fileinfo->getFilename() !== 'default') {
rename("{$path}/{$fileinfo->getFilename()}", "{$path}/default/{$fileinfo->getFilename()}");
}
}
}
do {
$documents = $this->projectDB->find('files', limit: $this->limit, cursor: $nextDocument);
$count = count($documents);
\Co\run(function (array $documents) {
foreach ($documents as $document) {
go(function (Document $document) {
$path = "/storage/uploads/app-{$this->project->getId()}";
$new = str_replace($path, "{$path}/default", $document->getAttribute('path'));
$document
->setAttribute('bucketId', 'default')
->setAttribute('path', $new);
$this->projectDB->createDocument('bucket_1', $document);
}, $document);
}
}, $documents);
if ($count !== $this->limit) {
$nextDocument = null;
} else {
$nextDocument = end($documents);
}
} while (!is_null($nextDocument));
}
break;
case 'functions':
/**
* Create deployments table if not exists.
*/
$this->createCollection('deployments');
/**
* Create builds table if not exists.
*/
$this->createCollection('builds');
break;
}
usleep(100000);
}
}
protected function createCollection(string $id, string $name = null)
{
$name ??= $id;
if (!$this->projectDB->exists(App::getEnv('_APP_DB_SCHEMA', 'appwrite'), $name)) {
$attributes = [];
$indexes = [];
$collection = $this->collections[$id];
foreach ($collection['attributes'] as $attribute) {
$attributes[] = new Document([
'$id' => $attribute['$id'],
'type' => $attribute['type'],
'size' => $attribute['size'],
'required' => $attribute['required'],
'signed' => $attribute['signed'],
'array' => $attribute['array'],
'filters' => $attribute['filters'],
]);
}
foreach ($collection['indexes'] as $index) {
$indexes[] = new Document([
'$id' => $index['$id'],
'type' => $index['type'],
'attributes' => $index['attributes'],
'lengths' => $index['lengths'],
'orders' => $index['orders'],
]);
}
try {
$this->projectDB->createCollection($name, $attributes, $indexes);
} catch (\Throwable $th) {
throw $th;
}
}
}
protected function migrateCustomCollections()
{
$nextCollection = null;
do {
$documents = $this->projectDB->find('collections', limit: $this->limit, cursor: $nextCollection);
$count = count($documents);
\Co\run(function (array $documents) {
foreach ($documents as $document) {
go(function (Document $collection) {
$id = $collection->getId();
$projectId = $this->project->getId();
$internalId = $collection->getInternalId();
$this->pdo->prepare("ALTER TABLE IF EXISTS _project_{$projectId}_collection_{$id} RENAME TO _{$projectId}_collection_{$internalId}")->execute();
$this->pdo->prepare("CREATE TABLE IF NOT EXISTS _{$projectId}_collection_{$internalId}_perms (
`_id` int(11) unsigned NOT NULL AUTO_INCREMENT,
`_type` VARCHAR(12) NOT NULL,
`_permission` VARCHAR(255) NOT NULL,
`_document` VARCHAR(255) NOT NULL,
PRIMARY KEY (`_id`),
UNIQUE INDEX `_index1` (`_type`,`_document`,`_permission`),
INDEX `_index2` (`_permission`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;")->execute();
$this->pdo->prepare("UPDATE _{$projectId}__metadata
SET
_uid = 'collection_{$internalId}',
name = 'collection_{$internalId}'
WHERE _uid = 'collection_{$id}';
")->execute();
$nextDocument = null;
do {
$documents = $this->projectDB->find('collection_' . $internalId, limit: $this->limit, cursor: $nextDocument);
$count = count($documents);
foreach ($documents as $document) {
go(function (Document $document, string $internalId) {
$sql = "SELECT _read, _write FROM `{$this->projectDB->getDefaultDatabase()}`.`{$this->projectDB->getNamespace()}_collection_{$internalId}` WHERE _uid = {$this->pdo->quote($document->getid())}";
$stmt = $this->pdo->prepare($sql);
$stmt->execute();
$permissions = $stmt->fetch();
$read = json_decode($permissions['_read'] ?? null) ?? [];
$write = json_decode($permissions['_write'] ?? null) ?? [];
$permissions = [];
foreach ($read as $permission) {
$permissions[] = "('read', '{$permission}', '{$document->getId()}')";
}
foreach ($write as $permission) {
$permissions[] = "('write', '{$permission}', '{$document->getId()}')";
}
if (!empty($permissions)) {
$queryPermissions = "INSERT IGNORE INTO `{$this->projectDB->getDefaultDatabase()}`.`{$this->projectDB->getNamespace()}_collection_{$internalId}_perms` (_type, _permission, _document) VALUES " . implode(', ', $permissions);
$stmtPermissions = $this->pdo->prepare($queryPermissions);
$stmtPermissions->execute();
}
}, $document, $internalId);
}
if ($count !== $this->limit) {
$nextDocument = null;
} else {
$nextDocument = end($documents);
}
} while (!is_null($nextDocument));
$this->pdo->prepare("
ALTER TABLE `{$this->projectDB->getDefaultDatabase()}`.`{$this->projectDB->getNamespace()}_collection_{$internalId}`
DROP COLUMN _read,
DROP COLUMN _write
")->execute();
}, $document);
}
}, $documents);
if ($count !== $this->limit) {
$nextCollection = null;
} else {
$nextCollection = end($documents);
}
} while (!is_null($nextCollection));
}
/**
* Migrate all Permission to new System with dedicated Table.
* @return void
@@ -146,6 +364,28 @@ class V12 extends Migration
{
foreach ($this->collections as $collection) {
$id = $collection['$id'];
if (in_array($id, ['buckets', 'deployments', 'builds'])) {
continue;
}
/**
* Check if permissions have already been migrated.
*/
try {
$stmtCheck = $this->pdo->prepare("SHOW COLUMNS from `{$this->projectDB->getDefaultDatabase()}`.`{$this->projectDB->getNamespace()}_{$id}` LIKE '_read'");
$stmtCheck->execute();
if (empty($stmtCheck->fetchAll())) {
continue;
}
} catch (\Throwable $th) {
if ($th->getCode() === "42S02") {
continue;
}
throw $th;
}
Console::log("- {$collection['$id']}");
$nextDocument = null;
@@ -189,6 +429,12 @@ class V12 extends Migration
$nextDocument = end($documents);
}
} while (!is_null($nextDocument));
$this->pdo->prepare("
ALTER TABLE `{$this->projectDB->getDefaultDatabase()}`.`{$this->projectDB->getNamespace()}_{$id}`
DROP COLUMN _read,
DROP COLUMN _write
")->execute();
}
/**