Merge pull request #8797 from appwrite/feat-migration

Feat migration
# Conflicts:
#	composer.lock
This commit is contained in:
Christy Jacob
2024-11-12 10:27:24 +01:00
committed by Jake Barnby
parent 0b11e0cc92
commit f490d4edaf
15 changed files with 152 additions and 83 deletions
+10 -5
View File
@@ -127,6 +127,11 @@ jobs:
Messaging,
Migrations
]
tables-mode: [
'Project',
'Shared V1',
'Shared V2',
]
steps:
- name: checkout
@@ -145,11 +150,11 @@ jobs:
docker compose up -d
sleep 30
- name: Run ${{matrix.service}} Tests
run: docker compose exec -T appwrite test /usr/src/code/tests/e2e/Services/${{matrix.service}} --debug
- name: Run ${{matrix.service}} Shared Tables Tests
run: _APP_DATABASE_SHARED_TABLES=database_db_main docker compose exec -T appwrite test /usr/src/code/tests/e2e/Services/${{matrix.service}} --debug
- name: Run ${{ matrix.service }} tests with ${{ matrix.tables-mode }} table mode
run: docker compose exec -T appwrite test /usr/src/code/tests/e2e/Services/${{ matrix.service }} --debug
env:
_APP_DATABASE_SHARED_TABLES: ${{ matrix.table_mode == 'Shared V1' || matrix.table_mode == 'Shared V2' && 'database_db_main' || '' }}
_APP_DATABASE_SHARED_TABLES_V1: ${{ matrix.table_mode == 'Shared V1' && 'database_db_main' || '' }}
benchmarking:
name: Benchmark
+4 -3
View File
@@ -114,8 +114,9 @@ CLI::setResource('getProjectDB', function (Group $pools, Database $dbForConsole,
if (isset($databases[$dsn->getHost()])) {
$database = $databases[$dsn->getHost()];
$sharedTables = \explode(',', System::getEnv('_APP_DATABASE_SHARED_TABLES', ''));
if ($dsn->getHost() === System::getEnv('_APP_DATABASE_SHARED_TABLES', '')) {
if (\in_array($dsn->getHost(), $sharedTables)) {
$database
->setSharedTables(true)
->setTenant($project->getInternalId())
@@ -136,10 +137,10 @@ CLI::setResource('getProjectDB', function (Group $pools, Database $dbForConsole,
->getResource();
$database = new Database($dbAdapter, $cache);
$databases[$dsn->getHost()] = $database;
$sharedTables = \explode(',', System::getEnv('_APP_DATABASE_SHARED_TABLES', ''));
if ($dsn->getHost() === System::getEnv('_APP_DATABASE_SHARED_TABLES', '')) {
if (\in_array($dsn->getHost(), $sharedTables)) {
$database
->setSharedTables(true)
->setTenant($project->getInternalId())
+1 -1
View File
@@ -686,7 +686,7 @@ return [
],
Exception::ATTRIBUTE_LIMIT_EXCEEDED => [
'name' => Exception::ATTRIBUTE_LIMIT_EXCEEDED,
'description' => 'The maximum number of attributes has been reached.',
'description' => 'The maximum number or size of attributes for this collection has been reached.',
'code' => 400,
],
Exception::ATTRIBUTE_VALUE_INVALID => [
+6 -3
View File
@@ -152,7 +152,7 @@ function createAttribute(string $databaseId, string $collectionId, Document $att
} catch (DuplicateException) {
throw new Exception(Exception::ATTRIBUTE_ALREADY_EXISTS);
} catch (LimitException) {
throw new Exception(Exception::ATTRIBUTE_LIMIT_EXCEEDED, 'Attribute limit exceeded');
throw new Exception(Exception::ATTRIBUTE_LIMIT_EXCEEDED);
} catch (\Throwable $e) {
$dbForProject->purgeCachedDocument('database_' . $db->getInternalId(), $collectionId);
$dbForProject->purgeCachedCollection('database_' . $db->getInternalId() . '_collection_' . $collection->getInternalId());
@@ -196,7 +196,7 @@ function createAttribute(string $databaseId, string $collectionId, Document $att
throw new Exception(Exception::ATTRIBUTE_ALREADY_EXISTS);
} catch (LimitException) {
$dbForProject->deleteDocument('attributes', $attribute->getId());
throw new Exception(Exception::ATTRIBUTE_LIMIT_EXCEEDED, 'Attribute limit exceeded');
throw new Exception(Exception::ATTRIBUTE_LIMIT_EXCEEDED);
} catch (\Throwable $e) {
$dbForProject->purgeCachedDocument('database_' . $db->getInternalId(), $relatedCollection->getId());
$dbForProject->purgeCachedCollection('database_' . $db->getInternalId() . '_collection_' . $relatedCollection->getInternalId());
@@ -392,6 +392,8 @@ function updateAttribute(
throw new Exception(Exception::ATTRIBUTE_INVALID_RESIZE);
} catch (NotFoundException) {
throw new Exception(Exception::ATTRIBUTE_NOT_FOUND);
} catch (LimitException) {
throw new Exception(Exception::ATTRIBUTE_LIMIT_EXCEEDED);
}
}
@@ -2631,7 +2633,8 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/indexes')
$validator = new IndexValidator(
$collection->getAttribute('attributes'),
$dbForProject->getAdapter()->getMaxIndexLength()
$dbForProject->getAdapter()->getMaxIndexLength(),
$dbForProject->getAdapter()->getInternalIndexesKeys(),
);
if (!$validator->isValid($index)) {
throw new Exception(Exception::INDEX_INVALID, $validator->getDescription());
+74 -29
View File
@@ -122,6 +122,10 @@ App::post('/v1/projects')
$projectId = ($projectId == 'unique()') ? ID::unique() : $projectId;
if ($projectId === 'console') {
throw new Exception(Exception::PROJECT_RESERVED_PROJECT, "'console' is a reserved project.");
}
$databases = Config::getParam('pools-database', []);
$databaseOverride = System::getEnv('_APP_DATABASE_OVERRIDE');
@@ -132,16 +136,14 @@ App::post('/v1/projects')
$dsn = $databases[array_rand($databases)];
}
if ($projectId === 'console') {
throw new Exception(Exception::PROJECT_RESERVED_PROJECT, "'console' is a reserved project.");
}
// TODO: Temporary until all projects are using shared tables.
if ($dsn === System::getEnv('_APP_DATABASE_SHARED_TABLES', '')) {
$sharedTables = \explode(',', System::getEnv('_APP_DATABASE_SHARED_TABLES', ''));
if (\in_array($dsn, $sharedTables)) {
$schema = 'appwrite';
$database = 'appwrite';
$namespace = System::getEnv('_APP_DATABASE_SHARED_NAMESPACE', '');
$dsn = $schema . '://' . System::getEnv('_APP_DATABASE_SHARED_TABLES', '') . '?database=' . $database;
$dsn = $schema . '://' . $dsn . '?database=' . $database;
if (!empty($namespace)) {
$dsn .= '&namespace=' . $namespace;
@@ -195,11 +197,18 @@ App::post('/v1/projects')
$adapter = $pools->get($dsn->getHost())->pop()->getResource();
$dbForProject = new Database($adapter, $cache);
$sharedTables = \explode(',', System::getEnv('_APP_DATABASE_SHARED_TABLES', ''));
$sharedTablesV1 = \explode(',', System::getEnv('_APP_DATABASE_SHARED_TABLES_V1', ''));
if ($dsn->getHost() === System::getEnv('_APP_DATABASE_SHARED_TABLES', '')) {
$projectTables = !\in_array($dsn->getHost(), $sharedTables);
$sharedTablesV1 = \in_array($dsn->getHost(), $sharedTablesV1);
$sharedTablesV2 = !$projectTables && !$sharedTablesV1;
$sharedTables = $sharedTablesV1 || $sharedTablesV2;
if ($sharedTables) {
$dbForProject
->setSharedTables(true)
->setTenant($project->getInternalId())
->setTenant($sharedTablesV1 ? $project->getInternalId() : null)
->setNamespace($dsn->getParam('namespace'));
} else {
$dbForProject
@@ -208,34 +217,70 @@ App::post('/v1/projects')
->setNamespace('_' . $project->getInternalId());
}
$dbForProject->create();
$create = true;
$audit = new Audit($dbForProject);
$audit->setup();
try {
$dbForProject->create();
} catch (Duplicate) {
$create = false;
}
$abuse = new TimeLimit('', 0, 1, $dbForProject);
$abuse->setup();
if ($create || $projectTables) {
$audit = new Audit($dbForProject);
$audit->setup();
/** @var array $collections */
$collections = Config::getParam('collections', [])['projects'] ?? [];
$abuse = new TimeLimit('', 0, 1, $dbForProject);
$abuse->setup();
}
foreach ($collections as $key => $collection) {
if (($collection['$collection'] ?? '') !== Database::METADATA) {
continue;
}
if (!$create && $sharedTablesV1) {
$attributes = \array_map(fn ($attribute) => new Document($attribute), Audit::ATTRIBUTES);
$indexes = \array_map(fn (array $index) => new Document($index), Audit::INDEXES);
$dbForProject->createDocument(Database::METADATA, new Document([
'$id' => ID::custom('audit'),
'$permissions' => [Permission::create(Role::any())],
'name' => 'audit',
'attributes' => $attributes,
'indexes' => $indexes,
'documentSecurity' => true
]));
$attributes = \array_map(function (array $attribute) {
return new Document($attribute);
}, $collection['attributes']);
$attributes = \array_map(fn ($attribute) => new Document($attribute), TimeLimit::ATTRIBUTES);
$indexes = \array_map(fn (array $index) => new Document($index), TimeLimit::INDEXES);
$dbForProject->createDocument(Database::METADATA, new Document([
'$id' => ID::custom('abuse'),
'$permissions' => [Permission::create(Role::any())],
'name' => 'abuse',
'attributes' => $attributes,
'indexes' => $indexes,
'documentSecurity' => true
]));
}
$indexes = \array_map(function (array $index) {
return new Document($index);
}, $collection['indexes']);
if ($create || $sharedTablesV1) {
/** @var array $collections */
$collections = Config::getParam('collections', [])['projects'] ?? [];
try {
$dbForProject->createCollection($key, $attributes, $indexes);
} catch (Duplicate) {
// Collection already exists
foreach ($collections as $key => $collection) {
if (($collection['$collection'] ?? '') !== Database::METADATA) {
continue;
}
$attributes = \array_map(fn ($attribute) => new Document($attribute), $collection['attributes']);
$indexes = \array_map(fn (array $index) => new Document($index), $collection['indexes']);
try {
$dbForProject->createCollection($key, $attributes, $indexes);
} catch (Duplicate) {
$dbForProject->createDocument(Database::METADATA, new Document([
'$id' => ID::custom($key),
'$permissions' => [Permission::create(Role::any())],
'name' => $key,
'attributes' => $attributes,
'indexes' => $indexes,
'documentSecurity' => true
]));
}
}
}
+3 -2
View File
@@ -16,6 +16,7 @@ use Utopia\CLI\Console;
use Utopia\Config\Config;
use Utopia\Database\Database;
use Utopia\Database\Document;
use Utopia\Database\Exception\Duplicate;
use Utopia\Database\Helpers\ID;
use Utopia\Database\Helpers\Permission;
use Utopia\Database\Helpers\Role;
@@ -93,7 +94,7 @@ $http->on(Constant::EVENT_START, function (Server $http) use ($payloadSize, $reg
try {
Console::success('[Setup] - Creating database: appwrite...');
$dbForConsole->create();
} catch (\Throwable $e) {
} catch (Duplicate) {
Console::success('[Setup] - Skip: metadata table already exists');
}
@@ -225,7 +226,7 @@ $http->on(Constant::EVENT_START, function (Server $http) use ($payloadSize, $reg
});
});
$http->on('request', function (SwooleRequest $swooleRequest, SwooleResponse $swooleResponse) use ($register) {
$http->on(Constant::EVENT_REQUEST, function (SwooleRequest $swooleRequest, SwooleResponse $swooleResponse) use ($register) {
App::setResource('swooleRequest', fn () => $swooleRequest);
App::setResource('swooleResponse', fn () => $swooleResponse);
+6 -2
View File
@@ -1432,7 +1432,9 @@ App::setResource('dbForProject', function (Group $pools, Database $dbForConsole,
$dsn = new DSN('mysql://' . $project->getAttribute('database'));
}
if ($dsn->getHost() === System::getEnv('_APP_DATABASE_SHARED_TABLES', '')) {
$sharedTables = \explode(',', System::getEnv('_APP_DATABASE_SHARED_TABLES', ''));
if (\in_array($dsn->getHost(), $sharedTables)) {
$database
->setSharedTables(true)
->setTenant($project->getInternalId())
@@ -1487,7 +1489,9 @@ App::setResource('getProjectDB', function (Group $pools, Database $dbForConsole,
->setTimeout(APP_DATABASE_TIMEOUT_MILLISECONDS)
->setMaxQueryValues(APP_DATABASE_QUERY_MAX_VALUES);
if ($dsn->getHost() === System::getEnv('_APP_DATABASE_SHARED_TABLES', '')) {
$sharedTables = \explode(',', System::getEnv('_APP_DATABASE_SHARED_TABLES', ''));
if (\in_array($dsn->getHost(), $sharedTables)) {
$database
->setSharedTables(true)
->setTenant($project->getInternalId())
+3 -1
View File
@@ -93,7 +93,9 @@ if (!function_exists('getProjectDB')) {
$database = new Database($adapter, getCache());
if ($dsn->getHost() === System::getEnv('_APP_DATABASE_SHARED_TABLES', '')) {
$sharedTables = \explode(',', System::getEnv('_APP_DATABASE_SHARED_TABLES', ''));
if (\in_array($dsn->getHost(), $sharedTables)) {
$database
->setSharedTables(true)
->setTenant($project->getInternalId())
+9 -3
View File
@@ -93,7 +93,9 @@ Server::setResource('dbForProject', function (Cache $cache, Registry $register,
$dsn = new DSN('mysql://' . $project->getAttribute('database'));
}
if ($dsn->getHost() === System::getEnv('_APP_DATABASE_SHARED_TABLES', '')) {
$sharedTables = \explode(',', System::getEnv('_APP_DATABASE_SHARED_TABLES', ''));
if (\in_array($dsn->getHost(), $sharedTables)) {
$database
->setSharedTables(true)
->setTenant($project->getInternalId())
@@ -126,7 +128,9 @@ Server::setResource('getProjectDB', function (Group $pools, Database $dbForConso
if (isset($databases[$dsn->getHost()])) {
$database = $databases[$dsn->getHost()];
if ($dsn->getHost() === System::getEnv('_APP_DATABASE_SHARED_TABLES', '')) {
$sharedTables = \explode(',', System::getEnv('_APP_DATABASE_SHARED_TABLES', ''));
if (\in_array($dsn->getHost(), $sharedTables)) {
$database
->setSharedTables(true)
->setTenant($project->getInternalId())
@@ -150,7 +154,9 @@ Server::setResource('getProjectDB', function (Group $pools, Database $dbForConso
$databases[$dsn->getHost()] = $database;
if ($dsn->getHost() === System::getEnv('_APP_DATABASE_SHARED_TABLES', '')) {
$sharedTables = \explode(',', System::getEnv('_APP_DATABASE_SHARED_TABLES', ''));
if (\in_array($dsn->getHost(), $sharedTables)) {
$database
->setSharedTables(true)
->setTenant($project->getInternalId())
+3 -3
View File
@@ -45,13 +45,13 @@
"ext-sockets": "*",
"appwrite/php-runtimes": "0.16.*",
"appwrite/php-clamav": "2.0.*",
"utopia-php/abuse": "0.43.0",
"utopia-php/abuse": "0.43.*",
"utopia-php/analytics": "0.10.*",
"utopia-php/audit": "0.43.0",
"utopia-php/audit": "0.43.*",
"utopia-php/cache": "0.11.*",
"utopia-php/cli": "0.15.*",
"utopia-php/config": "0.2.*",
"utopia-php/database": "0.53.16",
"utopia-php/database": "0.53.20",
"utopia-php/domains": "0.5.*",
"utopia-php/dsn": "0.2.1",
"utopia-php/framework": "0.33.*",
Generated
+26 -26
View File
@@ -4,7 +4,7 @@
"Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies",
"This file is @generated automatically"
],
"content-hash": "c56db8736d679aff6e2b275ec59f1dbf",
"content-hash": "ae3b9a491c9870a4897cdf712caba1e3",
"packages": [
{
"name": "adhocore/jwt",
@@ -3135,16 +3135,16 @@
},
{
"name": "utopia-php/abuse",
"version": "0.43.0",
"version": "0.43.1",
"source": {
"type": "git",
"url": "https://github.com/utopia-php/abuse.git",
"reference": "6346a3b4c5177a43160035a7289e30fdfb0790d6"
"reference": "e404c21e8dcf6a310bc83cf1d74e716b105598fa"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/utopia-php/abuse/zipball/6346a3b4c5177a43160035a7289e30fdfb0790d6",
"reference": "6346a3b4c5177a43160035a7289e30fdfb0790d6",
"url": "https://api.github.com/repos/utopia-php/abuse/zipball/e404c21e8dcf6a310bc83cf1d74e716b105598fa",
"reference": "e404c21e8dcf6a310bc83cf1d74e716b105598fa",
"shasum": ""
},
"require": {
@@ -3180,9 +3180,9 @@
],
"support": {
"issues": "https://github.com/utopia-php/abuse/issues",
"source": "https://github.com/utopia-php/abuse/tree/0.43.0"
"source": "https://github.com/utopia-php/abuse/tree/0.43.1"
},
"time": "2024-08-30T05:17:23+00:00"
"time": "2024-10-23T04:29:12+00:00"
},
{
"name": "utopia-php/analytics",
@@ -3232,16 +3232,16 @@
},
{
"name": "utopia-php/audit",
"version": "0.43.0",
"version": "0.43.1",
"source": {
"type": "git",
"url": "https://github.com/utopia-php/audit.git",
"reference": "cef22b5dc6a6d28fcd522f41c7bf7ded4a4dfd3e"
"reference": "04a47dd1f5f92e2d50e971a06bcc9e759325d277"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/utopia-php/audit/zipball/cef22b5dc6a6d28fcd522f41c7bf7ded4a4dfd3e",
"reference": "cef22b5dc6a6d28fcd522f41c7bf7ded4a4dfd3e",
"url": "https://api.github.com/repos/utopia-php/audit/zipball/04a47dd1f5f92e2d50e971a06bcc9e759325d277",
"reference": "04a47dd1f5f92e2d50e971a06bcc9e759325d277",
"shasum": ""
},
"require": {
@@ -3273,9 +3273,9 @@
],
"support": {
"issues": "https://github.com/utopia-php/audit/issues",
"source": "https://github.com/utopia-php/audit/tree/0.43.0"
"source": "https://github.com/utopia-php/audit/tree/0.43.1"
},
"time": "2024-08-30T05:17:36+00:00"
"time": "2024-10-23T04:27:59+00:00"
},
{
"name": "utopia-php/cache",
@@ -3475,16 +3475,16 @@
},
{
"name": "utopia-php/database",
"version": "0.53.16",
"version": "0.53.20",
"source": {
"type": "git",
"url": "https://github.com/utopia-php/database.git",
"reference": "6661edffeef05b59e16d102b989a72f7f78cf7de"
"reference": "e43f8ee26e06ee8812737e63642dbd7ee7c9dc04"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/utopia-php/database/zipball/6661edffeef05b59e16d102b989a72f7f78cf7de",
"reference": "6661edffeef05b59e16d102b989a72f7f78cf7de",
"url": "https://api.github.com/repos/utopia-php/database/zipball/e43f8ee26e06ee8812737e63642dbd7ee7c9dc04",
"reference": "e43f8ee26e06ee8812737e63642dbd7ee7c9dc04",
"shasum": ""
},
"require": {
@@ -3525,9 +3525,9 @@
],
"support": {
"issues": "https://github.com/utopia-php/database/issues",
"source": "https://github.com/utopia-php/database/tree/0.53.16"
"source": "https://github.com/utopia-php/database/tree/0.53.20"
},
"time": "2024-11-06T03:07:16+00:00"
"time": "2024-11-12T00:23:36+00:00"
},
{
"name": "utopia-php/domains",
@@ -3677,16 +3677,16 @@
},
{
"name": "utopia-php/framework",
"version": "0.33.13",
"version": "0.33.14",
"source": {
"type": "git",
"url": "https://github.com/utopia-php/http.git",
"reference": "4ecab88e424a136ffaa27cdf3db3c60f76c777e4"
"reference": "45a5a2db3602fa054096f378482c7da9936f5850"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/utopia-php/http/zipball/4ecab88e424a136ffaa27cdf3db3c60f76c777e4",
"reference": "4ecab88e424a136ffaa27cdf3db3c60f76c777e4",
"url": "https://api.github.com/repos/utopia-php/http/zipball/45a5a2db3602fa054096f378482c7da9936f5850",
"reference": "45a5a2db3602fa054096f378482c7da9936f5850",
"shasum": ""
},
"require": {
@@ -3718,9 +3718,9 @@
],
"support": {
"issues": "https://github.com/utopia-php/http/issues",
"source": "https://github.com/utopia-php/http/tree/0.33.13"
"source": "https://github.com/utopia-php/http/tree/0.33.14"
},
"time": "2024-11-15T08:37:31+00:00"
"time": "2024-11-20T12:39:10+00:00"
},
{
"name": "utopia-php/image",
@@ -8557,7 +8557,7 @@
],
"aliases": [],
"minimum-stability": "stable",
"stability-flags": [],
"stability-flags": {},
"prefer-stable": false,
"prefer-lowest": false,
"platform": {
+1
View File
@@ -193,6 +193,7 @@ services:
- _APP_EXPERIMENT_LOGGING_PROVIDER
- _APP_EXPERIMENT_LOGGING_CONFIG
- _APP_DATABASE_SHARED_TABLES
- _APP_DATABASE_SHARED_TABLES_V1
appwrite-console:
<<: *x-logging
+4 -3
View File
@@ -494,12 +494,13 @@ class Deletes extends Action
];
$limit = \count($projectCollectionIds) + 25;
$sharedTables = \explode(',', System::getEnv('_APP_DATABASE_SHARED_TABLES', ''));
while (true) {
$collections = $dbForProject->listCollections($limit);
foreach ($collections as $collection) {
if ($dsn->getHost() !== System::getEnv('_APP_DATABASE_SHARED_TABLES', '') || !\in_array($collection->getId(), $projectCollectionIds)) {
if (\in_array($dsn->getHost(), $sharedTables) || !\in_array($collection->getId(), $projectCollectionIds)) {
try {
$dbForProject->deleteCollection($collection->getId());
} catch (Throwable $e) {
@@ -517,7 +518,7 @@ class Deletes extends Action
}
}
if ($dsn->getHost() === System::getEnv('_APP_DATABASE_SHARED_TABLES', '')) {
if (\in_array($dsn->getHost(), $sharedTables)) {
$collectionsIds = \array_map(fn ($collection) => $collection->getId(), $collections);
if (empty(\array_diff($collectionsIds, $projectCollectionIds))) {
@@ -571,7 +572,7 @@ class Deletes extends Action
], $dbForConsole);
// Delete metadata table
if ($dsn->getHost() !== System::getEnv('_APP_DATABASE_SHARED_TABLES', '')) {
if (\in_array($dsn->getHost(), $sharedTables)) {
$dbForProject->deleteCollection('_metadata');
} else {
$this->deleteByGroup('_metadata', [], $dbForProject);
+1 -1
View File
@@ -94,7 +94,7 @@ class Func extends Model
])
->addRule('schedule', [
'type' => self::TYPE_STRING,
'description' => 'Function execution schedult in CRON format.',
'description' => 'Function execution schedule in CRON format.',
'default' => '',
'example' => '5 4 * * *',
])
@@ -1362,7 +1362,7 @@ class DatabasesCustomServerTest extends Scope
]);
$this->assertEquals(400, $tooWide['headers']['status-code']);
$this->assertEquals('Attribute limit exceeded', $tooWide['body']['message']);
$this->assertEquals('attribute_limit_exceeded', $tooWide['body']['type']);
}
public function testIndexLimitException()