feat: hack for multi region

feat fix web routes

.github/workflows/tests.yml

@@ -127,6 +127,11 @@ jobs:
             Messaging,
             Migrations
           ]
+        tables-mode: [
+          'Project',
+          'Shared V1',
+          'Shared V2',
+        ]
 
     steps:
       - name: checkout
@@ -145,11 +150,11 @@ jobs:
           docker compose up -d
           sleep 30
 
-      - name: Run ${{matrix.service}} Tests
-        run: docker compose exec -T appwrite test /usr/src/code/tests/e2e/Services/${{matrix.service}} --debug
-
-      - name: Run ${{matrix.service}} Shared Tables Tests
-        run: _APP_DATABASE_SHARED_TABLES=database_db_main docker compose exec -T appwrite test /usr/src/code/tests/e2e/Services/${{matrix.service}} --debug
+      - name: Run ${{ matrix.service }} tests with ${{ matrix.tables-mode }} table mode
+        run: docker compose exec -T appwrite test /usr/src/code/tests/e2e/Services/${{ matrix.service }} --debug
+        env:
+          _APP_DATABASE_SHARED_TABLES: ${{ matrix.table_mode == 'Shared V1' || matrix.table_mode == 'Shared V2' && 'database_db_main' || '' }}
+          _APP_DATABASE_SHARED_TABLES_V1: ${{ matrix.table_mode == 'Shared V1' && 'database_db_main' || '' }}
 
   benchmarking:
     name: Benchmark

app/cli.php

@@ -114,8 +114,9 @@ CLI::setResource('getProjectDB', function (Group $pools, Database $dbForConsole,
 
         if (isset($databases[$dsn->getHost()])) {
             $database = $databases[$dsn->getHost()];
+            $sharedTables = \explode(',', System::getEnv('_APP_DATABASE_SHARED_TABLES', ''));
 
-            if ($dsn->getHost() === System::getEnv('_APP_DATABASE_SHARED_TABLES', '')) {
+            if (\in_array($dsn->getHost(), $sharedTables)) {
                 $database
                     ->setSharedTables(true)
                     ->setTenant($project->getInternalId())
@@ -136,10 +137,10 @@ CLI::setResource('getProjectDB', function (Group $pools, Database $dbForConsole,
             ->getResource();
 
         $database = new Database($dbAdapter, $cache);
-
         $databases[$dsn->getHost()] = $database;
+        $sharedTables = \explode(',', System::getEnv('_APP_DATABASE_SHARED_TABLES', ''));
 
-        if ($dsn->getHost() === System::getEnv('_APP_DATABASE_SHARED_TABLES', '')) {
+        if (\in_array($dsn->getHost(), $sharedTables)) {
             $database
                 ->setSharedTables(true)
                 ->setTenant($project->getInternalId())

app/config/errors.php

@@ -686,7 +686,7 @@ return [
     ],
     Exception::ATTRIBUTE_LIMIT_EXCEEDED => [
         'name' => Exception::ATTRIBUTE_LIMIT_EXCEEDED,
-        'description' => 'The maximum number of attributes has been reached.',
+        'description' => 'The maximum number or size of attributes for this collection has been reached.',
         'code' => 400,
     ],
     Exception::ATTRIBUTE_VALUE_INVALID => [

app/controllers/api/databases.php

@@ -153,7 +153,7 @@ function createAttribute(string $databaseId, string $collectionId, Document $att
     } catch (DuplicateException) {
         throw new Exception(Exception::ATTRIBUTE_ALREADY_EXISTS);
     } catch (LimitException) {
-        throw new Exception(Exception::ATTRIBUTE_LIMIT_EXCEEDED, 'Attribute limit exceeded');
+        throw new Exception(Exception::ATTRIBUTE_LIMIT_EXCEEDED);
     } catch (\Throwable $e) {
         $dbForProject->purgeCachedDocument('database_' . $db->getInternalId(), $collectionId);
         $dbForProject->purgeCachedCollection('database_' . $db->getInternalId() . '_collection_' . $collection->getInternalId());
@@ -197,7 +197,7 @@ function createAttribute(string $databaseId, string $collectionId, Document $att
             throw new Exception(Exception::ATTRIBUTE_ALREADY_EXISTS);
         } catch (LimitException) {
             $dbForProject->deleteDocument('attributes', $attribute->getId());
-            throw new Exception(Exception::ATTRIBUTE_LIMIT_EXCEEDED, 'Attribute limit exceeded');
+            throw new Exception(Exception::ATTRIBUTE_LIMIT_EXCEEDED);
         } catch (\Throwable $e) {
             $dbForProject->purgeCachedDocument('database_' . $db->getInternalId(), $relatedCollection->getId());
             $dbForProject->purgeCachedCollection('database_' . $db->getInternalId() . '_collection_' . $relatedCollection->getInternalId());
@@ -393,6 +393,8 @@ function updateAttribute(
             throw new Exception(Exception::ATTRIBUTE_INVALID_RESIZE);
         } catch (NotFoundException) {
             throw new Exception(Exception::ATTRIBUTE_NOT_FOUND);
+        } catch (LimitException) {
+            throw new Exception(Exception::ATTRIBUTE_LIMIT_EXCEEDED);
         }
     }
 
@@ -2627,7 +2629,8 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/indexes')
 
         $validator = new IndexValidator(
             $collection->getAttribute('attributes'),
-            $dbForProject->getAdapter()->getMaxIndexLength()
+            $dbForProject->getAdapter()->getMaxIndexLength(),
+            $dbForProject->getAdapter()->getInternalIndexesKeys(),
         );
         if (!$validator->isValid($index)) {
             throw new Exception(Exception::INDEX_INVALID, $validator->getDescription());

app/controllers/api/projects.php

@@ -122,6 +122,10 @@ App::post('/v1/projects')
 
         $projectId = ($projectId == 'unique()') ? ID::unique() : $projectId;
 
+        if ($projectId === 'console') {
+            throw new Exception(Exception::PROJECT_RESERVED_PROJECT, "'console' is a reserved project.");
+        }
+
         $databases = Config::getParam('pools-database', []);
 
         $databaseOverride = System::getEnv('_APP_DATABASE_OVERRIDE');
@@ -132,16 +136,24 @@ App::post('/v1/projects')
             $dsn = $databases[array_rand($databases)];
         }
 
-        if ($projectId === 'console') {
-            throw new Exception(Exception::PROJECT_RESERVED_PROJECT, "'console' is a reserved project.");
+        $dsns = [
+            'fra' => 'database_db_fra1_self_hosted_0_0',
+            'syd' => 'database_db_syd1_self_hosted_0_0',
+            'nyc' => 'database_db_nyc1_self_hosted_0_0',
+        ];
+
+        if (isset($dsns[$region])) {
+            $dsn = $dsns[$region];
         }
 
         // TODO: Temporary until all projects are using shared tables.
-        if ($dsn === System::getEnv('_APP_DATABASE_SHARED_TABLES', '')) {
+        $sharedTables = \explode(',', System::getEnv('_APP_DATABASE_SHARED_TABLES', ''));
+
+        if (\in_array($dsn, $sharedTables)) {
             $schema = 'appwrite';
             $database = 'appwrite';
             $namespace = System::getEnv('_APP_DATABASE_SHARED_NAMESPACE', '');
-            $dsn = $schema . '://' . System::getEnv('_APP_DATABASE_SHARED_TABLES', '') . '?database=' . $database;
+            $dsn = $schema . '://' . $dsn . '?database=' . $database;
 
             if (!empty($namespace)) {
                 $dsn .= '&namespace=' . $namespace;
@@ -195,11 +207,18 @@ App::post('/v1/projects')
 
         $adapter = $pools->get($dsn->getHost())->pop()->getResource();
         $dbForProject = new Database($adapter, $cache);
+        $sharedTables = \explode(',', System::getEnv('_APP_DATABASE_SHARED_TABLES', ''));
+        $sharedTablesV1 = \explode(',', System::getEnv('_APP_DATABASE_SHARED_TABLES_V1', ''));
 
-        if ($dsn->getHost() === System::getEnv('_APP_DATABASE_SHARED_TABLES', '')) {
+        $projectTables = !\in_array($dsn->getHost(), $sharedTables);
+        $sharedTablesV1 = \in_array($dsn->getHost(), $sharedTablesV1);
+        $sharedTablesV2 = !$projectTables && !$sharedTablesV1;
+        $sharedTables = $sharedTablesV1 || $sharedTablesV2;
+
+        if ($sharedTables) {
             $dbForProject
                 ->setSharedTables(true)
-                ->setTenant($project->getInternalId())
+                ->setTenant($sharedTablesV1 ? $project->getInternalId() : null)
                 ->setNamespace($dsn->getParam('namespace'));
         } else {
             $dbForProject
@@ -208,34 +227,70 @@ App::post('/v1/projects')
                 ->setNamespace('_' . $project->getInternalId());
         }
 
-        $dbForProject->create();
+        $create = true;
 
-        $audit = new Audit($dbForProject);
-        $audit->setup();
+        try {
+            $dbForProject->create();
+        } catch (Duplicate) {
+            $create = false;
+        }
 
-        $abuse = new TimeLimit('', 0, 1, $dbForProject);
-        $abuse->setup();
+        if ($create || $projectTables) {
+            $audit = new Audit($dbForProject);
+            $audit->setup();
 
-        /** @var array $collections */
-        $collections = Config::getParam('collections', [])['projects'] ?? [];
+            $abuse = new TimeLimit('', 0, 1, $dbForProject);
+            $abuse->setup();
+        }
 
-        foreach ($collections as $key => $collection) {
-            if (($collection['$collection'] ?? '') !== Database::METADATA) {
-                continue;
-            }
+        if (!$create && $sharedTablesV1) {
+            $attributes = \array_map(fn ($attribute) => new Document($attribute), Audit::ATTRIBUTES);
+            $indexes = \array_map(fn (array $index) => new Document($index), Audit::INDEXES);
+            $dbForProject->createDocument(Database::METADATA, new Document([
+                '$id' => ID::custom('audit'),
+                '$permissions' => [Permission::create(Role::any())],
+                'name' => 'audit',
+                'attributes' => $attributes,
+                'indexes' => $indexes,
+                'documentSecurity' => true
+            ]));
 
-            $attributes = \array_map(function (array $attribute) {
-                return new Document($attribute);
-            }, $collection['attributes']);
+            $attributes = \array_map(fn ($attribute) => new Document($attribute), TimeLimit::ATTRIBUTES);
+            $indexes = \array_map(fn (array $index) => new Document($index), TimeLimit::INDEXES);
+            $dbForProject->createDocument(Database::METADATA, new Document([
+                '$id' => ID::custom('abuse'),
+                '$permissions' => [Permission::create(Role::any())],
+                'name' => 'abuse',
+                'attributes' => $attributes,
+                'indexes' => $indexes,
+                'documentSecurity' => true
+            ]));
+        }
 
-            $indexes = \array_map(function (array $index) {
-                return new Document($index);
-            }, $collection['indexes']);
+        if ($create || $sharedTablesV1) {
+            /** @var array $collections */
+            $collections = Config::getParam('collections', [])['projects'] ?? [];
 
-            try {
-                $dbForProject->createCollection($key, $attributes, $indexes);
-            } catch (Duplicate) {
-                // Collection already exists
+            foreach ($collections as $key => $collection) {
+                if (($collection['$collection'] ?? '') !== Database::METADATA) {
+                    continue;
+                }
+
+                $attributes = \array_map(fn ($attribute) => new Document($attribute), $collection['attributes']);
+                $indexes = \array_map(fn (array $index) => new Document($index), $collection['indexes']);
+
+                try {
+                    $dbForProject->createCollection($key, $attributes, $indexes);
+                } catch (Duplicate) {
+                    $dbForProject->createDocument(Database::METADATA, new Document([
+                        '$id' => ID::custom($key),
+                        '$permissions' => [Permission::create(Role::any())],
+                        'name' => $key,
+                        'attributes' => $attributes,
+                        'indexes' => $indexes,
+                        'documentSecurity' => true
+                    ]));
+                }
             }
         }
 

app/controllers/api/storage.php

@@ -68,19 +68,19 @@ App::post('/v1/storage/buckets')
     ->param('enabled', true, new Boolean(true), 'Is bucket enabled? When set to \'disabled\', users cannot access the files in this bucket but Server SDKs with and API key can still access the bucket. No files are lost when this is toggled.', true)
     ->param('maximumFileSize', fn (array $plan) => empty($plan['fileSize']) ? (int) System::getEnv('_APP_STORAGE_LIMIT', 0) : $plan['fileSize'] * 1000 * 1000, fn (array $plan) => new Range(1, empty($plan['fileSize']) ? (int) System::getEnv('_APP_STORAGE_LIMIT', 0) : $plan['fileSize'] * 1000 * 1000), 'Maximum file size allowed in bytes. Maximum allowed value is ' . Storage::human(System::getEnv('_APP_STORAGE_LIMIT', 0), 0) . '.', true, ['plan'])
     ->param('allowedFileExtensions', [], new ArrayList(new Text(64), APP_LIMIT_ARRAY_PARAMS_SIZE), 'Allowed file extensions. Maximum of ' . APP_LIMIT_ARRAY_PARAMS_SIZE . ' extensions are allowed, each 64 characters long.', true)
-    ->param('compression', Compression::NONE, new WhiteList([Compression::NONE, Compression::GZIP, Compression::ZSTD]), 'Compression algorithm choosen for compression. Can be one of ' . Compression::NONE . ',  [' . Compression::GZIP . '](https://en.wikipedia.org/wiki/Gzip), or [' . Compression::ZSTD . '](https://en.wikipedia.org/wiki/Zstd), For file size above ' . Storage::human(APP_STORAGE_READ_BUFFER, 0) . ' compression is skipped even if it\'s enabled', true)
+    ->param('compression', Compression::NONE, new WhiteList([Compression::NONE, Compression::GZIP, Compression::ZSTD], true), 'Compression algorithm choosen for compression. Can be one of ' . Compression::NONE . ',  [' . Compression::GZIP . '](https://en.wikipedia.org/wiki/Gzip), or [' . Compression::ZSTD . '](https://en.wikipedia.org/wiki/Zstd), For file size above ' . Storage::human(APP_STORAGE_READ_BUFFER, 0) . ' compression is skipped even if it\'s enabled', true)
     ->param('encryption', true, new Boolean(true), 'Is encryption enabled? For file size above ' . Storage::human(APP_STORAGE_READ_BUFFER, 0) . ' encryption is skipped even if it\'s enabled', true)
     ->param('antivirus', true, new Boolean(true), 'Is virus scanning enabled? For file size above ' . Storage::human(APP_LIMIT_ANTIVIRUS, 0) . ' AntiVirus scanning is skipped even if it\'s enabled', true)
     ->inject('response')
     ->inject('dbForProject')
     ->inject('queueForEvents')
-    ->action(function (string $bucketId, string $name, ?array $permissions, bool $fileSecurity, bool $enabled, int $maximumFileSize, array $allowedFileExtensions, string $compression, bool $encryption, bool $antivirus, Response $response, Database $dbForProject, Event $queueForEvents) {
+    ->action(function (string $bucketId, string $name, ?array $permissions, bool $fileSecurity, bool $enabled, int $maximumFileSize, array $allowedFileExtensions, ?string $compression, bool $encryption, bool $antivirus, Response $response, Database $dbForProject, Event $queueForEvents) {
 
         $bucketId = $bucketId === 'unique()' ? ID::unique() : $bucketId;
 
         // Map aggregate permissions into the multiple permissions they represent.
         $permissions = Permission::aggregate($permissions);
-
+        $compression ??= Compression::NONE;
         try {
             $files = (Config::getParam('collections', [])['buckets'] ?? [])['files'] ?? [];
             if (empty($files)) {
@@ -254,13 +254,13 @@ App::put('/v1/storage/buckets/:bucketId')
     ->param('enabled', true, new Boolean(true), 'Is bucket enabled? When set to \'disabled\', users cannot access the files in this bucket but Server SDKs with and API key can still access the bucket. No files are lost when this is toggled.', true)
     ->param('maximumFileSize', fn (array $plan) => empty($plan['fileSize']) ? (int) System::getEnv('_APP_STORAGE_LIMIT', 0) : $plan['fileSize'] * 1000 * 1000, fn (array $plan) => new Range(1, empty($plan['fileSize']) ? (int) System::getEnv('_APP_STORAGE_LIMIT', 0) : $plan['fileSize'] * 1000 * 1000), 'Maximum file size allowed in bytes. Maximum allowed value is ' . Storage::human(System::getEnv('_APP_STORAGE_LIMIT', 0), 0) . '.', true, ['plan'])
     ->param('allowedFileExtensions', [], new ArrayList(new Text(64), APP_LIMIT_ARRAY_PARAMS_SIZE), 'Allowed file extensions. Maximum of ' . APP_LIMIT_ARRAY_PARAMS_SIZE . ' extensions are allowed, each 64 characters long.', true)
-    ->param('compression', Compression::NONE, new WhiteList([Compression::NONE, Compression::GZIP, Compression::ZSTD]), 'Compression algorithm choosen for compression. Can be one of ' . Compression::NONE . ', [' . Compression::GZIP . '](https://en.wikipedia.org/wiki/Gzip), or [' . Compression::ZSTD . '](https://en.wikipedia.org/wiki/Zstd), For file size above ' . Storage::human(APP_STORAGE_READ_BUFFER, 0) . ' compression is skipped even if it\'s enabled', true)
+    ->param('compression', Compression::NONE, new WhiteList([Compression::NONE, Compression::GZIP, Compression::ZSTD], true), 'Compression algorithm choosen for compression. Can be one of ' . Compression::NONE . ', [' . Compression::GZIP . '](https://en.wikipedia.org/wiki/Gzip), or [' . Compression::ZSTD . '](https://en.wikipedia.org/wiki/Zstd), For file size above ' . Storage::human(APP_STORAGE_READ_BUFFER, 0) . ' compression is skipped even if it\'s enabled', true)
     ->param('encryption', true, new Boolean(true), 'Is encryption enabled? For file size above ' . Storage::human(APP_STORAGE_READ_BUFFER, 0) . ' encryption is skipped even if it\'s enabled', true)
     ->param('antivirus', true, new Boolean(true), 'Is virus scanning enabled? For file size above ' . Storage::human(APP_LIMIT_ANTIVIRUS, 0) . ' AntiVirus scanning is skipped even if it\'s enabled', true)
     ->inject('response')
     ->inject('dbForProject')
     ->inject('queueForEvents')
-    ->action(function (string $bucketId, string $name, ?array $permissions, bool $fileSecurity, bool $enabled, ?int $maximumFileSize, array $allowedFileExtensions, string $compression, bool $encryption, bool $antivirus, Response $response, Database $dbForProject, Event $queueForEvents) {
+    ->action(function (string $bucketId, string $name, ?array $permissions, bool $fileSecurity, bool $enabled, ?int $maximumFileSize, array $allowedFileExtensions, ?string $compression, bool $encryption, bool $antivirus, Response $response, Database $dbForProject, Event $queueForEvents) {
         $bucket = $dbForProject->getDocument('buckets', $bucketId);
 
         if ($bucket->isEmpty()) {
@@ -273,6 +273,7 @@ App::put('/v1/storage/buckets/:bucketId')
         $enabled ??= $bucket->getAttribute('enabled', true);
         $encryption ??= $bucket->getAttribute('encryption', true);
         $antivirus ??= $bucket->getAttribute('antivirus', true);
+        $compression ??= $bucket->getAttribute('compression', Compression::NONE);
 
         // Map aggregate permissions into the multiple permissions they represent.
         $permissions = Permission::aggregate($permissions);

app/controllers/general.php

@@ -63,7 +63,7 @@ function router(App $utopia, Database $dbForConsole, callable $getProjectDB, Swo
         }
 
         if (System::getEnv('_APP_OPTIONS_ROUTER_PROTECTION', 'disabled') === 'enabled') {
-            if ($host !== 'localhost' && $host !== APP_HOSTNAME_INTERNAL) { // localhost allowed for proxy, APP_HOSTNAME_INTERNAL allowed for migrations
+            if ($host !== 'localhost' && $host !== APP_HOSTNAME_INTERNAL && $host !== System::getEnv('_APP_CONSOLE_DOMAIN', '')) {
                 throw new AppwriteException(AppwriteException::GENERAL_ACCESS_FORBIDDEN, 'Router protection does not allow accessing Appwrite over this domain. Please add it as custom domain to your project or disable _APP_OPTIONS_ROUTER_PROTECTION environment variable.');
             }
         }
@@ -438,7 +438,7 @@ App::init()
     });
 
 App::init()
-    ->groups(['api', 'web'])
+    ->groups(['api'])
     ->inject('utopia')
     ->inject('swooleRequest')
     ->inject('request')

app/http.php

@@ -16,6 +16,7 @@ use Utopia\CLI\Console;
 use Utopia\Config\Config;
 use Utopia\Database\Database;
 use Utopia\Database\Document;
+use Utopia\Database\Exception\Duplicate;
 use Utopia\Database\Helpers\ID;
 use Utopia\Database\Helpers\Permission;
 use Utopia\Database\Helpers\Role;
@@ -60,8 +61,6 @@ include __DIR__ . '/controllers/general.php';
 
 $http->on(Constant::EVENT_START, function (Server $http) use ($payloadSize, $register) {
     $app = new App('UTC');
-    $app->setCompression(true);
-    $app->setCompressionMinSize(intval(System::getEnv('_APP_COMPRESSION_MIN_SIZE_BYTES', '1024'))); // 1KB
 
     go(function () use ($register, $app) {
         $pools = $register->get('pools');
@@ -93,7 +92,7 @@ $http->on(Constant::EVENT_START, function (Server $http) use ($payloadSize, $reg
         try {
             Console::success('[Setup] - Creating database: appwrite...');
             $dbForConsole->create();
-        } catch (\Throwable $e) {
+        } catch (Duplicate) {
             Console::success('[Setup] - Skip: metadata table already exists');
         }
 
@@ -225,7 +224,7 @@ $http->on(Constant::EVENT_START, function (Server $http) use ($payloadSize, $reg
     });
 });
 
-$http->on('request', function (SwooleRequest $swooleRequest, SwooleResponse $swooleResponse) use ($register) {
+$http->on(Constant::EVENT_REQUEST, function (SwooleRequest $swooleRequest, SwooleResponse $swooleResponse) use ($register) {
     App::setResource('swooleRequest', fn () => $swooleRequest);
     App::setResource('swooleResponse', fn () => $swooleResponse);
 

app/init.php

@@ -1432,7 +1432,9 @@ App::setResource('dbForProject', function (Group $pools, Database $dbForConsole,
         $dsn = new DSN('mysql://' . $project->getAttribute('database'));
     }
 
-    if ($dsn->getHost() === System::getEnv('_APP_DATABASE_SHARED_TABLES', '')) {
+    $sharedTables = \explode(',', System::getEnv('_APP_DATABASE_SHARED_TABLES', ''));
+
+    if (\in_array($dsn->getHost(), $sharedTables)) {
         $database
             ->setSharedTables(true)
             ->setTenant($project->getInternalId())
@@ -1487,7 +1489,9 @@ App::setResource('getProjectDB', function (Group $pools, Database $dbForConsole,
                 ->setTimeout(APP_DATABASE_TIMEOUT_MILLISECONDS)
                 ->setMaxQueryValues(APP_DATABASE_QUERY_MAX_VALUES);
 
-            if ($dsn->getHost() === System::getEnv('_APP_DATABASE_SHARED_TABLES', '')) {
+            $sharedTables = \explode(',', System::getEnv('_APP_DATABASE_SHARED_TABLES', ''));
+
+            if (\in_array($dsn->getHost(), $sharedTables)) {
                 $database
                     ->setSharedTables(true)
                     ->setTenant($project->getInternalId())

app/realtime.php

@@ -92,7 +92,9 @@ if (!function_exists('getProjectDB')) {
 
         $database = new Database($adapter, getCache());
 
-        if ($dsn->getHost() === System::getEnv('_APP_DATABASE_SHARED_TABLES', '')) {
+        $sharedTables = \explode(',', System::getEnv('_APP_DATABASE_SHARED_TABLES', ''));
+
+        if (\in_array($dsn->getHost(), $sharedTables)) {
             $database
                 ->setSharedTables(true)
                 ->setTenant($project->getInternalId())

app/worker.php

@@ -93,7 +93,9 @@ Server::setResource('dbForProject', function (Cache $cache, Registry $register,
         $dsn = new DSN('mysql://' . $project->getAttribute('database'));
     }
 
-    if ($dsn->getHost() === System::getEnv('_APP_DATABASE_SHARED_TABLES', '')) {
+    $sharedTables = \explode(',', System::getEnv('_APP_DATABASE_SHARED_TABLES', ''));
+
+    if (\in_array($dsn->getHost(), $sharedTables)) {
         $database
             ->setSharedTables(true)
             ->setTenant($project->getInternalId())
@@ -126,7 +128,9 @@ Server::setResource('getProjectDB', function (Group $pools, Database $dbForConso
         if (isset($databases[$dsn->getHost()])) {
             $database = $databases[$dsn->getHost()];
 
-            if ($dsn->getHost() === System::getEnv('_APP_DATABASE_SHARED_TABLES', '')) {
+            $sharedTables = \explode(',', System::getEnv('_APP_DATABASE_SHARED_TABLES', ''));
+
+            if (\in_array($dsn->getHost(), $sharedTables)) {
                 $database
                     ->setSharedTables(true)
                     ->setTenant($project->getInternalId())
@@ -150,7 +154,9 @@ Server::setResource('getProjectDB', function (Group $pools, Database $dbForConso
 
         $databases[$dsn->getHost()] = $database;
 
-        if ($dsn->getHost() === System::getEnv('_APP_DATABASE_SHARED_TABLES', '')) {
+        $sharedTables = \explode(',', System::getEnv('_APP_DATABASE_SHARED_TABLES', ''));
+
+        if (\in_array($dsn->getHost(), $sharedTables)) {
             $database
                 ->setSharedTables(true)
                 ->setTenant($project->getInternalId())

composer.json

@@ -45,13 +45,13 @@
         "ext-sockets": "*",
         "appwrite/php-runtimes": "0.16.*",
         "appwrite/php-clamav": "2.0.*",
-        "utopia-php/abuse": "0.43.0",
+        "utopia-php/abuse": "0.43.*",
         "utopia-php/analytics": "0.10.*",
-        "utopia-php/audit": "0.43.0",
+        "utopia-php/audit": "0.43.*",
         "utopia-php/cache": "0.11.*",
         "utopia-php/cli": "0.15.*",
         "utopia-php/config": "0.2.*",
-        "utopia-php/database": "0.53.16",
+        "utopia-php/database": "0.53.20",
         "utopia-php/domains": "0.5.*",
         "utopia-php/dsn": "0.2.1",
         "utopia-php/framework": "0.33.*",

composer.lock

@@ -4,7 +4,7 @@
         "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies",
         "This file is @generated automatically"
     ],
-    "content-hash": "b358198535c1867eabed7c0f99135a57",
+    "content-hash": "217b0c1b6c156d51bf5a2674f87a7630",
     "packages": [
         {
             "name": "adhocore/jwt",
@@ -1430,16 +1430,16 @@
         },
         {
             "name": "utopia-php/abuse",
-            "version": "0.43.0",
+            "version": "0.43.1",
             "source": {
                 "type": "git",
                 "url": "https://github.com/utopia-php/abuse.git",
-                "reference": "6346a3b4c5177a43160035a7289e30fdfb0790d6"
+                "reference": "e404c21e8dcf6a310bc83cf1d74e716b105598fa"
             },
             "dist": {
                 "type": "zip",
-                "url": "https://api.github.com/repos/utopia-php/abuse/zipball/6346a3b4c5177a43160035a7289e30fdfb0790d6",
-                "reference": "6346a3b4c5177a43160035a7289e30fdfb0790d6",
+                "url": "https://api.github.com/repos/utopia-php/abuse/zipball/e404c21e8dcf6a310bc83cf1d74e716b105598fa",
+                "reference": "e404c21e8dcf6a310bc83cf1d74e716b105598fa",
                 "shasum": ""
             },
             "require": {
@@ -1475,9 +1475,9 @@
             ],
             "support": {
                 "issues": "https://github.com/utopia-php/abuse/issues",
-                "source": "https://github.com/utopia-php/abuse/tree/0.43.0"
+                "source": "https://github.com/utopia-php/abuse/tree/0.43.1"
             },
-            "time": "2024-08-30T05:17:23+00:00"
+            "time": "2024-10-23T04:29:12+00:00"
         },
         {
             "name": "utopia-php/analytics",
@@ -1527,16 +1527,16 @@
         },
         {
             "name": "utopia-php/audit",
-            "version": "0.43.0",
+            "version": "0.43.1",
             "source": {
                 "type": "git",
                 "url": "https://github.com/utopia-php/audit.git",
-                "reference": "cef22b5dc6a6d28fcd522f41c7bf7ded4a4dfd3e"
+                "reference": "04a47dd1f5f92e2d50e971a06bcc9e759325d277"
             },
             "dist": {
                 "type": "zip",
-                "url": "https://api.github.com/repos/utopia-php/audit/zipball/cef22b5dc6a6d28fcd522f41c7bf7ded4a4dfd3e",
-                "reference": "cef22b5dc6a6d28fcd522f41c7bf7ded4a4dfd3e",
+                "url": "https://api.github.com/repos/utopia-php/audit/zipball/04a47dd1f5f92e2d50e971a06bcc9e759325d277",
+                "reference": "04a47dd1f5f92e2d50e971a06bcc9e759325d277",
                 "shasum": ""
             },
             "require": {
@@ -1568,9 +1568,9 @@
             ],
             "support": {
                 "issues": "https://github.com/utopia-php/audit/issues",
-                "source": "https://github.com/utopia-php/audit/tree/0.43.0"
+                "source": "https://github.com/utopia-php/audit/tree/0.43.1"
             },
-            "time": "2024-08-30T05:17:36+00:00"
+            "time": "2024-10-23T04:27:59+00:00"
         },
         {
             "name": "utopia-php/cache",
@@ -1770,16 +1770,16 @@
         },
         {
             "name": "utopia-php/database",
-            "version": "0.53.16",
+            "version": "0.53.20",
             "source": {
                 "type": "git",
                 "url": "https://github.com/utopia-php/database.git",
-                "reference": "6661edffeef05b59e16d102b989a72f7f78cf7de"
+                "reference": "e43f8ee26e06ee8812737e63642dbd7ee7c9dc04"
             },
             "dist": {
                 "type": "zip",
-                "url": "https://api.github.com/repos/utopia-php/database/zipball/6661edffeef05b59e16d102b989a72f7f78cf7de",
-                "reference": "6661edffeef05b59e16d102b989a72f7f78cf7de",
+                "url": "https://api.github.com/repos/utopia-php/database/zipball/e43f8ee26e06ee8812737e63642dbd7ee7c9dc04",
+                "reference": "e43f8ee26e06ee8812737e63642dbd7ee7c9dc04",
                 "shasum": ""
             },
             "require": {
@@ -1820,9 +1820,9 @@
             ],
             "support": {
                 "issues": "https://github.com/utopia-php/database/issues",
-                "source": "https://github.com/utopia-php/database/tree/0.53.16"
+                "source": "https://github.com/utopia-php/database/tree/0.53.20"
             },
-            "time": "2024-11-06T03:07:16+00:00"
+            "time": "2024-11-12T00:23:36+00:00"
         },
         {
             "name": "utopia-php/domains",
@@ -2222,16 +2222,16 @@
         },
         {
             "name": "utopia-php/migration",
-            "version": "0.6.11",
+            "version": "0.6.12",
             "source": {
                 "type": "git",
                 "url": "https://github.com/utopia-php/migration.git",
-                "reference": "4d167914d3f7fa1fe816b2b2c6f221e70166bfd7"
+                "reference": "9a8c905af4cece5c5ec9542a5b534befce067260"
             },
             "dist": {
                 "type": "zip",
-                "url": "https://api.github.com/repos/utopia-php/migration/zipball/4d167914d3f7fa1fe816b2b2c6f221e70166bfd7",
-                "reference": "4d167914d3f7fa1fe816b2b2c6f221e70166bfd7",
+                "url": "https://api.github.com/repos/utopia-php/migration/zipball/9a8c905af4cece5c5ec9542a5b534befce067260",
+                "reference": "9a8c905af4cece5c5ec9542a5b534befce067260",
                 "shasum": ""
             },
             "require": {
@@ -2272,9 +2272,9 @@
             ],
             "support": {
                 "issues": "https://github.com/utopia-php/migration/issues",
-                "source": "https://github.com/utopia-php/migration/tree/0.6.11"
+                "source": "https://github.com/utopia-php/migration/tree/0.6.12"
             },
-            "time": "2024-10-31T06:19:57+00:00"
+            "time": "2024-11-12T00:31:53+00:00"
         },
         {
             "name": "utopia-php/mongo",
@@ -2542,16 +2542,16 @@
         },
         {
             "name": "utopia-php/queue",
-            "version": "0.7.1",
+            "version": "0.7.2",
             "source": {
                 "type": "git",
                 "url": "https://github.com/utopia-php/queue.git",
-                "reference": "94c240d9f6383829807ce7b2d737f04b159fd3e8"
+                "reference": "40fdd9799d0a11dd33fca06f8223032a47dce2f6"
             },
             "dist": {
                 "type": "zip",
-                "url": "https://api.github.com/repos/utopia-php/queue/zipball/94c240d9f6383829807ce7b2d737f04b159fd3e8",
-                "reference": "94c240d9f6383829807ce7b2d737f04b159fd3e8",
+                "url": "https://api.github.com/repos/utopia-php/queue/zipball/40fdd9799d0a11dd33fca06f8223032a47dce2f6",
+                "reference": "40fdd9799d0a11dd33fca06f8223032a47dce2f6",
                 "shasum": ""
             },
             "require": {
@@ -2597,9 +2597,9 @@
             ],
             "support": {
                 "issues": "https://github.com/utopia-php/queue/issues",
-                "source": "https://github.com/utopia-php/queue/tree/0.7.1"
+                "source": "https://github.com/utopia-php/queue/tree/0.7.2"
             },
-            "time": "2024-11-05T17:00:38+00:00"
+            "time": "2024-11-11T10:04:02+00:00"
         },
         {
             "name": "utopia-php/registry",
@@ -2817,16 +2817,16 @@
         },
         {
             "name": "utopia-php/vcs",
-            "version": "0.8.3",
+            "version": "0.8.5",
             "source": {
                 "type": "git",
                 "url": "https://github.com/utopia-php/vcs.git",
-                "reference": "a032ed0611a8f4467aeaa9484f73223074457337"
+                "reference": "7622330628d53844a3873ca873338150756bab82"
             },
             "dist": {
                 "type": "zip",
-                "url": "https://api.github.com/repos/utopia-php/vcs/zipball/a032ed0611a8f4467aeaa9484f73223074457337",
-                "reference": "a032ed0611a8f4467aeaa9484f73223074457337",
+                "url": "https://api.github.com/repos/utopia-php/vcs/zipball/7622330628d53844a3873ca873338150756bab82",
+                "reference": "7622330628d53844a3873ca873338150756bab82",
                 "shasum": ""
             },
             "require": {
@@ -2860,9 +2860,9 @@
             ],
             "support": {
                 "issues": "https://github.com/utopia-php/vcs/issues",
-                "source": "https://github.com/utopia-php/vcs/tree/0.8.3"
+                "source": "https://github.com/utopia-php/vcs/tree/0.8.5"
             },
-            "time": "2024-11-05T17:10:09+00:00"
+            "time": "2024-11-11T18:33:10+00:00"
         },
         {
             "name": "utopia-php/websocket",
@@ -6923,16 +6923,16 @@
         },
         {
             "name": "twig/twig",
-            "version": "v3.14.1",
+            "version": "v3.14.2",
             "source": {
                 "type": "git",
                 "url": "https://github.com/twigphp/Twig.git",
-                "reference": "f405356d20fb43603bcadc8b09bfb676cb04a379"
+                "reference": "0b6f9d8370bb3b7f1ce5313ed8feb0fafd6e399a"
             },
             "dist": {
                 "type": "zip",
-                "url": "https://api.github.com/repos/twigphp/Twig/zipball/f405356d20fb43603bcadc8b09bfb676cb04a379",
-                "reference": "f405356d20fb43603bcadc8b09bfb676cb04a379",
+                "url": "https://api.github.com/repos/twigphp/Twig/zipball/0b6f9d8370bb3b7f1ce5313ed8feb0fafd6e399a",
+                "reference": "0b6f9d8370bb3b7f1ce5313ed8feb0fafd6e399a",
                 "shasum": ""
             },
             "require": {
@@ -6986,7 +6986,7 @@
             ],
             "support": {
                 "issues": "https://github.com/twigphp/Twig/issues",
-                "source": "https://github.com/twigphp/Twig/tree/v3.14.1"
+                "source": "https://github.com/twigphp/Twig/tree/v3.14.2"
             },
             "funding": [
                 {
@@ -6998,7 +6998,7 @@
                     "type": "tidelift"
                 }
             ],
-            "time": "2024-11-06T18:17:38+00:00"
+            "time": "2024-11-07T12:36:22+00:00"
         },
         {
             "name": "webmozart/glob",

docker-compose.yml

@@ -193,6 +193,7 @@ services:
       - _APP_EXPERIMENT_LOGGING_PROVIDER
       - _APP_EXPERIMENT_LOGGING_CONFIG
       - _APP_DATABASE_SHARED_TABLES
+      - _APP_DATABASE_SHARED_TABLES_V1
 
   appwrite-console:
     <<: *x-logging

src/Appwrite/Platform/Workers/Certificates.php

@@ -22,6 +22,7 @@ use Utopia\Database\Exception\Structure;
 use Utopia\Database\Helpers\ID;
 use Utopia\Database\Query;
 use Utopia\Domains\Domain;
+use Utopia\Fetch\Client;
 use Utopia\Locale\Locale;
 use Utopia\Logger\Log;
 use Utopia\Platform\Action;
@@ -43,16 +44,18 @@ class Certificates extends Action
         $this
             ->desc('Certificates worker')
             ->inject('message')
+            ->inject('project')
             ->inject('dbForConsole')
             ->inject('queueForMails')
             ->inject('queueForEvents')
             ->inject('queueForFunctions')
             ->inject('log')
-            ->callback(fn (Message $message, Database $dbForConsole, Mail $queueForMails, Event $queueForEvents, Func $queueForFunctions, Log $log) => $this->action($message, $dbForConsole, $queueForMails, $queueForEvents, $queueForFunctions, $log));
+            ->callback(fn (Message $message, Document $project, Database $dbForConsole, Mail $queueForMails, Event $queueForEvents, Func $queueForFunctions, Log $log) => $this->action($message, $project, $dbForConsole, $queueForMails, $queueForEvents, $queueForFunctions, $log));
     }
 
     /**
      * @param Message $message
+     * @param Document $project
      * @param Database $dbForConsole
      * @param Mail $queueForMails
      * @param Event $queueForEvents
@@ -62,7 +65,7 @@ class Certificates extends Action
      * @throws Throwable
      * @throws \Utopia\Database\Exception
      */
-    public function action(Message $message, Database $dbForConsole, Mail $queueForMails, Event $queueForEvents, Func $queueForFunctions, Log $log): void
+    public function action(Message $message, Document $project, Database $dbForConsole, Mail $queueForMails, Event $queueForEvents, Func $queueForFunctions, Log $log): void
     {
         $payload = $message->getPayload() ?? [];
 
@@ -76,7 +79,7 @@ class Certificates extends Action
 
         $log->addTag('domain', $domain->get());
 
-        $this->execute($domain, $dbForConsole, $queueForMails, $queueForEvents, $queueForFunctions, $log, $skipRenewCheck);
+        $this->execute($domain, $project, $dbForConsole, $queueForMails, $queueForEvents, $queueForFunctions, $log, $skipRenewCheck);
     }
 
     /**
@@ -90,7 +93,7 @@ class Certificates extends Action
      * @throws Throwable
      * @throws \Utopia\Database\Exception
      */
-    private function execute(Domain $domain, Database $dbForConsole, Mail $queueForMails, Event $queueForEvents, Func $queueForFunctions, Log $log, bool $skipRenewCheck = false): void
+    private function execute(Domain $domain, Document $project, Database $dbForConsole, Mail $queueForMails, Event $queueForEvents, Func $queueForFunctions, Log $log, bool $skipRenewCheck = false): void
     {
         /**
          * 1. Read arguments and validate domain
@@ -155,16 +158,26 @@ class Certificates extends Action
             // Prepare folder name for certbot. Using this helps prevent miss-match in LetsEncrypt configuration when renewing certificate
             $folder = ID::unique();
 
-            // Generate certificate files using Let's Encrypt
-            $letsEncryptData = $this->issueCertificate($folder, $domain->get(), $email);
+            try {
+                // Generate certificate files using Let's Encrypt
+                $letsEncryptData = $this->issueCertificate($folder, $domain->get(), $email);
+
+                // Give certificates to Traefik
+                $this->applyCertificateFiles($folder, $domain->get(), $letsEncryptData);
+            } catch (\Throwable $th) {
+                Console::error('Failed to generate Lets Encrypt certificate');
+            }
 
             // Command succeeded, store all data into document
             $logs = 'Certificate successfully generated.';
             $certificate->setAttribute('logs', \mb_strcut($logs, 0, 1000000));// Limit to 1MB
 
-
-            // Give certificates to Traefik
-            $this->applyCertificateFiles($folder, $domain->get(), $letsEncryptData);
+            try {
+                // TEMP: add custom hostnames to cloudflare
+                $this->addCustomHostnameToRegistrar($project, $domain->get());
+            } catch (\Throwable $th) {
+                Console::error('Failed to add custom hostname to registrar: ' . $th->getMessage());
+            }
 
             // Update certificate info stored in database
             $certificate->setAttribute('renewDate', $this->getRenewDate($domain->get()));
@@ -197,6 +210,35 @@ class Certificates extends Action
         }
     }
 
+    /**
+     * Add custom hostname to Cloudflare registrar
+     *
+     * @param Document $project
+     * @param string $hostname
+     * @return void
+     * @throws Exception
+     */
+    private function addCustomHostnameToRegistrar(Document $project, string $hostname): void
+    {
+        $client = new Client();
+        $client
+            ->addHeader('content-type', Client::CONTENT_TYPE_APPLICATION_JSON)
+             ->addHeader('Authorization', 'Bearer ' . System::getEnv('_APP_SYSTEM_CLOUDFLARE_TOKEN'));
+
+        $response = $client->fetch("https://api.cloudflare.com/client/v4/zones/b2d0e62383d3c0f6299efab107af2c7a/custom_hostnames", Client::METHOD_POST, [
+            'hostname' => $hostname,
+            'ssl' => [
+                "method" => "http",
+                "type" => "dv",
+                "wildcard" => false
+            ]
+        ]);
+
+        if ($response->getStatusCode() !== 201) {
+            throw new Exception('Failed to add custom hostname to Cloudflare: ' . $response->getBody());
+        }
+    }
+
     /**
      * Save certificate data into database.
      *

src/Appwrite/Platform/Workers/Deletes.php

@@ -494,12 +494,13 @@ class Deletes extends Action
         ];
 
         $limit = \count($projectCollectionIds) + 25;
+        $sharedTables = \explode(',', System::getEnv('_APP_DATABASE_SHARED_TABLES', ''));
 
         while (true) {
             $collections = $dbForProject->listCollections($limit);
 
             foreach ($collections as $collection) {
-                if ($dsn->getHost() !== System::getEnv('_APP_DATABASE_SHARED_TABLES', '') || !\in_array($collection->getId(), $projectCollectionIds)) {
+                if (\in_array($dsn->getHost(), $sharedTables) || !\in_array($collection->getId(), $projectCollectionIds)) {
                     try {
                         $dbForProject->deleteCollection($collection->getId());
                     } catch (Throwable $e) {
@@ -517,7 +518,7 @@ class Deletes extends Action
                 }
             }
 
-            if ($dsn->getHost() === System::getEnv('_APP_DATABASE_SHARED_TABLES', '')) {
+            if (\in_array($dsn->getHost(), $sharedTables)) {
                 $collectionsIds = \array_map(fn ($collection) => $collection->getId(), $collections);
 
                 if (empty(\array_diff($collectionsIds, $projectCollectionIds))) {
@@ -571,7 +572,7 @@ class Deletes extends Action
         ], $dbForConsole);
 
         // Delete metadata table
-        if ($dsn->getHost() !== System::getEnv('_APP_DATABASE_SHARED_TABLES', '')) {
+        if (\in_array($dsn->getHost(), $sharedTables)) {
             $dbForProject->deleteCollection('_metadata');
         } else {
             $this->deleteByGroup('_metadata', [], $dbForProject);

src/Appwrite/Utopia/Response/Model/Func.php

@@ -94,7 +94,7 @@ class Func extends Model
             ])
             ->addRule('schedule', [
                 'type' => self::TYPE_STRING,
-                'description' => 'Function execution schedult in CRON format.',
+                'description' => 'Function execution schedule in CRON format.',
                 'default' => '',
                 'example' => '5 4 * * *',
             ])

tests/e2e/Services/Databases/DatabasesCustomServerTest.php

@@ -1362,7 +1362,7 @@ class DatabasesCustomServerTest extends Scope
         ]);
 
         $this->assertEquals(400, $tooWide['headers']['status-code']);
-        $this->assertEquals('Attribute limit exceeded', $tooWide['body']['message']);
+        $this->assertEquals('attribute_limit_exceeded', $tooWide['body']['type']);
     }
 
     public function testIndexLimitException()