Merge remote-tracking branch 'origin/feat-db-pools' into feat-usage-refactor-fo-db-pools

.env

@@ -23,6 +23,11 @@ _APP_DB_SCHEMA=appwrite
 _APP_DB_USER=user
 _APP_DB_PASS=password
 _APP_DB_ROOT_PASS=rootsecretpassword
+_APP_CONNECTIONS_DB_PROJECT=db_fra1_02=mysql://user:password@mariadb:3306/appwrite
+_APP_CONNECTIONS_DB_CONSOLE=db_fra1_01=mysql://user:password@mariadb:3306/appwrite
+_APP_CONNECTIONS_CACHE=redis_fra1_01=redis://redis:6379
+_APP_CONNECTIONS_QUEUE=redis_fra1_01=redis://redis:6379
+_APP_CONNECTIONS_PUBSUB=redis_fra1_01=redis://redis:6379
 _APP_STORAGE_DEVICE=Local
 _APP_STORAGE_S3_ACCESS_KEY=
 _APP_STORAGE_S3_SECRET=

Dockerfile

@@ -35,6 +35,7 @@ ENV PHP_REDIS_VERSION=5.3.7 \
     PHP_IMAGICK_VERSION=3.7.0 \
     PHP_YAML_VERSION=2.2.2 \
     PHP_MAXMINDDB_VERSION=v1.11.0 \
+    PHP_MEMCACHED_VERSION=v3.2.0 \
     PHP_ZSTD_VERSION="4504e4186e79b197cfcb75d4d09aa47ef7d92fe9 "
 
 RUN \
@@ -52,6 +53,7 @@ RUN \
   imagemagick \
   imagemagick-dev \
   libmaxminddb-dev \
+  libmemcached-dev \
   zstd-dev
 
 RUN docker-php-ext-install sockets
@@ -125,6 +127,15 @@ RUN \
   ./configure && \
   make && make install
 
+# Memcached Extension
+FROM compile as memcached
+RUN \
+  git clone --depth 1 --branch $PHP_MEMCACHED_VERSION https://github.com/php-memcached-dev/php-memcached.git && \
+  cd php-memcached && \
+  phpize && \
+  ./configure && \
+  make && make install
+
 # Zstd Compression
 FROM compile as zstd
 RUN git clone --recursive -n https://github.com/kjdev/php-ext-zstd.git \
@@ -134,7 +145,6 @@ RUN git clone --recursive -n https://github.com/kjdev/php-ext-zstd.git \
   && ./configure --with-libzstd \
   && make && make install
 
-
 # Rust Extensions Compile Image
 FROM php:8.0.18-cli as rust_compile
 
@@ -304,6 +314,7 @@ COPY --from=imagick /usr/local/lib/php/extensions/no-debug-non-zts-20200930/imag
 COPY --from=yaml /usr/local/lib/php/extensions/no-debug-non-zts-20200930/yaml.so /usr/local/lib/php/extensions/no-debug-non-zts-20200930/
 COPY --from=maxmind /usr/local/lib/php/extensions/no-debug-non-zts-20200930/maxminddb.so /usr/local/lib/php/extensions/no-debug-non-zts-20200930/
 COPY --from=mongodb /usr/local/lib/php/extensions/no-debug-non-zts-20200930/mongodb.so /usr/local/lib/php/extensions/no-debug-non-zts-20200930/
+COPY --from=memcached /usr/local/lib/php/extensions/no-debug-non-zts-20200930/memcached.so /usr/local/lib/php/extensions/no-debug-non-zts-20200930/
 COPY --from=scrypt  /usr/local/lib/php/extensions/php-scrypt/target/libphp_scrypt.so /usr/local/lib/php/extensions/no-debug-non-zts-20200930/
 COPY --from=zstd /usr/local/lib/php/extensions/no-debug-non-zts-20200930/zstd.so /usr/local/lib/php/extensions/no-debug-non-zts-20200930/
 

app/cli.php

@@ -6,7 +6,78 @@ require_once __DIR__ . '/controllers/general.php';
 use Utopia\App;
 use Utopia\CLI\CLI;
 use Utopia\CLI\Console;
+use Utopia\Cache\Adapter\Sharding;
+use Utopia\Cache\Cache;
+use Utopia\Config\Config;
+use Utopia\Database\Database;
 use Utopia\Database\Validator\Authorization;
+use InfluxDB\Database as InfluxDatabase;
+
+function getInfluxDB(): InfluxDatabase
+{
+    global $register;
+
+    $client = $register->get('influxdb'); /** @var InfluxDB\Client $client */
+    $attempts = 0;
+    $max = 10;
+    $sleep = 1;
+
+    do { // check if telegraf database is ready
+        try {
+            $attempts++;
+            $database = $client->selectDB('telegraf');
+            if (in_array('telegraf', $client->listDatabases())) {
+                break; // leave the do-while if successful
+            }
+        } catch (\Throwable $th) {
+            Console::warning("InfluxDB not ready. Retrying connection ({$attempts})...");
+            if ($attempts >= $max) {
+                throw new \Exception('InfluxDB database not ready yet');
+            }
+            sleep($sleep);
+        }
+    } while ($attempts < $max);
+    return $database;
+}
+
+function getConsoleDB(): Database
+{
+    global $register;
+
+    $pools = $register->get('pools'); /** @var \Utopia\Pools\Group $pools */
+
+    $dbAdapter = $pools
+        ->get('console')
+        ->pop()
+        ->getResource()
+    ;
+
+    $database = new Database($dbAdapter, getCache());
+
+    $database->setNamespace('console');
+
+    return $database;
+}
+
+function getCache(): Cache
+{
+    global $register;
+
+    $pools = $register->get('pools'); /** @var \Utopia\Pools\Group $pools */
+
+    $list = Config::getParam('pools-cache', []);
+    $adapters = [];
+
+    foreach ($list as $value) {
+        $adapters[] = $pools
+            ->get($value)
+            ->pop()
+            ->getResource()
+        ;
+    }
+
+    return new Cache(new Sharding($adapters));
+}
 
 Authorization::disable();
 
@@ -29,4 +100,13 @@ $cli
         Console::log(App::getEnv('_APP_VERSION', 'UNKNOWN'));
     });
 
+$cli
+    ->error(function ($error) {
+        if (App::getEnv('_APP_ENV', 'development')) {
+            Console::error($error);
+        } else {
+            Console::error($error->getMessage());
+        }
+    });
+
 $cli->run();

app/config/collections.php

@@ -534,6 +534,17 @@ $collections = [
                 'array' => false,
                 'filters' => [],
             ],
+            [
+                '$id' => ID::custom('database'),
+                'type' => Database::VAR_STRING,
+                'format' => '',
+                'size' => 256,
+                'signed' => true,
+                'required' => true,
+                'default' => null,
+                'array' => false,
+                'filters' => [],
+            ],
             [
                 '$id' => ID::custom('logo'),
                 'type' => Database::VAR_STRING,
@@ -2205,69 +2216,69 @@ $collections = [
             ],
         ],
         'indexes' => [
-            [
-                '$id' => ID::custom('_key_search'),
-                'type' => Database::INDEX_FULLTEXT,
-                'attributes' => ['search'],
-                'lengths' => [2048],
-                'orders' => [Database::ORDER_ASC],
-            ],
-            [
-                '$id' => ID::custom('_key_name'),
-                'type' => Database::INDEX_KEY,
-                'attributes' => ['name'],
-                'lengths' => [2048],
-                'orders' => [Database::ORDER_ASC],
-            ],
-            [
-                '$id' => ID::custom('_key_enabled'),
-                'type' => Database::INDEX_KEY,
-                'attributes' => ['enabled'],
-                'lengths' => [],
-                'orders' => [Database::ORDER_ASC],
-            ],
-            [
-                '$id' => ID::custom('_key_runtime'),
-                'type' => Database::INDEX_KEY,
-                'attributes' => ['runtime'],
-                'lengths' => [2048],
-                'orders' => [Database::ORDER_ASC],
-            ],
-            [
-                '$id' => ID::custom('_key_deployment'),
-                'type' => Database::INDEX_KEY,
-                'attributes' => ['deployment'],
-                'lengths' => [Database::LENGTH_KEY],
-                'orders' => [Database::ORDER_ASC],
-            ],
-            [
-                '$id' => ID::custom('_key_schedule'),
-                'type' => Database::INDEX_KEY,
-                'attributes' => ['schedule'],
-                'lengths' => [128],
-                'orders' => [Database::ORDER_ASC],
-            ],
-            [
-                '$id' => ID::custom('_key_scheduleNext'),
-                'type' => Database::INDEX_KEY,
-                'attributes' => ['scheduleNext'],
-                'lengths' => [],
-                'orders' => [Database::ORDER_ASC],
-            ],
-            [
-                '$id' => ID::custom('_key_schedulePrevious'),
-                'type' => Database::INDEX_KEY,
-                'attributes' => ['schedulePrevious'],
-                'lengths' => [],
-                'orders' => [Database::ORDER_ASC],
-            ],
-            [
-                '$id' => ID::custom('_key_timeout'),
-                'type' => Database::INDEX_KEY,
-                'attributes' => ['timeout'],
-                'lengths' => [],
-                'orders' => [Database::ORDER_ASC],
-            ],
+        //     [
+        //         '$id' => ID::custom('_key_search'),
+        //         'type' => Database::INDEX_FULLTEXT,
+        //         'attributes' => ['search'],
+        //         'lengths' => [],
+        //         'orders' => [],
+        //     ],
+        //      [
+        //          '$id' => ID::custom('_key_name'),
+        //          'type' => Database::INDEX_KEY,
+        //          'attributes' => ['name'],
+        //          'lengths' => [700],
+        //          'orders' => [],
+        //      ],
+        //      [
+        //          '$id' => ID::custom('_key_enabled'),
+        //          'type' => Database::INDEX_KEY,
+        //          'attributes' => ['enabled'],
+        //          'lengths' => [],
+        //          'orders' => [],
+        //      ],
+        //      [
+        //          '$id' => ID::custom('_key_runtime'),
+        //          'type' => Database::INDEX_KEY,
+        //          'attributes' => ['runtime'],
+        //          'lengths' => [700],
+        //          'orders' => [],
+        //      ],
+        //      [
+        //          '$id' => ID::custom('_key_deployment'),
+        //          'type' => Database::INDEX_KEY,
+        //          'attributes' => ['deployment'],
+        //          'lengths' => [],
+        //          'orders' => [],
+        //      ],
+        //      [
+        //          '$id' => ID::custom('_key_schedule'),
+        //          'type' => Database::INDEX_KEY,
+        //          'attributes' => ['schedule'],
+        //          'lengths' => [],
+        //          'orders' => [],
+        //      ],
+        //      [
+        //          '$id' => ID::custom('_key_scheduleNext'),
+        //          'type' => Database::INDEX_KEY,
+        //          'attributes' => ['scheduleNext'],
+        //          'lengths' => [],
+        //          'orders' => [],
+        //      ],
+        //      [
+        //          '$id' => ID::custom('_key_schedulePrevious'),
+        //          'type' => Database::INDEX_KEY,
+        //          'attributes' => ['schedulePrevious'],
+        //          'lengths' => [],
+        //          'orders' => [],
+        //      ],
+        //      [
+        //          '$id' => ID::custom('_key_timeout'),
+        //          'type' => Database::INDEX_KEY,
+        //          'attributes' => ['timeout'],
+        //          'lengths' => [],
+        //          'orders' => [],
+        //      ],
         ],
     ],
 
@@ -2314,7 +2325,7 @@ $collections = [
                 '$id' => ID::custom('entrypoint'),
                 'type' => Database::VAR_STRING,
                 'format' => '',
-                'size' => 2048,
+                'size' => 16384,
                 'signed' => true,
                 'required' => false,
                 'default' => null,
@@ -2324,7 +2335,7 @@ $collections = [
                 '$id' => ID::custom('path'),
                 'type' => Database::VAR_STRING,
                 'format' => '',
-                'size' => 2048,
+                'size' => 16384,
                 'signed' => true,
                 'required' => false,
                 'default' => null,
@@ -2399,55 +2410,55 @@ $collections = [
             ]
         ],
         'indexes' => [
-            [
-                '$id' => ID::custom('_key_resource'),
-                'type' => Database::INDEX_KEY,
-                'attributes' => ['resourceId'],
-                'lengths' => [Database::LENGTH_KEY],
-                'orders' => [Database::ORDER_ASC],
-            ],
-            [
-                '$id' => ID::custom('_key_resource_type'),
-                'type' => Database::INDEX_KEY,
-                'attributes' => ['resourceType'],
-                'lengths' => [Database::LENGTH_KEY],
-                'orders' => [Database::ORDER_ASC],
-            ],
-            [
-                '$id' => ID::custom('_key_search'),
-                'type' => Database::INDEX_FULLTEXT,
-                'attributes' => ['search'],
-                'lengths' => [],
-                'orders' => [],
-            ],
-            [
-                '$id' => ID::custom('_key_entrypoint'),
-                'type' => Database::INDEX_KEY,
-                'attributes' => ['entrypoint'],
-                'lengths' => [2048],
-                'orders' => [Database::ORDER_ASC],
-            ],
-            [
-                '$id' => ID::custom('_key_size'),
-                'type' => Database::INDEX_KEY,
-                'attributes' => ['size'],
-                'lengths' => [],
-                'orders' => [Database::ORDER_ASC],
-            ],
-            [
-                '$id' => ID::custom('_key_buildId'),
-                'type' => Database::INDEX_KEY,
-                'attributes' => ['buildId'],
-                'lengths' => [Database::LENGTH_KEY],
-                'orders' => [Database::ORDER_ASC],
-            ],
-            [
-                '$id' => ID::custom('_key_activate'),
-                'type' => Database::INDEX_KEY,
-                'attributes' => ['activate'],
-                'lengths' => [],
-                'orders' => [Database::ORDER_ASC],
-            ],
+            // [
+            //     '$id' => ID::custom('_key_resource'),
+            //     'type' => Database::INDEX_KEY,
+            //     'attributes' => ['resourceId'],
+            //     'lengths' => [Database::LENGTH_KEY],
+            //     'orders' => [Database::ORDER_ASC],
+            // ],
+            // [
+            //     '$id' => ID::custom('_key_resource_type'),
+            //     'type' => Database::INDEX_KEY,
+            //     'attributes' => ['resourceType'],
+            //     'lengths' => [Database::LENGTH_KEY],
+            //     'orders' => [Database::ORDER_ASC],
+            // ],
+            // [
+            //     '$id' => ID::custom('_key_search'),
+            //     'type' => Database::INDEX_FULLTEXT,
+            //     'attributes' => ['search'],
+            //     'lengths' => [],
+            //     'orders' => [],
+            // ],
+            // [
+            //     '$id' => ID::custom('_key_entrypoint'),
+            //     'type' => Database::INDEX_KEY,
+            //     'attributes' => ['entrypoint'],
+            //     'lengths' => [Database::LENGTH_KEY],
+            //     'orders' => [Database::ORDER_ASC],
+            // ],
+            // [
+            //     '$id' => ID::custom('_key_size'),
+            //     'type' => Database::INDEX_KEY,
+            //     'attributes' => ['size'],
+            //     'lengths' => [],
+            //     'orders' => [Database::ORDER_ASC],
+            // ],
+            // [
+            //     '$id' => ID::custom('_key_buildId'),
+            //     'type' => Database::INDEX_KEY,
+            //     'attributes' => ['buildId'],
+            //     'lengths' => [Database::LENGTH_KEY],
+            //     'orders' => [Database::ORDER_ASC],
+            // ],
+            // [
+            //     '$id' => ID::custom('_key_activate'),
+            //     'type' => Database::INDEX_KEY,
+            //     'attributes' => ['activate'],
+            //     'lengths' => [],
+            //     'orders' => [Database::ORDER_ASC],
+            // ],
         ],
     ],
 
@@ -2896,7 +2907,7 @@ $collections = [
                 '$id' => 'compression',
                 'type' => Database::VAR_STRING,
                 'signed' => true,
-                'size' => 10,
+                'size' => 128,
                 'format' => '',
                 'filters' => [],
                 'required' => true,
@@ -2935,62 +2946,62 @@ $collections = [
             ],
         ],
         'indexes' => [
-            [
-                '$id' => ID::custom('_fulltext_name'),
-                'type' => Database::INDEX_FULLTEXT,
-                'attributes' => ['name'],
-                'lengths' => [1024],
-                'orders' => [Database::ORDER_ASC],
-            ],
-            [
-                '$id' => ID::custom('_key_search'),
-                'type' => Database::INDEX_FULLTEXT,
-                'attributes' => ['search'],
-                'lengths' => [2048],
-                'orders' => [Database::ORDER_ASC],
-            ],
-            [
-                '$id' => ID::custom('_key_enabled'),
-                'type' => Database::INDEX_KEY,
-                'attributes' => ['enabled'],
-                'lengths' => [],
-                'orders' => [Database::ORDER_ASC],
-            ],
-            [
-                '$id' => ID::custom('_key_name'),
-                'type' => Database::INDEX_KEY,
-                'attributes' => ['name'],
-                'lengths' => [128],
-                'orders' => [Database::ORDER_ASC],
-            ],
-            [
-                '$id' => ID::custom('_key_fileSecurity'),
-                'type' => Database::INDEX_KEY,
-                'attributes' => ['fileSecurity'],
-                'lengths' => [],
-                'orders' => [Database::ORDER_ASC],
-            ],
-            [
-                '$id' => ID::custom('_key_maximumFileSize'),
-                'type' => Database::INDEX_KEY,
-                'attributes' => ['maximumFileSize'],
-                'lengths' => [],
-                'orders' => [Database::ORDER_ASC],
-            ],
-            [
-                '$id' => ID::custom('_key_encryption'),
-                'type' => Database::INDEX_KEY,
-                'attributes' => ['encryption'],
-                'lengths' => [],
-                'orders' => [Database::ORDER_ASC],
-            ],
-            [
-                '$id' => ID::custom('_key_antivirus'),
-                'type' => Database::INDEX_KEY,
-                'attributes' => ['antivirus'],
-                'lengths' => [],
-                'orders' => [Database::ORDER_ASC],
-            ],
+            // [
+            //     '$id' => ID::custom('_fulltext_name'),
+            //     'type' => Database::INDEX_FULLTEXT,
+            //     'attributes' => ['name'],
+            //     'lengths' => [1024],
+            //     'orders' => [Database::ORDER_ASC],
+            // ],
+            // [
+            //     '$id' => ID::custom('_key_search'),
+            //     'type' => Database::INDEX_FULLTEXT,
+            //     'attributes' => ['search'],
+            //     'lengths' => [],
+            //     'orders' => [Database::ORDER_ASC],
+            // ],
+            // [
+            //     '$id' => ID::custom('_key_enabled'),
+            //     'type' => Database::INDEX_KEY,
+            //     'attributes' => ['enabled'],
+            //     'lengths' => [],
+            //     'orders' => [Database::ORDER_ASC],
+            // ],
+            // [
+            //     '$id' => ID::custom('_key_name'),
+            //     'type' => Database::INDEX_KEY,
+            //     'attributes' => ['name'],
+            //     'lengths' => [128],
+            //     'orders' => [Database::ORDER_ASC],
+            // ],
+            // [
+            //     '$id' => ID::custom('_key_fileSecurity'),
+            //     'type' => Database::INDEX_KEY,
+            //     'attributes' => ['fileSecurity'],
+            //     'lengths' => [],
+            //     'orders' => [Database::ORDER_ASC],
+            // ],
+            // [
+            //     '$id' => ID::custom('_key_maximumFileSize'),
+            //     'type' => Database::INDEX_KEY,
+            //     'attributes' => ['maximumFileSize'],
+            //     'lengths' => [],
+            //     'orders' => [Database::ORDER_ASC],
+            // ],
+            // [
+            //     '$id' => ID::custom('_key_encryption'),
+            //     'type' => Database::INDEX_KEY,
+            //     'attributes' => ['encryption'],
+            //     'lengths' => [],
+            //     'orders' => [Database::ORDER_ASC],
+            // ],
+            // [
+            //     '$id' => ID::custom('_key_antivirus'),
+            //     'type' => Database::INDEX_KEY,
+            //     'attributes' => ['antivirus'],
+            //     'lengths' => [],
+            //     'orders' => [Database::ORDER_ASC],
+            // ],
         ]
     ],
 
@@ -3379,62 +3390,62 @@ $collections = [
             ],
         ],
         'indexes' => [
-            [
-                '$id' => ID::custom('_key_search'),
-                'type' => Database::INDEX_FULLTEXT,
-                'attributes' => ['search'],
-                'lengths' => [2048],
-                'orders' => [Database::ORDER_ASC],
-            ],
-            [
-                '$id' => ID::custom('_key_bucket'),
-                'type' => Database::INDEX_KEY,
-                'attributes' => ['bucketId'],
-                'lengths' => [Database::LENGTH_KEY],
-                'orders' => [Database::ORDER_ASC],
-            ],
-            [
-                '$id' => ID::custom('_key_name'),
-                'type' => Database::INDEX_KEY,
-                'attributes' => ['name'],
-                'lengths' => [2048],
-                'orders' => [Database::ORDER_ASC],
-            ],
-            [
-                '$id' => ID::custom('_key_signature'),
-                'type' => Database::INDEX_KEY,
-                'attributes' => ['signature'],
-                'lengths' => [2048],
-                'orders' => [Database::ORDER_ASC],
-            ],
-            [
-                '$id' => ID::custom('_key_mimeType'),
-                'type' => Database::INDEX_KEY,
-                'attributes' => ['mimeType'],
-                'lengths' => [127],
-                'orders' => [Database::ORDER_ASC],
-            ],
-            [
-                '$id' => ID::custom('_key_sizeOriginal'),
-                'type' => Database::INDEX_KEY,
-                'attributes' => ['sizeOriginal'],
-                'lengths' => [],
-                'orders' => [Database::ORDER_ASC],
-            ],
-            [
-                '$id' => ID::custom('_key_chunksTotal'),
-                'type' => Database::INDEX_KEY,
-                'attributes' => ['chunksTotal'],
-                'lengths' => [],
-                'orders' => [Database::ORDER_ASC],
-            ],
-            [
-                '$id' => ID::custom('_key_chunksUploaded'),
-                'type' => Database::INDEX_KEY,
-                'attributes' => ['chunksUploaded'],
-                'lengths' => [],
-                'orders' => [Database::ORDER_ASC],
-            ],
+            // [
+            //     '$id' => ID::custom('_key_search'),
+            //     'type' => Database::INDEX_FULLTEXT,
+            //     'attributes' => ['search'],
+            //     'lengths' => [],
+            //     'orders' => [Database::ORDER_ASC],
+            // ],
+            // [
+            //     '$id' => ID::custom('_key_bucket'),
+            //     'type' => Database::INDEX_KEY,
+            //     'attributes' => ['bucketId'],
+            //     'lengths' => [Database::LENGTH_KEY],
+            //     'orders' => [Database::ORDER_ASC],
+            // ],
+            // [
+            //     '$id' => ID::custom('_key_name'),
+            //     'type' => Database::INDEX_KEY,
+            //     'attributes' => ['name'],
+            //     'lengths' => [],
+            //     'orders' => [Database::ORDER_ASC],
+            // ],
+            // [
+            //     '$id' => ID::custom('_key_signature'),
+            //     'type' => Database::INDEX_KEY,
+            //     'attributes' => ['signature'],
+            //     'lengths' => [],
+            //     'orders' => [Database::ORDER_ASC],
+            // ],
+            // [
+            //     '$id' => ID::custom('_key_mimeType'),
+            //     'type' => Database::INDEX_KEY,
+            //     'attributes' => ['mimeType'],
+            //     'lengths' => [127],
+            //     'orders' => [Database::ORDER_ASC],
+            // ],
+            // [
+            //     '$id' => ID::custom('_key_sizeOriginal'),
+            //     'type' => Database::INDEX_KEY,
+            //     'attributes' => ['sizeOriginal'],
+            //     'lengths' => [],
+            //     'orders' => [Database::ORDER_ASC],
+            // ],
+            // [
+            //     '$id' => ID::custom('_key_chunksTotal'),
+            //     'type' => Database::INDEX_KEY,
+            //     'attributes' => ['chunksTotal'],
+            //     'lengths' => [],
+            //     'orders' => [Database::ORDER_ASC],
+            // ],
+            // [
+            //     '$id' => ID::custom('_key_chunksUploaded'),
+            //     'type' => Database::INDEX_KEY,
+            //     'attributes' => ['chunksUploaded'],
+            //     'lengths' => [],
+            //     'orders' => [Database::ORDER_ASC],
+            // ],
         ]
     ],
 

app/config/variables.php

@@ -306,6 +306,24 @@ return [
                 'question' => '',
                 'filter' => 'password'
             ],
+            // [
+            //     'name' => '_APP_CONNECTIONS_DB_PROJECT',
+            //     'description' => 'A list of comma-separated key value pairs representing Project DBs where key is the database name and value is the DSN connection string.',
+            //     'introduction' => 'TBD',
+            //     'default' => 'db_fra1_01=mysql://user:password@mariadb:3306/appwrite',
+            //     'required' => true,
+            //     'question' => '',
+            //     'filter' => ''
+            // ],
+            // [
+            //     'name' => '_APP_CONNECTIONS_DB_CONSOLE',
+            //     'description' => 'A key value pair representing the Console DB where key is the database name and value is the DSN connection string.',
+            //     'introduction' => 'TBD',
+            //     'default' => 'db_fra1_01=mysql://user:password@mariadb:3306/appwrite',
+            //     'required' => true,
+            //     'question' => '',
+            //     'filter' => ''
+            // ]
         ],
     ],
     [

app/controllers/api/account.php

@@ -5,7 +5,6 @@ use Appwrite\Auth\Auth;
 use Appwrite\Auth\Validator\Password;
 use Appwrite\Auth\Validator\Phone;
 use Appwrite\Detector\Detector;
-use Appwrite\Event\Audit;
 use Appwrite\Event\Event;
 use Appwrite\Event\Mail;
 use Appwrite\Event\Phone as EventPhone;
@@ -39,7 +38,6 @@ use Utopia\Database\Validator\UID;
 use Utopia\Locale\Locale;
 use Utopia\Validator\ArrayList;
 use Utopia\Validator\Assoc;
-use Utopia\Validator\Range;
 use Utopia\Validator\Text;
 use Utopia\Validator\WhiteList;
 

app/controllers/api/databases.php

@@ -20,7 +20,7 @@ use Utopia\Database\Database;
 use Utopia\Database\Document;
 use Utopia\Database\DateTime;
 use Utopia\Database\Query;
-use Utopia\Database\Adapter\MariaDB;
+use Utopia\Database\Adapter\MySQL;
 use Utopia\Database\Validator\Authorization;
 use Utopia\Database\Validator\Key;
 use Utopia\Database\Validator\Permissions;

app/controllers/api/functions.php

@@ -25,7 +25,6 @@ use Appwrite\Task\Validator\Cron;
 use Appwrite\Utopia\Database\Validator\Queries\Deployments;
 use Appwrite\Utopia\Database\Validator\Queries\Executions;
 use Appwrite\Utopia\Database\Validator\Queries\Functions;
-use Appwrite\Utopia\Database\Validator\Queries\Variables;
 use Utopia\App;
 use Utopia\Database\Database;
 use Utopia\Database\Document;
@@ -33,7 +32,6 @@ use Utopia\Database\DateTime;
 use Utopia\Database\Query;
 use Utopia\Database\Validator\Authorization;
 use Utopia\Validator\ArrayList;
-use Utopia\Validator\Assoc;
 use Utopia\Validator\Text;
 use Utopia\Validator\Range;
 use Utopia\Validator\WhiteList;

app/controllers/api/health.php

@@ -5,7 +5,9 @@ use Appwrite\Event\Event;
 use Appwrite\Extend\Exception;
 use Appwrite\Utopia\Response;
 use Utopia\App;
+use Utopia\Config\Config;
 use Utopia\Database\Document;
+use Utopia\Pools\Group;
 use Utopia\Registry\Registry;
 use Utopia\Storage\Device;
 use Utopia\Storage\Device\Local;
@@ -26,6 +28,7 @@ App::get('/v1/health')
     ->action(function (Response $response) {
 
         $output = [
+            'name' => 'http',
             'status' => 'pass',
             'ping' => 0
         ];
@@ -42,7 +45,6 @@ App::get('/v1/health/version')
     ->label('sdk.response.model', Response::MODEL_HEALTH_VERSION)
     ->inject('response')
     ->action(function (Response $response) {
-
         $response->dynamic(new Document([ 'version' => APP_VERSION_STABLE ]), Response::MODEL_HEALTH_VERSION);
     });
 
@@ -58,30 +60,50 @@ App::get('/v1/health/db')
     ->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
     ->label('sdk.response.model', Response::MODEL_HEALTH_STATUS)
     ->inject('response')
-    ->inject('utopia')
-    ->action(function (Response $response, App $utopia) {
+    ->inject('pools')
+    ->action(function (Response $response, Group $pools) {
 
-        $checkStart = \microtime(true);
+        $output = [];
 
-        try {
-            $db = $utopia->getResource('db'); /* @var $db PDO */
-
-            // Run a small test to check the connection
-            $statement = $db->prepare("SELECT 1;");
-
-            $statement->closeCursor();
-
-            $statement->execute();
-        } catch (Exception $_e) {
-            throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Database is not available');
-        }
-
-        $output = [
-            'status' => 'pass',
-            'ping' => \round((\microtime(true) - $checkStart) / 1000)
+        $configs = [
+            'Console.DB' => Config::getParam('pools-console'),
+            'Projects.DB' => Config::getParam('pools-database'),
         ];
 
-        $response->dynamic(new Document($output), Response::MODEL_HEALTH_STATUS);
+        foreach ($configs as $key => $config) {
+            foreach ($config as $database) {
+                try {
+                    $adapter = $pools->get($database)->pop()->getResource();
+
+                    $checkStart = \microtime(true);
+
+                    if ($adapter->ping()) {
+                        $output[] = new Document([
+                            'name' => $key . " ($database)",
+                            'status' => 'pass',
+                            'ping' => \round((\microtime(true) - $checkStart) / 1000)
+                        ]);
+                    } else {
+                        $output[] = new Document([
+                            'name' => $key . " ($database)",
+                            'status' => 'fail',
+                            'ping' => \round((\microtime(true) - $checkStart) / 1000)
+                        ]);
+                    }
+                } catch (\Throwable $th) {
+                    $output[] = new Document([
+                        'name' => $key . " ($database)",
+                        'status' => 'fail',
+                        'ping' => \round((\microtime(true) - $checkStart) / 1000)
+                    ]);
+                }
+            }
+        }
+
+        $response->dynamic(new Document([
+            'statuses' => $output,
+            'total' => count($output),
+        ]), Response::MODEL_HEALTH_STATUS_LIST);
     });
 
 App::get('/v1/health/cache')
@@ -96,23 +118,163 @@ App::get('/v1/health/cache')
     ->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
     ->label('sdk.response.model', Response::MODEL_HEALTH_STATUS)
     ->inject('response')
-    ->inject('utopia')
-    ->action(function (Response $response, App $utopia) {
+    ->inject('pools')
+    ->action(function (Response $response, Group $pools) {
 
-        $checkStart = \microtime(true);
+        $output = [];
 
-        $redis = $utopia->getResource('cache');
-
-        if (!$redis->ping(true)) {
-            throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Cache is not available');
-        }
-
-        $output = [
-            'status' => 'pass',
-            'ping' => \round((\microtime(true) - $checkStart) / 1000)
+        $configs = [
+            'Cache' => Config::getParam('pools-cache'),
         ];
 
-        $response->dynamic(new Document($output), Response::MODEL_HEALTH_STATUS);
+        foreach ($configs as $key => $config) {
+            foreach ($config as $database) {
+                try {
+                    $adapter = $pools->get($database)->pop()->getResource();
+
+                    $checkStart = \microtime(true);
+
+                    if ($adapter->ping()) {
+                        $output[] = new Document([
+                            'name' => $key . " ($database)",
+                            'status' => 'pass',
+                            'ping' => \round((\microtime(true) - $checkStart) / 1000)
+                        ]);
+                    } else {
+                        $output[] = new Document([
+                            'name' => $key . " ($database)",
+                            'status' => 'fail',
+                            'ping' => \round((\microtime(true) - $checkStart) / 1000)
+                        ]);
+                    }
+                } catch (\Throwable $th) {
+                    $output[] = new Document([
+                        'name' => $key . " ($database)",
+                        'status' => 'fail',
+                        'ping' => \round((\microtime(true) - $checkStart) / 1000)
+                    ]);
+                }
+            }
+        }
+
+        $response->dynamic(new Document([
+            'statuses' => $output,
+            'total' => count($output),
+        ]), Response::MODEL_HEALTH_STATUS_LIST);
+    });
+
+App::get('/v1/health/queue')
+    ->desc('Get Queue')
+    ->groups(['api', 'health'])
+    ->label('scope', 'health.read')
+    ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
+    ->label('sdk.namespace', 'health')
+    ->label('sdk.method', 'getQueue')
+    ->label('sdk.description', '/docs/references/health/get-queue.md')
+    ->label('sdk.response.code', Response::STATUS_CODE_OK)
+    ->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
+    ->label('sdk.response.model', Response::MODEL_HEALTH_STATUS)
+    ->inject('response')
+    ->inject('pools')
+    ->action(function (Response $response, Group $pools) {
+
+        $output = [];
+
+        $configs = [
+            'Queue' => Config::getParam('pools-queue'),
+        ];
+
+        foreach ($configs as $key => $config) {
+            foreach ($config as $database) {
+                try {
+                    $adapter = $pools->get($database)->pop()->getResource();
+
+                    $checkStart = \microtime(true);
+
+                    if ($adapter->ping()) {
+                        $output[] = new Document([
+                            'name' => $key . " ($database)",
+                            'status' => 'pass',
+                            'ping' => \round((\microtime(true) - $checkStart) / 1000)
+                        ]);
+                    } else {
+                        $output[] = new Document([
+                            'name' => $key . " ($database)",
+                            'status' => 'fail',
+                            'ping' => \round((\microtime(true) - $checkStart) / 1000)
+                        ]);
+                    }
+                } catch (\Throwable $th) {
+                    $output[] = new Document([
+                        'name' => $key . " ($database)",
+                        'status' => 'fail',
+                        'ping' => \round((\microtime(true) - $checkStart) / 1000)
+                    ]);
+                }
+            }
+        }
+
+        $response->dynamic(new Document([
+            'statuses' => $output,
+            'total' => count($output),
+        ]), Response::MODEL_HEALTH_STATUS_LIST);
+    });
+
+App::get('/v1/health/pubsub')
+    ->desc('Get PubSub')
+    ->groups(['api', 'health'])
+    ->label('scope', 'health.read')
+    ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
+    ->label('sdk.namespace', 'health')
+    ->label('sdk.method', 'getPubSub')
+    ->label('sdk.description', '/docs/references/health/get-pubsub.md')
+    ->label('sdk.response.code', Response::STATUS_CODE_OK)
+    ->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
+    ->label('sdk.response.model', Response::MODEL_HEALTH_STATUS)
+    ->inject('response')
+    ->inject('pools')
+    ->action(function (Response $response, Group $pools) {
+
+        $output = [];
+
+        $configs = [
+            'PubSub' => Config::getParam('pools-pubsub'),
+        ];
+
+        foreach ($configs as $key => $config) {
+            foreach ($config as $database) {
+                try {
+                    $adapter = $pools->get($database)->pop()->getResource();
+
+                    $checkStart = \microtime(true);
+
+                    if ($adapter->ping()) {
+                        $output[] = new Document([
+                            'name' => $key . " ($database)",
+                            'status' => 'pass',
+                            'ping' => \round((\microtime(true) - $checkStart) / 1000)
+                        ]);
+                    } else {
+                        $output[] = new Document([
+                            'name' => $key . " ($database)",
+                            'status' => 'fail',
+                            'ping' => \round((\microtime(true) - $checkStart) / 1000)
+                        ]);
+                    }
+                } catch (\Throwable $th) {
+                    $output[] = new Document([
+                        'name' => $key . " ($database)",
+                        'status' => 'fail',
+                        'ping' => \round((\microtime(true) - $checkStart) / 1000)
+                    ]);
+                }
+            }
+        }
+
+        $response->dynamic(new Document([
+            'statuses' => $output,
+            'total' => count($output),
+        ]), Response::MODEL_HEALTH_STATUS_LIST);
     });
 
 App::get('/v1/health/time')

app/controllers/api/projects.php

@@ -29,6 +29,8 @@ use Utopia\Domains\Domain;
 use Utopia\Registry\Registry;
 use Appwrite\Extend\Exception;
 use Appwrite\Utopia\Database\Validator\Queries\Projects;
+use Utopia\Cache\Cache;
+use Utopia\Pools\Group;
 use Utopia\Validator\ArrayList;
 use Utopia\Validator\Boolean;
 use Utopia\Validator\Hostname;
@@ -69,8 +71,10 @@ App::post('/v1/projects')
     ->param('legalTaxId', '', new Text(256), 'Project legal Tax ID. Max length: 256 chars.', true)
     ->inject('response')
     ->inject('dbForConsole')
-    ->inject('dbForProject')
-    ->action(function (string $projectId, string $name, string $teamId, string $description, string $logo, string $url, string $legalName, string $legalCountry, string $legalState, string $legalCity, string $legalAddress, string $legalTaxId, Response $response, Database $dbForConsole, Database $dbForProject) {
+    ->inject('cache')
+    ->inject('pools')
+    ->action(function (string $projectId, string $name, string $teamId, string $description, string $logo, string $url, string $legalName, string $legalCountry, string $legalState, string $legalCity, string $legalAddress, string $legalTaxId, Response $response, Database $dbForConsole, Cache $cache, Group $pools) {
+
 
         $team = $dbForConsole->getDocument('teams', $teamId);
 
@@ -85,6 +89,8 @@ App::post('/v1/projects')
         }
 
         $projectId = ($projectId == 'unique()') ? ID::unique() : $projectId;
+        $databases = Config::getParam('pools-database', []);
+        $database = $databases[array_rand($databases)];
 
         if ($projectId === 'console') {
             throw new Exception(Exception::PROJECT_RESERVED_PROJECT, "'console' is a reserved project.");
@@ -120,12 +126,12 @@ App::post('/v1/projects')
             'domains' => null,
             'auths' => $auths,
             'search' => implode(' ', [$projectId, $name]),
+            'database' => $database,
         ]));
-        /** @var array $collections */
-        $collections = Config::getParam('collections', []);
 
+        $dbForProject = new Database($pools->get($database)->pop()->getResource(), $cache);
         $dbForProject->setNamespace("_{$project->getInternalId()}");
-        $dbForProject->create(App::getEnv('_APP_DB_SCHEMA', 'appwrite'));
+        $dbForProject->create();
 
         $audit = new Audit($dbForProject);
         $audit->setup();
@@ -133,6 +139,9 @@ App::post('/v1/projects')
         $adapter = new TimeLimit('', 0, 1, $dbForProject);
         $adapter->setup();
 
+        /** @var array $collections */
+        $collections = Config::getParam('collections', []);
+
         foreach ($collections as $key => $collection) {
             if (($collection['$collection'] ?? '') !== Database::METADATA) {
                 continue;

app/controllers/api/teams.php

@@ -20,7 +20,6 @@ use Appwrite\Utopia\Response;
 use MaxMind\Db\Reader;
 use Utopia\App;
 use Utopia\Audit\Audit;
-use Utopia\CLI\Console;
 use Utopia\Config\Config;
 use Utopia\Database\Database;
 use Utopia\Database\Document;
@@ -36,9 +35,7 @@ use Utopia\Database\Validator\Key;
 use Utopia\Database\Validator\UID;
 use Utopia\Locale\Locale;
 use Utopia\Validator\Text;
-use Utopia\Validator\Range;
 use Utopia\Validator\ArrayList;
-use Utopia\Validator\WhiteList;
 
 App::post('/v1/teams')
     ->desc('Create Team')

app/http.php

@@ -22,6 +22,7 @@ use Utopia\Swoole\Files;
 use Appwrite\Utopia\Request;
 use Utopia\Logger\Log;
 use Utopia\Logger\Log\User;
+use Utopia\Pools\Group;
 
 $http = new Server("0.0.0.0", App::getEnv('PORT', 80));
 
@@ -60,6 +61,10 @@ $http->on('start', function (Server $http) use ($payloadSize, $register) {
     $app = new App('UTC');
 
     go(function () use ($register, $app) {
+
+        $pools = $register->get('pools'); /** @var Group $pools */
+        App::setResource('pools', fn() => $pools);
+
         // wait for database to be ready
         $attempts = 0;
         $max = 10;
@@ -68,8 +73,7 @@ $http->on('start', function (Server $http) use ($payloadSize, $register) {
         do {
             try {
                 $attempts++;
-                $db = $register->get('dbPool')->get();
-                $redis = $register->get('redisPool')->get();
+                $dbForConsole = $app->getResource('dbForConsole'); /** @var Utopia\Database\Database $dbForConsole */
                 break; // leave the do-while if successful
             } catch (\Exception $e) {
                 Console::warning("Database not ready. Retrying connection ({$attempts})...");
@@ -80,21 +84,16 @@ $http->on('start', function (Server $http) use ($payloadSize, $register) {
             }
         } while ($attempts < $max);
 
-        App::setResource('db', fn () => $db);
-        App::setResource('cache', fn () => $redis);
-
-        /** @var Utopia\Database\Database $dbForConsole */
-        $dbForConsole = $app->getResource('dbForConsole');
-
         Console::success('[Setup] - Server database init started...');
 
         /** @var array $collections */
         $collections = Config::getParam('collections', []);
 
         try {
-            $redis->flushAll();
+            $cache = $app->getResource('cache'); /** @var Utopia\Cache\Cache $cache */
+            $cache->flush();
             Console::success('[Setup] - Creating database: appwrite...');
-            $dbForConsole->create(App::getEnv('_APP_DB_SCHEMA', 'appwrite'));
+            $dbForConsole->create();
         } catch (\Exception $e) {
             Console::success('[Setup] - Skip: metadata table already exists');
         }
@@ -116,6 +115,7 @@ $http->on('start', function (Server $http) use ($payloadSize, $register) {
             if (!$dbForConsole->getCollection($key)->isEmpty()) {
                 continue;
             }
+
             /**
              * Skip to prevent 0.16 migration issues.
              */
@@ -215,6 +215,8 @@ $http->on('start', function (Server $http) use ($payloadSize, $register) {
             $dbForConsole->createCollection('bucket_' . $bucket->getInternalId(), $attributes, $indexes);
         }
 
+        $pools->reclaim();
+
         Console::success('[Setup] - Server database init completed...');
     });
 
@@ -246,11 +248,8 @@ $http->on('request', function (SwooleRequest $swooleRequest, SwooleResponse $swo
 
     $app = new App('UTC');
 
-    $db = $register->get('dbPool')->get();
-    $redis = $register->get('redisPool')->get();
-
-    App::setResource('db', fn () => $db);
-    App::setResource('cache', fn () => $redis);
+    $pools = $register->get('pools');
+    App::setResource('pools', fn() => $pools);
 
     try {
         Authorization::cleanRoles();
@@ -317,13 +316,6 @@ $http->on('request', function (SwooleRequest $swooleRequest, SwooleResponse $swo
         Console::error('[Error] File: ' . $th->getFile());
         Console::error('[Error] Line: ' . $th->getLine());
 
-        /**
-         * Reset Database connection if PDOException was thrown.
-         */
-        if ($th instanceof PDOException) {
-            $db = null;
-        }
-
         $swooleResponse->setStatusCode(500);
 
         $output = ((App::isDevelopment())) ? [
@@ -341,13 +333,7 @@ $http->on('request', function (SwooleRequest $swooleRequest, SwooleResponse $swo
 
         $swooleResponse->end(\json_encode($output));
     } finally {
-        /** @var PDOPool $dbPool */
-        $dbPool = $register->get('dbPool');
-        $dbPool->put($db);
-
-        /** @var RedisPool $redisPool */
-        $redisPool = $register->get('redisPool');
-        $redisPool->put($redis);
+        $pools->reclaim();
     }
 });
 

app/init.php

@@ -18,9 +18,6 @@ ini_set('display_startup_errors', 1);
 ini_set('default_socket_timeout', -1);
 error_reporting(E_ALL);
 
-use Appwrite\Extend\PDO;
-use Ahc\Jwt\JWT;
-use Ahc\Jwt\JWTException;
 use Appwrite\Extend\Exception;
 use Appwrite\Auth\Auth;
 use Appwrite\SMS\Adapter\Mock;
@@ -32,39 +29,31 @@ use Appwrite\SMS\Adapter\Vonage;
 use Appwrite\DSN\DSN;
 use Appwrite\Event\Audit;
 use Appwrite\Event\Database as EventDatabase;
-use Appwrite\Event\Delete;
 use Appwrite\Event\Event;
 use Appwrite\Event\Mail;
 use Appwrite\Event\Phone;
+use Appwrite\Event\Delete;
 use Appwrite\Network\Validator\Email;
 use Appwrite\Network\Validator\IP;
 use Appwrite\Network\Validator\URL;
 use Appwrite\OpenSSL\OpenSSL;
+use Appwrite\URL\URL as AppwriteURL;
 use Appwrite\Usage\Stats;
 use Appwrite\Utopia\View;
 use Utopia\App;
+use Utopia\Validator\Range;
+use Utopia\Validator\WhiteList;
 use Utopia\Database\ID;
+use Utopia\Database\Document;
+use Utopia\Database\Database;
+use Utopia\Database\Query;
+use Utopia\Database\Validator\Authorization;
+use Utopia\Database\Validator\DatetimeValidator;
+use Utopia\Database\Validator\Structure;
 use Utopia\Logger\Logger;
 use Utopia\Config\Config;
 use Utopia\Locale\Locale;
 use Utopia\Registry\Registry;
-use MaxMind\Db\Reader;
-use PHPMailer\PHPMailer\PHPMailer;
-use Utopia\Cache\Adapter\Redis as RedisCache;
-use Utopia\Cache\Cache;
-use Utopia\Database\Adapter\MariaDB;
-use Utopia\Database\Document;
-use Utopia\Database\Database;
-use Utopia\Database\Validator\Structure;
-use Utopia\Database\Validator\Authorization;
-use Utopia\Validator\Range;
-use Utopia\Validator\WhiteList;
-use Swoole\Database\PDOConfig;
-use Swoole\Database\PDOPool;
-use Swoole\Database\RedisConfig;
-use Swoole\Database\RedisPool;
-use Utopia\Database\Query;
-use Utopia\Database\Validator\DatetimeValidator;
 use Utopia\Storage\Device;
 use Utopia\Storage\Storage;
 use Utopia\Storage\Device\Backblaze;
@@ -73,6 +62,19 @@ use Utopia\Storage\Device\Local;
 use Utopia\Storage\Device\S3;
 use Utopia\Storage\Device\Linode;
 use Utopia\Storage\Device\Wasabi;
+use Utopia\Cache\Adapter\Redis as RedisCache;
+use Utopia\Cache\Adapter\Sharding;
+use Utopia\Cache\Cache;
+use Utopia\CLI\Console;
+use Utopia\Database\Adapter\MariaDB;
+use Utopia\Database\Adapter\MySQL;
+use Utopia\Pools\Group;
+use Utopia\Pools\Pool;
+use Ahc\Jwt\JWT;
+use Ahc\Jwt\JWTException;
+use MaxMind\Db\Reader;
+use PHPMailer\PHPMailer\PHPMailer;
+use Swoole\Database\PDOProxy;
 
 const APP_NAME = 'Appwrite';
 const APP_DOMAIN = 'appwrite.io';
@@ -495,56 +497,181 @@ $register->set('logger', function () {
     $adapter = new $classname($providerConfig);
     return new Logger($adapter);
 });
-$register->set('dbPool', function () {
- // Register DB connection
-    $dbHost = App::getEnv('_APP_DB_HOST', '');
-    $dbPort = App::getEnv('_APP_DB_PORT', '');
-    $dbUser = App::getEnv('_APP_DB_USER', '');
-    $dbPass = App::getEnv('_APP_DB_PASS', '');
-    $dbScheme = App::getEnv('_APP_DB_SCHEMA', '');
+$register->set('pools', function () {
 
-    $pool = new PDOPool(
-        (new PDOConfig())
-        ->withHost($dbHost)
-        ->withPort($dbPort)
-        ->withDbName($dbScheme)
-        ->withCharset('utf8mb4')
-        ->withUsername($dbUser)
-        ->withPassword($dbPass)
-        ->withOptions([
-            PDO::ATTR_ERRMODE => App::isDevelopment() ? PDO::ERRMODE_WARNING : PDO::ERRMODE_SILENT, // If in production mode, warnings are not displayed
-            PDO::ATTR_TIMEOUT => 3, // Seconds
-            PDO::ATTR_PERSISTENT => true,
-            PDO::ATTR_DEFAULT_FETCH_MODE => PDO::FETCH_ASSOC,
-            PDO::ATTR_EMULATE_PREPARES => true,
-            PDO::ATTR_STRINGIFY_FETCHES => true,
-        ]),
-        64
-    );
+    $group = new Group();
 
-    return $pool;
-});
-$register->set('redisPool', function () {
-    $redisHost = App::getEnv('_APP_REDIS_HOST', '');
-    $redisPort = App::getEnv('_APP_REDIS_PORT', '');
-    $redisUser = App::getEnv('_APP_REDIS_USER', '');
-    $redisPass = App::getEnv('_APP_REDIS_PASS', '');
-    $redisAuth = '';
+    $fallbackForDB = AppwriteURL::unparse([
+        'scheme' => 'mariadb',
+        'host' => App::getEnv('_APP_DB_HOST', 'mariadb'),
+        'port' => App::getEnv('_APP_DB_PORT', '3306'),
+        'user' => App::getEnv('_APP_DB_USER', ''),
+        'pass' => App::getEnv('_APP_DB_PASS', ''),
+    ]);
+    $fallbackForRedis = AppwriteURL::unparse([
+        'scheme' => 'redis',
+        'host' => App::getEnv('_APP_REDIS_HOST', 'redis'),
+        'port' => App::getEnv('_APP_REDIS_PORT', '6379'),
+        'user' => App::getEnv('_APP_REDIS_USER', ''),
+        'pass' => App::getEnv('_APP_REDIS_PASS', ''),
+    ]);
 
-    if ($redisUser && $redisPass) {
-        $redisAuth = $redisUser . ':' . $redisPass;
+    $connections = [
+        'console' => [
+            'type' => 'database',
+            'dsns' => App::getEnv('_APP_CONNECTIONS_DB_CONSOLE', $fallbackForDB),
+            'multiple' => false,
+            'schemes' => ['mariadb', 'mysql'],
+        ],
+        'database' => [
+            'type' => 'database',
+            'dsns' => App::getEnv('_APP_CONNECTIONS_DB_PROJECT', $fallbackForDB),
+            'multiple' => true,
+            'schemes' => ['mariadb', 'mysql'],
+        ],
+        'queue' => [
+            'type' => 'queue',
+            'dsns' => App::getEnv('_APP_CONNECTIONS_QUEUE', $fallbackForRedis),
+            'multiple' => false,
+            'schemes' => ['redis'],
+        ],
+        'pubsub' => [
+            'type' => 'pubsub',
+            'dsns' => App::getEnv('_APP_CONNECTIONS_PUBSUB', $fallbackForRedis),
+            'multiple' => false,
+            'schemes' => ['redis'],
+        ],
+        'cache' => [
+            'type' => 'cache',
+            'dsns' => App::getEnv('_APP_CONNECTIONS_CACHE', $fallbackForRedis),
+            'multiple' => true,
+            'schemes' => ['redis'],
+        ],
+    ];
+
+    foreach ($connections as $key => $connection) {
+        $type = $connection['type'] ?? '';
+        $dsns = $connection['dsns'] ?? '';
+        $multipe = $connection['multiple'] ?? false;
+        $schemes = $connection['schemes'] ?? [];
+        $config = [];
+        $dsns = explode(',', $connection['dsns'] ?? '');
+
+        foreach ($dsns as &$dsn) {
+            $dsn = explode('=', $dsn);
+            $name = ($multipe) ? $key . '_' . $dsn[0] : $key;
+            $dsn = $dsn[1] ?? '';
+            $config[] = $name;
+
+            if (empty($dsn)) {
+                //throw new Exception(Exception::GENERAL_SERVER_ERROR, "Missing value for DSN connection in {$key}");
+                continue;
+            }
+
+            $dsn = new DSN($dsn);
+            $dsnHost = $dsn->getHost();
+            $dsnPort = $dsn->getPort();
+            $dsnUser = $dsn->getUser();
+            $dsnPass = $dsn->getPassword();
+            $dsnScheme = $dsn->getScheme();
+            $dsnDatabase = $dsn->getDatabase();
+
+            if (!in_array($dsnScheme, $schemes)) {
+                throw new Exception(Exception::GENERAL_SERVER_ERROR, "Invalid console database scheme");
+            }
+
+            /**
+             * Get Resource
+             *
+             * Creation could be reused accross connection types like database, cache, queue, etc.
+             *
+             * Resource assignment to an adapter will happen below.
+             */
+
+            switch ($dsnScheme) {
+                case 'mysql':
+                case 'mariadb':
+                    $resource = function () use ($dsnHost, $dsnPort, $dsnUser, $dsnPass, $dsnDatabase) {
+                        return new PDOProxy(function () use ($dsnHost, $dsnPort, $dsnUser, $dsnPass, $dsnDatabase) {
+                            return new PDO("mysql:host={$dsnHost};port={$dsnPort};dbname={$dsnDatabase};charset=utf8mb4", $dsnUser, $dsnPass, array(
+                                PDO::ATTR_TIMEOUT => 3, // Seconds
+                                PDO::ATTR_PERSISTENT => true,
+                                PDO::ATTR_DEFAULT_FETCH_MODE => PDO::FETCH_ASSOC,
+                                PDO::ATTR_ERRMODE => App::isDevelopment() ? PDO::ERRMODE_WARNING : PDO::ERRMODE_SILENT, // If in production mode, warnings are not displayed
+                                PDO::ATTR_EMULATE_PREPARES => true,
+                                PDO::ATTR_STRINGIFY_FETCHES => true
+                            ));
+                        });
+                    };
+                    break;
+                case 'redis':
+                    $resource = function () use ($dsnHost, $dsnPort, $dsnPass) {
+                        $redis = new Redis();
+                        @$redis->pconnect($dsnHost, (int)$dsnPort);
+                        if ($dsnPass) {
+                            $redis->auth($dsnPass);
+                        }
+                        $redis->setOption(Redis::OPT_READ_TIMEOUT, -1);
+
+                        return $redis;
+                    };
+                    break;
+
+                default:
+                    throw new Exception(Exception::GENERAL_SERVER_ERROR, "Invalid scheme");
+                    break;
+            }
+
+            $pool = new Pool($name, 64, function () use ($type, $resource, $dsn) {
+                // Get Adapter
+                $adapter = null;
+
+                switch ($type) {
+                    case 'database':
+                        $adapter = match ($dsn->getScheme()) {
+                            'mariadb' => new MariaDB($resource()),
+                            'mysql' => new MySQL($resource()),
+                            default => null
+                        };
+
+                        var_dump($dsn->getDatabase());
+                        $adapter->setDefaultDatabase($dsn->getDatabase());
+
+                        break;
+                    case 'queue':
+                        $adapter = $resource();
+                        break;
+                    case 'pubsub':
+                        $adapter = $resource();
+                        break;
+                    case 'cache':
+                        $adapter = match ($dsn->getScheme()) {
+                            'redis' => new RedisCache($resource()),
+                            default => null
+                        };
+                        break;
+
+                    default:
+                        throw new Exception(Exception::GENERAL_SERVER_ERROR, "Server error: Missing adapter implementation.");
+                        break;
+                }
+
+                return $adapter;
+            });
+
+            $group->add($pool);
+        }
+
+        Config::setParam('pools-' . $key, $config);
     }
 
-    $pool = new RedisPool(
-        (new RedisConfig())
-        ->withHost($redisHost)
-        ->withPort($redisPort)
-        ->withAuth($redisAuth)
-        ->withDbIndex(0),
-        64
-    );
+    try {
+        $group->fill();
+    } catch (\Throwable $th) {
+        Console::error('Connection failure: ' . $th->getMessage());
+    }
 
-    return $pool;
+    return $group;
 });
 $register->set('influxdb', function () {
  // Register DB connection
@@ -561,7 +688,7 @@ $register->set('influxdb', function () {
     return $client;
 });
 $register->set('statsd', function () {
- // Register DB connection
+    // Register DB connection
     $host = App::getEnv('_APP_STATSD_HOST', 'telegraf');
     $port = App::getEnv('_APP_STATSD_PORT', 8125);
 
@@ -601,33 +728,6 @@ $register->set('smtp', function () {
 $register->set('geodb', function () {
     return new Reader(__DIR__ . '/db/DBIP/dbip-country-lite-2022-06.mmdb');
 });
-$register->set('db', function () {
- // This is usually for our workers or CLI commands scope
-    $dbHost = App::getEnv('_APP_DB_HOST', '');
-    $dbPort = App::getEnv('_APP_DB_PORT', '');
-    $dbUser = App::getEnv('_APP_DB_USER', '');
-    $dbPass = App::getEnv('_APP_DB_PASS', '');
-    $dbScheme = App::getEnv('_APP_DB_SCHEMA', '');
-
-    $pdo = new PDO("mysql:host={$dbHost};port={$dbPort};dbname={$dbScheme};charset=utf8mb4", $dbUser, $dbPass, array(
-        PDO::ATTR_TIMEOUT => 3, // Seconds
-        PDO::ATTR_PERSISTENT => true,
-        PDO::ATTR_DEFAULT_FETCH_MODE => PDO::FETCH_ASSOC,
-        PDO::ATTR_ERRMODE => PDO::ERRMODE_EXCEPTION,
-        PDO::ATTR_EMULATE_PREPARES => true,
-        PDO::ATTR_STRINGIFY_FETCHES => true,
-    ));
-
-    return $pdo;
-});
-$register->set('cache', function () {
- // This is usually for our workers or CLI commands scope
-    $redis = new Redis();
-    $redis->pconnect(App::getEnv('_APP_REDIS_HOST', ''), App::getEnv('_APP_REDIS_PORT', ''));
-    $redis->setOption(Redis::OPT_READ_TIMEOUT, -1);
-
-    return $redis;
-});
 
 /*
  * Localization
@@ -925,26 +1025,51 @@ App::setResource('console', function () {
     ]);
 }, []);
 
-App::setResource('dbForProject', function ($db, $cache, Document $project) {
-    $cache = new Cache(new RedisCache($cache));
+App::setResource('dbForProject', function (Group $pools, Database $dbForConsole, Cache $cache, Document $project) {
+    if ($project->isEmpty() || $project->getId() === 'console') {
+        return $dbForConsole;
+    }
 
-    $database = new Database(new MariaDB($db), $cache);
-    $database->setDefaultDatabase(App::getEnv('_APP_DB_SCHEMA', 'appwrite'));
-    $database->setNamespace("_{$project->getInternalId()}");
+    $dbAdapter = $pools
+        ->get($project->getAttribute('database'))
+        ->pop()
+        ->getResource()
+    ;
+
+    $database = new Database($dbAdapter, $cache);
+    $database->setNamespace('_' . $project->getInternalId());
 
     return $database;
-}, ['db', 'cache', 'project']);
+}, ['pools', 'dbForConsole', 'cache', 'project']);
 
-App::setResource('dbForConsole', function ($db, $cache) {
-    $cache = new Cache(new RedisCache($cache));
+App::setResource('dbForConsole', function (Group $pools, Cache $cache) {
+    $dbAdapter = $pools
+        ->get('console')
+        ->pop()
+        ->getResource()
+    ;
 
-    $database = new Database(new MariaDB($db), $cache);
-    $database->setDefaultDatabase(App::getEnv('_APP_DB_SCHEMA', 'appwrite'));
-    $database->setNamespace('_console');
+    $database = new Database($dbAdapter, $cache);
+
+    $database->setNamespace('console');
 
     return $database;
-}, ['db', 'cache']);
+}, ['pools', 'cache']);
 
+App::setResource('cache', function (Group $pools) {
+    $list = Config::getParam('pools-cache', []);
+    $adapters = [];
+
+    foreach ($list as $value) {
+        $adapters[] = $pools
+            ->get($value)
+            ->pop()
+            ->getResource()
+        ;
+    }
+
+    return new Cache(new Sharding($adapters));
+}, ['pools']);
 
 App::setResource('deviceLocal', function () {
     return new Local();

app/preload.php

@@ -35,6 +35,7 @@ foreach (
     realpath(__DIR__ . '/../vendor/symfony'),
     realpath(__DIR__ . '/../vendor/mongodb'),
     realpath(__DIR__ . '/../vendor/utopia-php/websocket'), // TODO: remove workerman autoload
+    realpath(__DIR__ . '/../vendor/utopia-php/cache'), // TODO: remove memcached autoload
     ] as $key => $value
 ) {
     if ($value !== false) {

app/realtime.php

@@ -16,16 +16,15 @@ use Utopia\CLI\Console;
 use Utopia\Database\ID;
 use Utopia\Database\Role;
 use Utopia\Logger\Log;
-use Utopia\Database\Database;
 use Utopia\Database\DateTime;
-use Utopia\Cache\Adapter\Redis as RedisCache;
-use Utopia\Cache\Cache;
-use Utopia\Database\Adapter\MariaDB;
 use Utopia\Database\Document;
 use Utopia\Database\Query;
 use Utopia\Database\Validator\Authorization;
-use Utopia\Registry\Registry;
 use Appwrite\Utopia\Request;
+use Utopia\Cache\Adapter\Sharding;
+use Utopia\Cache\Cache;
+use Utopia\Config\Config;
+use Utopia\Database\Database;
 use Utopia\WebSocket\Server;
 use Utopia\WebSocket\Adapter;
 
@@ -33,6 +32,68 @@ require_once __DIR__ . '/init.php';
 
 Runtime::enableCoroutine(SWOOLE_HOOK_ALL);
 
+
+function getConsoleDB(): Database
+{
+    global $register;
+
+    $pools = $register->get('pools'); /** @var \Utopia\Pools\Group $pools */
+
+    $dbAdapter = $pools
+        ->get('console')
+        ->pop()
+        ->getResource()
+    ;
+
+    $database = new Database($dbAdapter, getCache());
+
+    $database->setNamespace('console');
+
+    return $database;
+}
+
+function getProjectDB(Document $project): Database
+{
+    global $register;
+
+    $pools = $register->get('pools'); /** @var \Utopia\Pools\Group $pools */
+
+    if ($project->isEmpty() || $project->getId() === 'console') {
+        return getConsoleDB();
+    }
+
+    $dbAdapter = $pools
+        ->get($project->getAttribute('database'))
+        ->pop()
+        ->getResource()
+    ;
+
+    $database = new Database($dbAdapter, getCache());
+    $database->setNamespace('_' . $project->getInternalId());
+
+    return $database;
+}
+
+function getCache(): Cache
+{
+    global $register;
+
+    $pools = $register->get('pools'); /** @var \Utopia\Pools\Group $pools */
+
+    $list = Config::getParam('pools-cache', []);
+    $adapters = [];
+
+    foreach ($list as $value) {
+        $adapters[] = $pools
+            ->get($value)
+            ->pop()
+            ->getResource()
+        ;
+    }
+
+    return new Cache(new Sharding($adapters));
+}
+
 $realtime = new Realtime();
 
 /**
@@ -95,45 +156,6 @@ $logError = function (Throwable $error, string $action) use ($register) {
 
 $server->error($logError);
 
-function getDatabase(Registry &$register, string $namespace)
-{
-    $attempts = 0;
-
-    do {
-        try {
-            $attempts++;
-
-            $db = $register->get('dbPool')->get();
-            $redis = $register->get('redisPool')->get();
-
-            $cache = new Cache(new RedisCache($redis));
-            $database = new Database(new MariaDB($db), $cache);
-            $database->setDefaultDatabase(App::getEnv('_APP_DB_SCHEMA', 'appwrite'));
-            $database->setNamespace($namespace);
-
-            if (!$database->exists($database->getDefaultDatabase(), 'realtime')) {
-                throw new Exception('Collection not ready');
-            }
-
-            break; // leave loop if successful
-        } catch (\Throwable $e) {
-            Console::warning("Database not ready. Retrying connection ({$attempts})...");
-            if ($attempts >= DATABASE_RECONNECT_MAX_ATTEMPTS) {
-                throw new \Exception('Failed to connect to database: ' . $e->getMessage());
-            }
-            sleep(DATABASE_RECONNECT_SLEEP);
-        }
-    } while ($attempts < DATABASE_RECONNECT_MAX_ATTEMPTS);
-
-    return [
-        $database,
-        function () use ($register, $db, $redis) {
-            $register->get('dbPool')->put($db);
-            $register->get('redisPool')->put($redis);
-        }
-    ];
-}
-
 $server->onStart(function () use ($stats, $register, $containerId, &$statsDocument, $logError) {
     sleep(5); // wait for the initial database schema to be ready
     Console::success('Server started successfully');
@@ -143,7 +165,8 @@ $server->onStart(function () use ($stats, $register, $containerId, &$statsDocume
      */
     go(function () use ($register, $containerId, &$statsDocument) {
         $attempts = 0;
-        [$database, $returnDatabase] = getDatabase($register, '_console');
+        $database = getConsoleDB();
+
         do {
             try {
                 $attempts++;
@@ -163,7 +186,7 @@ $server->onStart(function () use ($stats, $register, $containerId, &$statsDocume
                 sleep(DATABASE_RECONNECT_SLEEP);
             }
         } while (true);
-        call_user_func($returnDatabase);
+        $register->get('pools')->reclaim();
     });
 
     /**
@@ -179,7 +202,7 @@ $server->onStart(function () use ($stats, $register, $containerId, &$statsDocume
         }
 
         try {
-            [$database, $returnDatabase] = getDatabase($register, '_console');
+            $database = getConsoleDB();
 
             $statsDocument
                 ->setAttribute('timestamp', DateTime::now())
@@ -189,7 +212,7 @@ $server->onStart(function () use ($stats, $register, $containerId, &$statsDocume
         } catch (\Throwable $th) {
             call_user_func($logError, $th, "updateWorkerDocument");
         } finally {
-            call_user_func($returnDatabase);
+            $register->get('pools')->reclaim();
         }
     });
 });
@@ -205,7 +228,7 @@ $server->onWorkerStart(function (int $workerId) use ($server, $register, $stats,
          * Sending current connections to project channels on the console project every 5 seconds.
          */
         if ($realtime->hasSubscriber('console', Role::users()->toString(), 'project')) {
-            [$database, $returnDatabase] = getDatabase($register, '_console');
+            $database = getConsoleDB();
 
             $payload = [];
 
@@ -250,7 +273,7 @@ $server->onWorkerStart(function (int $workerId) use ($server, $register, $stats,
                 ]));
             }
 
-            call_user_func($returnDatabase);
+            $register->get('pools')->reclaim();
         }
         /**
          * Sending test message for SDK E2E tests every 5 seconds.
@@ -285,8 +308,7 @@ $server->onWorkerStart(function (int $workerId) use ($server, $register, $stats,
             }
             $start = time();
 
-            /** @var Redis $redis */
-            $redis = $register->get('redisPool')->get();
+            $redis = $register->get('pools')->get('pubsub')->pop()->getResource(); /** @var Redis $redis */
             $redis->setOption(Redis::OPT_READ_TIMEOUT, -1);
 
             if ($redis->ping(true)) {
@@ -305,9 +327,9 @@ $server->onWorkerStart(function (int $workerId) use ($server, $register, $stats,
 
                     if ($realtime->hasSubscriber($projectId, 'user:' . $userId)) {
                         $connection = array_key_first(reset($realtime->subscriptions[$projectId]['user:' . $userId]));
-                        [$consoleDatabase, $returnConsoleDatabase] = getDatabase($register, '_console');
+                        $consoleDatabase = getConsoleDB();
                         $project = Authorization::skip(fn() => $consoleDatabase->getDocument('projects', $projectId));
-                        [$database, $returnDatabase] = getDatabase($register, "_{$project->getInternalId()}");
+                        $database = getProjectDB($project);
 
                         $user = $database->getDocument('users', $userId);
 
@@ -315,8 +337,7 @@ $server->onWorkerStart(function (int $workerId) use ($server, $register, $stats,
 
                         $realtime->subscribe($projectId, $connection, $roles, $realtime->connections[$connection]['channels']);
 
-                        call_user_func($returnDatabase);
-                        call_user_func($returnConsoleDatabase);
+                        $register->get('pools')->reclaim();
                     }
                 }
 
@@ -344,7 +365,7 @@ $server->onWorkerStart(function (int $workerId) use ($server, $register, $stats,
             call_user_func($logError, $th, "pubSubConnection");
 
             Console::error('Pub/sub error: ' . $th->getMessage());
-            $register->get('redisPool')->put($redis);
+            $register->get('pools')->reclaim();
             $attempts++;
             sleep(DATABASE_RECONNECT_SLEEP);
             continue;
@@ -359,33 +380,16 @@ $server->onOpen(function (int $connection, SwooleRequest $request) use ($server,
     $request = new Request($request);
     $response = new Response(new SwooleResponse());
 
-    /** @var PDO $db */
-    $db = $register->get('dbPool')->get();
-    /** @var Redis $redis */
-    $redis = $register->get('redisPool')->get();
-
     Console::info("Connection open (user: {$connection})");
 
-    App::setResource('db', fn () => $db);
-    App::setResource('cache', fn () => $redis);
-    App::setResource('request', fn () => $request);
-    App::setResource('response', fn () => $response);
+    App::setResource('pools', fn() => $register->get('pools'));
+    App::setResource('request', fn() => $request);
+    App::setResource('response', fn() => $response);
 
     try {
-        /** @var \Utopia\Database\Document $user */
-        $user = $app->getResource('user');
-
         /** @var \Utopia\Database\Document $project */
         $project = $app->getResource('project');
 
-        /** @var \Utopia\Database\Document $console */
-        $console = $app->getResource('console');
-
-        $cache = new Cache(new RedisCache($redis));
-        $database = new Database(new MariaDB($db), $cache);
-        $database->setDefaultDatabase(App::getEnv('_APP_DB_SCHEMA', 'appwrite'));
-        $database->setNamespace("_{$project->getInternalId()}");
-
         /*
          *  Project Check
          */
@@ -393,12 +397,16 @@ $server->onOpen(function (int $connection, SwooleRequest $request) use ($server,
             throw new Exception('Missing or unknown project ID', 1008);
         }
 
+        $dbForProject = getProjectDB($project);
+        $console = $app->getResource('console'); /** @var \Utopia\Database\Document $console */
+        $user = $app->getResource('user'); /** @var \Utopia\Database\Document $user */
+
         /*
          * Abuse Check
          *
          * Abuse limits are connecting 128 times per minute and ip address.
          */
-        $timeLimit = new TimeLimit('url:{url},ip:{ip}', 128, 60, $database);
+        $timeLimit = new TimeLimit('url:{url},ip:{ip}', 128, 60, $dbForProject);
         $timeLimit
             ->setParam('{ip}', $request->getIP())
             ->setParam('{url}', $request->getURI());
@@ -469,34 +477,20 @@ $server->onOpen(function (int $connection, SwooleRequest $request) use ($server,
             Console::error('[Error] Code: ' . $response['data']['code']);
             Console::error('[Error] Message: ' . $response['data']['message']);
         }
-
-        if ($th instanceof PDOException) {
-            $db = null;
-        }
     } finally {
-        /**
-         * Put used PDO and Redis Connections back into their pools.
-         */
-        $register->get('dbPool')->put($db);
-        $register->get('redisPool')->put($redis);
+        $register->get('pools')->reclaim();
     }
 });
 
 $server->onMessage(function (int $connection, string $message) use ($server, $register, $realtime, $containerId) {
     try {
         $response = new Response(new SwooleResponse());
-        $db = $register->get('dbPool')->get();
-        $redis = $register->get('redisPool')->get();
-
-        $cache = new Cache(new RedisCache($redis));
-        $database = new Database(new MariaDB($db), $cache);
-        $database->setDefaultDatabase(App::getEnv('_APP_DB_SCHEMA', 'appwrite'));
-        $database->setNamespace("_console");
         $projectId = $realtime->connections[$connection]['projectId'];
+        $database = getConsoleDB();
 
         if ($projectId !== 'console') {
             $project = Authorization::skip(fn() => $database->getDocument('projects', $projectId));
-            $database->setNamespace("_{$project->getInternalId()}");
+            $database = getProjectDB($project);
         }
 
         /*
@@ -580,8 +574,7 @@ $server->onMessage(function (int $connection, string $message) use ($server, $re
             $server->close($connection, $th->getCode());
         }
     } finally {
-        $register->get('dbPool')->put($db);
-        $register->get('redisPool')->put($redis);
+        $register->get('pools')->reclaim();
     }
 });
 

app/tasks/doctor.php

@@ -8,6 +8,7 @@ use Utopia\Storage\Device\Local;
 use Utopia\Storage\Storage;
 use Utopia\App;
 use Utopia\CLI\Console;
+use Utopia\Config\Config;
 use Utopia\Domains\Domain;
 
 $cli
@@ -21,7 +22,7 @@ $cli
 
         Console::log("\n" . '👩‍⚕️ Running ' . APP_NAME . ' Doctor for version ' . App::getEnv('_APP_VERSION', 'UNKNOWN') . ' ...' . "\n");
 
-        Console::log('Checking for production best practices...');
+        Console::log('[Settings]');
 
         $domain = new Domain(App::getEnv('_APP_DOMAIN'));
 
@@ -77,7 +78,6 @@ $cli
             Console::log('🟢 HTTPS force option is enabled');
         }
 
-
         $providerName = App::getEnv('_APP_LOGGING_PROVIDER', '');
         $providerConfig = App::getEnv('_APP_LOGGING_CONFIG', '');
 
@@ -90,30 +90,55 @@ $cli
         \sleep(0.2);
 
         try {
-            Console::log("\n" . 'Checking connectivity...');
+            Console::log("\n" . '[Connectivity]');
         } catch (\Throwable $th) {
             //throw $th;
         }
 
-        try {
-            $register->get('db'); /* @var $db PDO */
-            Console::success('Database............connected 👍');
-        } catch (\Throwable $th) {
-            Console::error('Database.........disconnected 👎');
+        $pools = $register->get('pools'); /** @var \Utopia\Pools\Group $pools */
+
+        $configs = [
+            'Console.DB' => Config::getParam('pools-console'),
+            'Projects.DB' => Config::getParam('pools-database'),
+        ];
+
+        foreach ($configs as $key => $config) {
+            foreach ($config as $database) {
+                try {
+                    $adapter = $pools->get($database)->pop()->getResource();
+
+                    if ($adapter->ping()) {
+                        Console::success('🟢 ' . str_pad("{$key}({$database})", 50, '.') . 'connected');
+                    } else {
+                        Console::error('🔴 ' . str_pad("{$key}({$database})", 47, '.') . 'disconnected');
+                    }
+                } catch (\Throwable $th) {
+                    Console::error('🔴 ' . str_pad("{$key}.({$database})", 47, '.') . 'disconnected');
+                }
+            }
         }
 
-        try {
-            $register->get('cache');
-            Console::success('Queue...............connected 👍');
-        } catch (\Throwable $th) {
-            Console::error('Queue............disconnected 👎');
-        }
+        $pools = $register->get('pools'); /** @var \Utopia\Pools\Group $pools */
+        $configs = [
+            'Cache' => Config::getParam('pools-cache'),
+            'Queue' => Config::getParam('pools-queue'),
+            'PubSub' => Config::getParam('pools-pubsub'),
+        ];
 
-        try {
-            $register->get('cache');
-            Console::success('Cache...............connected 👍');
-        } catch (\Throwable $th) {
-            Console::error('Cache............disconnected 👎');
+        foreach ($configs as $key => $config) {
+            foreach ($config as $pool) {
+                try {
+                    $adapter = $pools->get($pool)->pop()->getResource();
+
+                    if ($adapter->ping()) {
+                        Console::success('🟢 ' . str_pad("{$key}({$pool})", 50, '.') . 'connected');
+                    } else {
+                        Console::error('🔴 ' . str_pad("{$key}({$pool})", 47, '.') . 'disconnected');
+                    }
+                } catch (\Throwable $th) {
+                    Console::error('🔴 ' . str_pad("{$key}({$pool})", 47, '.') . 'disconnected');
+                }
+            }
         }
 
         if (App::getEnv('_APP_STORAGE_ANTIVIRUS') === 'enabled') { // Check if scans are enabled
@@ -124,12 +149,12 @@ $cli
                 );
 
                 if ((@$antivirus->ping())) {
-                    Console::success('Antivirus...........connected 👍');
+                    Console::success('🟢 ' . str_pad("Antivirus", 50, '.') . 'connected');
                 } else {
-                    Console::error('Antivirus........disconnected 👎');
+                    Console::error('🔴 ' . str_pad("Antivirus", 47, '.') . 'disconnected');
                 }
             } catch (\Throwable $th) {
-                Console::error('Antivirus........disconnected 👎');
+                Console::error('🔴 ' . str_pad("Antivirus", 47, '.') . 'disconnected');
             }
         }
 
@@ -142,35 +167,35 @@ $cli
             $mail->AltBody = 'Hello World';
 
             $mail->send();
-            Console::success('SMTP................connected 👍');
+            Console::success('🟢 ' . str_pad("SMTP", 50, '.') . 'connected');
         } catch (\Throwable $th) {
-            Console::error('SMTP.............disconnected 👎');
+            Console::error('🔴 ' . str_pad("SMTP", 47, '.') . 'disconnected');
         }
 
         $host = App::getEnv('_APP_STATSD_HOST', 'telegraf');
         $port = App::getEnv('_APP_STATSD_PORT', 8125);
 
         if ($fp = @\fsockopen('udp://' . $host, $port, $errCode, $errStr, 2)) {
-            Console::success('StatsD..............connected 👍');
+            Console::success('🟢 ' . str_pad("StatsD", 50, '.') . 'connected');
             \fclose($fp);
         } else {
-            Console::error('StatsD...........disconnected 👎');
+            Console::error('🔴 ' . str_pad("StatsD", 47, '.') . 'disconnected');
         }
 
         $host = App::getEnv('_APP_INFLUXDB_HOST', '');
         $port = App::getEnv('_APP_INFLUXDB_PORT', '');
 
         if ($fp = @\fsockopen($host, $port, $errCode, $errStr, 2)) {
-            Console::success('InfluxDB............connected 👍');
+            Console::success('🟢 ' . str_pad("InfluxDB", 50, '.') . 'connected');
             \fclose($fp);
         } else {
-            Console::error('InfluxDB.........disconnected 👎');
+            Console::error('🔴 ' . str_pad("InfluxDB", 47, '.') . 'disconnected');
         }
 
         \sleep(0.2);
 
         Console::log('');
-        Console::log('Checking volumes...');
+        Console::log('[Volumes]');
 
         foreach (
             [
@@ -198,7 +223,7 @@ $cli
         \sleep(0.2);
 
         Console::log('');
-        Console::log('Checking disk space usage...');
+        Console::log('[Disk Space]');
 
         foreach (
             [

app/tasks/maintenance.php

@@ -1,52 +1,16 @@
 <?php
 
 global $cli;
-global $register;
 
 use Appwrite\Auth\Auth;
 use Appwrite\Event\Certificate;
 use Appwrite\Event\Delete;
 use Utopia\App;
-use Utopia\Cache\Cache;
 use Utopia\CLI\Console;
-use Utopia\Database\Adapter\MariaDB;
-use Utopia\Database\Database;
-use Utopia\Database\DateTime;
-use Utopia\Cache\Adapter\Redis as RedisCache;
 use Utopia\Database\Document;
+use Utopia\Database\DateTime;
 use Utopia\Database\Query;
 
-function getConsoleDB(): Database
-{
-    global $register;
-
-    $attempts = 0;
-
-    do {
-        try {
-            $attempts++;
-            $cache = new Cache(new RedisCache($register->get('cache')));
-            $database = new Database(new MariaDB($register->get('db')), $cache);
-            $database->setDefaultDatabase(App::getEnv('_APP_DB_SCHEMA', 'appwrite'));
-            $database->setNamespace('_console'); // Main DB
-
-            if (!$database->exists($database->getDefaultDatabase(), 'certificates')) {
-                throw new \Exception('Console project not ready');
-            }
-
-            break; // leave loop if successful
-        } catch (\Exception $e) {
-            Console::warning("Database not ready. Retrying connection ({$attempts})...");
-            if ($attempts >= DATABASE_RECONNECT_MAX_ATTEMPTS) {
-                throw new \Exception('Failed to connect to database: ' . $e->getMessage());
-            }
-            sleep(DATABASE_RECONNECT_SLEEP);
-        }
-    } while ($attempts < DATABASE_RECONNECT_MAX_ATTEMPTS);
-
-    return $database;
-}
-
 $cli
     ->task('maintenance')
     ->desc('Schedules maintenance tasks and publishes them to resque')

app/tasks/migrate.php

@@ -7,9 +7,6 @@ use Appwrite\Migration\Migration;
 use Utopia\App;
 use Utopia\Cache\Cache;
 use Utopia\Cache\Adapter\Redis as RedisCache;
-use Utopia\Database\Adapter\MariaDB;
-use Utopia\Database\Database;
-use Utopia\Database\Query;
 use Utopia\Database\Validator\Authorization;
 use Utopia\Validator\Text;
 
@@ -28,17 +25,13 @@ $cli
 
         Console::success('Starting Data Migration to version ' . $version);
 
-        $db = $register->get('db', true);
+        $dbPool = $register->get('dbPool', true);
         $redis = $register->get('cache', true);
         $redis->flushAll();
         $cache = new Cache(new RedisCache($redis));
 
-        $projectDB = new Database(new MariaDB($db), $cache);
-        $projectDB->setDefaultDatabase(App::getEnv('_APP_DB_SCHEMA', 'appwrite'));
-
-        $consoleDB = new Database(new MariaDB($db), $cache);
-        $consoleDB->setDefaultDatabase(App::getEnv('_APP_DB_SCHEMA', 'appwrite'));
-        $consoleDB->setNamespace('_project_console');
+        $dbForConsole = $dbPool->getDB('console', $cache);
+        $dbForConsole->setNamespace('_project_console');
 
         $console = $app->getResource('console');
 
@@ -52,10 +45,10 @@ $cli
         $count = 0;
 
         try {
-            $totalProjects = $consoleDB->count('projects') + 1;
+            $totalProjects = $dbForConsole->count('projects') + 1;
         } catch (\Throwable $th) {
-            $consoleDB->setNamespace('_console');
-            $totalProjects = $consoleDB->count('projects') + 1;
+            $dbForConsole->setNamespace('_console');
+            $totalProjects = $dbForConsole->count('projects') + 1;
         }
 
         $class = 'Appwrite\\Migration\\Version\\' . Migration::$versions[$version];
@@ -71,8 +64,10 @@ $cli
                 }
 
                 try {
+                    // TODO: Iterate through all project DBs
+                    $projectDB = $dbPool->getDB($project->getId(), $cache);
                     $migration
-                        ->setProject($project, $projectDB, $consoleDB)
+                        ->setProject($project, $projectDB, $dbForConsole)
                         ->execute();
                 } catch (\Throwable $th) {
                     throw $th;
@@ -81,7 +76,7 @@ $cli
             }
 
             $sum = \count($projects);
-            $projects = $consoleDB->find('projects', [Query::limit($limit), Query::offset($offset)]);
+            $projects = $dbForConsole->find('projects', limit: $limit, offset: $offset);
 
             $offset = $offset + $limit;
             $count = $count + $sum;

app/tasks/specs.php

@@ -9,8 +9,12 @@ use Appwrite\Specification\Specification;
 use Appwrite\Utopia\Response;
 use Swoole\Http\Response as HttpResponse;
 use Utopia\App;
+use Utopia\Cache\Adapter\None;
+use Utopia\Cache\Cache;
 use Utopia\CLI\Console;
 use Utopia\Config\Config;
+use Utopia\Database\Adapter\MySQL;
+use Utopia\Database\Database;
 use Utopia\Request;
 use Utopia\Validator\WhiteList;
 
@@ -19,16 +23,15 @@ $cli
     ->param('version', 'latest', new Text(16), 'Spec version', true)
     ->param('mode', 'normal', new WhiteList(['normal', 'mocks']), 'Spec Mode', true)
     ->action(function ($version, $mode) use ($register) {
-        $db = $register->get('db');
-        $redis = $register->get('cache');
         $appRoutes = App::getRoutes();
         $response = new Response(new HttpResponse());
         $mocks = ($mode === 'mocks');
 
+        // Mock dependencies
         App::setResource('request', fn () => new Request());
         App::setResource('response', fn () => $response);
-        App::setResource('db', fn () => $db);
-        App::setResource('cache', fn () => $redis);
+        App::setResource('dbForConsole', fn () => new Database(new MySQL(''), new Cache(new None())));
+        App::setResource('dbForProject', fn () => new Database(new MySQL(''), new Cache(new None())));
 
         $platforms = [
             'client' => APP_PLATFORM_CLIENT,

app/tasks/usage.php

@@ -5,77 +5,15 @@ global $cli, $register;
 use Appwrite\Usage\Calculators\TimeSeries;
 use InfluxDB\Database as InfluxDatabase;
 use Utopia\App;
-use Utopia\Cache\Adapter\Redis as RedisCache;
-use Utopia\Cache\Cache;
 use Utopia\CLI\Console;
-use Utopia\Database\Adapter\MariaDB;
 use Utopia\Database\Database as UtopiaDatabase;
 use Utopia\Database\Validator\Authorization;
-use Utopia\Registry\Registry;
 use Utopia\Logger\Log;
 use Utopia\Validator\WhiteList;
 
 Authorization::disable();
 Authorization::setDefaultStatus(false);
 
-function getDatabase(Registry &$register, string $namespace): UtopiaDatabase
-{
-    $attempts = 0;
-
-    do {
-        try {
-            $attempts++;
-
-            $db = $register->get('db');
-            $redis = $register->get('cache');
-
-            $cache = new Cache(new RedisCache($redis));
-            $database = new UtopiaDatabase(new MariaDB($db), $cache);
-            $database->setDefaultDatabase(App::getEnv('_APP_DB_SCHEMA', 'appwrite'));
-            $database->setNamespace($namespace);
-
-            if (!$database->exists($database->getDefaultDatabase(), 'projects')) {
-                throw new Exception('Projects collection not ready');
-            }
-            break; // leave loop if successful
-        } catch (\Exception $e) {
-            Console::warning("Database not ready. Retrying connection ({$attempts})...");
-            if ($attempts >= DATABASE_RECONNECT_MAX_ATTEMPTS) {
-                throw new \Exception('Failed to connect to database: ' . $e->getMessage());
-            }
-            sleep(DATABASE_RECONNECT_SLEEP);
-        }
-    } while ($attempts < DATABASE_RECONNECT_MAX_ATTEMPTS);
-
-    return $database;
-}
-
-function getInfluxDB(Registry &$register): InfluxDatabase
-{
-    /** @var InfluxDB\Client $client */
-    $client = $register->get('influxdb');
-    $attempts = 0;
-    $max = 10;
-    $sleep = 1;
-
-    do { // check if telegraf database is ready
-        try {
-            $attempts++;
-            $database = $client->selectDB('telegraf');
-            if (in_array('telegraf', $client->listDatabases())) {
-                break; // leave the do-while if successful
-            }
-        } catch (\Throwable $th) {
-            Console::warning("InfluxDB not ready. Retrying connection ({$attempts})...");
-            if ($attempts >= $max) {
-                throw new \Exception('InfluxDB database not ready yet');
-            }
-            sleep($sleep);
-        }
-    } while ($attempts < $max);
-    return $database;
-}
-
 $logError = function (Throwable $error, string $action = 'syncUsageStats') use ($register) {
     $logger = $register->get('logger');
 
@@ -113,12 +51,12 @@ $logError = function (Throwable $error, string $action = 'syncUsageStats') use (
 $cli
     ->task('usage')
     ->desc('Schedules syncing data from influxdb to Appwrite console db')
-    ->action(function () use ($register, $logError) {
+    ->action(function () use ($logError) {
         Console::title('Usage Aggregation V1');
         Console::success(APP_NAME . ' usage aggregation process v1 has started');
 
-        $database = getDatabase($register, '_console');
-        $influxDB = getInfluxDB($register);
+        $database = getConsoleDB();
+        $influxDB = getInfluxDB();
 
         $interval = (int) App::getEnv('_APP_USAGE_AGGREGATION_INTERVAL', '30'); // 30 seconds (by default)
         $usage = new TimeSeries($database, $influxDB, $logError);

app/views/install/compose.phtml

@@ -88,15 +88,15 @@ services:
       - _APP_OPENSSL_KEY_V1
       - _APP_DOMAIN
       - _APP_DOMAIN_TARGET
-      - _APP_REDIS_HOST
-      - _APP_REDIS_PORT
-      - _APP_REDIS_USER
-      - _APP_REDIS_PASS
       - _APP_DB_HOST
       - _APP_DB_PORT
       - _APP_DB_SCHEMA
       - _APP_DB_USER
       - _APP_DB_PASS
+      - _APP_REDIS_HOST
+      - _APP_REDIS_PORT
+      - _APP_REDIS_USER
+      - _APP_REDIS_PASS
       - _APP_SMTP_HOST
       - _APP_SMTP_PORT
       - _APP_SMTP_SECURE
@@ -185,13 +185,15 @@ services:
       - _APP_WORKER_PER_CORE
       - _APP_OPTIONS_ABUSE
       - _APP_OPENSSL_KEY_V1
-      - _APP_REDIS_HOST
-      - _APP_REDIS_PORT
       - _APP_DB_HOST
       - _APP_DB_PORT
       - _APP_DB_SCHEMA
       - _APP_DB_USER
       - _APP_DB_PASS
+      - _APP_REDIS_HOST
+      - _APP_REDIS_PORT
+      - _APP_REDIS_USER
+      - _APP_REDIS_PASS
       - _APP_USAGE_STATS
       - _APP_LOGGING_PROVIDER
       - _APP_LOGGING_CONFIG
@@ -210,15 +212,15 @@ services:
     environment:
       - _APP_ENV
       - _APP_OPENSSL_KEY_V1
-      - _APP_REDIS_HOST
-      - _APP_REDIS_PORT
-      - _APP_REDIS_USER
-      - _APP_REDIS_PASS
       - _APP_DB_HOST
       - _APP_DB_PORT
       - _APP_DB_SCHEMA
       - _APP_DB_USER
       - _APP_DB_PASS
+      - _APP_REDIS_HOST
+      - _APP_REDIS_PORT
+      - _APP_REDIS_USER
+      - _APP_REDIS_PASS
       - _APP_LOGGING_PROVIDER
       - _APP_LOGGING_CONFIG
 
@@ -264,15 +266,15 @@ services:
     environment:
       - _APP_ENV
       - _APP_OPENSSL_KEY_V1
-      - _APP_REDIS_HOST
-      - _APP_REDIS_PORT
-      - _APP_REDIS_USER
-      - _APP_REDIS_PASS
       - _APP_DB_HOST
       - _APP_DB_PORT
       - _APP_DB_SCHEMA
       - _APP_DB_USER
       - _APP_DB_PASS
+      - _APP_REDIS_HOST
+      - _APP_REDIS_PORT
+      - _APP_REDIS_USER
+      - _APP_REDIS_PASS
       - _APP_STORAGE_DEVICE
       - _APP_STORAGE_S3_ACCESS_KEY
       - _APP_STORAGE_S3_SECRET
@@ -313,15 +315,15 @@ services:
     environment:
       - _APP_ENV
       - _APP_OPENSSL_KEY_V1
-      - _APP_REDIS_HOST
-      - _APP_REDIS_PORT
-      - _APP_REDIS_USER
-      - _APP_REDIS_PASS
       - _APP_DB_HOST
       - _APP_DB_PORT
       - _APP_DB_SCHEMA
       - _APP_DB_USER
       - _APP_DB_PASS
+      - _APP_REDIS_HOST
+      - _APP_REDIS_PORT
+      - _APP_REDIS_USER
+      - _APP_REDIS_PASS
       - _APP_LOGGING_PROVIDER
       - _APP_LOGGING_CONFIG
 
@@ -341,15 +343,15 @@ services:
       - _APP_OPENSSL_KEY_V1
       - _APP_EXECUTOR_SECRET
       - _APP_EXECUTOR_HOST
-      - _APP_REDIS_HOST
-      - _APP_REDIS_PORT
-      - _APP_REDIS_USER
-      - _APP_REDIS_PASS
       - _APP_DB_HOST
       - _APP_DB_PORT
       - _APP_DB_SCHEMA
       - _APP_DB_USER
       - _APP_DB_PASS
+      - _APP_REDIS_HOST
+      - _APP_REDIS_PORT
+      - _APP_REDIS_USER
+      - _APP_REDIS_PASS
       - _APP_LOGGING_PROVIDER
       - _APP_LOGGING_CONFIG
 
@@ -373,15 +375,15 @@ services:
       - _APP_DOMAIN
       - _APP_DOMAIN_TARGET
       - _APP_SYSTEM_SECURITY_EMAIL_ADDRESS
-      - _APP_REDIS_HOST
-      - _APP_REDIS_PORT
-      - _APP_REDIS_USER
-      - _APP_REDIS_PASS
       - _APP_DB_HOST
       - _APP_DB_PORT
       - _APP_DB_SCHEMA
       - _APP_DB_USER
       - _APP_DB_PASS
+      - _APP_REDIS_HOST
+      - _APP_REDIS_PORT
+      - _APP_REDIS_USER
+      - _APP_REDIS_PASS
       - _APP_LOGGING_PROVIDER
       - _APP_LOGGING_CONFIG
 
@@ -400,15 +402,15 @@ services:
     environment:
       - _APP_ENV
       - _APP_OPENSSL_KEY_V1
-      - _APP_REDIS_HOST
-      - _APP_REDIS_PORT
-      - _APP_REDIS_USER
-      - _APP_REDIS_PASS
       - _APP_DB_HOST
       - _APP_DB_PORT
       - _APP_DB_SCHEMA
       - _APP_DB_USER
       - _APP_DB_PASS
+      - _APP_REDIS_HOST
+      - _APP_REDIS_PORT
+      - _APP_REDIS_USER
+      - _APP_REDIS_PASS
       - _APP_FUNCTIONS_TIMEOUT
       - _APP_EXECUTOR_SECRET
       - _APP_EXECUTOR_HOST
@@ -536,15 +538,15 @@ services:
       - _APP_OPENSSL_KEY_V1
       - _APP_DOMAIN
       - _APP_DOMAIN_TARGET
-      - _APP_REDIS_HOST
-      - _APP_REDIS_PORT
-      - _APP_REDIS_USER
-      - _APP_REDIS_PASS
       - _APP_DB_HOST
       - _APP_DB_PORT
       - _APP_DB_SCHEMA
       - _APP_DB_USER
       - _APP_DB_PASS
+      - _APP_REDIS_HOST
+      - _APP_REDIS_PORT
+      - _APP_REDIS_USER
+      - _APP_REDIS_PASS
       - _APP_MAINTENANCE_INTERVAL
       - _APP_MAINTENANCE_RETENTION_EXECUTION
       - _APP_MAINTENANCE_RETENTION_CACHE

app/workers/audits.php

@@ -1,6 +1,5 @@
 <?php
 
-use Appwrite\Event\Event;
 use Appwrite\Resque\Worker;
 use Utopia\Audit\Audit;
 use Utopia\CLI\Console;
@@ -37,7 +36,7 @@ class AuditsV1 extends Worker
         $userName = $user->getAttribute('name', '');
         $userEmail = $user->getAttribute('email', '');
 
-        $dbForProject = $this->getProjectDB($project->getId());
+        $dbForProject = $this->getProjectDB($project);
         $audit = new Audit($dbForProject);
         $audit->log(
             userId: $user->getId(),

app/workers/builds.php

@@ -7,7 +7,6 @@ use Appwrite\Utopia\Response\Model\Deployment;
 use Cron\CronExpression;
 use Executor\Executor;
 use Appwrite\Usage\Stats;
-use Utopia\Database\Database;
 use Utopia\Database\DateTime;
 use Utopia\App;
 use Utopia\CLI\Console;
@@ -15,7 +14,6 @@ use Utopia\Database\ID;
 use Utopia\Storage\Storage;
 use Utopia\Database\Document;
 use Utopia\Config\Config;
-use Utopia\Database\Query;
 
 require_once __DIR__ . '/../init.php';
 
@@ -59,7 +57,7 @@ class BuildsV1 extends Worker
 
     protected function buildDeployment(Document $project, Document $function, Document $deployment)
     {
-        $dbForProject = $this->getProjectDB($project->getId());
+        $dbForProject = $this->getProjectDB($project);
 
         $function = $dbForProject->getDocument('functions', $function->getId());
         if ($function->isEmpty()) {

app/workers/certificates.php

@@ -1,6 +1,5 @@
 <?php
 
-use Appwrite\Event\Event;
 use Appwrite\Event\Mail;
 use Appwrite\Network\Validator\CNAME;
 use Appwrite\Resque\Worker;

app/workers/databases.php

@@ -35,16 +35,16 @@ class DatabaseV1 extends Worker
 
         switch (strval($type)) {
             case DATABASE_TYPE_CREATE_ATTRIBUTE:
-                $this->createAttribute($database, $collection, $document, $project->getId());
+                $this->createAttribute($database, $collection, $document, $project);
                 break;
             case DATABASE_TYPE_DELETE_ATTRIBUTE:
-                $this->deleteAttribute($database, $collection, $document, $project->getId());
+                $this->deleteAttribute($database, $collection, $document, $project);
                 break;
             case DATABASE_TYPE_CREATE_INDEX:
-                $this->createIndex($database, $collection, $document, $project->getId());
+                $this->createIndex($database, $collection, $document, $project);
                 break;
             case DATABASE_TYPE_DELETE_INDEX:
-                $this->deleteIndex($database, $collection, $document, $project->getId());
+                $this->deleteIndex($database, $collection, $document, $project);
                 break;
 
             default:
@@ -61,12 +61,13 @@ class DatabaseV1 extends Worker
      * @param Document $database
      * @param Document $collection
      * @param Document $attribute
-     * @param string $projectId
+     * @param Document $project
      */
-    protected function createAttribute(Document $database, Document $collection, Document $attribute, string $projectId): void
+    protected function createAttribute(Document $database, Document $collection, Document $attribute, Document $project): void
     {
+        $projectId = $project->getId();
         $dbForConsole = $this->getConsoleDB();
-        $dbForProject = $this->getProjectDB($projectId);
+        $dbForProject = $this->getProjectDB($project);
 
         $events = Event::generateEvents('databases.[databaseId].collections.[collectionId].attributes.[attributeId].update', [
             'databaseId' => $database->getId(),
@@ -128,12 +129,13 @@ class DatabaseV1 extends Worker
      * @param Document $database
      * @param Document $collection
      * @param Document $attribute
-     * @param string $projectId
+     * @param Document $project
      */
-    protected function deleteAttribute(Document $database, Document $collection, Document $attribute, string $projectId): void
+    protected function deleteAttribute(Document $database, Document $collection, Document $attribute, Document $project): void
     {
+        $projectId = $project->getId();
         $dbForConsole = $this->getConsoleDB();
-        $dbForProject = $this->getProjectDB($projectId);
+        $dbForProject = $this->getProjectDB($project);
 
         $events = Event::generateEvents('databases.[databaseId].collections.[collectionId].attributes.[attributeId].delete', [
             'databaseId' => $database->getId(),
@@ -225,7 +227,7 @@ class DatabaseV1 extends Worker
                     }
 
                     if ($exists) { // Delete the duplicate if created, else update in db
-                        $this->deleteIndex($database, $collection, $index, $projectId);
+                        $this->deleteIndex($database, $collection, $index, $project);
                     } else {
                         $dbForProject->updateDocument('indexes', $index->getId(), $index);
                     }
@@ -241,12 +243,13 @@ class DatabaseV1 extends Worker
      * @param Document $database
      * @param Document $collection
      * @param Document $index
-     * @param string $projectId
+     * @param Document $project
      */
-    protected function createIndex(Document $database, Document $collection, Document $index, string $projectId): void
+    protected function createIndex(Document $database, Document $collection, Document $index, Document $project): void
     {
+        $projectId = $project->getId();
         $dbForConsole = $this->getConsoleDB();
-        $dbForProject = $this->getProjectDB($projectId);
+        $dbForProject = $this->getProjectDB($project);
 
         $events = Event::generateEvents('databases.[databaseId].collections.[collectionId].indexes.[indexId].update', [
             'databaseId' => $database->getId(),
@@ -298,12 +301,13 @@ class DatabaseV1 extends Worker
      * @param Document $database
      * @param Document $collection
      * @param Document $index
-     * @param string $projectId
+     * @param Document $project
      */
-    protected function deleteIndex(Document $database, Document $collection, Document $index, string $projectId): void
+    protected function deleteIndex(Document $database, Document $collection, Document $index, Document $project): void
     {
+        $projectId = $project->getId();
         $dbForConsole = $this->getConsoleDB();
-        $dbForProject = $this->getProjectDB($projectId);
+        $dbForProject = $this->getProjectDB($project);
 
         $events = Event::generateEvents('databases.[databaseId].collections.[collectionId].indexes.[indexId].delete', [
             'databaseId' => $database->getId(),

app/workers/deletes.php

@@ -40,28 +40,28 @@ class DeletesV1 extends Worker
 
                 switch ($document->getCollection()) {
                     case DELETE_TYPE_DATABASES:
-                        $this->deleteDatabase($document, $project->getId());
+                        $this->deleteDatabase($document, $project);
                         break;
                     case DELETE_TYPE_COLLECTIONS:
-                        $this->deleteCollection($document, $project->getId());
+                        $this->deleteCollection($document, $project);
                         break;
                     case DELETE_TYPE_PROJECTS:
                         $this->deleteProject($document);
                         break;
                     case DELETE_TYPE_FUNCTIONS:
-                        $this->deleteFunction($document, $project->getId());
+                        $this->deleteFunction($document, $project);
                         break;
                     case DELETE_TYPE_DEPLOYMENTS:
-                        $this->deleteDeployment($document, $project->getId());
+                        $this->deleteDeployment($document, $project);
                         break;
                     case DELETE_TYPE_USERS:
-                        $this->deleteUser($document, $project->getId());
+                        $this->deleteUser($document, $project);
                         break;
                     case DELETE_TYPE_TEAMS:
-                        $this->deleteMemberships($document, $project->getId());
+                        $this->deleteMemberships($document, $project);
                         break;
                     case DELETE_TYPE_BUCKETS:
-                        $this->deleteBucket($document, $project->getId());
+                        $this->deleteBucket($document, $project);
                         break;
                     default:
                         Console::error('No lazy delete operation available for document of type: ' . $document->getCollection());
@@ -82,7 +82,7 @@ class DeletesV1 extends Worker
                 $document = new Document($this->args['document'] ?? []);
 
                 if (!$document->isEmpty()) {
-                    $this->deleteAuditLogsByResource('document/' . $document->getId(), $project->getId());
+                    $this->deleteAuditLogsByResource('document/' . $document->getId(), $project);
                 }
 
                 break;
@@ -109,7 +109,7 @@ class DeletesV1 extends Worker
                 break;
 
             case DELETE_TYPE_CACHE_BY_RESOURCE:
-                $this->deleteCacheByResource($project->getId());
+                $this->deleteCacheByResource($this->args['resource']);
                 break;
             case DELETE_TYPE_CACHE_BY_TIMESTAMP:
                 $this->deleteCacheByDate();
@@ -125,12 +125,12 @@ class DeletesV1 extends Worker
     }
 
     /**
-     * @param string $projectId
+     * @param string $resource
      */
-    protected function deleteCacheByResource(string $projectId): void
+    protected function deleteCacheByResource(string $resource): void
     {
         $this->deleteCacheFiles([
-            Query::equal('resource', [$this->args['resource']]),
+            Query::equal('resource', [$resource]),
         ]);
     }
 
@@ -143,9 +143,10 @@ class DeletesV1 extends Worker
 
     protected function deleteCacheFiles($query): void
     {
-        $this->deleteForProjectIds(function (string $projectId) use ($query) {
+        $this->deleteForProjectIds(function (Document $project) use ($query) {
 
-            $dbForProject = $this->getProjectDB($projectId);
+            $projectId = $project->getId();
+            $dbForProject = $this->getProjectDB($project);
             $cache = new Cache(
                 new Filesystem(APP_STORAGE_CACHE . DIRECTORY_SEPARATOR . 'app-' . $projectId)
             );
@@ -169,34 +170,35 @@ class DeletesV1 extends Worker
 
     /**
      * @param Document $document database document
-     * @param string $projectId
+     * @param Document $projectId
      */
-    protected function deleteDatabase(Document $document, string $projectId): void
+    protected function deleteDatabase(Document $document, Document $project): void
     {
         $databaseId = $document->getId();
+        $projectId = $project->getId();
 
-        $dbForProject = $this->getProjectDB($projectId);
+        $dbForProject = $this->getProjectDB($project);
 
-        $this->deleteByGroup('database_' . $document->getInternalId(), [], $dbForProject, function ($document) use ($projectId) {
-            $this->deleteCollection($document, $projectId);
+        $this->deleteByGroup('database_' . $document->getInternalId(), [], $dbForProject, function ($document) use ($project) {
+            $this->deleteCollection($document, $project);
         });
 
         $dbForProject->deleteCollection('database_' . $document->getInternalId());
 
-        $this->deleteAuditLogsByResource('database/' . $databaseId, $projectId);
+        $this->deleteAuditLogsByResource('database/' . $databaseId, $project);
     }
 
     /**
      * @param Document $document teams document
-     * @param string $projectId
+     * @param Document $project
      */
-    protected function deleteCollection(Document $document, string $projectId): void
+    protected function deleteCollection(Document $document, Document $project): void
     {
         $collectionId = $document->getId();
         $databaseId = $document->getAttribute('databaseId');
         $databaseInternalId = $document->getAttribute('databaseInternalId');
 
-        $dbForProject = $this->getProjectDB($projectId);
+        $dbForProject = $this->getProjectDB($project);
 
         $dbForProject->deleteCollection('database_' . $databaseInternalId . '_collection_' . $document->getInternalId());
 
@@ -210,7 +212,7 @@ class DeletesV1 extends Worker
             Query::equal('collectionId', [$collectionId])
         ], $dbForProject);
 
-        $this->deleteAuditLogsByResource('database/' . $databaseId . '/collection/' . $collectionId, $projectId);
+        $this->deleteAuditLogsByResource('database/' . $databaseId . '/collection/' . $collectionId, $project);
     }
 
     /**
@@ -219,8 +221,9 @@ class DeletesV1 extends Worker
      */
     protected function deleteUsageStats(string $hourlyUsageRetentionDatetime)
     {
-        $this->deleteForProjectIds(function (string $projectId) use ($hourlyUsageRetentionDatetime) {
-            $dbForProject = $this->getProjectDB($projectId);
+        $this->deleteForProjectIds(function (Document $project) use ($hourlyUsageRetentionDatetime) {
+            $dbForProject = $this->getProjectDB($project);
+            // Delete Usage stats
             $this->deleteByGroup('stats', [
                 Query::lessThan('time', $hourlyUsageRetentionDatetime),
                 Query::equal('period', ['1h']),
@@ -230,16 +233,16 @@ class DeletesV1 extends Worker
 
     /**
      * @param Document $document teams document
-     * @param string $projectId
+     * @param Document $project
      */
-    protected function deleteMemberships(Document $document, string $projectId): void
+    protected function deleteMemberships(Document $document, Document $project): void
     {
         $teamId = $document->getAttribute('teamId', '');
 
         // Delete Memberships
         $this->deleteByGroup('memberships', [
             Query::equal('teamId', [$teamId])
-        ], $this->getProjectDB($projectId));
+        ], $this->getProjectDB($project));
     }
 
     /**
@@ -250,7 +253,7 @@ class DeletesV1 extends Worker
         $projectId = $document->getId();
 
         // Delete all DBs
-        $this->getProjectDB($projectId)->delete($projectId);
+        $this->getProjectDB($document)->delete($projectId);
 
         // Delete all storage directories
         $uploads = new Local(APP_STORAGE_UPLOADS . '/app-' . $document->getId());
@@ -262,30 +265,30 @@ class DeletesV1 extends Worker
 
     /**
      * @param Document $document user document
-     * @param string $projectId
+     * @param Document $project
      */
-    protected function deleteUser(Document $document, string $projectId): void
+    protected function deleteUser(Document $document, Document $project): void
     {
         $userId = $document->getId();
 
         // Delete all sessions of this user from the sessions table and update the sessions field of the user record
         $this->deleteByGroup('sessions', [
             Query::equal('userId', [$userId])
-        ], $this->getProjectDB($projectId));
+        ], $this->getProjectDB($project));
 
-        $this->getProjectDB($projectId)->deleteCachedDocument('users', $userId);
+        $this->getProjectDB($project)->deleteCachedDocument('users', $userId);
 
         // Delete Memberships and decrement team membership counts
         $this->deleteByGroup('memberships', [
             Query::equal('userId', [$userId])
-        ], $this->getProjectDB($projectId), function (Document $document) use ($projectId) {
+        ], $this->getProjectDB($project), function (Document $document) use ($project) {
 
             if ($document->getAttribute('confirm')) { // Count only confirmed members
                 $teamId = $document->getAttribute('teamId');
-                $team = $this->getProjectDB($projectId)->getDocument('teams', $teamId);
+                $team = $this->getProjectDB($project)->getDocument('teams', $teamId);
                 if (!$team->isEmpty()) {
                     $team = $this
-                        ->getProjectDB($projectId)
+                        ->getProjectDB($project)
                         ->updateDocument(
                             'teams',
                             $teamId,
@@ -299,7 +302,7 @@ class DeletesV1 extends Worker
         // Delete tokens
         $this->deleteByGroup('tokens', [
             Query::equal('userId', [$userId])
-        ], $this->getProjectDB($projectId));
+        ], $this->getProjectDB($project));
     }
 
     /**
@@ -307,8 +310,8 @@ class DeletesV1 extends Worker
      */
     protected function deleteExecutionLogs(string $datetime): void
     {
-        $this->deleteForProjectIds(function (string $projectId) use ($datetime) {
-            $dbForProject = $this->getProjectDB($projectId);
+        $this->deleteForProjectIds(function (Document $project) use ($datetime) {
+            $dbForProject = $this->getProjectDB($project);
             // Delete Executions
             $this->deleteByGroup('executions', [
                 Query::lessThan('$createdAt', $datetime)
@@ -321,8 +324,8 @@ class DeletesV1 extends Worker
      */
     protected function deleteExpiredSessions(string $datetime): void
     {
-        $this->deleteForProjectIds(function (string $projectId) use ($datetime) {
-            $dbForProject = $this->getProjectDB($projectId);
+        $this->deleteForProjectIds(function (Document $project) use ($datetime) {
+            $dbForProject = $this->getProjectDB($project);
             // Delete Sessions
             $this->deleteByGroup('sessions', [
                 Query::lessThan('expire', $datetime)
@@ -335,8 +338,8 @@ class DeletesV1 extends Worker
      */
     protected function deleteRealtimeUsage(string $datetime): void
     {
-        $this->deleteForProjectIds(function (string $projectId) use ($datetime) {
-            $dbForProject = $this->getProjectDB($projectId);
+        $this->deleteForProjectIds(function (Document $project) use ($datetime) {
+            $dbForProject = $this->getProjectDB($project);
             // Delete Dead Realtime Logs
             $this->deleteByGroup('realtime', [
                 Query::lessThan('timestamp', $datetime)
@@ -354,8 +357,9 @@ class DeletesV1 extends Worker
             throw new Exception('Failed to delete audit logs. No datetime provided');
         }
 
-        $this->deleteForProjectIds(function (string $projectId) use ($datetime) {
-            $dbForProject = $this->getProjectDB($projectId);
+        $this->deleteForProjectIds(function (Document $project) use ($datetime) {
+            $projectId = $project->getId();
+            $dbForProject = $this->getProjectDB($project);
             $timeLimit = new TimeLimit("", 0, 1, $dbForProject);
             $abuse = new Abuse($timeLimit);
             $status = $abuse->cleanup($datetime);
@@ -375,8 +379,9 @@ class DeletesV1 extends Worker
             throw new Exception('Failed to delete audit logs. No datetime provided');
         }
 
-        $this->deleteForProjectIds(function (string $projectId) use ($datetime) {
-            $dbForProject = $this->getProjectDB($projectId);
+        $this->deleteForProjectIds(function (Document $project) use ($datetime) {
+            $projectId = $project->getId();
+            $dbForProject = $this->getProjectDB($project);
             $audit = new Audit($dbForProject);
             $status = $audit->cleanup($datetime);
             if (!$status) {
@@ -387,11 +392,11 @@ class DeletesV1 extends Worker
 
     /**
      * @param string $resource
-     * @param string $projectId
+     * @param Document $project
      */
-    protected function deleteAuditLogsByResource(string $resource, string $projectId): void
+    protected function deleteAuditLogsByResource(string $resource, Document $project): void
     {
-        $dbForProject = $this->getProjectDB($projectId);
+        $dbForProject = $this->getProjectDB($project);
 
         $this->deleteByGroup(Audit::COLLECTION, [
             Query::equal('resource', [$resource])
@@ -400,11 +405,12 @@ class DeletesV1 extends Worker
 
     /**
      * @param Document $document function document
-     * @param string $projectId
+     * @param Document $project
      */
-    protected function deleteFunction(Document $document, string $projectId): void
+    protected function deleteFunction(Document $document, Document $project): void
     {
-        $dbForProject = $this->getProjectDB($projectId);
+        $projectId = $project->getId();
+        $dbForProject = $this->getProjectDB($project);
         $functionId = $document->getId();
 
         /**
@@ -473,11 +479,12 @@ class DeletesV1 extends Worker
 
     /**
      * @param Document $document deployment document
-     * @param string $projectId
+     * @param Document $project
      */
-    protected function deleteDeployment(Document $document, string $projectId): void
+    protected function deleteDeployment(Document $document, Document $project): void
     {
-        $dbForProject = $this->getProjectDB($projectId);
+        $projectId = $project->getId();
+        $dbForProject = $this->getProjectDB($project);
         $deploymentId = $document->getId();
         $functionId = $document->getAttribute('resourceId');
 
@@ -562,13 +569,11 @@ class DeletesV1 extends Worker
             $chunk++;
 
             /** @var string[] $projectIds */
-            $projectIds = array_map(fn (Document $project) => $project->getId(), $projects);
-
             $sum = count($projects);
 
             Console::info('Executing delete function for chunk #' . $chunk . '. Found ' . $sum . ' projects');
-            foreach ($projectIds as $projectId) {
-                $callback($projectId);
+            foreach ($projects as $project) {
+                $callback($project);
                 $count++;
             }
         }
@@ -660,9 +665,10 @@ class DeletesV1 extends Worker
         }
     }
 
-    protected function deleteBucket(Document $document, string $projectId)
+    protected function deleteBucket(Document $document, Document $project)
     {
-        $dbForProject = $this->getProjectDB($projectId);
+        $projectId = $project->getId();
+        $dbForProject = $this->getProjectDB($project);
         $dbForProject->deleteCollection('bucket_' . $document->getInternalId());
 
         $device = $this->getDevice(APP_STORAGE_UPLOADS . '/app-' . $projectId);

app/workers/functions.php

@@ -52,7 +52,7 @@ class FunctionsV1 extends Worker
             return;
         }
 
-        $database = $this->getProjectDB($project->getId());
+        $database = $this->getProjectDB($project);
 
         /**
          * Handle Event execution.

composer.json

@@ -48,10 +48,10 @@
         "utopia-php/abuse": "0.15.*",
         "utopia-php/analytics": "0.2.*",
         "utopia-php/audit": "0.16.*",
-        "utopia-php/cache": "0.6.*",
+        "utopia-php/cache": "0.8.*",
         "utopia-php/cli": "0.13.*",
         "utopia-php/config": "0.2.*",
-        "utopia-php/database": "0.27.0",
+        "utopia-php/database": "dev-feat-update-create as 0.27.0",
         "utopia-php/locale": "0.4.*",
         "utopia-php/registry": "0.5.*",
         "utopia-php/preloader": "0.2.*",
@@ -61,6 +61,7 @@
         "utopia-php/websocket": "0.1.0",
         "utopia-php/image": "0.5.*",
         "utopia-php/orchestration": "0.6.*",
+        "utopia-php/pools": "0.1.*",
         "resque/php-resque": "1.3.6",
         "matomo/device-detector": "6.0.0",
         "dragonmantank/cron-expression": "3.3.1",

composer.lock

@@ -4,7 +4,7 @@
         "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies",
         "This file is @generated automatically"
     ],
-    "content-hash": "fe23114c0eec940a582137c2ecac8e6a",
+    "content-hash": "c33b697a2f1422eb6fccfd860a7e0169",
     "packages": [
         {
             "name": "adhocore/jwt",
@@ -345,79 +345,6 @@
             },
             "time": "2022-06-20T22:56:59+00:00"
         },
-        {
-            "name": "composer/package-versions-deprecated",
-            "version": "1.11.99.5",
-            "source": {
-                "type": "git",
-                "url": "https://github.com/composer/package-versions-deprecated.git",
-                "reference": "b4f54f74ef3453349c24a845d22392cd31e65f1d"
-            },
-            "dist": {
-                "type": "zip",
-                "url": "https://api.github.com/repos/composer/package-versions-deprecated/zipball/b4f54f74ef3453349c24a845d22392cd31e65f1d",
-                "reference": "b4f54f74ef3453349c24a845d22392cd31e65f1d",
-                "shasum": ""
-            },
-            "require": {
-                "composer-plugin-api": "^1.1.0 || ^2.0",
-                "php": "^7 || ^8"
-            },
-            "replace": {
-                "ocramius/package-versions": "1.11.99"
-            },
-            "require-dev": {
-                "composer/composer": "^1.9.3 || ^2.0@dev",
-                "ext-zip": "^1.13",
-                "phpunit/phpunit": "^6.5 || ^7"
-            },
-            "type": "composer-plugin",
-            "extra": {
-                "class": "PackageVersions\\Installer",
-                "branch-alias": {
-                    "dev-master": "1.x-dev"
-                }
-            },
-            "autoload": {
-                "psr-4": {
-                    "PackageVersions\\": "src/PackageVersions"
-                }
-            },
-            "notification-url": "https://packagist.org/downloads/",
-            "license": [
-                "MIT"
-            ],
-            "authors": [
-                {
-                    "name": "Marco Pivetta",
-                    "email": "ocramius@gmail.com"
-                },
-                {
-                    "name": "Jordi Boggiano",
-                    "email": "j.boggiano@seld.be"
-                }
-            ],
-            "description": "Composer plugin that provides efficient querying for installed package versions (no runtime IO)",
-            "support": {
-                "issues": "https://github.com/composer/package-versions-deprecated/issues",
-                "source": "https://github.com/composer/package-versions-deprecated/tree/1.11.99.5"
-            },
-            "funding": [
-                {
-                    "url": "https://packagist.com",
-                    "type": "custom"
-                },
-                {
-                    "url": "https://github.com/composer",
-                    "type": "github"
-                },
-                {
-                    "url": "https://tidelift.com/funding/github/packagist/composer/composer",
-                    "type": "tidelift"
-                }
-            ],
-            "time": "2022-01-17T14:14:24+00:00"
-        },
         {
             "name": "dragonmantank/cron-expression",
             "version": "v3.3.1",
@@ -876,61 +803,6 @@
             },
             "time": "2020-12-26T17:45:17+00:00"
         },
-        {
-            "name": "jean85/pretty-package-versions",
-            "version": "1.6.0",
-            "source": {
-                "type": "git",
-                "url": "https://github.com/Jean85/pretty-package-versions.git",
-                "reference": "1e0104b46f045868f11942aea058cd7186d6c303"
-            },
-            "dist": {
-                "type": "zip",
-                "url": "https://api.github.com/repos/Jean85/pretty-package-versions/zipball/1e0104b46f045868f11942aea058cd7186d6c303",
-                "reference": "1e0104b46f045868f11942aea058cd7186d6c303",
-                "shasum": ""
-            },
-            "require": {
-                "composer/package-versions-deprecated": "^1.8.0",
-                "php": "^7.0|^8.0"
-            },
-            "require-dev": {
-                "phpunit/phpunit": "^6.0|^8.5|^9.2"
-            },
-            "type": "library",
-            "extra": {
-                "branch-alias": {
-                    "dev-master": "1.x-dev"
-                }
-            },
-            "autoload": {
-                "psr-4": {
-                    "Jean85\\": "src/"
-                }
-            },
-            "notification-url": "https://packagist.org/downloads/",
-            "license": [
-                "MIT"
-            ],
-            "authors": [
-                {
-                    "name": "Alessandro Lai",
-                    "email": "alessandro.lai85@gmail.com"
-                }
-            ],
-            "description": "A wrapper for ocramius/package-versions to get pretty versions strings",
-            "keywords": [
-                "composer",
-                "package",
-                "release",
-                "versions"
-            ],
-            "support": {
-                "issues": "https://github.com/Jean85/pretty-package-versions/issues",
-                "source": "https://github.com/Jean85/pretty-package-versions/tree/1.6.0"
-            },
-            "time": "2021-02-04T16:20:16+00:00"
-        },
         {
             "name": "matomo/device-detector",
             "version": "6.0.0",
@@ -1000,74 +872,6 @@
             },
             "time": "2022-04-11T09:58:17+00:00"
         },
-        {
-            "name": "mongodb/mongodb",
-            "version": "1.8.0",
-            "source": {
-                "type": "git",
-                "url": "https://github.com/mongodb/mongo-php-library.git",
-                "reference": "953dbc19443aa9314c44b7217a16873347e6840d"
-            },
-            "dist": {
-                "type": "zip",
-                "url": "https://api.github.com/repos/mongodb/mongo-php-library/zipball/953dbc19443aa9314c44b7217a16873347e6840d",
-                "reference": "953dbc19443aa9314c44b7217a16873347e6840d",
-                "shasum": ""
-            },
-            "require": {
-                "ext-hash": "*",
-                "ext-json": "*",
-                "ext-mongodb": "^1.8.1",
-                "jean85/pretty-package-versions": "^1.2",
-                "php": "^7.0 || ^8.0",
-                "symfony/polyfill-php80": "^1.19"
-            },
-            "require-dev": {
-                "squizlabs/php_codesniffer": "^3.5, <3.5.5",
-                "symfony/phpunit-bridge": "5.x-dev"
-            },
-            "type": "library",
-            "extra": {
-                "branch-alias": {
-                    "dev-master": "1.8.x-dev"
-                }
-            },
-            "autoload": {
-                "files": [
-                    "src/functions.php"
-                ],
-                "psr-4": {
-                    "MongoDB\\": "src/"
-                }
-            },
-            "notification-url": "https://packagist.org/downloads/",
-            "license": [
-                "Apache-2.0"
-            ],
-            "authors": [
-                {
-                    "name": "Andreas Braun",
-                    "email": "andreas.braun@mongodb.com"
-                },
-                {
-                    "name": "Jeremy Mikola",
-                    "email": "jmikola@gmail.com"
-                }
-            ],
-            "description": "MongoDB driver library",
-            "homepage": "https://jira.mongodb.org/browse/PHPLIB",
-            "keywords": [
-                "database",
-                "driver",
-                "mongodb",
-                "persistence"
-            ],
-            "support": {
-                "issues": "https://github.com/mongodb/mongo-php-library/issues",
-                "source": "https://github.com/mongodb/mongo-php-library/tree/1.8.0"
-            },
-            "time": "2020-11-25T12:26:02+00:00"
-        },
         {
             "name": "mustangostang/spyc",
             "version": "0.6.3",
@@ -1656,89 +1460,6 @@
             ],
             "time": "2022-02-25T11:15:52+00:00"
         },
-        {
-            "name": "symfony/polyfill-php80",
-            "version": "v1.26.0",
-            "source": {
-                "type": "git",
-                "url": "https://github.com/symfony/polyfill-php80.git",
-                "reference": "cfa0ae98841b9e461207c13ab093d76b0fa7bace"
-            },
-            "dist": {
-                "type": "zip",
-                "url": "https://api.github.com/repos/symfony/polyfill-php80/zipball/cfa0ae98841b9e461207c13ab093d76b0fa7bace",
-                "reference": "cfa0ae98841b9e461207c13ab093d76b0fa7bace",
-                "shasum": ""
-            },
-            "require": {
-                "php": ">=7.1"
-            },
-            "type": "library",
-            "extra": {
-                "branch-alias": {
-                    "dev-main": "1.26-dev"
-                },
-                "thanks": {
-                    "name": "symfony/polyfill",
-                    "url": "https://github.com/symfony/polyfill"
-                }
-            },
-            "autoload": {
-                "files": [
-                    "bootstrap.php"
-                ],
-                "psr-4": {
-                    "Symfony\\Polyfill\\Php80\\": ""
-                },
-                "classmap": [
-                    "Resources/stubs"
-                ]
-            },
-            "notification-url": "https://packagist.org/downloads/",
-            "license": [
-                "MIT"
-            ],
-            "authors": [
-                {
-                    "name": "Ion Bazan",
-                    "email": "ion.bazan@gmail.com"
-                },
-                {
-                    "name": "Nicolas Grekas",
-                    "email": "p@tchwork.com"
-                },
-                {
-                    "name": "Symfony Community",
-                    "homepage": "https://symfony.com/contributors"
-                }
-            ],
-            "description": "Symfony polyfill backporting some PHP 8.0+ features to lower PHP versions",
-            "homepage": "https://symfony.com",
-            "keywords": [
-                "compatibility",
-                "polyfill",
-                "portable",
-                "shim"
-            ],
-            "support": {
-                "source": "https://github.com/symfony/polyfill-php80/tree/v1.26.0"
-            },
-            "funding": [
-                {
-                    "url": "https://symfony.com/sponsor",
-                    "type": "custom"
-                },
-                {
-                    "url": "https://github.com/fabpot",
-                    "type": "github"
-                },
-                {
-                    "url": "https://tidelift.com/funding/github/packagist/symfony/symfony",
-                    "type": "tidelift"
-                }
-            ],
-            "time": "2022-05-10T07:21:04+00:00"
-        },
         {
             "name": "utopia-php/abuse",
             "version": "0.15.0",
@@ -1897,24 +1618,26 @@
         },
         {
             "name": "utopia-php/cache",
-            "version": "0.6.1",
+            "version": "0.8.0",
             "source": {
                 "type": "git",
                 "url": "https://github.com/utopia-php/cache.git",
-                "reference": "9889235a6d3da6cbb1f435201529da4d27c30e79"
+                "reference": "212e66100a1f32e674fca5d9bc317cc998303089"
             },
             "dist": {
                 "type": "zip",
-                "url": "https://api.github.com/repos/utopia-php/cache/zipball/9889235a6d3da6cbb1f435201529da4d27c30e79",
-                "reference": "9889235a6d3da6cbb1f435201529da4d27c30e79",
+                "url": "https://api.github.com/repos/utopia-php/cache/zipball/212e66100a1f32e674fca5d9bc317cc998303089",
+                "reference": "212e66100a1f32e674fca5d9bc317cc998303089",
                 "shasum": ""
             },
             "require": {
                 "ext-json": "*",
+                "ext-memcached": "*",
                 "ext-redis": "*",
                 "php": ">=8.0"
             },
             "require-dev": {
+                "laravel/pint": "1.2.*",
                 "phpunit/phpunit": "^9.3",
                 "vimeo/psalm": "4.13.1"
             },
@@ -1928,12 +1651,6 @@
             "license": [
                 "MIT"
             ],
-            "authors": [
-                {
-                    "name": "Eldad Fux",
-                    "email": "eldad@appwrite.io"
-                }
-            ],
             "description": "A simple cache library to manage application cache storing, loading and purging",
             "keywords": [
                 "cache",
@@ -1944,9 +1661,9 @@
             ],
             "support": {
                 "issues": "https://github.com/utopia-php/cache/issues",
-                "source": "https://github.com/utopia-php/cache/tree/0.6.1"
+                "source": "https://github.com/utopia-php/cache/tree/0.8.0"
             },
-            "time": "2022-08-10T08:12:46+00:00"
+            "time": "2022-10-16T16:48:09+00:00"
         },
         {
             "name": "utopia-php/cli",
@@ -2054,29 +1771,29 @@
         },
         {
             "name": "utopia-php/database",
-            "version": "0.27.0",
+            "version": "dev-feat-update-create",
             "source": {
                 "type": "git",
                 "url": "https://github.com/utopia-php/database.git",
-                "reference": "0644cd8a7efee2fc745ed70dbf2b73a78eeefa52"
+                "reference": "8ba52f4ce40674ed9f6e8f72e300d5d9a4900590"
             },
             "dist": {
                 "type": "zip",
-                "url": "https://api.github.com/repos/utopia-php/database/zipball/0644cd8a7efee2fc745ed70dbf2b73a78eeefa52",
-                "reference": "0644cd8a7efee2fc745ed70dbf2b73a78eeefa52",
+                "url": "https://api.github.com/repos/utopia-php/database/zipball/8ba52f4ce40674ed9f6e8f72e300d5d9a4900590",
+                "reference": "8ba52f4ce40674ed9f6e8f72e300d5d9a4900590",
                 "shasum": ""
             },
             "require": {
-                "ext-mongodb": "*",
-                "ext-pdo": "*",
-                "ext-redis": "*",
-                "mongodb/mongodb": "1.8.0",
                 "php": ">=8.0",
-                "utopia-php/cache": "0.6.*",
+                "utopia-php/cache": "0.8.*",
                 "utopia-php/framework": "0.*.*"
             },
             "require-dev": {
+                "ext-mongodb": "*",
+                "ext-pdo": "*",
+                "ext-redis": "*",
                 "fakerphp/faker": "^1.14",
+                "mongodb/mongodb": "1.8.0",
                 "phpunit/phpunit": "^9.4",
                 "swoole/ide-helper": "4.8.0",
                 "utopia-php/cli": "^0.11.0",
@@ -2102,9 +1819,9 @@
             ],
             "support": {
                 "issues": "https://github.com/utopia-php/database/issues",
-                "source": "https://github.com/utopia-php/database/tree/0.27.0"
+                "source": "https://github.com/utopia-php/database/tree/feat-update-create"
             },
-            "time": "2022-10-16T07:39:26+00:00"
+            "time": "2022-10-28T12:01:54+00:00"
         },
         {
             "name": "utopia-php/domains",
@@ -2433,6 +2150,59 @@
             },
             "time": "2022-07-13T16:47:18+00:00"
         },
+        {
+            "name": "utopia-php/pools",
+            "version": "0.1.0",
+            "source": {
+                "type": "git",
+                "url": "https://github.com/utopia-php/pools.git",
+                "reference": "5a467a569a80aefc846a97dc195b4adc2fd71805"
+            },
+            "dist": {
+                "type": "zip",
+                "url": "https://api.github.com/repos/utopia-php/pools/zipball/5a467a569a80aefc846a97dc195b4adc2fd71805",
+                "reference": "5a467a569a80aefc846a97dc195b4adc2fd71805",
+                "shasum": ""
+            },
+            "require": {
+                "ext-mongodb": "*",
+                "ext-pdo": "*",
+                "ext-redis": "*",
+                "php": ">=8.0"
+            },
+            "require-dev": {
+                "phpunit/phpunit": "^9.4",
+                "vimeo/psalm": "4.0.1"
+            },
+            "type": "library",
+            "autoload": {
+                "psr-4": {
+                    "Utopia\\Pools\\": "src/Pools"
+                }
+            },
+            "notification-url": "https://packagist.org/downloads/",
+            "license": [
+                "MIT"
+            ],
+            "authors": [
+                {
+                    "name": "Team Appwrite",
+                    "email": "team@appwrite.io"
+                }
+            ],
+            "description": "A simple library to manage connection pools",
+            "keywords": [
+                "framework",
+                "php",
+                "pools",
+                "utopia"
+            ],
+            "support": {
+                "issues": "https://github.com/utopia-php/pools/issues",
+                "source": "https://github.com/utopia-php/pools/tree/0.1.0"
+            },
+            "time": "2022-10-11T19:31:07+00:00"
+        },
         {
             "name": "utopia-php/preloader",
             "version": "0.2.4",
@@ -5347,9 +5117,18 @@
             "time": "2022-09-28T08:42:51+00:00"
         }
     ],
-    "aliases": [],
+    "aliases": [
+        {
+            "package": "utopia-php/database",
+            "version": "dev-feat-update-create",
+            "alias": "0.27.0",
+            "alias_normalized": "0.27.0.0"
+        }
+    ],
     "minimum-stability": "stable",
-    "stability-flags": [],
+    "stability-flags": {
+        "utopia-php/database": 20
+    },
     "prefer-stable": false,
     "prefer-lowest": false,
     "platform": {

docker-compose.yml

@@ -84,6 +84,7 @@ services:
       - ./public:/usr/src/code/public
       - ./src:/usr/src/code/src
       - ./dev:/usr/local/dev
+      - ./vendor/utopia-php/framework:/usr/src/code/vendor/utopia-php/framework
     depends_on:
       - mariadb
       - redis
@@ -109,15 +110,20 @@ services:
       - _APP_OPENSSL_KEY_V1
       - _APP_DOMAIN
       - _APP_DOMAIN_TARGET
-      - _APP_REDIS_HOST
-      - _APP_REDIS_PORT
-      - _APP_REDIS_USER
-      - _APP_REDIS_PASS
       - _APP_DB_HOST
       - _APP_DB_PORT
       - _APP_DB_SCHEMA
       - _APP_DB_USER
       - _APP_DB_PASS
+      - _APP_REDIS_HOST
+      - _APP_REDIS_PORT
+      - _APP_REDIS_USER
+      - _APP_REDIS_PASS
+      - _APP_CONNECTIONS_DB_PROJECT
+      - _APP_CONNECTIONS_DB_CONSOLE
+      - _APP_CONNECTIONS_CACHE
+      - _APP_CONNECTIONS_QUEUE
+      - _APP_CONNECTIONS_PUBSUB
       - _APP_SMTP_HOST
       - _APP_SMTP_PORT
       - _APP_SMTP_SECURE
@@ -211,13 +217,19 @@ services:
       - _APP_WORKER_PER_CORE
       - _APP_OPTIONS_ABUSE
       - _APP_OPENSSL_KEY_V1
-      - _APP_REDIS_HOST
-      - _APP_REDIS_PORT
       - _APP_DB_HOST
       - _APP_DB_PORT
       - _APP_DB_SCHEMA
       - _APP_DB_USER
       - _APP_DB_PASS
+      - _APP_REDIS_HOST
+      - _APP_REDIS_PORT
+      - _APP_REDIS_USER
+      - _APP_REDIS_PASS
+      - _APP_CONNECTIONS_DB_CONSOLE
+      - _APP_CONNECTIONS_DB_PROJECT
+      - _APP_CONNECTIONS_CACHE
+      - _APP_CONNECTIONS_PUBSUB
       - _APP_USAGE_STATS
       - _APP_LOGGING_PROVIDER
       - _APP_LOGGING_CONFIG
@@ -238,15 +250,19 @@ services:
     environment:
       - _APP_ENV
       - _APP_OPENSSL_KEY_V1
-      - _APP_REDIS_HOST
-      - _APP_REDIS_PORT
-      - _APP_REDIS_USER
-      - _APP_REDIS_PASS
       - _APP_DB_HOST
       - _APP_DB_PORT
       - _APP_DB_SCHEMA
       - _APP_DB_USER
       - _APP_DB_PASS
+      - _APP_REDIS_HOST
+      - _APP_REDIS_PORT
+      - _APP_REDIS_USER
+      - _APP_REDIS_PASS
+      - _APP_CONNECTIONS_DB_CONSOLE
+      - _APP_CONNECTIONS_DB_PROJECT
+      - _APP_CONNECTIONS_CACHE
+      - _APP_CONNECTIONS_QUEUE
       - _APP_LOGGING_PROVIDER
       - _APP_LOGGING_CONFIG
 
@@ -272,6 +288,7 @@ services:
       - _APP_REDIS_PORT
       - _APP_REDIS_USER
       - _APP_REDIS_PASS
+      - _APP_CONNECTIONS_QUEUE
       - _APP_LOGGING_PROVIDER
       - _APP_LOGGING_CONFIG
 
@@ -296,15 +313,19 @@ services:
     environment:
       - _APP_ENV
       - _APP_OPENSSL_KEY_V1
-      - _APP_REDIS_HOST
-      - _APP_REDIS_PORT
-      - _APP_REDIS_USER
-      - _APP_REDIS_PASS
       - _APP_DB_HOST
       - _APP_DB_PORT
       - _APP_DB_SCHEMA
       - _APP_DB_USER
       - _APP_DB_PASS
+      - _APP_REDIS_HOST
+      - _APP_REDIS_PORT
+      - _APP_REDIS_USER
+      - _APP_REDIS_PASS
+      - _APP_CONNECTIONS_DB_CONSOLE
+      - _APP_CONNECTIONS_DB_PROJECT
+      - _APP_CONNECTIONS_CACHE
+      - _APP_CONNECTIONS_QUEUE
       - _APP_STORAGE_DEVICE
       - _APP_STORAGE_S3_ACCESS_KEY
       - _APP_STORAGE_S3_SECRET
@@ -348,15 +369,19 @@ services:
     environment:
       - _APP_ENV
       - _APP_OPENSSL_KEY_V1
-      - _APP_REDIS_HOST
-      - _APP_REDIS_PORT
-      - _APP_REDIS_USER
-      - _APP_REDIS_PASS
       - _APP_DB_HOST
       - _APP_DB_PORT
       - _APP_DB_SCHEMA
       - _APP_DB_USER
       - _APP_DB_PASS
+      - _APP_REDIS_HOST
+      - _APP_REDIS_PORT
+      - _APP_REDIS_USER
+      - _APP_REDIS_PASS
+      - _APP_CONNECTIONS_DB_CONSOLE
+      - _APP_CONNECTIONS_DB_PROJECT
+      - _APP_CONNECTIONS_CACHE
+      - _APP_CONNECTIONS_QUEUE
       - _APP_LOGGING_PROVIDER
       - _APP_LOGGING_CONFIG
 
@@ -378,15 +403,19 @@ services:
       - _APP_OPENSSL_KEY_V1
       - _APP_EXECUTOR_SECRET
       - _APP_EXECUTOR_HOST
-      - _APP_REDIS_HOST
-      - _APP_REDIS_PORT
-      - _APP_REDIS_USER
-      - _APP_REDIS_PASS
       - _APP_DB_HOST
       - _APP_DB_PORT
       - _APP_DB_SCHEMA
       - _APP_DB_USER
       - _APP_DB_PASS
+      - _APP_REDIS_HOST
+      - _APP_REDIS_PORT
+      - _APP_REDIS_USER
+      - _APP_REDIS_PASS
+      - _APP_CONNECTIONS_DB_CONSOLE
+      - _APP_CONNECTIONS_DB_PROJECT
+      - _APP_CONNECTIONS_CACHE
+      - _APP_CONNECTIONS_QUEUE
       - _APP_LOGGING_PROVIDER
       - _APP_LOGGING_CONFIG
 
@@ -411,15 +440,19 @@ services:
       - _APP_DOMAIN
       - _APP_DOMAIN_TARGET
       - _APP_SYSTEM_SECURITY_EMAIL_ADDRESS
-      - _APP_REDIS_HOST
-      - _APP_REDIS_PORT
-      - _APP_REDIS_USER
-      - _APP_REDIS_PASS
       - _APP_DB_HOST
       - _APP_DB_PORT
       - _APP_DB_SCHEMA
       - _APP_DB_USER
       - _APP_DB_PASS
+      - _APP_REDIS_HOST
+      - _APP_REDIS_PORT
+      - _APP_REDIS_USER
+      - _APP_REDIS_PASS
+      - _APP_CONNECTIONS_DB_CONSOLE
+      - _APP_CONNECTIONS_DB_PROJECT
+      - _APP_CONNECTIONS_CACHE
+      - _APP_CONNECTIONS_QUEUE
       - _APP_LOGGING_PROVIDER
       - _APP_LOGGING_CONFIG
 
@@ -440,15 +473,19 @@ services:
     environment:
       - _APP_ENV
       - _APP_OPENSSL_KEY_V1
-      - _APP_REDIS_HOST
-      - _APP_REDIS_PORT
-      - _APP_REDIS_USER
-      - _APP_REDIS_PASS
       - _APP_DB_HOST
       - _APP_DB_PORT
       - _APP_DB_SCHEMA
       - _APP_DB_USER
       - _APP_DB_PASS
+      - _APP_REDIS_HOST
+      - _APP_REDIS_PORT
+      - _APP_REDIS_USER
+      - _APP_REDIS_PASS
+      - _APP_CONNECTIONS_DB_CONSOLE
+      - _APP_CONNECTIONS_DB_PROJECT
+      - _APP_CONNECTIONS_CACHE
+      - _APP_CONNECTIONS_QUEUE
       - _APP_FUNCTIONS_TIMEOUT
       - _APP_EXECUTOR_SECRET
       - _APP_EXECUTOR_HOST
@@ -540,6 +577,7 @@ services:
       - _APP_REDIS_PORT
       - _APP_REDIS_USER
       - _APP_REDIS_PASS
+      - _APP_CONNECTIONS_QUEUE
       - _APP_SMTP_HOST
       - _APP_SMTP_PORT
       - _APP_SMTP_SECURE
@@ -566,6 +604,7 @@ services:
       - _APP_REDIS_PORT
       - _APP_REDIS_USER
       - _APP_REDIS_PASS
+      - _APP_CONNECTIONS_QUEUE
       - _APP_SMS_PROVIDER
       - _APP_SMS_FROM
       - _APP_LOGGING_PROVIDER
@@ -589,15 +628,18 @@ services:
       - _APP_DOMAIN
       - _APP_DOMAIN_TARGET
       - _APP_OPENSSL_KEY_V1
-      - _APP_REDIS_HOST
-      - _APP_REDIS_PORT
-      - _APP_REDIS_USER
-      - _APP_REDIS_PASS
       - _APP_DB_HOST
       - _APP_DB_PORT
       - _APP_DB_SCHEMA
       - _APP_DB_USER
       - _APP_DB_PASS
+      - _APP_REDIS_HOST
+      - _APP_REDIS_PORT
+      - _APP_REDIS_USER
+      - _APP_REDIS_PASS
+      - _APP_CONNECTIONS_DB_CONSOLE
+      - _APP_CONNECTIONS_DB_PROJECT
+      - _APP_CONNECTIONS_CACHE
       - _APP_MAINTENANCE_INTERVAL
       - _APP_MAINTENANCE_RETENTION_EXECUTION
       - _APP_MAINTENANCE_RETENTION_CACHE
@@ -634,6 +676,9 @@ services:
       - _APP_REDIS_PORT
       - _APP_REDIS_USER
       - _APP_REDIS_PASS
+      - _APP_CONNECTIONS_DB_CONSOLE
+      - _APP_CONNECTIONS_DB_PROJECT
+      - _APP_CONNECTIONS_CACHE
       - _APP_LOGGING_PROVIDER
       - _APP_LOGGING_CONFIG
 
@@ -655,6 +700,7 @@ services:
       - _APP_REDIS_PORT
       - _APP_REDIS_USER
       - _APP_REDIS_PASS
+      - _APP_CONNECTIONS_QUEUE
 
   mariadb:
     image: mariadb:10.7 # fix issues when upgrading using: mysql_upgrade -u root -p
@@ -677,7 +723,6 @@ services:
   # smtp:
   #   image: appwrite/smtp:1.2.0
   #   container_name: appwrite-smtp
-  #   restart: unless-stopped
   #   networks:
   #     - appwrite
   #   environment:
@@ -764,7 +809,6 @@ services:
     image: adminer
     container_name: appwrite-adminer
     <<: *x-logging
-    restart: always
     ports:
       - 9506:8080
     networks:
@@ -772,7 +816,6 @@ services:
 
   # redis-commander:
   #   image: rediscommander/redis-commander:latest
-  #   restart: unless-stopped
   #   networks:
   #   - appwrite
   #   environment:
@@ -782,7 +825,6 @@ services:
 
   # resque:
   #   image: appwrite/resque-web:1.1.0
-  #   restart: unless-stopped
   #   networks:
   #     - appwrite
   #   ports:
@@ -796,7 +838,6 @@ services:
   # chronograf:
   #   image: chronograf:1.6
   #   container_name: appwrite-chronograf
-  #   restart: unless-stopped
   #   networks:
   #   - appwrite
   #   volumes:

docs/references/health/get-cache.md

@@ -1 +1 @@
-Check the Appwrite in-memory cache server is up and connection is successful.
+Check the Appwrite in-memory cache servers are up and connection is successful.

docs/references/health/get-db.md

@@ -1 +1 @@
-Check the Appwrite database server is up and connection is successful.
+Check the Appwrite database servers are up and connection is successful.

docs/references/health/get-pubsub.md

@@ -0,0 +1 @@
+Check the Appwrite pub-sub servers are up and connection is successful.

docs/references/health/get-queue.md

@@ -0,0 +1 @@
+Check the Appwrite queue messaging servers are up and connection is successful.

phpunit.xml

@@ -7,7 +7,7 @@
          convertWarningsToExceptions="true"
          processIsolation="false"
          stopOnFailure="false"
->
+        >
     <extensions>
         <extension class="Appwrite\Tests\TestHook" />
     </extensions>

src/Appwrite/Extend/PDO.php

@@ -1,110 +0,0 @@
-<?php
-
-namespace Appwrite\Extend;
-
-use PDO as PDONative;
-
-class PDO extends PDONative
-{
-    /**
-     * @var PDONative
-     */
-    protected $pdo;
-
-    /**
-     * @var mixed
-     */
-    protected $dsn;
-
-    /**
-     * @var mixed
-     */
-    protected $username;
-
-    /**
-     * @var mixed
-     */
-    protected $passwd;
-
-    /**
-     * @var mixed
-     */
-    protected $options;
-
-    /**
-     * Create A Proxy PDO Object
-     */
-    public function __construct($dsn, $username = null, $passwd = null, $options = null)
-    {
-        $this->dsn = $dsn;
-        $this->username = $username;
-        $this->passwd = $passwd;
-        $this->options = $options;
-
-        $this->pdo = new PDONative($dsn, $username, $passwd, $options);
-    }
-
-    public function setAttribute($attribute, $value)
-    {
-        return $this->pdo->setAttribute($attribute, $value);
-    }
-
-    public function prepare($statement, $driver_options = null)
-    {
-        return new PDOStatement($this, $this->pdo->prepare($statement, []));
-    }
-
-    public function quote($string, $parameter_type = PDONative::PARAM_STR)
-    {
-        return $this->pdo->quote($string, $parameter_type);
-    }
-
-    public function beginTransaction()
-    {
-        try {
-            $result = $this->pdo->beginTransaction();
-        } catch (\Throwable $th) {
-            $this->pdo = $this->reconnect();
-            $result = $this->pdo->beginTransaction();
-        }
-
-        return $result;
-    }
-
-    public function rollBack()
-    {
-        try {
-            $result = $this->pdo->rollBack();
-        } catch (\Throwable $th) {
-            $this->pdo = $this->reconnect();
-            return false;
-        }
-
-        return $result;
-    }
-
-    public function commit()
-    {
-        try {
-            $result = $this->pdo->commit();
-        } catch (\Throwable $th) {
-            $this->pdo = $this->reconnect();
-            $result = $this->pdo->commit();
-        }
-
-        return $result;
-    }
-
-    public function reconnect(): PDONative
-    {
-        $this->pdo = new PDONative($this->dsn, $this->username, $this->passwd, $this->options);
-
-        echo '[PDO] MySQL connection restarted' . PHP_EOL;
-
-        // Connection settings
-        $this->pdo->setAttribute(PDONative::ATTR_DEFAULT_FETCH_MODE, PDONative::FETCH_ASSOC);   // Return arrays
-        $this->pdo->setAttribute(PDONative::ATTR_ERRMODE, PDONative::ERRMODE_EXCEPTION);        // Handle all errors with exceptions
-
-        return $this->pdo;
-    }
-}

src/Appwrite/Extend/PDOStatement.php

@@ -1,115 +0,0 @@
-<?php
-
-namespace Appwrite\Extend;
-
-use PDO as PDONative;
-use PDOStatement as PDOStatementNative;
-
-class PDOStatement extends PDOStatementNative
-{
-    /**
-     * @var PDO
-     */
-    protected $pdo;
-
-    /**
-     * Params
-     */
-    protected $params = [];
-
-    /**
-     * Values
-     */
-    protected $values = [];
-
-    /**
-     * Columns
-     */
-    protected $columns = [];
-
-    /**
-     * @var PDOStatementNative
-     */
-    protected $PDOStatement;
-
-    public function __construct(PDO &$pdo, PDOStatementNative $PDOStatement)
-    {
-        $this->pdo = &$pdo;
-        $this->PDOStatement = $PDOStatement;
-    }
-
-    public function bindValue($parameter, $value, $data_type = PDONative::PARAM_STR)
-    {
-        $this->values[$parameter] = ['value' => $value, 'data_type' => $data_type];
-
-        $result = $this->PDOStatement->bindValue($parameter, $value, $data_type);
-
-        return $result;
-    }
-
-    public function bindParam($parameter, &$variable, $data_type = PDONative::PARAM_STR, $length = null, $driver_options = null)
-    {
-        $this->params[$parameter] = ['value' => &$variable, 'data_type' => $data_type, 'length' => $length, 'driver_options' => $driver_options];
-
-        $result = $this->PDOStatement->bindParam($parameter, $variable, $data_type, $length, $driver_options);
-
-        return $result;
-    }
-
-    public function bindColumn($column, &$param, $type = null, $maxlen = null, $driverdata = null)
-    {
-        $this->columns[$column] = ['param' => &$param, 'type' => $type, 'maxlen' => $maxlen, 'driverdata' => $driverdata];
-
-        $result = $this->PDOStatement->bindColumn($column, $param, $type, $maxlen, $driverdata);
-
-        return $result;
-    }
-
-    public function execute($input_parameters = null)
-    {
-        try {
-            $result = $this->PDOStatement->execute($input_parameters);
-        } catch (\Throwable $th) {
-            $this->pdo = $this->pdo->reconnect();
-            $this->PDOStatement = $this->pdo->prepare($this->PDOStatement->queryString, []);
-
-            foreach ($this->values as $key => $set) {
-                $this->PDOStatement->bindValue($key, $set['value'], $set['data_type']);
-            }
-
-            foreach ($this->params as $key => $set) {
-                $this->PDOStatement->bindParam($key, $set['variable'], $set['data_type'], $set['length'], $set['driver_options']);
-            }
-
-            foreach ($this->columns as $key => $set) {
-                $this->PDOStatement->bindColumn($key, $set['param'], $set['type'], $set['maxlen'], $set['driverdata']);
-            }
-
-            $result = $this->PDOStatement->execute($input_parameters);
-        }
-
-        return $result;
-    }
-
-    public function fetch($fetch_style = PDONative::FETCH_ASSOC, $cursor_orientation = PDONative::FETCH_ORI_NEXT, $cursor_offset = 0)
-    {
-        $result = $this->PDOStatement->fetch($fetch_style, $cursor_orientation, $cursor_offset);
-
-        return $result;
-    }
-
-    /**
-     * Fetch All
-     *
-     * @param int $fetch_style
-     * @param mixed $fetch_args
-     *
-     * @return array|false
-     */
-    public function fetchAll(int $fetch_style = PDO::FETCH_BOTH, mixed ...$fetch_args)
-    {
-        $result = $this->PDOStatement->fetchAll();
-
-        return $result;
-    }
-}

src/Appwrite/Migration/Migration.php

@@ -86,8 +86,6 @@ abstract class Migration
     {
         $this->project = $project;
         $this->projectDB = $projectDB;
-        $this->projectDB->setNamespace('_' . $this->project->getId());
-
         $this->consoleDB = $consoleDB;
 
         return $this;

src/Appwrite/Resque/Worker.php

@@ -2,12 +2,12 @@
 
 namespace Appwrite\Resque;
 
+use Exception;
 use Utopia\App;
 use Utopia\Cache\Cache;
-use Utopia\Cache\Adapter\Redis as RedisCache;
-use Utopia\CLI\Console;
+use Utopia\Config\Config;
+use Utopia\Cache\Adapter\Sharding;
 use Utopia\Database\Database;
-use Utopia\Database\Adapter\MariaDB;
 use Utopia\Storage\Device;
 use Utopia\Storage\Storage;
 use Utopia\Storage\Device\Local;
@@ -16,7 +16,7 @@ use Utopia\Storage\Device\Linode;
 use Utopia\Storage\Device\Wasabi;
 use Utopia\Storage\Device\Backblaze;
 use Utopia\Storage\Device\S3;
-use Exception;
+use Utopia\Database\Document;
 use Utopia\Database\Validator\Authorization;
 
 abstract class Worker
@@ -136,7 +136,12 @@ abstract class Worker
      */
     public function tearDown(): void
     {
+        global $register;
+
         try {
+            $pools = $register->get('pools'); /** @var \Utopia\Pools\Group $pools */
+            $pools->reclaim();
+
             $this->shutdown();
         } catch (\Throwable $error) {
             foreach (self::$errorCallbacks as $errorCallback) {
@@ -158,23 +163,32 @@ abstract class Worker
     {
         \array_push(self::$errorCallbacks, $callback);
     }
+
     /**
      * Get internal project database
-     * @param string $projectId
+     * @param Document $project
      * @return Database
      */
-    protected function getProjectDB(string $projectId): Database
+    protected function getProjectDB(Document $project): Database
     {
-        $consoleDB = $this->getConsoleDB();
+        global $register;
 
-        if ($projectId === 'console') {
-            return $consoleDB;
+        $pools = $register->get('pools'); /** @var \Utopia\Pools\Group $pools */
+
+        if ($project->isEmpty() || $project->getId() === 'console') {
+            return $this->getConsoleDB();
         }
 
-        /** @var Document $project */
-        $project = Authorization::skip(fn() => $consoleDB->getDocument('projects', $projectId));
+        $dbAdapter = $pools
+            ->get($project->getAttribute('database'))
+            ->pop()
+            ->getResource()
+        ;
 
-        return $this->getDB(self::DATABASE_PROJECT, $projectId, $project->getInternalId());
+        $database = new Database($dbAdapter, $this->getCache());
+        $database->setNamespace('_' . $project->getInternalId());
+
+        return $database;
     }
 
     /**
@@ -183,67 +197,46 @@ abstract class Worker
      */
     protected function getConsoleDB(): Database
     {
-        return $this->getDB(self::DATABASE_CONSOLE);
+        global $register;
+
+        $pools = $register->get('pools'); /** @var \Utopia\Pools\Group $pools */
+
+        $dbAdapter = $pools
+            ->get('console')
+            ->pop()
+            ->getResource()
+        ;
+
+        $database = new Database($dbAdapter, $this->getCache());
+
+        $database->setNamespace('console');
+
+        return $database;
     }
 
+
     /**
-     * Get console database
-     * @param string $type One of (internal, external, console)
-     * @param string $projectId of internal or external DB
-     * @return Database
+     * Get Cache
+     * @return Cache
      */
-    private function getDB(string $type, string $projectId = '', string $projectInternalId = ''): Database
+    protected function getCache(): Cache
     {
         global $register;
 
-        $namespace = '';
-        $sleep = DATABASE_RECONNECT_SLEEP; // overwritten when necessary
+        $pools = $register->get('pools'); /** @var \Utopia\Pools\Group $pools */
 
-        switch ($type) {
-            case self::DATABASE_PROJECT:
-                if (!$projectId) {
-                    throw new \Exception('ProjectID not provided - cannot get database');
-                }
-                $namespace = "_{$projectInternalId}";
-                break;
-            case self::DATABASE_CONSOLE:
-                $namespace = "_console";
-                $sleep = 5; // ConsoleDB needs extra sleep time to ensure tables are created
-                break;
-            default:
-                throw new \Exception('Unknown database type: ' . $type);
-                break;
+        $list = Config::getParam('pools-cache', []);
+        $adapters = [];
+
+        foreach ($list as $value) {
+            $adapters[] = $pools
+                ->get($value)
+                ->pop()
+                ->getResource()
+            ;
         }
 
-        $attempts = 0;
-
-        do {
-            try {
-                $attempts++;
-                $cache = new Cache(new RedisCache($register->get('cache')));
-                $database = new Database(new MariaDB($register->get('db')), $cache);
-                $database->setDefaultDatabase(App::getEnv('_APP_DB_SCHEMA', 'appwrite'));
-                $database->setNamespace($namespace); // Main DB
-
-                if (!empty($projectId) && !$database->getDocument('projects', $projectId)->isEmpty()) {
-                    throw new \Exception("Project does not exist: {$projectId}");
-                }
-
-                if ($type === self::DATABASE_CONSOLE && !$database->exists($database->getDefaultDatabase(), Database::METADATA)) {
-                    throw new \Exception('Console project not ready');
-                }
-
-                break; // leave loop if successful
-            } catch (\Exception $e) {
-                Console::warning("Database not ready. Retrying connection ({$attempts})...");
-                if ($attempts >= DATABASE_RECONNECT_MAX_ATTEMPTS) {
-                    throw new \Exception('Failed to connect to database: ' . $e->getMessage());
-                }
-                sleep($sleep);
-            }
-        } while ($attempts < DATABASE_RECONNECT_MAX_ATTEMPTS);
-
-        return $database;
+        return new Cache(new Sharding($adapters));
     }
 
     /**
@@ -266,7 +259,6 @@ abstract class Worker
         return $this->getDevice(APP_STORAGE_UPLOADS . '/app-' . $projectId);
     }
 
-
     /**
      * Get Builds Storage Device
      * @param string $projectId of the project

src/Appwrite/Utopia/Response.php

@@ -207,6 +207,7 @@ class Response extends SwooleResponse
     public const MODEL_HEALTH_QUEUE = 'healthQueue';
     public const MODEL_HEALTH_TIME = 'healthTime';
     public const MODEL_HEALTH_ANTIVIRUS = 'healthAntivirus';
+    public const MODEL_HEALTH_STATUS_LIST = 'healthStatusList';
 
     // Deprecated
     public const MODEL_PERMISSIONS = 'permissions';
@@ -268,6 +269,7 @@ class Response extends SwooleResponse
             ->setModel(new BaseList('Phones List', self::MODEL_PHONE_LIST, 'phones', self::MODEL_PHONE))
             ->setModel(new BaseList('Metric List', self::MODEL_METRIC_LIST, 'metrics', self::MODEL_METRIC, true, false))
             ->setModel(new BaseList('Variables List', self::MODEL_VARIABLE_LIST, 'variables', self::MODEL_VARIABLE))
+            ->setModel(new BaseList('Status List', self::MODEL_HEALTH_STATUS_LIST, 'statuses', self::MODEL_HEALTH_STATUS))
             // Entities
             ->setModel(new Database())
             ->setModel(new Collection())

src/Appwrite/Utopia/Response/Model/HealthStatus.php

@@ -10,6 +10,12 @@ class HealthStatus extends Model
     public function __construct()
     {
         $this
+            ->addRule('name', [
+                'type' => self::TYPE_STRING,
+                'description' => 'Name of the service.',
+                'default' => '',
+                'example' => 'database',
+            ])
             ->addRule('ping', [
                 'type' => self::TYPE_INTEGER,
                 'description' => 'Duration in milliseconds how long the health check took.',

tests/e2e/Services/Health/HealthCustomServerTest.php

@@ -47,9 +47,9 @@ class HealthCustomServerTest extends Scope
         ], $this->getHeaders()), []);
 
         $this->assertEquals(200, $response['headers']['status-code']);
-        $this->assertEquals('pass', $response['body']['status']);
-        $this->assertIsInt($response['body']['ping']);
-        $this->assertLessThan(100, $response['body']['ping']);
+        $this->assertEquals('pass', $response['body']['statuses'][0]['status']);
+        $this->assertIsInt($response['body']['statuses'][0]['ping']);
+        $this->assertLessThan(100, $response['body']['statuses'][0]['ping']);
 
         /**
          * Test for FAILURE
@@ -69,9 +69,53 @@ class HealthCustomServerTest extends Scope
         ], $this->getHeaders()), []);
 
         $this->assertEquals(200, $response['headers']['status-code']);
-        $this->assertEquals('pass', $response['body']['status']);
-        $this->assertIsInt($response['body']['ping']);
-        $this->assertLessThan(100, $response['body']['ping']);
+        $this->assertEquals('pass', $response['body']['statuses'][0]['status']);
+        $this->assertIsInt($response['body']['statuses'][0]['ping']);
+        $this->assertLessThan(100, $response['body']['statuses'][0]['ping']);
+
+        /**
+         * Test for FAILURE
+         */
+
+        return [];
+    }
+
+    public function testQueueSuccess(): array
+    {
+        /**
+         * Test for SUCCESS
+         */
+        $response = $this->client->call(Client::METHOD_GET, '/health/queue', array_merge([
+            'content-type' => 'application/json',
+            'x-appwrite-project' => $this->getProject()['$id'],
+        ], $this->getHeaders()), []);
+
+        $this->assertEquals(200, $response['headers']['status-code']);
+        $this->assertEquals('pass', $response['body']['statuses'][0]['status']);
+        $this->assertIsInt($response['body']['statuses'][0]['ping']);
+        $this->assertLessThan(100, $response['body']['statuses'][0]['ping']);
+
+        /**
+         * Test for FAILURE
+         */
+
+        return [];
+    }
+
+    public function testPubSubSuccess(): array
+    {
+        /**
+         * Test for SUCCESS
+         */
+        $response = $this->client->call(Client::METHOD_GET, '/health/pubsub', array_merge([
+            'content-type' => 'application/json',
+            'x-appwrite-project' => $this->getProject()['$id'],
+        ], $this->getHeaders()), []);
+
+        $this->assertEquals(200, $response['headers']['status-code']);
+        $this->assertEquals('pass', $response['body']['statuses'][0]['status']);
+        $this->assertIsInt($response['body']['statuses'][0]['ping']);
+        $this->assertLessThan(100, $response['body']['statuses'][0]['ping']);
 
         /**
          * Test for FAILURE