Pin each project to a specific database worker and scale to 4 workers

- Hash project ID (crc32) modulo _APP_DATABASE_WORKERS to determine
  which database worker queue to use for each project
- Add _APP_DATABASE_WORKERS env var (default 4)
- Replace single appwrite-worker-databases service with 4 partitioned
  workers (appwrite-worker-databases-0 through 3)
- Update health check endpoint default queue name
- Update install template with 4 database worker services
- Update health check test for new queue naming

Co-authored-by: abnegate <5857008+abnegate@users.noreply.github.com>

.env

@@ -133,6 +133,7 @@ _APP_WEBHOOK_MAX_FAILED_ATTEMPTS=10
 _APP_PROJECT_REGIONS=default
 _APP_FUNCTIONS_CREATION_ABUSE_LIMIT=5000
 _APP_STATS_USAGE_DUAL_WRITING_DBS=database_db_main
+_APP_DATABASE_WORKERS=4
 _APP_TRUSTED_HEADERS=x-forwarded-for
 _APP_POOL_ADAPTER=stack
 _APP_WORKER_SCREENSHOTS_ROUTER=http://appwrite

app/views/install/compose.phtml

@@ -180,6 +180,7 @@ $dbService = $this->getParam('database');
       - _APP_MIGRATIONS_FIREBASE_CLIENT_ID
       - _APP_MIGRATIONS_FIREBASE_CLIENT_SECRET
       - _APP_ASSISTANT_OPENAI_API_KEY
+      - _APP_DATABASE_WORKERS
   appwrite-console:
     <<: *x-logging
     container_name: appwrite-console
@@ -366,11 +367,12 @@ $dbService = $this->getParam('database');
       - _APP_SYSTEM_SECURITY_EMAIL_ADDRESS
       - _APP_EMAIL_CERTIFICATES
 
-  appwrite-worker-databases:
+<?php for ($i = 0; $i < 4; $i++) { ?>
+  appwrite-worker-databases-<?= $i ?>:
     image: <?php echo $organization; ?>/<?php echo $image; ?>:<?php echo $version."\n"; ?>
     entrypoint: worker-databases
     <<: *x-logging
-    container_name: appwrite-worker-databases
+    container_name: appwrite-worker-databases-<?= $i."\n" ?>
     restart: unless-stopped
     networks:
       - appwrite
@@ -392,8 +394,9 @@ $dbService = $this->getParam('database');
       - _APP_DB_PASS
       - _APP_DB_ADAPTER
       - _APP_LOGGING_CONFIG
+      - _APP_QUEUE_NAME=database_db_main-<?= $i."\n" ?>
 
-  appwrite-worker-builds:
+<?php } ?>  appwrite-worker-builds:
     image: <?php echo $organization; ?>/<?php echo $image; ?>:<?php echo $version."\n"; ?>
     entrypoint: worker-builds
     <<: *x-logging

app/worker.php

@@ -518,7 +518,7 @@ if (!isset($args[1])) {
 $workerName = $args[0];
 
 if (\str_starts_with($workerName, 'databases')) {
-    $queueName = System::getEnv('_APP_QUEUE_NAME', 'database_db_main');
+    $queueName = System::getEnv('_APP_QUEUE_NAME', 'database_db_main-0');
 } else {
     $queueName = System::getEnv('_APP_QUEUE_NAME', 'v1-' . strtolower($workerName));
 }

docker-compose.yml

@@ -232,6 +232,7 @@ services:
       - _APP_CUSTOM_DOMAIN_DENY_LIST
       - _APP_TRUSTED_HEADERS
       - _APP_MIGRATION_HOST
+      - _APP_DATABASE_WORKERS
     extra_hosts:
       - "host.docker.internal:host-gateway"
 
@@ -440,10 +441,10 @@ services:
       - _APP_MAINTENANCE_RETENTION_AUDIT
       - _APP_MAINTENANCE_RETENTION_AUDIT_CONSOLE
 
-  appwrite-worker-databases:
+  appwrite-worker-databases-0:
     entrypoint: worker-databases
     <<: *x-logging
-    container_name: appwrite-worker-databases
+    container_name: appwrite-worker-databases-0
     image: appwrite-dev
     networks:
       - appwrite
@@ -470,7 +471,106 @@ services:
       - _APP_DB_ADAPTER
       - _APP_LOGGING_CONFIG
       - _APP_WORKERS_NUM
-      - _APP_QUEUE_NAME
+      - _APP_QUEUE_NAME=database_db_main-0
+      - _APP_DATABASE_SHARED_TABLES
+
+  appwrite-worker-databases-1:
+    entrypoint: worker-databases
+    <<: *x-logging
+    container_name: appwrite-worker-databases-1
+    image: appwrite-dev
+    networks:
+      - appwrite
+    volumes:
+      - ./app:/usr/src/code/app
+      - ./src:/usr/src/code/src
+    depends_on:
+      - redis
+      - ${_APP_DB_HOST:-mariadb}
+    environment:
+      - _APP_ENV
+      - _APP_WORKER_PER_CORE
+      - _APP_POOL_ADAPTER
+      - _APP_OPENSSL_KEY_V1
+      - _APP_REDIS_HOST
+      - _APP_REDIS_PORT
+      - _APP_REDIS_USER
+      - _APP_REDIS_PASS
+      - _APP_DB_HOST
+      - _APP_DB_PORT
+      - _APP_DB_SCHEMA
+      - _APP_DB_USER
+      - _APP_DB_PASS
+      - _APP_DB_ADAPTER
+      - _APP_LOGGING_CONFIG
+      - _APP_WORKERS_NUM
+      - _APP_QUEUE_NAME=database_db_main-1
+      - _APP_DATABASE_SHARED_TABLES
+
+  appwrite-worker-databases-2:
+    entrypoint: worker-databases
+    <<: *x-logging
+    container_name: appwrite-worker-databases-2
+    image: appwrite-dev
+    networks:
+      - appwrite
+    volumes:
+      - ./app:/usr/src/code/app
+      - ./src:/usr/src/code/src
+    depends_on:
+      - redis
+      - ${_APP_DB_HOST:-mariadb}
+    environment:
+      - _APP_ENV
+      - _APP_WORKER_PER_CORE
+      - _APP_POOL_ADAPTER
+      - _APP_OPENSSL_KEY_V1
+      - _APP_REDIS_HOST
+      - _APP_REDIS_PORT
+      - _APP_REDIS_USER
+      - _APP_REDIS_PASS
+      - _APP_DB_HOST
+      - _APP_DB_PORT
+      - _APP_DB_SCHEMA
+      - _APP_DB_USER
+      - _APP_DB_PASS
+      - _APP_DB_ADAPTER
+      - _APP_LOGGING_CONFIG
+      - _APP_WORKERS_NUM
+      - _APP_QUEUE_NAME=database_db_main-2
+      - _APP_DATABASE_SHARED_TABLES
+
+  appwrite-worker-databases-3:
+    entrypoint: worker-databases
+    <<: *x-logging
+    container_name: appwrite-worker-databases-3
+    image: appwrite-dev
+    networks:
+      - appwrite
+    volumes:
+      - ./app:/usr/src/code/app
+      - ./src:/usr/src/code/src
+    depends_on:
+      - redis
+      - ${_APP_DB_HOST:-mariadb}
+    environment:
+      - _APP_ENV
+      - _APP_WORKER_PER_CORE
+      - _APP_POOL_ADAPTER
+      - _APP_OPENSSL_KEY_V1
+      - _APP_REDIS_HOST
+      - _APP_REDIS_PORT
+      - _APP_REDIS_USER
+      - _APP_REDIS_PASS
+      - _APP_DB_HOST
+      - _APP_DB_PORT
+      - _APP_DB_SCHEMA
+      - _APP_DB_USER
+      - _APP_DB_PASS
+      - _APP_DB_ADAPTER
+      - _APP_LOGGING_CONFIG
+      - _APP_WORKERS_NUM
+      - _APP_QUEUE_NAME=database_db_main-3
       - _APP_DATABASE_SHARED_TABLES
 
   appwrite-worker-builds:

src/Appwrite/Event/Database.php

@@ -172,7 +172,10 @@ class Database extends Event
                 // TODO: Temporary until all projects are using shared tables
                 $dsn = new DSN("mysql://$database");
             }
-            $this->queue = $dsn->getHost();
+
+            $numWorkers = (int) System::getEnv('_APP_DATABASE_WORKERS', 4);
+            $workerIndex = \abs(\crc32($project->getId())) % $numWorkers;
+            $this->queue = $dsn->getHost() . '-' . $workerIndex;
         }
 
         return parent::setProject($project);

src/Appwrite/Platform/Modules/Health/Http/Health/Queue/Databases/Get.php

@@ -42,7 +42,7 @@ class Get extends Base
                 ],
                 contentType: ContentType::JSON
             ))
-            ->param('name', 'database_db_main', new Text(256), 'Queue name for which to check the queue size', true)
+            ->param('name', 'database_db_main-0', new Text(256), 'Queue name for which to check the queue size', true)
             ->param('threshold', 5000, new Integer(true), 'Queue size threshold. When hit (equal or higher), endpoint returns server error. Default value is 5000.', true)
             ->inject('queueForDatabase')
             ->inject('response')

tests/e2e/Services/Health/DatabasesQueueTest.php

@@ -6,12 +6,12 @@ class DatabasesQueueTest extends HealthBase
 {
     public function testDatabasesQueue(): void
     {
-        $response = $this->callGet('/health/queue/databases', ['name' => 'database_db_main']);
+        $response = $this->callGet('/health/queue/databases', ['name' => 'database_db_main-0']);
         $this->assertEquals(200, $response['headers']['status-code']);
         $this->assertIsInt($response['body']['size']);
         $this->assertLessThan(100, $response['body']['size']);
 
-        $failure = $this->callGet('/health/queue/databases', ['name' => 'database_db_main', 'threshold' => '0']);
+        $failure = $this->callGet('/health/queue/databases', ['name' => 'database_db_main-0', 'threshold' => '0']);
         $this->assertEquals(503, $failure['headers']['status-code']);
     }
 }