From 7e25b4fad9c9df579af3432388fb4e2b11cd2b26 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Thu, 30 Apr 2026 00:41:09 +1200 Subject: [PATCH] feat: wire SQLite database adapter for tests SQLite gives e2e tests a single shared file across all containers without needing a heavyweight MariaDB/MongoDB/PostgreSQL service. WAL + busy_timeout keep multi-process writers tolerable. Memory adapter was considered but its per-process PHP arrays don't survive Swoole worker fan-out or cross-container queue handoffs, so file-backed SQLite is the practical choice. - registers: add 'sqlite' scheme to pool factory and worker db register, configured with PRAGMA journal_mode=WAL, busy_timeout=5000, foreign_keys=ON - compose: pass _APP_DB_SQLITE_PATH and mount appwrite-sqlite:/storage/sqlite on every service that touches the main DB - .env: switch _APP_DB_ADAPTER to sqlite, drop mariadb from COMPOSE_PROFILES - ci.yml: add SQLite to the e2e matrix alongside MongoDB so each service is timed against both adapters in the same workflow run Co-Authored-By: Claude Opus 4.7 (1M context) --- .env | 13 ++++--- .github/workflows/ci.yml | 10 ++++- app/init/registers.php | 29 ++++++++++++-- docker-compose.yml | 47 ++++++++++++++++++++++- tests/resources/docker/docker-compose.yml | 15 ++++++++ 5 files changed, 102 insertions(+), 12 deletions(-) diff --git a/.env b/.env index 3dc7afe34a..8840493591 100644 --- a/.env +++ b/.env @@ -39,14 +39,15 @@ _APP_REDIS_HOST=redis _APP_REDIS_PORT=6379 _APP_REDIS_PASS= _APP_REDIS_USER= -COMPOSE_PROFILES=mariadb,mongodb,postgresql -_APP_DB_ADAPTER=mongodb +COMPOSE_PROFILES=mongodb,postgresql +_APP_DB_ADAPTER=sqlite _APP_DB_HOST=mongodb -_APP_DB_PORT=27017 +_APP_DB_PORT=0 _APP_DB_SCHEMA=appwrite -_APP_DB_USER=user -_APP_DB_PASS=password -_APP_DB_ROOT_PASS=rootsecretpassword +_APP_DB_USER= +_APP_DB_PASS= +_APP_DB_ROOT_PASS= +_APP_DB_SQLITE_PATH=/storage/sqlite/appwrite.db _APP_DB_ADAPTER_DOCUMENTSDB=mongodb _APP_DB_HOST_DOCUMENTSDB=mongodb _APP_DB_PORT_DOCUMENTSDB=27017 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e521ac3771..cf0ecf833c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -210,10 +210,10 @@ jobs: uses: actions/github-script@v8 with: script: | - const allDatabases = ['MariaDB', 'PostgreSQL', 'MongoDB']; + const allDatabases = ['MariaDB', 'PostgreSQL', 'MongoDB', 'SQLite']; const allModes = ['dedicated', 'shared']; - const defaultDatabases = ['MongoDB']; + const defaultDatabases = ['MongoDB', 'SQLite']; const defaultModes = ['dedicated']; const pr = context.payload.pull_request; @@ -475,6 +475,12 @@ jobs: echo "_APP_DB_ADAPTER=postgresql" >> $GITHUB_ENV echo "_APP_DB_HOST=postgresql" >> $GITHUB_ENV echo "_APP_DB_PORT=5432" >> $GITHUB_ENV + elif [ "${{ matrix.database }}" = "SQLite" ]; then + echo "COMPOSE_PROFILES=mongodb" >> $GITHUB_ENV + echo "_APP_DB_ADAPTER=sqlite" >> $GITHUB_ENV + echo "_APP_DB_HOST=mongodb" >> $GITHUB_ENV + echo "_APP_DB_PORT=0" >> $GITHUB_ENV + echo "_APP_DB_SQLITE_PATH=/storage/sqlite/appwrite.db" >> $GITHUB_ENV fi - name: Login to Docker Hub diff --git a/app/init/registers.php b/app/init/registers.php index 54c0053a33..c7ea32d863 100644 --- a/app/init/registers.php +++ b/app/init/registers.php @@ -15,6 +15,7 @@ use Utopia\Database\Adapter\Mongo; use Utopia\Database\Adapter\MySQL; use Utopia\Database\Adapter\Postgres; use Utopia\Database\Adapter\SQL; +use Utopia\Database\Adapter\SQLite; use Utopia\Database\PDO; use Utopia\Domains\Validator\PublicDomain; use Utopia\DSN\DSN; @@ -190,13 +191,13 @@ $register->set('pools', function () { 'type' => 'database', 'dsns' => $fallbackForDB, 'multiple' => false, - 'schemes' => ['mariadb', 'mongodb', 'mysql', 'postgresql'], + 'schemes' => ['mariadb', 'mongodb', 'mysql', 'postgresql', 'sqlite'], ], 'database' => [ 'type' => 'database', 'dsns' => $fallbackForDB, 'multiple' => true, - 'schemes' => ['mongodb','mariadb', 'mysql','postgresql'], + 'schemes' => ['mongodb', 'mariadb', 'mysql', 'postgresql', 'sqlite'], ], 'documentsdb' => [ 'type' => 'database', @@ -214,7 +215,7 @@ $register->set('pools', function () { 'type' => 'database', 'dsns' => System::getEnv('_APP_CONNECTIONS_DB_LOGS', $fallbackForDB), 'multiple' => false, - 'schemes' => ['mongodb','mariadb', 'mysql','postgresql'], + 'schemes' => ['mongodb', 'mariadb', 'mysql', 'postgresql', 'sqlite'], ], 'publisher' => [ 'type' => 'publisher', @@ -318,6 +319,20 @@ $register->set('pools', function () { )); }); }, + 'sqlite' => function () { + $path = System::getEnv('_APP_DB_SQLITE_PATH', '/tmp/appwrite.db'); + return new PDOProxy(function () use ($path) { + $pdo = new PDO("sqlite:{$path}", null, null, [ + \PDO::ATTR_DEFAULT_FETCH_MODE => \PDO::FETCH_ASSOC, + \PDO::ATTR_EMULATE_PREPARES => true, + \PDO::ATTR_STRINGIFY_FETCHES => true, + ]); + $pdo->query('PRAGMA journal_mode=WAL'); + $pdo->query('PRAGMA busy_timeout=5000'); + $pdo->query('PRAGMA foreign_keys=ON'); + return $pdo; + }); + }, default => function () use ($dsnHost, $dsnPort, $dsnPass) { $redis = new \Redis(); @$redis->pconnect($dsnHost, (int)$dsnPort); @@ -341,6 +356,7 @@ $register->set('pools', function () { 'mysql' => new MySQL($resource()), 'mongodb' => new Mongo($resource()), 'postgresql' => new Postgres($resource()), + 'sqlite' => new SQLite($resource()), default => null }; @@ -415,6 +431,13 @@ $register->set('db', function () { case 'postgresql': $dsn = "pgsql:host={$dbHost};port={$dbPort};dbname={$dbSchema};connect_timeout=3"; return new PDO($dsn, $dbUser, $dbPass, SQL::getPDOAttributes()); + case 'sqlite': + $path = System::getEnv('_APP_DB_SQLITE_PATH', '/tmp/appwrite.db'); + $pdo = new PDO("sqlite:{$path}", null, null, SQL::getPDOAttributes()); + $pdo->query('PRAGMA journal_mode=WAL'); + $pdo->query('PRAGMA busy_timeout=5000'); + $pdo->query('PRAGMA foreign_keys=ON'); + return $pdo; default: throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Invalid database adapter'); } diff --git a/docker-compose.yml b/docker-compose.yml index da5efac438..4b5f8d1da9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -105,6 +105,7 @@ services: - ./docs:/usr/src/code/docs - ./public:/usr/src/code/public - ./src:/usr/src/code/src + - appwrite-sqlite:/storage/sqlite:rw - ./dev:/usr/src/code/dev depends_on: @@ -156,6 +157,7 @@ services: - _APP_REDIS_USER - _APP_REDIS_PASS - _APP_DB_ADAPTER + - _APP_DB_SQLITE_PATH - _APP_DB_HOST - _APP_DB_PORT - _APP_DB_SCHEMA @@ -301,6 +303,7 @@ services: volumes: - ./app:/usr/src/code/app - ./src:/usr/src/code/src + - appwrite-sqlite:/storage/sqlite:rw depends_on: - ${_APP_DB_HOST:-mongodb} - redis @@ -316,6 +319,7 @@ services: - _APP_REDIS_USER - _APP_REDIS_PASS - _APP_DB_ADAPTER + - _APP_DB_SQLITE_PATH - _APP_DB_HOST - _APP_DB_PORT - _APP_DB_SCHEMA @@ -343,6 +347,7 @@ services: volumes: - ./app:/usr/src/code/app - ./src:/usr/src/code/src + - appwrite-sqlite:/storage/sqlite:rw depends_on: - redis - ${_APP_DB_HOST:-mongodb} @@ -357,6 +362,7 @@ services: - _APP_REDIS_USER - _APP_REDIS_PASS - _APP_DB_ADAPTER + - _APP_DB_SQLITE_PATH - _APP_DB_HOST - _APP_DB_PORT - _APP_DB_SCHEMA @@ -375,6 +381,7 @@ services: volumes: - ./app:/usr/src/code/app - ./src:/usr/src/code/src + - appwrite-sqlite:/storage/sqlite:rw depends_on: - redis - ${_APP_DB_HOST:-mongodb} @@ -388,6 +395,7 @@ services: - _APP_OPENSSL_KEY_V1 - _APP_EMAIL_SECURITY - _APP_DB_ADAPTER + - _APP_DB_SQLITE_PATH - _APP_DB_HOST - _APP_DB_PORT - _APP_DB_SCHEMA @@ -421,6 +429,7 @@ services: - appwrite-certificates:/storage/certificates:rw - ./app:/usr/src/code/app - ./src:/usr/src/code/src + - appwrite-sqlite:/storage/sqlite:rw environment: - _APP_ENV @@ -432,6 +441,7 @@ services: - _APP_REDIS_USER - _APP_REDIS_PASS - _APP_DB_ADAPTER + - _APP_DB_SQLITE_PATH - _APP_DB_HOST - _APP_DB_PORT - _APP_DB_SCHEMA @@ -477,6 +487,7 @@ services: volumes: - ./app:/usr/src/code/app - ./src:/usr/src/code/src + - appwrite-sqlite:/storage/sqlite:rw depends_on: - redis @@ -492,6 +503,7 @@ services: - _APP_REDIS_USER - _APP_REDIS_PASS - _APP_DB_ADAPTER + - _APP_DB_SQLITE_PATH - _APP_DB_HOST - _APP_DB_PORT - _APP_DB_SCHEMA @@ -521,6 +533,7 @@ services: - appwrite-builds:/storage/builds:rw - ./app:/usr/src/code/app - ./src:/usr/src/code/src + - appwrite-sqlite:/storage/sqlite:rw depends_on: - redis - ${_APP_DB_HOST:-mongodb} @@ -537,6 +550,7 @@ services: - _APP_REDIS_USER - _APP_REDIS_PASS - _APP_DB_ADAPTER + - _APP_DB_SQLITE_PATH - _APP_DB_HOST - _APP_DB_PORT - _APP_DB_SCHEMA @@ -595,6 +609,7 @@ services: - appwrite-uploads:/storage/uploads:rw - ./app:/usr/src/code/app - ./src:/usr/src/code/src + - appwrite-sqlite:/storage/sqlite:rw depends_on: - redis - ${_APP_DB_HOST:-mongodb} @@ -615,6 +630,7 @@ services: - _APP_REDIS_USER - _APP_REDIS_PASS - _APP_DB_ADAPTER + - _APP_DB_SQLITE_PATH - _APP_DB_HOST - _APP_DB_PORT - _APP_DB_SCHEMA @@ -663,6 +679,7 @@ services: - appwrite-certificates:/storage/certificates:rw - ./app:/usr/src/code/app - ./src:/usr/src/code/src + - appwrite-sqlite:/storage/sqlite:rw environment: - _APP_ENV - _APP_WORKER_PER_CORE @@ -682,6 +699,7 @@ services: - _APP_REDIS_USER - _APP_REDIS_PASS - _APP_DB_ADAPTER + - _APP_DB_SQLITE_PATH - _APP_DB_HOST - _APP_DB_PORT - _APP_DB_SCHEMA @@ -700,6 +718,7 @@ services: volumes: - ./app:/usr/src/code/app - ./src:/usr/src/code/src + - appwrite-sqlite:/storage/sqlite:rw depends_on: - redis - ${_APP_DB_HOST:-mongodb} @@ -712,6 +731,7 @@ services: - _APP_REDIS_USER - _APP_REDIS_PASS - _APP_DB_ADAPTER + - _APP_DB_SQLITE_PATH - _APP_DB_HOST - _APP_DB_PORT - _APP_DB_SCHEMA @@ -730,6 +750,7 @@ services: volumes: - ./app:/usr/src/code/app - ./src:/usr/src/code/src + - appwrite-sqlite:/storage/sqlite:rw depends_on: - ${_APP_DB_HOST:-mongodb} - redis @@ -745,6 +766,7 @@ services: - _APP_REDIS_USER - _APP_REDIS_PASS - _APP_DB_ADAPTER + - _APP_DB_SQLITE_PATH - _APP_DB_HOST - _APP_DB_PORT - _APP_DB_SCHEMA @@ -774,6 +796,7 @@ services: volumes: - ./app:/usr/src/code/app - ./src:/usr/src/code/src + - appwrite-sqlite:/storage/sqlite:rw depends_on: - redis - maildev @@ -796,6 +819,7 @@ services: - _APP_DB_USER - _APP_DB_PASS - _APP_DB_ADAPTER + - _APP_DB_SQLITE_PATH - _APP_SMTP_HOST - _APP_SMTP_PORT - _APP_SMTP_SECURE @@ -818,6 +842,7 @@ services: - appwrite-uploads:/storage/uploads:rw - ./app:/usr/src/code/app - ./src:/usr/src/code/src + - appwrite-sqlite:/storage/sqlite:rw depends_on: - redis environment: @@ -830,6 +855,7 @@ services: - _APP_REDIS_USER - _APP_REDIS_PASS - _APP_DB_ADAPTER + - _APP_DB_SQLITE_PATH - _APP_DB_HOST - _APP_DB_PORT - _APP_DB_SCHEMA @@ -876,6 +902,7 @@ services: - appwrite-uploads:/storage/uploads:rw - ./app:/usr/src/code/app - ./src:/usr/src/code/src + - appwrite-sqlite:/storage/sqlite:rw - ./tests:/usr/src/code/tests depends_on: @@ -897,6 +924,7 @@ services: - _APP_REDIS_USER - _APP_REDIS_PASS - _APP_DB_ADAPTER + - _APP_DB_SQLITE_PATH - _APP_DB_HOST - _APP_DB_PORT - _APP_DB_SCHEMA @@ -919,6 +947,7 @@ services: volumes: - ./app:/usr/src/code/app - ./src:/usr/src/code/src + - appwrite-sqlite:/storage/sqlite:rw depends_on: - ${_APP_DB_HOST:-mongodb} - redis @@ -939,6 +968,7 @@ services: - _APP_REDIS_USER - _APP_REDIS_PASS - _APP_DB_ADAPTER + - _APP_DB_SQLITE_PATH - _APP_DB_HOST - _APP_DB_PORT - _APP_DB_SCHEMA @@ -965,6 +995,7 @@ services: volumes: - ./app:/usr/src/code/app - ./src:/usr/src/code/src + - appwrite-sqlite:/storage/sqlite:rw depends_on: - ${_APP_DB_HOST:-mongodb} - redis @@ -986,6 +1017,7 @@ services: - _APP_REDIS_USER - _APP_REDIS_PASS - _APP_DB_ADAPTER + - _APP_DB_SQLITE_PATH - _APP_DB_HOST - _APP_DB_PORT - _APP_DB_SCHEMA @@ -1005,6 +1037,7 @@ services: volumes: - ./app:/usr/src/code/app - ./src:/usr/src/code/src + - appwrite-sqlite:/storage/sqlite:rw depends_on: - redis - ${_APP_DB_HOST:-mongodb} @@ -1014,6 +1047,7 @@ services: - _APP_POOL_ADAPTER - _APP_OPENSSL_KEY_V1 - _APP_DB_ADAPTER + - _APP_DB_SQLITE_PATH - _APP_DB_HOST - _APP_DB_PORT - _APP_DB_SCHEMA @@ -1038,6 +1072,7 @@ services: volumes: - ./app:/usr/src/code/app - ./src:/usr/src/code/src + - appwrite-sqlite:/storage/sqlite:rw depends_on: - redis - ${_APP_DB_HOST:-mongodb} @@ -1047,6 +1082,7 @@ services: - _APP_POOL_ADAPTER - _APP_OPENSSL_KEY_V1 - _APP_DB_ADAPTER + - _APP_DB_SQLITE_PATH - _APP_DB_HOST - _APP_DB_PORT - _APP_DB_SCHEMA @@ -1071,6 +1107,7 @@ services: volumes: - ./app:/usr/src/code/app - ./src:/usr/src/code/src + - appwrite-sqlite:/storage/sqlite:rw depends_on: - redis - ${_APP_DB_HOST:-mongodb} @@ -1081,6 +1118,7 @@ services: - _APP_POOL_ADAPTER - _APP_OPENSSL_KEY_V1 - _APP_DB_ADAPTER + - _APP_DB_SQLITE_PATH - _APP_DB_HOST - _APP_DB_PORT - _APP_DB_SCHEMA @@ -1105,6 +1143,7 @@ services: volumes: - ./app:/usr/src/code/app - ./src:/usr/src/code/src + - appwrite-sqlite:/storage/sqlite:rw depends_on: - ${_APP_DB_HOST:-mongodb} - redis @@ -1119,6 +1158,7 @@ services: - _APP_REDIS_USER - _APP_REDIS_PASS - _APP_DB_ADAPTER + - _APP_DB_SQLITE_PATH - _APP_DB_HOST - _APP_DB_PORT - _APP_DB_SCHEMA @@ -1136,6 +1176,7 @@ services: volumes: - ./app:/usr/src/code/app - ./src:/usr/src/code/src + - appwrite-sqlite:/storage/sqlite:rw depends_on: - ${_APP_DB_HOST:-mongodb} - redis @@ -1150,6 +1191,7 @@ services: - _APP_REDIS_USER - _APP_REDIS_PASS - _APP_DB_ADAPTER + - _APP_DB_SQLITE_PATH - _APP_DB_HOST - _APP_DB_PORT - _APP_DB_SCHEMA @@ -1167,6 +1209,7 @@ services: volumes: - ./app:/usr/src/code/app - ./src:/usr/src/code/src + - appwrite-sqlite:/storage/sqlite:rw depends_on: - ${_APP_DB_HOST:-mongodb} - redis @@ -1181,6 +1224,7 @@ services: - _APP_REDIS_USER - _APP_REDIS_PASS - _APP_DB_ADAPTER + - _APP_DB_SQLITE_PATH - _APP_DB_HOST - _APP_DB_PORT - _APP_DB_SCHEMA @@ -1478,4 +1522,5 @@ volumes: appwrite-sites: appwrite-builds: appwrite-config: - appwrite-models: \ No newline at end of file + appwrite-models: + appwrite-sqlite: diff --git a/tests/resources/docker/docker-compose.yml b/tests/resources/docker/docker-compose.yml index 47a69f077b..abdac6ab03 100644 --- a/tests/resources/docker/docker-compose.yml +++ b/tests/resources/docker/docker-compose.yml @@ -55,6 +55,7 @@ services: - ./docs:/usr/src/code/docs - ./public:/usr/src/code/public - ./src:/usr/src/code/src + - appwrite-sqlite:/storage/sqlite:rw depends_on: - mariadb - redis @@ -77,6 +78,7 @@ services: - _APP_DB_USER - _APP_DB_PASS - _APP_DB_ADAPTER + - _APP_DB_SQLITE_PATH - _APP_USAGE_STATS - _APP_STORAGE_ANTIVIRUS=disabled - _APP_STORAGE_LIMIT @@ -133,6 +135,8 @@ services: depends_on: - redis - mariadb + volumes: + - appwrite-sqlite:/storage/sqlite:rw environment: - _APP_ENV - _APP_REDIS_HOST @@ -143,6 +147,7 @@ services: - _APP_DB_USER - _APP_DB_PASS - _APP_DB_ADAPTER + - _APP_DB_SQLITE_PATH appwrite-worker-tasks: entrypoint: worker-tasks @@ -155,6 +160,8 @@ services: depends_on: - redis - mariadb + volumes: + - appwrite-sqlite:/storage/sqlite:rw environment: - _APP_ENV - _APP_REDIS_HOST @@ -165,6 +172,7 @@ services: - _APP_DB_USER - _APP_DB_PASS - _APP_DB_ADAPTER + - _APP_DB_SQLITE_PATH appwrite-worker-deletes: entrypoint: worker-deletes @@ -180,12 +188,14 @@ services: volumes: - appwrite-uploads:/storage/uploads:rw - appwrite-cache:/storage/cache:rw + - appwrite-sqlite:/storage/sqlite:rw environment: - _APP_ENV - _APP_REDIS_HOST - _APP_REDIS_PORT - _APP_DB_HOST - _APP_DB_ADAPTER + - _APP_DB_SQLITE_PATH - _APP_DB_PORT - _APP_DB_SCHEMA - _APP_DB_USER @@ -279,6 +289,8 @@ services: - redis - maildev # - smtp + volumes: + - appwrite-sqlite:/storage/sqlite:rw environment: - _APP_ENV - _APP_REDIS_HOST @@ -289,6 +301,7 @@ services: - _APP_DB_USER - _APP_DB_PASS - _APP_DB_ADAPTER + - _APP_DB_SQLITE_PATH - _APP_SMTP_HOST - _APP_SMTP_PORT @@ -302,6 +315,7 @@ services: volumes: - ./app:/usr/src/code/app - ./src:/usr/src/code/src + - appwrite-sqlite:/storage/sqlite:rw depends_on: - redis - mariadb @@ -393,3 +407,4 @@ volumes: appwrite-certificates: appwrite-functions: appwrite-config: + appwrite-sqlite: