diff --git a/.env b/.env index c616312e50..cca7c460e7 100644 --- a/.env +++ b/.env @@ -23,6 +23,11 @@ _APP_DB_SCHEMA=appwrite _APP_DB_USER=user _APP_DB_PASS=password _APP_DB_ROOT_PASS=rootsecretpassword +_APP_CONNECTIONS_DB_PROJECT=db_fra1_02=mysql://user:password@mariadb:3306/appwrite +_APP_CONNECTIONS_DB_CONSOLE=db_fra1_01=mysql://user:password@mariadb:3306/appwrite +_APP_CONNECTIONS_CACHE=redis_fra1_01=redis://redis:6379 +_APP_CONNECTIONS_QUEUE=redis_fra1_01=redis://redis:6379 +_APP_CONNECTIONS_PUBSUB=redis_fra1_01=redis://redis:6379 _APP_STORAGE_DEVICE=Local _APP_STORAGE_S3_ACCESS_KEY= _APP_STORAGE_S3_SECRET= diff --git a/Dockerfile b/Dockerfile index 725fcaa46f..f1997160d9 100755 --- a/Dockerfile +++ b/Dockerfile @@ -35,6 +35,7 @@ ENV PHP_REDIS_VERSION=5.3.7 \ PHP_IMAGICK_VERSION=3.7.0 \ PHP_YAML_VERSION=2.2.2 \ PHP_MAXMINDDB_VERSION=v1.11.0 \ + PHP_MEMCACHED_VERSION=v3.2.0 \ PHP_ZSTD_VERSION="4504e4186e79b197cfcb75d4d09aa47ef7d92fe9 " RUN \ @@ -52,6 +53,7 @@ RUN \ imagemagick \ imagemagick-dev \ libmaxminddb-dev \ + libmemcached-dev \ zstd-dev RUN docker-php-ext-install sockets @@ -125,6 +127,15 @@ RUN \ ./configure && \ make && make install +# Memcached Extension +FROM compile as memcached +RUN \ + git clone --depth 1 --branch $PHP_MEMCACHED_VERSION https://github.com/php-memcached-dev/php-memcached.git && \ + cd php-memcached && \ + phpize && \ + ./configure && \ + make && make install + # Zstd Compression FROM compile as zstd RUN git clone --recursive -n https://github.com/kjdev/php-ext-zstd.git \ @@ -134,7 +145,6 @@ RUN git clone --recursive -n https://github.com/kjdev/php-ext-zstd.git \ && ./configure --with-libzstd \ && make && make install - # Rust Extensions Compile Image FROM php:8.0.18-cli as rust_compile @@ -304,6 +314,7 @@ COPY --from=imagick /usr/local/lib/php/extensions/no-debug-non-zts-20200930/imag COPY --from=yaml /usr/local/lib/php/extensions/no-debug-non-zts-20200930/yaml.so /usr/local/lib/php/extensions/no-debug-non-zts-20200930/ COPY --from=maxmind /usr/local/lib/php/extensions/no-debug-non-zts-20200930/maxminddb.so /usr/local/lib/php/extensions/no-debug-non-zts-20200930/ COPY --from=mongodb /usr/local/lib/php/extensions/no-debug-non-zts-20200930/mongodb.so /usr/local/lib/php/extensions/no-debug-non-zts-20200930/ +COPY --from=memcached /usr/local/lib/php/extensions/no-debug-non-zts-20200930/memcached.so /usr/local/lib/php/extensions/no-debug-non-zts-20200930/ COPY --from=scrypt /usr/local/lib/php/extensions/php-scrypt/target/libphp_scrypt.so /usr/local/lib/php/extensions/no-debug-non-zts-20200930/ COPY --from=zstd /usr/local/lib/php/extensions/no-debug-non-zts-20200930/zstd.so /usr/local/lib/php/extensions/no-debug-non-zts-20200930/ diff --git a/app/cli.php b/app/cli.php index 09b3dc9413..3a62c80816 100644 --- a/app/cli.php +++ b/app/cli.php @@ -6,7 +6,78 @@ require_once __DIR__ . '/controllers/general.php'; use Utopia\App; use Utopia\CLI\CLI; use Utopia\CLI\Console; +use Utopia\Cache\Adapter\Sharding; +use Utopia\Cache\Cache; +use Utopia\Config\Config; +use Utopia\Database\Database; use Utopia\Database\Validator\Authorization; +use InfluxDB\Database as InfluxDatabase; + +function getInfluxDB(): InfluxDatabase +{ + global $register; + + $client = $register->get('influxdb'); /** @var InfluxDB\Client $client */ + $attempts = 0; + $max = 10; + $sleep = 1; + + do { // check if telegraf database is ready + try { + $attempts++; + $database = $client->selectDB('telegraf'); + if (in_array('telegraf', $client->listDatabases())) { + break; // leave the do-while if successful + } + } catch (\Throwable $th) { + Console::warning("InfluxDB not ready. Retrying connection ({$attempts})..."); + if ($attempts >= $max) { + throw new \Exception('InfluxDB database not ready yet'); + } + sleep($sleep); + } + } while ($attempts < $max); + return $database; +} + +function getConsoleDB(): Database +{ + global $register; + + $pools = $register->get('pools'); /** @var \Utopia\Pools\Group $pools */ + + $dbAdapter = $pools + ->get('console') + ->pop() + ->getResource() + ; + + $database = new Database($dbAdapter, getCache()); + + $database->setNamespace('console'); + + return $database; +} + +function getCache(): Cache +{ + global $register; + + $pools = $register->get('pools'); /** @var \Utopia\Pools\Group $pools */ + + $list = Config::getParam('pools-cache', []); + $adapters = []; + + foreach ($list as $value) { + $adapters[] = $pools + ->get($value) + ->pop() + ->getResource() + ; + } + + return new Cache(new Sharding($adapters)); +} Authorization::disable(); @@ -29,4 +100,13 @@ $cli Console::log(App::getEnv('_APP_VERSION', 'UNKNOWN')); }); +$cli + ->error(function ($error) { + if (App::getEnv('_APP_ENV', 'development')) { + Console::error($error); + } else { + Console::error($error->getMessage()); + } + }); + $cli->run(); diff --git a/app/config/collections.php b/app/config/collections.php index d8f65da788..4d88518737 100644 --- a/app/config/collections.php +++ b/app/config/collections.php @@ -534,6 +534,17 @@ $collections = [ 'array' => false, 'filters' => [], ], + [ + '$id' => ID::custom('database'), + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => 256, + 'signed' => true, + 'required' => true, + 'default' => null, + 'array' => false, + 'filters' => [], + ], [ '$id' => ID::custom('logo'), 'type' => Database::VAR_STRING, @@ -2205,69 +2216,69 @@ $collections = [ ], ], 'indexes' => [ - [ - '$id' => ID::custom('_key_search'), - 'type' => Database::INDEX_FULLTEXT, - 'attributes' => ['search'], - 'lengths' => [2048], - 'orders' => [Database::ORDER_ASC], - ], - [ - '$id' => ID::custom('_key_name'), - 'type' => Database::INDEX_KEY, - 'attributes' => ['name'], - 'lengths' => [2048], - 'orders' => [Database::ORDER_ASC], - ], - [ - '$id' => ID::custom('_key_enabled'), - 'type' => Database::INDEX_KEY, - 'attributes' => ['enabled'], - 'lengths' => [], - 'orders' => [Database::ORDER_ASC], - ], - [ - '$id' => ID::custom('_key_runtime'), - 'type' => Database::INDEX_KEY, - 'attributes' => ['runtime'], - 'lengths' => [2048], - 'orders' => [Database::ORDER_ASC], - ], - [ - '$id' => ID::custom('_key_deployment'), - 'type' => Database::INDEX_KEY, - 'attributes' => ['deployment'], - 'lengths' => [Database::LENGTH_KEY], - 'orders' => [Database::ORDER_ASC], - ], - [ - '$id' => ID::custom('_key_schedule'), - 'type' => Database::INDEX_KEY, - 'attributes' => ['schedule'], - 'lengths' => [128], - 'orders' => [Database::ORDER_ASC], - ], - [ - '$id' => ID::custom('_key_scheduleNext'), - 'type' => Database::INDEX_KEY, - 'attributes' => ['scheduleNext'], - 'lengths' => [], - 'orders' => [Database::ORDER_ASC], - ], - [ - '$id' => ID::custom('_key_schedulePrevious'), - 'type' => Database::INDEX_KEY, - 'attributes' => ['schedulePrevious'], - 'lengths' => [], - 'orders' => [Database::ORDER_ASC], - ], - [ - '$id' => ID::custom('_key_timeout'), - 'type' => Database::INDEX_KEY, - 'attributes' => ['timeout'], - 'lengths' => [], - 'orders' => [Database::ORDER_ASC], - ], + // [ + // '$id' => ID::custom('_key_search'), + // 'type' => Database::INDEX_FULLTEXT, + // 'attributes' => ['search'], + // 'lengths' => [], + // 'orders' => [], + // ], + // [ + // '$id' => ID::custom('_key_name'), + // 'type' => Database::INDEX_KEY, + // 'attributes' => ['name'], + // 'lengths' => [700], + // 'orders' => [], + // ], + // [ + // '$id' => ID::custom('_key_enabled'), + // 'type' => Database::INDEX_KEY, + // 'attributes' => ['enabled'], + // 'lengths' => [], + // 'orders' => [], + // ], + // [ + // '$id' => ID::custom('_key_runtime'), + // 'type' => Database::INDEX_KEY, + // 'attributes' => ['runtime'], + // 'lengths' => [700], + // 'orders' => [], + // ], + // [ + // '$id' => ID::custom('_key_deployment'), + // 'type' => Database::INDEX_KEY, + // 'attributes' => ['deployment'], + // 'lengths' => [], + // 'orders' => [], + // ], + // [ + // '$id' => ID::custom('_key_schedule'), + // 'type' => Database::INDEX_KEY, + // 'attributes' => ['schedule'], + // 'lengths' => [], + // 'orders' => [], + // ], + // [ + // '$id' => ID::custom('_key_scheduleNext'), + // 'type' => Database::INDEX_KEY, + // 'attributes' => ['scheduleNext'], + // 'lengths' => [], + // 'orders' => [], + // ], + // [ + // '$id' => ID::custom('_key_schedulePrevious'), + // 'type' => Database::INDEX_KEY, + // 'attributes' => ['schedulePrevious'], + // 'lengths' => [], + // 'orders' => [], + // ], + // [ + // '$id' => ID::custom('_key_timeout'), + // 'type' => Database::INDEX_KEY, + // 'attributes' => ['timeout'], + // 'lengths' => [], + // 'orders' => [], + // ], ], ], @@ -2314,7 +2325,7 @@ $collections = [ '$id' => ID::custom('entrypoint'), 'type' => Database::VAR_STRING, 'format' => '', - 'size' => 2048, + 'size' => 16384, 'signed' => true, 'required' => false, 'default' => null, @@ -2324,7 +2335,7 @@ $collections = [ '$id' => ID::custom('path'), 'type' => Database::VAR_STRING, 'format' => '', - 'size' => 2048, + 'size' => 16384, 'signed' => true, 'required' => false, 'default' => null, @@ -2399,55 +2410,55 @@ $collections = [ ] ], 'indexes' => [ - [ - '$id' => ID::custom('_key_resource'), - 'type' => Database::INDEX_KEY, - 'attributes' => ['resourceId'], - 'lengths' => [Database::LENGTH_KEY], - 'orders' => [Database::ORDER_ASC], - ], - [ - '$id' => ID::custom('_key_resource_type'), - 'type' => Database::INDEX_KEY, - 'attributes' => ['resourceType'], - 'lengths' => [Database::LENGTH_KEY], - 'orders' => [Database::ORDER_ASC], - ], - [ - '$id' => ID::custom('_key_search'), - 'type' => Database::INDEX_FULLTEXT, - 'attributes' => ['search'], - 'lengths' => [], - 'orders' => [], - ], - [ - '$id' => ID::custom('_key_entrypoint'), - 'type' => Database::INDEX_KEY, - 'attributes' => ['entrypoint'], - 'lengths' => [2048], - 'orders' => [Database::ORDER_ASC], - ], - [ - '$id' => ID::custom('_key_size'), - 'type' => Database::INDEX_KEY, - 'attributes' => ['size'], - 'lengths' => [], - 'orders' => [Database::ORDER_ASC], - ], - [ - '$id' => ID::custom('_key_buildId'), - 'type' => Database::INDEX_KEY, - 'attributes' => ['buildId'], - 'lengths' => [Database::LENGTH_KEY], - 'orders' => [Database::ORDER_ASC], - ], - [ - '$id' => ID::custom('_key_activate'), - 'type' => Database::INDEX_KEY, - 'attributes' => ['activate'], - 'lengths' => [], - 'orders' => [Database::ORDER_ASC], - ], + // [ + // '$id' => ID::custom('_key_resource'), + // 'type' => Database::INDEX_KEY, + // 'attributes' => ['resourceId'], + // 'lengths' => [Database::LENGTH_KEY], + // 'orders' => [Database::ORDER_ASC], + // ], + // [ + // '$id' => ID::custom('_key_resource_type'), + // 'type' => Database::INDEX_KEY, + // 'attributes' => ['resourceType'], + // 'lengths' => [Database::LENGTH_KEY], + // 'orders' => [Database::ORDER_ASC], + // ], + // [ + // '$id' => ID::custom('_key_search'), + // 'type' => Database::INDEX_FULLTEXT, + // 'attributes' => ['search'], + // 'lengths' => [], + // 'orders' => [], + // ], + // [ + // '$id' => ID::custom('_key_entrypoint'), + // 'type' => Database::INDEX_KEY, + // 'attributes' => ['entrypoint'], + // 'lengths' => [Database::LENGTH_KEY], + // 'orders' => [Database::ORDER_ASC], + // ], + // [ + // '$id' => ID::custom('_key_size'), + // 'type' => Database::INDEX_KEY, + // 'attributes' => ['size'], + // 'lengths' => [], + // 'orders' => [Database::ORDER_ASC], + // ], + // [ + // '$id' => ID::custom('_key_buildId'), + // 'type' => Database::INDEX_KEY, + // 'attributes' => ['buildId'], + // 'lengths' => [Database::LENGTH_KEY], + // 'orders' => [Database::ORDER_ASC], + // ], + // [ + // '$id' => ID::custom('_key_activate'), + // 'type' => Database::INDEX_KEY, + // 'attributes' => ['activate'], + // 'lengths' => [], + // 'orders' => [Database::ORDER_ASC], + // ], ], ], @@ -2896,7 +2907,7 @@ $collections = [ '$id' => 'compression', 'type' => Database::VAR_STRING, 'signed' => true, - 'size' => 10, + 'size' => 128, 'format' => '', 'filters' => [], 'required' => true, @@ -2935,62 +2946,62 @@ $collections = [ ], ], 'indexes' => [ - [ - '$id' => ID::custom('_fulltext_name'), - 'type' => Database::INDEX_FULLTEXT, - 'attributes' => ['name'], - 'lengths' => [1024], - 'orders' => [Database::ORDER_ASC], - ], - [ - '$id' => ID::custom('_key_search'), - 'type' => Database::INDEX_FULLTEXT, - 'attributes' => ['search'], - 'lengths' => [2048], - 'orders' => [Database::ORDER_ASC], - ], - [ - '$id' => ID::custom('_key_enabled'), - 'type' => Database::INDEX_KEY, - 'attributes' => ['enabled'], - 'lengths' => [], - 'orders' => [Database::ORDER_ASC], - ], - [ - '$id' => ID::custom('_key_name'), - 'type' => Database::INDEX_KEY, - 'attributes' => ['name'], - 'lengths' => [128], - 'orders' => [Database::ORDER_ASC], - ], - [ - '$id' => ID::custom('_key_fileSecurity'), - 'type' => Database::INDEX_KEY, - 'attributes' => ['fileSecurity'], - 'lengths' => [], - 'orders' => [Database::ORDER_ASC], - ], - [ - '$id' => ID::custom('_key_maximumFileSize'), - 'type' => Database::INDEX_KEY, - 'attributes' => ['maximumFileSize'], - 'lengths' => [], - 'orders' => [Database::ORDER_ASC], - ], - [ - '$id' => ID::custom('_key_encryption'), - 'type' => Database::INDEX_KEY, - 'attributes' => ['encryption'], - 'lengths' => [], - 'orders' => [Database::ORDER_ASC], - ], - [ - '$id' => ID::custom('_key_antivirus'), - 'type' => Database::INDEX_KEY, - 'attributes' => ['antivirus'], - 'lengths' => [], - 'orders' => [Database::ORDER_ASC], - ], + // [ + // '$id' => ID::custom('_fulltext_name'), + // 'type' => Database::INDEX_FULLTEXT, + // 'attributes' => ['name'], + // 'lengths' => [1024], + // 'orders' => [Database::ORDER_ASC], + // ], + // [ + // '$id' => ID::custom('_key_search'), + // 'type' => Database::INDEX_FULLTEXT, + // 'attributes' => ['search'], + // 'lengths' => [], + // 'orders' => [Database::ORDER_ASC], + // ], + // [ + // '$id' => ID::custom('_key_enabled'), + // 'type' => Database::INDEX_KEY, + // 'attributes' => ['enabled'], + // 'lengths' => [], + // 'orders' => [Database::ORDER_ASC], + // ], + // [ + // '$id' => ID::custom('_key_name'), + // 'type' => Database::INDEX_KEY, + // 'attributes' => ['name'], + // 'lengths' => [128], + // 'orders' => [Database::ORDER_ASC], + // ], + // [ + // '$id' => ID::custom('_key_fileSecurity'), + // 'type' => Database::INDEX_KEY, + // 'attributes' => ['fileSecurity'], + // 'lengths' => [], + // 'orders' => [Database::ORDER_ASC], + // ], + // [ + // '$id' => ID::custom('_key_maximumFileSize'), + // 'type' => Database::INDEX_KEY, + // 'attributes' => ['maximumFileSize'], + // 'lengths' => [], + // 'orders' => [Database::ORDER_ASC], + // ], + // [ + // '$id' => ID::custom('_key_encryption'), + // 'type' => Database::INDEX_KEY, + // 'attributes' => ['encryption'], + // 'lengths' => [], + // 'orders' => [Database::ORDER_ASC], + // ], + // [ + // '$id' => ID::custom('_key_antivirus'), + // 'type' => Database::INDEX_KEY, + // 'attributes' => ['antivirus'], + // 'lengths' => [], + // 'orders' => [Database::ORDER_ASC], + // ], ] ], @@ -3379,62 +3390,62 @@ $collections = [ ], ], 'indexes' => [ - [ - '$id' => ID::custom('_key_search'), - 'type' => Database::INDEX_FULLTEXT, - 'attributes' => ['search'], - 'lengths' => [2048], - 'orders' => [Database::ORDER_ASC], - ], - [ - '$id' => ID::custom('_key_bucket'), - 'type' => Database::INDEX_KEY, - 'attributes' => ['bucketId'], - 'lengths' => [Database::LENGTH_KEY], - 'orders' => [Database::ORDER_ASC], - ], - [ - '$id' => ID::custom('_key_name'), - 'type' => Database::INDEX_KEY, - 'attributes' => ['name'], - 'lengths' => [2048], - 'orders' => [Database::ORDER_ASC], - ], - [ - '$id' => ID::custom('_key_signature'), - 'type' => Database::INDEX_KEY, - 'attributes' => ['signature'], - 'lengths' => [2048], - 'orders' => [Database::ORDER_ASC], - ], - [ - '$id' => ID::custom('_key_mimeType'), - 'type' => Database::INDEX_KEY, - 'attributes' => ['mimeType'], - 'lengths' => [127], - 'orders' => [Database::ORDER_ASC], - ], - [ - '$id' => ID::custom('_key_sizeOriginal'), - 'type' => Database::INDEX_KEY, - 'attributes' => ['sizeOriginal'], - 'lengths' => [], - 'orders' => [Database::ORDER_ASC], - ], - [ - '$id' => ID::custom('_key_chunksTotal'), - 'type' => Database::INDEX_KEY, - 'attributes' => ['chunksTotal'], - 'lengths' => [], - 'orders' => [Database::ORDER_ASC], - ], - [ - '$id' => ID::custom('_key_chunksUploaded'), - 'type' => Database::INDEX_KEY, - 'attributes' => ['chunksUploaded'], - 'lengths' => [], - 'orders' => [Database::ORDER_ASC], - ], + // [ + // '$id' => ID::custom('_key_search'), + // 'type' => Database::INDEX_FULLTEXT, + // 'attributes' => ['search'], + // 'lengths' => [], + // 'orders' => [Database::ORDER_ASC], + // ], + // [ + // '$id' => ID::custom('_key_bucket'), + // 'type' => Database::INDEX_KEY, + // 'attributes' => ['bucketId'], + // 'lengths' => [Database::LENGTH_KEY], + // 'orders' => [Database::ORDER_ASC], + // ], + // [ + // '$id' => ID::custom('_key_name'), + // 'type' => Database::INDEX_KEY, + // 'attributes' => ['name'], + // 'lengths' => [], + // 'orders' => [Database::ORDER_ASC], + // ], + // [ + // '$id' => ID::custom('_key_signature'), + // 'type' => Database::INDEX_KEY, + // 'attributes' => ['signature'], + // 'lengths' => [], + // 'orders' => [Database::ORDER_ASC], + // ], + // [ + // '$id' => ID::custom('_key_mimeType'), + // 'type' => Database::INDEX_KEY, + // 'attributes' => ['mimeType'], + // 'lengths' => [127], + // 'orders' => [Database::ORDER_ASC], + // ], + // [ + // '$id' => ID::custom('_key_sizeOriginal'), + // 'type' => Database::INDEX_KEY, + // 'attributes' => ['sizeOriginal'], + // 'lengths' => [], + // 'orders' => [Database::ORDER_ASC], + // ], + // [ + // '$id' => ID::custom('_key_chunksTotal'), + // 'type' => Database::INDEX_KEY, + // 'attributes' => ['chunksTotal'], + // 'lengths' => [], + // 'orders' => [Database::ORDER_ASC], + // ], + // [ + // '$id' => ID::custom('_key_chunksUploaded'), + // 'type' => Database::INDEX_KEY, + // 'attributes' => ['chunksUploaded'], + // 'lengths' => [], + // 'orders' => [Database::ORDER_ASC], + // ], ] ], diff --git a/app/config/variables.php b/app/config/variables.php index 40bb99f8f2..7f0944ff1c 100644 --- a/app/config/variables.php +++ b/app/config/variables.php @@ -306,6 +306,24 @@ return [ 'question' => '', 'filter' => 'password' ], + // [ + // 'name' => '_APP_CONNECTIONS_DB_PROJECT', + // 'description' => 'A list of comma-separated key value pairs representing Project DBs where key is the database name and value is the DSN connection string.', + // 'introduction' => 'TBD', + // 'default' => 'db_fra1_01=mysql://user:password@mariadb:3306/appwrite', + // 'required' => true, + // 'question' => '', + // 'filter' => '' + // ], + // [ + // 'name' => '_APP_CONNECTIONS_DB_CONSOLE', + // 'description' => 'A key value pair representing the Console DB where key is the database name and value is the DSN connection string.', + // 'introduction' => 'TBD', + // 'default' => 'db_fra1_01=mysql://user:password@mariadb:3306/appwrite', + // 'required' => true, + // 'question' => '', + // 'filter' => '' + // ] ], ], [ diff --git a/app/controllers/api/account.php b/app/controllers/api/account.php index fb91a4517d..8bebf307b9 100644 --- a/app/controllers/api/account.php +++ b/app/controllers/api/account.php @@ -5,7 +5,6 @@ use Appwrite\Auth\Auth; use Appwrite\Auth\Validator\Password; use Appwrite\Auth\Validator\Phone; use Appwrite\Detector\Detector; -use Appwrite\Event\Audit; use Appwrite\Event\Event; use Appwrite\Event\Mail; use Appwrite\Event\Phone as EventPhone; @@ -39,7 +38,6 @@ use Utopia\Database\Validator\UID; use Utopia\Locale\Locale; use Utopia\Validator\ArrayList; use Utopia\Validator\Assoc; -use Utopia\Validator\Range; use Utopia\Validator\Text; use Utopia\Validator\WhiteList; diff --git a/app/controllers/api/databases.php b/app/controllers/api/databases.php index acec695233..6dbb607ce4 100644 --- a/app/controllers/api/databases.php +++ b/app/controllers/api/databases.php @@ -20,7 +20,7 @@ use Utopia\Database\Database; use Utopia\Database\Document; use Utopia\Database\DateTime; use Utopia\Database\Query; -use Utopia\Database\Adapter\MariaDB; +use Utopia\Database\Adapter\MySQL; use Utopia\Database\Validator\Authorization; use Utopia\Database\Validator\Key; use Utopia\Database\Validator\Permissions; diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index a8df29582e..097d8f39e2 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -25,7 +25,6 @@ use Appwrite\Task\Validator\Cron; use Appwrite\Utopia\Database\Validator\Queries\Deployments; use Appwrite\Utopia\Database\Validator\Queries\Executions; use Appwrite\Utopia\Database\Validator\Queries\Functions; -use Appwrite\Utopia\Database\Validator\Queries\Variables; use Utopia\App; use Utopia\Database\Database; use Utopia\Database\Document; @@ -33,7 +32,6 @@ use Utopia\Database\DateTime; use Utopia\Database\Query; use Utopia\Database\Validator\Authorization; use Utopia\Validator\ArrayList; -use Utopia\Validator\Assoc; use Utopia\Validator\Text; use Utopia\Validator\Range; use Utopia\Validator\WhiteList; diff --git a/app/controllers/api/health.php b/app/controllers/api/health.php index 4384fcbcd5..f65e65ba23 100644 --- a/app/controllers/api/health.php +++ b/app/controllers/api/health.php @@ -5,7 +5,9 @@ use Appwrite\Event\Event; use Appwrite\Extend\Exception; use Appwrite\Utopia\Response; use Utopia\App; +use Utopia\Config\Config; use Utopia\Database\Document; +use Utopia\Pools\Group; use Utopia\Registry\Registry; use Utopia\Storage\Device; use Utopia\Storage\Device\Local; @@ -26,6 +28,7 @@ App::get('/v1/health') ->action(function (Response $response) { $output = [ + 'name' => 'http', 'status' => 'pass', 'ping' => 0 ]; @@ -42,7 +45,6 @@ App::get('/v1/health/version') ->label('sdk.response.model', Response::MODEL_HEALTH_VERSION) ->inject('response') ->action(function (Response $response) { - $response->dynamic(new Document([ 'version' => APP_VERSION_STABLE ]), Response::MODEL_HEALTH_VERSION); }); @@ -58,30 +60,50 @@ App::get('/v1/health/db') ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) ->label('sdk.response.model', Response::MODEL_HEALTH_STATUS) ->inject('response') - ->inject('utopia') - ->action(function (Response $response, App $utopia) { + ->inject('pools') + ->action(function (Response $response, Group $pools) { - $checkStart = \microtime(true); + $output = []; - try { - $db = $utopia->getResource('db'); /* @var $db PDO */ - - // Run a small test to check the connection - $statement = $db->prepare("SELECT 1;"); - - $statement->closeCursor(); - - $statement->execute(); - } catch (Exception $_e) { - throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Database is not available'); - } - - $output = [ - 'status' => 'pass', - 'ping' => \round((\microtime(true) - $checkStart) / 1000) + $configs = [ + 'Console.DB' => Config::getParam('pools-console'), + 'Projects.DB' => Config::getParam('pools-database'), ]; - $response->dynamic(new Document($output), Response::MODEL_HEALTH_STATUS); + foreach ($configs as $key => $config) { + foreach ($config as $database) { + try { + $adapter = $pools->get($database)->pop()->getResource(); + + $checkStart = \microtime(true); + + if ($adapter->ping()) { + $output[] = new Document([ + 'name' => $key . " ($database)", + 'status' => 'pass', + 'ping' => \round((\microtime(true) - $checkStart) / 1000) + ]); + } else { + $output[] = new Document([ + 'name' => $key . " ($database)", + 'status' => 'fail', + 'ping' => \round((\microtime(true) - $checkStart) / 1000) + ]); + } + } catch (\Throwable $th) { + $output[] = new Document([ + 'name' => $key . " ($database)", + 'status' => 'fail', + 'ping' => \round((\microtime(true) - $checkStart) / 1000) + ]); + } + } + } + + $response->dynamic(new Document([ + 'statuses' => $output, + 'total' => count($output), + ]), Response::MODEL_HEALTH_STATUS_LIST); }); App::get('/v1/health/cache') @@ -96,23 +118,163 @@ App::get('/v1/health/cache') ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) ->label('sdk.response.model', Response::MODEL_HEALTH_STATUS) ->inject('response') - ->inject('utopia') - ->action(function (Response $response, App $utopia) { + ->inject('pools') + ->action(function (Response $response, Group $pools) { - $checkStart = \microtime(true); + $output = []; - $redis = $utopia->getResource('cache'); - - if (!$redis->ping(true)) { - throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Cache is not available'); - } - - $output = [ - 'status' => 'pass', - 'ping' => \round((\microtime(true) - $checkStart) / 1000) + $configs = [ + 'Cache' => Config::getParam('pools-cache'), ]; - $response->dynamic(new Document($output), Response::MODEL_HEALTH_STATUS); + foreach ($configs as $key => $config) { + foreach ($config as $database) { + try { + $adapter = $pools->get($database)->pop()->getResource(); + + $checkStart = \microtime(true); + + if ($adapter->ping()) { + $output[] = new Document([ + 'name' => $key . " ($database)", + 'status' => 'pass', + 'ping' => \round((\microtime(true) - $checkStart) / 1000) + ]); + } else { + $output[] = new Document([ + 'name' => $key . " ($database)", + 'status' => 'fail', + 'ping' => \round((\microtime(true) - $checkStart) / 1000) + ]); + } + } catch (\Throwable $th) { + $output[] = new Document([ + 'name' => $key . " ($database)", + 'status' => 'fail', + 'ping' => \round((\microtime(true) - $checkStart) / 1000) + ]); + } + } + } + + $response->dynamic(new Document([ + 'statuses' => $output, + 'total' => count($output), + ]), Response::MODEL_HEALTH_STATUS_LIST); + }); + +App::get('/v1/health/queue') + ->desc('Get Queue') + ->groups(['api', 'health']) + ->label('scope', 'health.read') + ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) + ->label('sdk.namespace', 'health') + ->label('sdk.method', 'getQueue') + ->label('sdk.description', '/docs/references/health/get-queue.md') + ->label('sdk.response.code', Response::STATUS_CODE_OK) + ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) + ->label('sdk.response.model', Response::MODEL_HEALTH_STATUS) + ->inject('response') + ->inject('pools') + ->action(function (Response $response, Group $pools) { + + $output = []; + + $configs = [ + 'Queue' => Config::getParam('pools-queue'), + ]; + + foreach ($configs as $key => $config) { + foreach ($config as $database) { + try { + $adapter = $pools->get($database)->pop()->getResource(); + + $checkStart = \microtime(true); + + if ($adapter->ping()) { + $output[] = new Document([ + 'name' => $key . " ($database)", + 'status' => 'pass', + 'ping' => \round((\microtime(true) - $checkStart) / 1000) + ]); + } else { + $output[] = new Document([ + 'name' => $key . " ($database)", + 'status' => 'fail', + 'ping' => \round((\microtime(true) - $checkStart) / 1000) + ]); + } + } catch (\Throwable $th) { + $output[] = new Document([ + 'name' => $key . " ($database)", + 'status' => 'fail', + 'ping' => \round((\microtime(true) - $checkStart) / 1000) + ]); + } + } + } + + $response->dynamic(new Document([ + 'statuses' => $output, + 'total' => count($output), + ]), Response::MODEL_HEALTH_STATUS_LIST); + }); + +App::get('/v1/health/pubsub') + ->desc('Get PubSub') + ->groups(['api', 'health']) + ->label('scope', 'health.read') + ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) + ->label('sdk.namespace', 'health') + ->label('sdk.method', 'getPubSub') + ->label('sdk.description', '/docs/references/health/get-pubsub.md') + ->label('sdk.response.code', Response::STATUS_CODE_OK) + ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) + ->label('sdk.response.model', Response::MODEL_HEALTH_STATUS) + ->inject('response') + ->inject('pools') + ->action(function (Response $response, Group $pools) { + + $output = []; + + $configs = [ + 'PubSub' => Config::getParam('pools-pubsub'), + ]; + + foreach ($configs as $key => $config) { + foreach ($config as $database) { + try { + $adapter = $pools->get($database)->pop()->getResource(); + + $checkStart = \microtime(true); + + if ($adapter->ping()) { + $output[] = new Document([ + 'name' => $key . " ($database)", + 'status' => 'pass', + 'ping' => \round((\microtime(true) - $checkStart) / 1000) + ]); + } else { + $output[] = new Document([ + 'name' => $key . " ($database)", + 'status' => 'fail', + 'ping' => \round((\microtime(true) - $checkStart) / 1000) + ]); + } + } catch (\Throwable $th) { + $output[] = new Document([ + 'name' => $key . " ($database)", + 'status' => 'fail', + 'ping' => \round((\microtime(true) - $checkStart) / 1000) + ]); + } + } + } + + $response->dynamic(new Document([ + 'statuses' => $output, + 'total' => count($output), + ]), Response::MODEL_HEALTH_STATUS_LIST); }); App::get('/v1/health/time') diff --git a/app/controllers/api/projects.php b/app/controllers/api/projects.php index 1ed6bd62d0..e0c50c9b09 100644 --- a/app/controllers/api/projects.php +++ b/app/controllers/api/projects.php @@ -29,6 +29,8 @@ use Utopia\Domains\Domain; use Utopia\Registry\Registry; use Appwrite\Extend\Exception; use Appwrite\Utopia\Database\Validator\Queries\Projects; +use Utopia\Cache\Cache; +use Utopia\Pools\Group; use Utopia\Validator\ArrayList; use Utopia\Validator\Boolean; use Utopia\Validator\Hostname; @@ -69,8 +71,10 @@ App::post('/v1/projects') ->param('legalTaxId', '', new Text(256), 'Project legal Tax ID. Max length: 256 chars.', true) ->inject('response') ->inject('dbForConsole') - ->inject('dbForProject') - ->action(function (string $projectId, string $name, string $teamId, string $description, string $logo, string $url, string $legalName, string $legalCountry, string $legalState, string $legalCity, string $legalAddress, string $legalTaxId, Response $response, Database $dbForConsole, Database $dbForProject) { + ->inject('cache') + ->inject('pools') + ->action(function (string $projectId, string $name, string $teamId, string $description, string $logo, string $url, string $legalName, string $legalCountry, string $legalState, string $legalCity, string $legalAddress, string $legalTaxId, Response $response, Database $dbForConsole, Cache $cache, Group $pools) { + $team = $dbForConsole->getDocument('teams', $teamId); @@ -85,6 +89,8 @@ App::post('/v1/projects') } $projectId = ($projectId == 'unique()') ? ID::unique() : $projectId; + $databases = Config::getParam('pools-database', []); + $database = $databases[array_rand($databases)]; if ($projectId === 'console') { throw new Exception(Exception::PROJECT_RESERVED_PROJECT, "'console' is a reserved project."); @@ -120,12 +126,12 @@ App::post('/v1/projects') 'domains' => null, 'auths' => $auths, 'search' => implode(' ', [$projectId, $name]), + 'database' => $database, ])); - /** @var array $collections */ - $collections = Config::getParam('collections', []); + $dbForProject = new Database($pools->get($database)->pop()->getResource(), $cache); $dbForProject->setNamespace("_{$project->getInternalId()}"); - $dbForProject->create(App::getEnv('_APP_DB_SCHEMA', 'appwrite')); + $dbForProject->create(); $audit = new Audit($dbForProject); $audit->setup(); @@ -133,6 +139,9 @@ App::post('/v1/projects') $adapter = new TimeLimit('', 0, 1, $dbForProject); $adapter->setup(); + /** @var array $collections */ + $collections = Config::getParam('collections', []); + foreach ($collections as $key => $collection) { if (($collection['$collection'] ?? '') !== Database::METADATA) { continue; diff --git a/app/controllers/api/teams.php b/app/controllers/api/teams.php index a42b9d317e..9f1aedf483 100644 --- a/app/controllers/api/teams.php +++ b/app/controllers/api/teams.php @@ -20,7 +20,6 @@ use Appwrite\Utopia\Response; use MaxMind\Db\Reader; use Utopia\App; use Utopia\Audit\Audit; -use Utopia\CLI\Console; use Utopia\Config\Config; use Utopia\Database\Database; use Utopia\Database\Document; @@ -36,9 +35,7 @@ use Utopia\Database\Validator\Key; use Utopia\Database\Validator\UID; use Utopia\Locale\Locale; use Utopia\Validator\Text; -use Utopia\Validator\Range; use Utopia\Validator\ArrayList; -use Utopia\Validator\WhiteList; App::post('/v1/teams') ->desc('Create Team') diff --git a/app/http.php b/app/http.php index fe0778bb15..7b4ac39cef 100644 --- a/app/http.php +++ b/app/http.php @@ -22,6 +22,7 @@ use Utopia\Swoole\Files; use Appwrite\Utopia\Request; use Utopia\Logger\Log; use Utopia\Logger\Log\User; +use Utopia\Pools\Group; $http = new Server("0.0.0.0", App::getEnv('PORT', 80)); @@ -60,6 +61,10 @@ $http->on('start', function (Server $http) use ($payloadSize, $register) { $app = new App('UTC'); go(function () use ($register, $app) { + + $pools = $register->get('pools'); /** @var Group $pools */ + App::setResource('pools', fn() => $pools); + // wait for database to be ready $attempts = 0; $max = 10; @@ -68,8 +73,7 @@ $http->on('start', function (Server $http) use ($payloadSize, $register) { do { try { $attempts++; - $db = $register->get('dbPool')->get(); - $redis = $register->get('redisPool')->get(); + $dbForConsole = $app->getResource('dbForConsole'); /** @var Utopia\Database\Database $dbForConsole */ break; // leave the do-while if successful } catch (\Exception $e) { Console::warning("Database not ready. Retrying connection ({$attempts})..."); @@ -80,21 +84,16 @@ $http->on('start', function (Server $http) use ($payloadSize, $register) { } } while ($attempts < $max); - App::setResource('db', fn () => $db); - App::setResource('cache', fn () => $redis); - - /** @var Utopia\Database\Database $dbForConsole */ - $dbForConsole = $app->getResource('dbForConsole'); - Console::success('[Setup] - Server database init started...'); /** @var array $collections */ $collections = Config::getParam('collections', []); try { - $redis->flushAll(); + $cache = $app->getResource('cache'); /** @var Utopia\Cache\Cache $cache */ + $cache->flush(); Console::success('[Setup] - Creating database: appwrite...'); - $dbForConsole->create(App::getEnv('_APP_DB_SCHEMA', 'appwrite')); + $dbForConsole->create(); } catch (\Exception $e) { Console::success('[Setup] - Skip: metadata table already exists'); } @@ -116,6 +115,7 @@ $http->on('start', function (Server $http) use ($payloadSize, $register) { if (!$dbForConsole->getCollection($key)->isEmpty()) { continue; } + /** * Skip to prevent 0.16 migration issues. */ @@ -215,6 +215,8 @@ $http->on('start', function (Server $http) use ($payloadSize, $register) { $dbForConsole->createCollection('bucket_' . $bucket->getInternalId(), $attributes, $indexes); } + $pools->reclaim(); + Console::success('[Setup] - Server database init completed...'); }); @@ -246,11 +248,8 @@ $http->on('request', function (SwooleRequest $swooleRequest, SwooleResponse $swo $app = new App('UTC'); - $db = $register->get('dbPool')->get(); - $redis = $register->get('redisPool')->get(); - - App::setResource('db', fn () => $db); - App::setResource('cache', fn () => $redis); + $pools = $register->get('pools'); + App::setResource('pools', fn() => $pools); try { Authorization::cleanRoles(); @@ -317,13 +316,6 @@ $http->on('request', function (SwooleRequest $swooleRequest, SwooleResponse $swo Console::error('[Error] File: ' . $th->getFile()); Console::error('[Error] Line: ' . $th->getLine()); - /** - * Reset Database connection if PDOException was thrown. - */ - if ($th instanceof PDOException) { - $db = null; - } - $swooleResponse->setStatusCode(500); $output = ((App::isDevelopment())) ? [ @@ -341,13 +333,7 @@ $http->on('request', function (SwooleRequest $swooleRequest, SwooleResponse $swo $swooleResponse->end(\json_encode($output)); } finally { - /** @var PDOPool $dbPool */ - $dbPool = $register->get('dbPool'); - $dbPool->put($db); - - /** @var RedisPool $redisPool */ - $redisPool = $register->get('redisPool'); - $redisPool->put($redis); + $pools->reclaim(); } }); diff --git a/app/init.php b/app/init.php index 054721d439..3a05ea5ddf 100644 --- a/app/init.php +++ b/app/init.php @@ -18,9 +18,6 @@ ini_set('display_startup_errors', 1); ini_set('default_socket_timeout', -1); error_reporting(E_ALL); -use Appwrite\Extend\PDO; -use Ahc\Jwt\JWT; -use Ahc\Jwt\JWTException; use Appwrite\Extend\Exception; use Appwrite\Auth\Auth; use Appwrite\SMS\Adapter\Mock; @@ -32,39 +29,31 @@ use Appwrite\SMS\Adapter\Vonage; use Appwrite\DSN\DSN; use Appwrite\Event\Audit; use Appwrite\Event\Database as EventDatabase; -use Appwrite\Event\Delete; use Appwrite\Event\Event; use Appwrite\Event\Mail; use Appwrite\Event\Phone; +use Appwrite\Event\Delete; use Appwrite\Network\Validator\Email; use Appwrite\Network\Validator\IP; use Appwrite\Network\Validator\URL; use Appwrite\OpenSSL\OpenSSL; +use Appwrite\URL\URL as AppwriteURL; use Appwrite\Usage\Stats; use Appwrite\Utopia\View; use Utopia\App; +use Utopia\Validator\Range; +use Utopia\Validator\WhiteList; use Utopia\Database\ID; +use Utopia\Database\Document; +use Utopia\Database\Database; +use Utopia\Database\Query; +use Utopia\Database\Validator\Authorization; +use Utopia\Database\Validator\DatetimeValidator; +use Utopia\Database\Validator\Structure; use Utopia\Logger\Logger; use Utopia\Config\Config; use Utopia\Locale\Locale; use Utopia\Registry\Registry; -use MaxMind\Db\Reader; -use PHPMailer\PHPMailer\PHPMailer; -use Utopia\Cache\Adapter\Redis as RedisCache; -use Utopia\Cache\Cache; -use Utopia\Database\Adapter\MariaDB; -use Utopia\Database\Document; -use Utopia\Database\Database; -use Utopia\Database\Validator\Structure; -use Utopia\Database\Validator\Authorization; -use Utopia\Validator\Range; -use Utopia\Validator\WhiteList; -use Swoole\Database\PDOConfig; -use Swoole\Database\PDOPool; -use Swoole\Database\RedisConfig; -use Swoole\Database\RedisPool; -use Utopia\Database\Query; -use Utopia\Database\Validator\DatetimeValidator; use Utopia\Storage\Device; use Utopia\Storage\Storage; use Utopia\Storage\Device\Backblaze; @@ -73,6 +62,19 @@ use Utopia\Storage\Device\Local; use Utopia\Storage\Device\S3; use Utopia\Storage\Device\Linode; use Utopia\Storage\Device\Wasabi; +use Utopia\Cache\Adapter\Redis as RedisCache; +use Utopia\Cache\Adapter\Sharding; +use Utopia\Cache\Cache; +use Utopia\CLI\Console; +use Utopia\Database\Adapter\MariaDB; +use Utopia\Database\Adapter\MySQL; +use Utopia\Pools\Group; +use Utopia\Pools\Pool; +use Ahc\Jwt\JWT; +use Ahc\Jwt\JWTException; +use MaxMind\Db\Reader; +use PHPMailer\PHPMailer\PHPMailer; +use Swoole\Database\PDOProxy; const APP_NAME = 'Appwrite'; const APP_DOMAIN = 'appwrite.io'; @@ -495,56 +497,181 @@ $register->set('logger', function () { $adapter = new $classname($providerConfig); return new Logger($adapter); }); -$register->set('dbPool', function () { - // Register DB connection - $dbHost = App::getEnv('_APP_DB_HOST', ''); - $dbPort = App::getEnv('_APP_DB_PORT', ''); - $dbUser = App::getEnv('_APP_DB_USER', ''); - $dbPass = App::getEnv('_APP_DB_PASS', ''); - $dbScheme = App::getEnv('_APP_DB_SCHEMA', ''); +$register->set('pools', function () { - $pool = new PDOPool( - (new PDOConfig()) - ->withHost($dbHost) - ->withPort($dbPort) - ->withDbName($dbScheme) - ->withCharset('utf8mb4') - ->withUsername($dbUser) - ->withPassword($dbPass) - ->withOptions([ - PDO::ATTR_ERRMODE => App::isDevelopment() ? PDO::ERRMODE_WARNING : PDO::ERRMODE_SILENT, // If in production mode, warnings are not displayed - PDO::ATTR_TIMEOUT => 3, // Seconds - PDO::ATTR_PERSISTENT => true, - PDO::ATTR_DEFAULT_FETCH_MODE => PDO::FETCH_ASSOC, - PDO::ATTR_EMULATE_PREPARES => true, - PDO::ATTR_STRINGIFY_FETCHES => true, - ]), - 64 - ); + $group = new Group(); - return $pool; -}); -$register->set('redisPool', function () { - $redisHost = App::getEnv('_APP_REDIS_HOST', ''); - $redisPort = App::getEnv('_APP_REDIS_PORT', ''); - $redisUser = App::getEnv('_APP_REDIS_USER', ''); - $redisPass = App::getEnv('_APP_REDIS_PASS', ''); - $redisAuth = ''; + $fallbackForDB = AppwriteURL::unparse([ + 'scheme' => 'mariadb', + 'host' => App::getEnv('_APP_DB_HOST', 'mariadb'), + 'port' => App::getEnv('_APP_DB_PORT', '3306'), + 'user' => App::getEnv('_APP_DB_USER', ''), + 'pass' => App::getEnv('_APP_DB_PASS', ''), + ]); + $fallbackForRedis = AppwriteURL::unparse([ + 'scheme' => 'redis', + 'host' => App::getEnv('_APP_REDIS_HOST', 'redis'), + 'port' => App::getEnv('_APP_REDIS_PORT', '6379'), + 'user' => App::getEnv('_APP_REDIS_USER', ''), + 'pass' => App::getEnv('_APP_REDIS_PASS', ''), + ]); - if ($redisUser && $redisPass) { - $redisAuth = $redisUser . ':' . $redisPass; + $connections = [ + 'console' => [ + 'type' => 'database', + 'dsns' => App::getEnv('_APP_CONNECTIONS_DB_CONSOLE', $fallbackForDB), + 'multiple' => false, + 'schemes' => ['mariadb', 'mysql'], + ], + 'database' => [ + 'type' => 'database', + 'dsns' => App::getEnv('_APP_CONNECTIONS_DB_PROJECT', $fallbackForDB), + 'multiple' => true, + 'schemes' => ['mariadb', 'mysql'], + ], + 'queue' => [ + 'type' => 'queue', + 'dsns' => App::getEnv('_APP_CONNECTIONS_QUEUE', $fallbackForRedis), + 'multiple' => false, + 'schemes' => ['redis'], + ], + 'pubsub' => [ + 'type' => 'pubsub', + 'dsns' => App::getEnv('_APP_CONNECTIONS_PUBSUB', $fallbackForRedis), + 'multiple' => false, + 'schemes' => ['redis'], + ], + 'cache' => [ + 'type' => 'cache', + 'dsns' => App::getEnv('_APP_CONNECTIONS_CACHE', $fallbackForRedis), + 'multiple' => true, + 'schemes' => ['redis'], + ], + ]; + + foreach ($connections as $key => $connection) { + $type = $connection['type'] ?? ''; + $dsns = $connection['dsns'] ?? ''; + $multipe = $connection['multiple'] ?? false; + $schemes = $connection['schemes'] ?? []; + $config = []; + $dsns = explode(',', $connection['dsns'] ?? ''); + + foreach ($dsns as &$dsn) { + $dsn = explode('=', $dsn); + $name = ($multipe) ? $key . '_' . $dsn[0] : $key; + $dsn = $dsn[1] ?? ''; + $config[] = $name; + + if (empty($dsn)) { + //throw new Exception(Exception::GENERAL_SERVER_ERROR, "Missing value for DSN connection in {$key}"); + continue; + } + + $dsn = new DSN($dsn); + $dsnHost = $dsn->getHost(); + $dsnPort = $dsn->getPort(); + $dsnUser = $dsn->getUser(); + $dsnPass = $dsn->getPassword(); + $dsnScheme = $dsn->getScheme(); + $dsnDatabase = $dsn->getDatabase(); + + if (!in_array($dsnScheme, $schemes)) { + throw new Exception(Exception::GENERAL_SERVER_ERROR, "Invalid console database scheme"); + } + + /** + * Get Resource + * + * Creation could be reused accross connection types like database, cache, queue, etc. + * + * Resource assignment to an adapter will happen below. + */ + + switch ($dsnScheme) { + case 'mysql': + case 'mariadb': + $resource = function () use ($dsnHost, $dsnPort, $dsnUser, $dsnPass, $dsnDatabase) { + return new PDOProxy(function () use ($dsnHost, $dsnPort, $dsnUser, $dsnPass, $dsnDatabase) { + return new PDO("mysql:host={$dsnHost};port={$dsnPort};dbname={$dsnDatabase};charset=utf8mb4", $dsnUser, $dsnPass, array( + PDO::ATTR_TIMEOUT => 3, // Seconds + PDO::ATTR_PERSISTENT => true, + PDO::ATTR_DEFAULT_FETCH_MODE => PDO::FETCH_ASSOC, + PDO::ATTR_ERRMODE => App::isDevelopment() ? PDO::ERRMODE_WARNING : PDO::ERRMODE_SILENT, // If in production mode, warnings are not displayed + PDO::ATTR_EMULATE_PREPARES => true, + PDO::ATTR_STRINGIFY_FETCHES => true + )); + }); + }; + break; + case 'redis': + $resource = function () use ($dsnHost, $dsnPort, $dsnPass) { + $redis = new Redis(); + @$redis->pconnect($dsnHost, (int)$dsnPort); + if ($dsnPass) { + $redis->auth($dsnPass); + } + $redis->setOption(Redis::OPT_READ_TIMEOUT, -1); + + return $redis; + }; + break; + + default: + throw new Exception(Exception::GENERAL_SERVER_ERROR, "Invalid scheme"); + break; + } + + $pool = new Pool($name, 64, function () use ($type, $resource, $dsn) { + // Get Adapter + $adapter = null; + + switch ($type) { + case 'database': + $adapter = match ($dsn->getScheme()) { + 'mariadb' => new MariaDB($resource()), + 'mysql' => new MySQL($resource()), + default => null + }; + + var_dump($dsn->getDatabase()); + $adapter->setDefaultDatabase($dsn->getDatabase()); + + break; + case 'queue': + $adapter = $resource(); + break; + case 'pubsub': + $adapter = $resource(); + break; + case 'cache': + $adapter = match ($dsn->getScheme()) { + 'redis' => new RedisCache($resource()), + default => null + }; + break; + + default: + throw new Exception(Exception::GENERAL_SERVER_ERROR, "Server error: Missing adapter implementation."); + break; + } + + return $adapter; + }); + + $group->add($pool); + } + + Config::setParam('pools-' . $key, $config); } - $pool = new RedisPool( - (new RedisConfig()) - ->withHost($redisHost) - ->withPort($redisPort) - ->withAuth($redisAuth) - ->withDbIndex(0), - 64 - ); + try { + $group->fill(); + } catch (\Throwable $th) { + Console::error('Connection failure: ' . $th->getMessage()); + } - return $pool; + return $group; }); $register->set('influxdb', function () { // Register DB connection @@ -561,7 +688,7 @@ $register->set('influxdb', function () { return $client; }); $register->set('statsd', function () { - // Register DB connection + // Register DB connection $host = App::getEnv('_APP_STATSD_HOST', 'telegraf'); $port = App::getEnv('_APP_STATSD_PORT', 8125); @@ -601,33 +728,6 @@ $register->set('smtp', function () { $register->set('geodb', function () { return new Reader(__DIR__ . '/db/DBIP/dbip-country-lite-2022-06.mmdb'); }); -$register->set('db', function () { - // This is usually for our workers or CLI commands scope - $dbHost = App::getEnv('_APP_DB_HOST', ''); - $dbPort = App::getEnv('_APP_DB_PORT', ''); - $dbUser = App::getEnv('_APP_DB_USER', ''); - $dbPass = App::getEnv('_APP_DB_PASS', ''); - $dbScheme = App::getEnv('_APP_DB_SCHEMA', ''); - - $pdo = new PDO("mysql:host={$dbHost};port={$dbPort};dbname={$dbScheme};charset=utf8mb4", $dbUser, $dbPass, array( - PDO::ATTR_TIMEOUT => 3, // Seconds - PDO::ATTR_PERSISTENT => true, - PDO::ATTR_DEFAULT_FETCH_MODE => PDO::FETCH_ASSOC, - PDO::ATTR_ERRMODE => PDO::ERRMODE_EXCEPTION, - PDO::ATTR_EMULATE_PREPARES => true, - PDO::ATTR_STRINGIFY_FETCHES => true, - )); - - return $pdo; -}); -$register->set('cache', function () { - // This is usually for our workers or CLI commands scope - $redis = new Redis(); - $redis->pconnect(App::getEnv('_APP_REDIS_HOST', ''), App::getEnv('_APP_REDIS_PORT', '')); - $redis->setOption(Redis::OPT_READ_TIMEOUT, -1); - - return $redis; -}); /* * Localization @@ -925,26 +1025,51 @@ App::setResource('console', function () { ]); }, []); -App::setResource('dbForProject', function ($db, $cache, Document $project) { - $cache = new Cache(new RedisCache($cache)); +App::setResource('dbForProject', function (Group $pools, Database $dbForConsole, Cache $cache, Document $project) { + if ($project->isEmpty() || $project->getId() === 'console') { + return $dbForConsole; + } - $database = new Database(new MariaDB($db), $cache); - $database->setDefaultDatabase(App::getEnv('_APP_DB_SCHEMA', 'appwrite')); - $database->setNamespace("_{$project->getInternalId()}"); + $dbAdapter = $pools + ->get($project->getAttribute('database')) + ->pop() + ->getResource() + ; + + $database = new Database($dbAdapter, $cache); + $database->setNamespace('_' . $project->getInternalId()); return $database; -}, ['db', 'cache', 'project']); +}, ['pools', 'dbForConsole', 'cache', 'project']); -App::setResource('dbForConsole', function ($db, $cache) { - $cache = new Cache(new RedisCache($cache)); +App::setResource('dbForConsole', function (Group $pools, Cache $cache) { + $dbAdapter = $pools + ->get('console') + ->pop() + ->getResource() + ; - $database = new Database(new MariaDB($db), $cache); - $database->setDefaultDatabase(App::getEnv('_APP_DB_SCHEMA', 'appwrite')); - $database->setNamespace('_console'); + $database = new Database($dbAdapter, $cache); + + $database->setNamespace('console'); return $database; -}, ['db', 'cache']); +}, ['pools', 'cache']); +App::setResource('cache', function (Group $pools) { + $list = Config::getParam('pools-cache', []); + $adapters = []; + + foreach ($list as $value) { + $adapters[] = $pools + ->get($value) + ->pop() + ->getResource() + ; + } + + return new Cache(new Sharding($adapters)); +}, ['pools']); App::setResource('deviceLocal', function () { return new Local(); diff --git a/app/preload.php b/app/preload.php index bf8b0bfd1d..4935db3da4 100644 --- a/app/preload.php +++ b/app/preload.php @@ -35,6 +35,7 @@ foreach ( realpath(__DIR__ . '/../vendor/symfony'), realpath(__DIR__ . '/../vendor/mongodb'), realpath(__DIR__ . '/../vendor/utopia-php/websocket'), // TODO: remove workerman autoload + realpath(__DIR__ . '/../vendor/utopia-php/cache'), // TODO: remove memcached autoload ] as $key => $value ) { if ($value !== false) { diff --git a/app/realtime.php b/app/realtime.php index be87c3d6e6..8cd28c193f 100644 --- a/app/realtime.php +++ b/app/realtime.php @@ -16,16 +16,15 @@ use Utopia\CLI\Console; use Utopia\Database\ID; use Utopia\Database\Role; use Utopia\Logger\Log; -use Utopia\Database\Database; use Utopia\Database\DateTime; -use Utopia\Cache\Adapter\Redis as RedisCache; -use Utopia\Cache\Cache; -use Utopia\Database\Adapter\MariaDB; use Utopia\Database\Document; use Utopia\Database\Query; use Utopia\Database\Validator\Authorization; -use Utopia\Registry\Registry; use Appwrite\Utopia\Request; +use Utopia\Cache\Adapter\Sharding; +use Utopia\Cache\Cache; +use Utopia\Config\Config; +use Utopia\Database\Database; use Utopia\WebSocket\Server; use Utopia\WebSocket\Adapter; @@ -33,6 +32,68 @@ require_once __DIR__ . '/init.php'; Runtime::enableCoroutine(SWOOLE_HOOK_ALL); + +function getConsoleDB(): Database +{ + global $register; + + $pools = $register->get('pools'); /** @var \Utopia\Pools\Group $pools */ + + $dbAdapter = $pools + ->get('console') + ->pop() + ->getResource() + ; + + $database = new Database($dbAdapter, getCache()); + + $database->setNamespace('console'); + + return $database; +} + +function getProjectDB(Document $project): Database +{ + global $register; + + $pools = $register->get('pools'); /** @var \Utopia\Pools\Group $pools */ + + if ($project->isEmpty() || $project->getId() === 'console') { + return getConsoleDB(); + } + + $dbAdapter = $pools + ->get($project->getAttribute('database')) + ->pop() + ->getResource() + ; + + $database = new Database($dbAdapter, getCache()); + $database->setNamespace('_' . $project->getInternalId()); + + return $database; +} + +function getCache(): Cache +{ + global $register; + + $pools = $register->get('pools'); /** @var \Utopia\Pools\Group $pools */ + + $list = Config::getParam('pools-cache', []); + $adapters = []; + + foreach ($list as $value) { + $adapters[] = $pools + ->get($value) + ->pop() + ->getResource() + ; + } + + return new Cache(new Sharding($adapters)); +} + $realtime = new Realtime(); /** @@ -95,45 +156,6 @@ $logError = function (Throwable $error, string $action) use ($register) { $server->error($logError); -function getDatabase(Registry &$register, string $namespace) -{ - $attempts = 0; - - do { - try { - $attempts++; - - $db = $register->get('dbPool')->get(); - $redis = $register->get('redisPool')->get(); - - $cache = new Cache(new RedisCache($redis)); - $database = new Database(new MariaDB($db), $cache); - $database->setDefaultDatabase(App::getEnv('_APP_DB_SCHEMA', 'appwrite')); - $database->setNamespace($namespace); - - if (!$database->exists($database->getDefaultDatabase(), 'realtime')) { - throw new Exception('Collection not ready'); - } - - break; // leave loop if successful - } catch (\Throwable $e) { - Console::warning("Database not ready. Retrying connection ({$attempts})..."); - if ($attempts >= DATABASE_RECONNECT_MAX_ATTEMPTS) { - throw new \Exception('Failed to connect to database: ' . $e->getMessage()); - } - sleep(DATABASE_RECONNECT_SLEEP); - } - } while ($attempts < DATABASE_RECONNECT_MAX_ATTEMPTS); - - return [ - $database, - function () use ($register, $db, $redis) { - $register->get('dbPool')->put($db); - $register->get('redisPool')->put($redis); - } - ]; -} - $server->onStart(function () use ($stats, $register, $containerId, &$statsDocument, $logError) { sleep(5); // wait for the initial database schema to be ready Console::success('Server started successfully'); @@ -143,7 +165,8 @@ $server->onStart(function () use ($stats, $register, $containerId, &$statsDocume */ go(function () use ($register, $containerId, &$statsDocument) { $attempts = 0; - [$database, $returnDatabase] = getDatabase($register, '_console'); + $database = getConsoleDB(); + do { try { $attempts++; @@ -163,7 +186,7 @@ $server->onStart(function () use ($stats, $register, $containerId, &$statsDocume sleep(DATABASE_RECONNECT_SLEEP); } } while (true); - call_user_func($returnDatabase); + $register->get('pools')->reclaim(); }); /** @@ -179,7 +202,7 @@ $server->onStart(function () use ($stats, $register, $containerId, &$statsDocume } try { - [$database, $returnDatabase] = getDatabase($register, '_console'); + $database = getConsoleDB(); $statsDocument ->setAttribute('timestamp', DateTime::now()) @@ -189,7 +212,7 @@ $server->onStart(function () use ($stats, $register, $containerId, &$statsDocume } catch (\Throwable $th) { call_user_func($logError, $th, "updateWorkerDocument"); } finally { - call_user_func($returnDatabase); + $register->get('pools')->reclaim(); } }); }); @@ -205,7 +228,7 @@ $server->onWorkerStart(function (int $workerId) use ($server, $register, $stats, * Sending current connections to project channels on the console project every 5 seconds. */ if ($realtime->hasSubscriber('console', Role::users()->toString(), 'project')) { - [$database, $returnDatabase] = getDatabase($register, '_console'); + $database = getConsoleDB(); $payload = []; @@ -250,7 +273,7 @@ $server->onWorkerStart(function (int $workerId) use ($server, $register, $stats, ])); } - call_user_func($returnDatabase); + $register->get('pools')->reclaim(); } /** * Sending test message for SDK E2E tests every 5 seconds. @@ -285,8 +308,7 @@ $server->onWorkerStart(function (int $workerId) use ($server, $register, $stats, } $start = time(); - /** @var Redis $redis */ - $redis = $register->get('redisPool')->get(); + $redis = $register->get('pools')->get('pubsub')->pop()->getResource(); /** @var Redis $redis */ $redis->setOption(Redis::OPT_READ_TIMEOUT, -1); if ($redis->ping(true)) { @@ -305,9 +327,9 @@ $server->onWorkerStart(function (int $workerId) use ($server, $register, $stats, if ($realtime->hasSubscriber($projectId, 'user:' . $userId)) { $connection = array_key_first(reset($realtime->subscriptions[$projectId]['user:' . $userId])); - [$consoleDatabase, $returnConsoleDatabase] = getDatabase($register, '_console'); + $consoleDatabase = getConsoleDB(); $project = Authorization::skip(fn() => $consoleDatabase->getDocument('projects', $projectId)); - [$database, $returnDatabase] = getDatabase($register, "_{$project->getInternalId()}"); + $database = getProjectDB($project); $user = $database->getDocument('users', $userId); @@ -315,8 +337,7 @@ $server->onWorkerStart(function (int $workerId) use ($server, $register, $stats, $realtime->subscribe($projectId, $connection, $roles, $realtime->connections[$connection]['channels']); - call_user_func($returnDatabase); - call_user_func($returnConsoleDatabase); + $register->get('pools')->reclaim(); } } @@ -344,7 +365,7 @@ $server->onWorkerStart(function (int $workerId) use ($server, $register, $stats, call_user_func($logError, $th, "pubSubConnection"); Console::error('Pub/sub error: ' . $th->getMessage()); - $register->get('redisPool')->put($redis); + $register->get('pools')->reclaim(); $attempts++; sleep(DATABASE_RECONNECT_SLEEP); continue; @@ -359,33 +380,16 @@ $server->onOpen(function (int $connection, SwooleRequest $request) use ($server, $request = new Request($request); $response = new Response(new SwooleResponse()); - /** @var PDO $db */ - $db = $register->get('dbPool')->get(); - /** @var Redis $redis */ - $redis = $register->get('redisPool')->get(); - Console::info("Connection open (user: {$connection})"); - App::setResource('db', fn () => $db); - App::setResource('cache', fn () => $redis); - App::setResource('request', fn () => $request); - App::setResource('response', fn () => $response); + App::setResource('pools', fn() => $register->get('pools')); + App::setResource('request', fn() => $request); + App::setResource('response', fn() => $response); try { - /** @var \Utopia\Database\Document $user */ - $user = $app->getResource('user'); - /** @var \Utopia\Database\Document $project */ $project = $app->getResource('project'); - /** @var \Utopia\Database\Document $console */ - $console = $app->getResource('console'); - - $cache = new Cache(new RedisCache($redis)); - $database = new Database(new MariaDB($db), $cache); - $database->setDefaultDatabase(App::getEnv('_APP_DB_SCHEMA', 'appwrite')); - $database->setNamespace("_{$project->getInternalId()}"); - /* * Project Check */ @@ -393,12 +397,16 @@ $server->onOpen(function (int $connection, SwooleRequest $request) use ($server, throw new Exception('Missing or unknown project ID', 1008); } + $dbForProject = getProjectDB($project); + $console = $app->getResource('console'); /** @var \Utopia\Database\Document $console */ + $user = $app->getResource('user'); /** @var \Utopia\Database\Document $user */ + /* * Abuse Check * * Abuse limits are connecting 128 times per minute and ip address. */ - $timeLimit = new TimeLimit('url:{url},ip:{ip}', 128, 60, $database); + $timeLimit = new TimeLimit('url:{url},ip:{ip}', 128, 60, $dbForProject); $timeLimit ->setParam('{ip}', $request->getIP()) ->setParam('{url}', $request->getURI()); @@ -469,34 +477,20 @@ $server->onOpen(function (int $connection, SwooleRequest $request) use ($server, Console::error('[Error] Code: ' . $response['data']['code']); Console::error('[Error] Message: ' . $response['data']['message']); } - - if ($th instanceof PDOException) { - $db = null; - } } finally { - /** - * Put used PDO and Redis Connections back into their pools. - */ - $register->get('dbPool')->put($db); - $register->get('redisPool')->put($redis); + $register->get('pools')->reclaim(); } }); $server->onMessage(function (int $connection, string $message) use ($server, $register, $realtime, $containerId) { try { $response = new Response(new SwooleResponse()); - $db = $register->get('dbPool')->get(); - $redis = $register->get('redisPool')->get(); - - $cache = new Cache(new RedisCache($redis)); - $database = new Database(new MariaDB($db), $cache); - $database->setDefaultDatabase(App::getEnv('_APP_DB_SCHEMA', 'appwrite')); - $database->setNamespace("_console"); $projectId = $realtime->connections[$connection]['projectId']; + $database = getConsoleDB(); if ($projectId !== 'console') { $project = Authorization::skip(fn() => $database->getDocument('projects', $projectId)); - $database->setNamespace("_{$project->getInternalId()}"); + $database = getProjectDB($project); } /* @@ -580,8 +574,7 @@ $server->onMessage(function (int $connection, string $message) use ($server, $re $server->close($connection, $th->getCode()); } } finally { - $register->get('dbPool')->put($db); - $register->get('redisPool')->put($redis); + $register->get('pools')->reclaim(); } }); diff --git a/app/tasks/doctor.php b/app/tasks/doctor.php index d79e8d530f..daeaaa3f52 100644 --- a/app/tasks/doctor.php +++ b/app/tasks/doctor.php @@ -8,6 +8,7 @@ use Utopia\Storage\Device\Local; use Utopia\Storage\Storage; use Utopia\App; use Utopia\CLI\Console; +use Utopia\Config\Config; use Utopia\Domains\Domain; $cli @@ -21,7 +22,7 @@ $cli Console::log("\n" . '👩‍⚕️ Running ' . APP_NAME . ' Doctor for version ' . App::getEnv('_APP_VERSION', 'UNKNOWN') . ' ...' . "\n"); - Console::log('Checking for production best practices...'); + Console::log('[Settings]'); $domain = new Domain(App::getEnv('_APP_DOMAIN')); @@ -77,7 +78,6 @@ $cli Console::log('🟢 HTTPS force option is enabled'); } - $providerName = App::getEnv('_APP_LOGGING_PROVIDER', ''); $providerConfig = App::getEnv('_APP_LOGGING_CONFIG', ''); @@ -90,30 +90,55 @@ $cli \sleep(0.2); try { - Console::log("\n" . 'Checking connectivity...'); + Console::log("\n" . '[Connectivity]'); } catch (\Throwable $th) { //throw $th; } - try { - $register->get('db'); /* @var $db PDO */ - Console::success('Database............connected 👍'); - } catch (\Throwable $th) { - Console::error('Database.........disconnected 👎'); + $pools = $register->get('pools'); /** @var \Utopia\Pools\Group $pools */ + + $configs = [ + 'Console.DB' => Config::getParam('pools-console'), + 'Projects.DB' => Config::getParam('pools-database'), + ]; + + foreach ($configs as $key => $config) { + foreach ($config as $database) { + try { + $adapter = $pools->get($database)->pop()->getResource(); + + if ($adapter->ping()) { + Console::success('🟢 ' . str_pad("{$key}({$database})", 50, '.') . 'connected'); + } else { + Console::error('🔴 ' . str_pad("{$key}({$database})", 47, '.') . 'disconnected'); + } + } catch (\Throwable $th) { + Console::error('🔴 ' . str_pad("{$key}.({$database})", 47, '.') . 'disconnected'); + } + } } - try { - $register->get('cache'); - Console::success('Queue...............connected 👍'); - } catch (\Throwable $th) { - Console::error('Queue............disconnected 👎'); - } + $pools = $register->get('pools'); /** @var \Utopia\Pools\Group $pools */ + $configs = [ + 'Cache' => Config::getParam('pools-cache'), + 'Queue' => Config::getParam('pools-queue'), + 'PubSub' => Config::getParam('pools-pubsub'), + ]; - try { - $register->get('cache'); - Console::success('Cache...............connected 👍'); - } catch (\Throwable $th) { - Console::error('Cache............disconnected 👎'); + foreach ($configs as $key => $config) { + foreach ($config as $pool) { + try { + $adapter = $pools->get($pool)->pop()->getResource(); + + if ($adapter->ping()) { + Console::success('🟢 ' . str_pad("{$key}({$pool})", 50, '.') . 'connected'); + } else { + Console::error('🔴 ' . str_pad("{$key}({$pool})", 47, '.') . 'disconnected'); + } + } catch (\Throwable $th) { + Console::error('🔴 ' . str_pad("{$key}({$pool})", 47, '.') . 'disconnected'); + } + } } if (App::getEnv('_APP_STORAGE_ANTIVIRUS') === 'enabled') { // Check if scans are enabled @@ -124,12 +149,12 @@ $cli ); if ((@$antivirus->ping())) { - Console::success('Antivirus...........connected 👍'); + Console::success('🟢 ' . str_pad("Antivirus", 50, '.') . 'connected'); } else { - Console::error('Antivirus........disconnected 👎'); + Console::error('🔴 ' . str_pad("Antivirus", 47, '.') . 'disconnected'); } } catch (\Throwable $th) { - Console::error('Antivirus........disconnected 👎'); + Console::error('🔴 ' . str_pad("Antivirus", 47, '.') . 'disconnected'); } } @@ -142,35 +167,35 @@ $cli $mail->AltBody = 'Hello World'; $mail->send(); - Console::success('SMTP................connected 👍'); + Console::success('🟢 ' . str_pad("SMTP", 50, '.') . 'connected'); } catch (\Throwable $th) { - Console::error('SMTP.............disconnected 👎'); + Console::error('🔴 ' . str_pad("SMTP", 47, '.') . 'disconnected'); } $host = App::getEnv('_APP_STATSD_HOST', 'telegraf'); $port = App::getEnv('_APP_STATSD_PORT', 8125); if ($fp = @\fsockopen('udp://' . $host, $port, $errCode, $errStr, 2)) { - Console::success('StatsD..............connected 👍'); + Console::success('🟢 ' . str_pad("StatsD", 50, '.') . 'connected'); \fclose($fp); } else { - Console::error('StatsD...........disconnected 👎'); + Console::error('🔴 ' . str_pad("StatsD", 47, '.') . 'disconnected'); } $host = App::getEnv('_APP_INFLUXDB_HOST', ''); $port = App::getEnv('_APP_INFLUXDB_PORT', ''); if ($fp = @\fsockopen($host, $port, $errCode, $errStr, 2)) { - Console::success('InfluxDB............connected 👍'); + Console::success('🟢 ' . str_pad("InfluxDB", 50, '.') . 'connected'); \fclose($fp); } else { - Console::error('InfluxDB.........disconnected 👎'); + Console::error('🔴 ' . str_pad("InfluxDB", 47, '.') . 'disconnected'); } \sleep(0.2); Console::log(''); - Console::log('Checking volumes...'); + Console::log('[Volumes]'); foreach ( [ @@ -198,7 +223,7 @@ $cli \sleep(0.2); Console::log(''); - Console::log('Checking disk space usage...'); + Console::log('[Disk Space]'); foreach ( [ diff --git a/app/tasks/maintenance.php b/app/tasks/maintenance.php index 92f260ce46..8a102b4b0e 100644 --- a/app/tasks/maintenance.php +++ b/app/tasks/maintenance.php @@ -1,52 +1,16 @@ get('cache'))); - $database = new Database(new MariaDB($register->get('db')), $cache); - $database->setDefaultDatabase(App::getEnv('_APP_DB_SCHEMA', 'appwrite')); - $database->setNamespace('_console'); // Main DB - - if (!$database->exists($database->getDefaultDatabase(), 'certificates')) { - throw new \Exception('Console project not ready'); - } - - break; // leave loop if successful - } catch (\Exception $e) { - Console::warning("Database not ready. Retrying connection ({$attempts})..."); - if ($attempts >= DATABASE_RECONNECT_MAX_ATTEMPTS) { - throw new \Exception('Failed to connect to database: ' . $e->getMessage()); - } - sleep(DATABASE_RECONNECT_SLEEP); - } - } while ($attempts < DATABASE_RECONNECT_MAX_ATTEMPTS); - - return $database; -} - $cli ->task('maintenance') ->desc('Schedules maintenance tasks and publishes them to resque') diff --git a/app/tasks/migrate.php b/app/tasks/migrate.php index f0ab71a964..bf3d1ff602 100644 --- a/app/tasks/migrate.php +++ b/app/tasks/migrate.php @@ -7,9 +7,6 @@ use Appwrite\Migration\Migration; use Utopia\App; use Utopia\Cache\Cache; use Utopia\Cache\Adapter\Redis as RedisCache; -use Utopia\Database\Adapter\MariaDB; -use Utopia\Database\Database; -use Utopia\Database\Query; use Utopia\Database\Validator\Authorization; use Utopia\Validator\Text; @@ -28,17 +25,13 @@ $cli Console::success('Starting Data Migration to version ' . $version); - $db = $register->get('db', true); + $dbPool = $register->get('dbPool', true); $redis = $register->get('cache', true); $redis->flushAll(); $cache = new Cache(new RedisCache($redis)); - $projectDB = new Database(new MariaDB($db), $cache); - $projectDB->setDefaultDatabase(App::getEnv('_APP_DB_SCHEMA', 'appwrite')); - - $consoleDB = new Database(new MariaDB($db), $cache); - $consoleDB->setDefaultDatabase(App::getEnv('_APP_DB_SCHEMA', 'appwrite')); - $consoleDB->setNamespace('_project_console'); + $dbForConsole = $dbPool->getDB('console', $cache); + $dbForConsole->setNamespace('_project_console'); $console = $app->getResource('console'); @@ -52,10 +45,10 @@ $cli $count = 0; try { - $totalProjects = $consoleDB->count('projects') + 1; + $totalProjects = $dbForConsole->count('projects') + 1; } catch (\Throwable $th) { - $consoleDB->setNamespace('_console'); - $totalProjects = $consoleDB->count('projects') + 1; + $dbForConsole->setNamespace('_console'); + $totalProjects = $dbForConsole->count('projects') + 1; } $class = 'Appwrite\\Migration\\Version\\' . Migration::$versions[$version]; @@ -71,8 +64,10 @@ $cli } try { + // TODO: Iterate through all project DBs + $projectDB = $dbPool->getDB($project->getId(), $cache); $migration - ->setProject($project, $projectDB, $consoleDB) + ->setProject($project, $projectDB, $dbForConsole) ->execute(); } catch (\Throwable $th) { throw $th; @@ -81,7 +76,7 @@ $cli } $sum = \count($projects); - $projects = $consoleDB->find('projects', [Query::limit($limit), Query::offset($offset)]); + $projects = $dbForConsole->find('projects', limit: $limit, offset: $offset); $offset = $offset + $limit; $count = $count + $sum; diff --git a/app/tasks/specs.php b/app/tasks/specs.php index 57c6bf29e1..8d1bc5039f 100644 --- a/app/tasks/specs.php +++ b/app/tasks/specs.php @@ -9,8 +9,12 @@ use Appwrite\Specification\Specification; use Appwrite\Utopia\Response; use Swoole\Http\Response as HttpResponse; use Utopia\App; +use Utopia\Cache\Adapter\None; +use Utopia\Cache\Cache; use Utopia\CLI\Console; use Utopia\Config\Config; +use Utopia\Database\Adapter\MySQL; +use Utopia\Database\Database; use Utopia\Request; use Utopia\Validator\WhiteList; @@ -19,16 +23,15 @@ $cli ->param('version', 'latest', new Text(16), 'Spec version', true) ->param('mode', 'normal', new WhiteList(['normal', 'mocks']), 'Spec Mode', true) ->action(function ($version, $mode) use ($register) { - $db = $register->get('db'); - $redis = $register->get('cache'); $appRoutes = App::getRoutes(); $response = new Response(new HttpResponse()); $mocks = ($mode === 'mocks'); + // Mock dependencies App::setResource('request', fn () => new Request()); App::setResource('response', fn () => $response); - App::setResource('db', fn () => $db); - App::setResource('cache', fn () => $redis); + App::setResource('dbForConsole', fn () => new Database(new MySQL(''), new Cache(new None()))); + App::setResource('dbForProject', fn () => new Database(new MySQL(''), new Cache(new None()))); $platforms = [ 'client' => APP_PLATFORM_CLIENT, diff --git a/app/tasks/usage.php b/app/tasks/usage.php index b28be1de17..68543d0748 100644 --- a/app/tasks/usage.php +++ b/app/tasks/usage.php @@ -5,77 +5,15 @@ global $cli, $register; use Appwrite\Usage\Calculators\TimeSeries; use InfluxDB\Database as InfluxDatabase; use Utopia\App; -use Utopia\Cache\Adapter\Redis as RedisCache; -use Utopia\Cache\Cache; use Utopia\CLI\Console; -use Utopia\Database\Adapter\MariaDB; use Utopia\Database\Database as UtopiaDatabase; use Utopia\Database\Validator\Authorization; -use Utopia\Registry\Registry; use Utopia\Logger\Log; use Utopia\Validator\WhiteList; Authorization::disable(); Authorization::setDefaultStatus(false); -function getDatabase(Registry &$register, string $namespace): UtopiaDatabase -{ - $attempts = 0; - - do { - try { - $attempts++; - - $db = $register->get('db'); - $redis = $register->get('cache'); - - $cache = new Cache(new RedisCache($redis)); - $database = new UtopiaDatabase(new MariaDB($db), $cache); - $database->setDefaultDatabase(App::getEnv('_APP_DB_SCHEMA', 'appwrite')); - $database->setNamespace($namespace); - - if (!$database->exists($database->getDefaultDatabase(), 'projects')) { - throw new Exception('Projects collection not ready'); - } - break; // leave loop if successful - } catch (\Exception $e) { - Console::warning("Database not ready. Retrying connection ({$attempts})..."); - if ($attempts >= DATABASE_RECONNECT_MAX_ATTEMPTS) { - throw new \Exception('Failed to connect to database: ' . $e->getMessage()); - } - sleep(DATABASE_RECONNECT_SLEEP); - } - } while ($attempts < DATABASE_RECONNECT_MAX_ATTEMPTS); - - return $database; -} - -function getInfluxDB(Registry &$register): InfluxDatabase -{ - /** @var InfluxDB\Client $client */ - $client = $register->get('influxdb'); - $attempts = 0; - $max = 10; - $sleep = 1; - - do { // check if telegraf database is ready - try { - $attempts++; - $database = $client->selectDB('telegraf'); - if (in_array('telegraf', $client->listDatabases())) { - break; // leave the do-while if successful - } - } catch (\Throwable $th) { - Console::warning("InfluxDB not ready. Retrying connection ({$attempts})..."); - if ($attempts >= $max) { - throw new \Exception('InfluxDB database not ready yet'); - } - sleep($sleep); - } - } while ($attempts < $max); - return $database; -} - $logError = function (Throwable $error, string $action = 'syncUsageStats') use ($register) { $logger = $register->get('logger'); @@ -113,12 +51,12 @@ $logError = function (Throwable $error, string $action = 'syncUsageStats') use ( $cli ->task('usage') ->desc('Schedules syncing data from influxdb to Appwrite console db') - ->action(function () use ($register, $logError) { + ->action(function () use ($logError) { Console::title('Usage Aggregation V1'); Console::success(APP_NAME . ' usage aggregation process v1 has started'); - $database = getDatabase($register, '_console'); - $influxDB = getInfluxDB($register); + $database = getConsoleDB(); + $influxDB = getInfluxDB(); $interval = (int) App::getEnv('_APP_USAGE_AGGREGATION_INTERVAL', '30'); // 30 seconds (by default) $usage = new TimeSeries($database, $influxDB, $logError); diff --git a/app/views/install/compose.phtml b/app/views/install/compose.phtml index e14e434a84..02b030a0a6 100644 --- a/app/views/install/compose.phtml +++ b/app/views/install/compose.phtml @@ -88,15 +88,15 @@ services: - _APP_OPENSSL_KEY_V1 - _APP_DOMAIN - _APP_DOMAIN_TARGET - - _APP_REDIS_HOST - - _APP_REDIS_PORT - - _APP_REDIS_USER - - _APP_REDIS_PASS - _APP_DB_HOST - _APP_DB_PORT - _APP_DB_SCHEMA - _APP_DB_USER - _APP_DB_PASS + - _APP_REDIS_HOST + - _APP_REDIS_PORT + - _APP_REDIS_USER + - _APP_REDIS_PASS - _APP_SMTP_HOST - _APP_SMTP_PORT - _APP_SMTP_SECURE @@ -185,13 +185,15 @@ services: - _APP_WORKER_PER_CORE - _APP_OPTIONS_ABUSE - _APP_OPENSSL_KEY_V1 - - _APP_REDIS_HOST - - _APP_REDIS_PORT - _APP_DB_HOST - _APP_DB_PORT - _APP_DB_SCHEMA - _APP_DB_USER - _APP_DB_PASS + - _APP_REDIS_HOST + - _APP_REDIS_PORT + - _APP_REDIS_USER + - _APP_REDIS_PASS - _APP_USAGE_STATS - _APP_LOGGING_PROVIDER - _APP_LOGGING_CONFIG @@ -210,15 +212,15 @@ services: environment: - _APP_ENV - _APP_OPENSSL_KEY_V1 - - _APP_REDIS_HOST - - _APP_REDIS_PORT - - _APP_REDIS_USER - - _APP_REDIS_PASS - _APP_DB_HOST - _APP_DB_PORT - _APP_DB_SCHEMA - _APP_DB_USER - _APP_DB_PASS + - _APP_REDIS_HOST + - _APP_REDIS_PORT + - _APP_REDIS_USER + - _APP_REDIS_PASS - _APP_LOGGING_PROVIDER - _APP_LOGGING_CONFIG @@ -264,15 +266,15 @@ services: environment: - _APP_ENV - _APP_OPENSSL_KEY_V1 - - _APP_REDIS_HOST - - _APP_REDIS_PORT - - _APP_REDIS_USER - - _APP_REDIS_PASS - _APP_DB_HOST - _APP_DB_PORT - _APP_DB_SCHEMA - _APP_DB_USER - _APP_DB_PASS + - _APP_REDIS_HOST + - _APP_REDIS_PORT + - _APP_REDIS_USER + - _APP_REDIS_PASS - _APP_STORAGE_DEVICE - _APP_STORAGE_S3_ACCESS_KEY - _APP_STORAGE_S3_SECRET @@ -313,15 +315,15 @@ services: environment: - _APP_ENV - _APP_OPENSSL_KEY_V1 - - _APP_REDIS_HOST - - _APP_REDIS_PORT - - _APP_REDIS_USER - - _APP_REDIS_PASS - _APP_DB_HOST - _APP_DB_PORT - _APP_DB_SCHEMA - _APP_DB_USER - _APP_DB_PASS + - _APP_REDIS_HOST + - _APP_REDIS_PORT + - _APP_REDIS_USER + - _APP_REDIS_PASS - _APP_LOGGING_PROVIDER - _APP_LOGGING_CONFIG @@ -341,15 +343,15 @@ services: - _APP_OPENSSL_KEY_V1 - _APP_EXECUTOR_SECRET - _APP_EXECUTOR_HOST - - _APP_REDIS_HOST - - _APP_REDIS_PORT - - _APP_REDIS_USER - - _APP_REDIS_PASS - _APP_DB_HOST - _APP_DB_PORT - _APP_DB_SCHEMA - _APP_DB_USER - _APP_DB_PASS + - _APP_REDIS_HOST + - _APP_REDIS_PORT + - _APP_REDIS_USER + - _APP_REDIS_PASS - _APP_LOGGING_PROVIDER - _APP_LOGGING_CONFIG @@ -373,15 +375,15 @@ services: - _APP_DOMAIN - _APP_DOMAIN_TARGET - _APP_SYSTEM_SECURITY_EMAIL_ADDRESS - - _APP_REDIS_HOST - - _APP_REDIS_PORT - - _APP_REDIS_USER - - _APP_REDIS_PASS - _APP_DB_HOST - _APP_DB_PORT - _APP_DB_SCHEMA - _APP_DB_USER - _APP_DB_PASS + - _APP_REDIS_HOST + - _APP_REDIS_PORT + - _APP_REDIS_USER + - _APP_REDIS_PASS - _APP_LOGGING_PROVIDER - _APP_LOGGING_CONFIG @@ -400,15 +402,15 @@ services: environment: - _APP_ENV - _APP_OPENSSL_KEY_V1 - - _APP_REDIS_HOST - - _APP_REDIS_PORT - - _APP_REDIS_USER - - _APP_REDIS_PASS - _APP_DB_HOST - _APP_DB_PORT - _APP_DB_SCHEMA - _APP_DB_USER - _APP_DB_PASS + - _APP_REDIS_HOST + - _APP_REDIS_PORT + - _APP_REDIS_USER + - _APP_REDIS_PASS - _APP_FUNCTIONS_TIMEOUT - _APP_EXECUTOR_SECRET - _APP_EXECUTOR_HOST @@ -536,15 +538,15 @@ services: - _APP_OPENSSL_KEY_V1 - _APP_DOMAIN - _APP_DOMAIN_TARGET - - _APP_REDIS_HOST - - _APP_REDIS_PORT - - _APP_REDIS_USER - - _APP_REDIS_PASS - _APP_DB_HOST - _APP_DB_PORT - _APP_DB_SCHEMA - _APP_DB_USER - _APP_DB_PASS + - _APP_REDIS_HOST + - _APP_REDIS_PORT + - _APP_REDIS_USER + - _APP_REDIS_PASS - _APP_MAINTENANCE_INTERVAL - _APP_MAINTENANCE_RETENTION_EXECUTION - _APP_MAINTENANCE_RETENTION_CACHE diff --git a/app/workers/audits.php b/app/workers/audits.php index b86649543b..90ac020536 100644 --- a/app/workers/audits.php +++ b/app/workers/audits.php @@ -1,6 +1,5 @@ getAttribute('name', ''); $userEmail = $user->getAttribute('email', ''); - $dbForProject = $this->getProjectDB($project->getId()); + $dbForProject = $this->getProjectDB($project); $audit = new Audit($dbForProject); $audit->log( userId: $user->getId(), diff --git a/app/workers/builds.php b/app/workers/builds.php index bf780c6464..235ad0da72 100644 --- a/app/workers/builds.php +++ b/app/workers/builds.php @@ -7,7 +7,6 @@ use Appwrite\Utopia\Response\Model\Deployment; use Cron\CronExpression; use Executor\Executor; use Appwrite\Usage\Stats; -use Utopia\Database\Database; use Utopia\Database\DateTime; use Utopia\App; use Utopia\CLI\Console; @@ -15,7 +14,6 @@ use Utopia\Database\ID; use Utopia\Storage\Storage; use Utopia\Database\Document; use Utopia\Config\Config; -use Utopia\Database\Query; require_once __DIR__ . '/../init.php'; @@ -59,7 +57,7 @@ class BuildsV1 extends Worker protected function buildDeployment(Document $project, Document $function, Document $deployment) { - $dbForProject = $this->getProjectDB($project->getId()); + $dbForProject = $this->getProjectDB($project); $function = $dbForProject->getDocument('functions', $function->getId()); if ($function->isEmpty()) { diff --git a/app/workers/certificates.php b/app/workers/certificates.php index e7885bc669..b4f0701c46 100644 --- a/app/workers/certificates.php +++ b/app/workers/certificates.php @@ -1,6 +1,5 @@ createAttribute($database, $collection, $document, $project->getId()); + $this->createAttribute($database, $collection, $document, $project); break; case DATABASE_TYPE_DELETE_ATTRIBUTE: - $this->deleteAttribute($database, $collection, $document, $project->getId()); + $this->deleteAttribute($database, $collection, $document, $project); break; case DATABASE_TYPE_CREATE_INDEX: - $this->createIndex($database, $collection, $document, $project->getId()); + $this->createIndex($database, $collection, $document, $project); break; case DATABASE_TYPE_DELETE_INDEX: - $this->deleteIndex($database, $collection, $document, $project->getId()); + $this->deleteIndex($database, $collection, $document, $project); break; default: @@ -61,12 +61,13 @@ class DatabaseV1 extends Worker * @param Document $database * @param Document $collection * @param Document $attribute - * @param string $projectId + * @param Document $project */ - protected function createAttribute(Document $database, Document $collection, Document $attribute, string $projectId): void + protected function createAttribute(Document $database, Document $collection, Document $attribute, Document $project): void { + $projectId = $project->getId(); $dbForConsole = $this->getConsoleDB(); - $dbForProject = $this->getProjectDB($projectId); + $dbForProject = $this->getProjectDB($project); $events = Event::generateEvents('databases.[databaseId].collections.[collectionId].attributes.[attributeId].update', [ 'databaseId' => $database->getId(), @@ -128,12 +129,13 @@ class DatabaseV1 extends Worker * @param Document $database * @param Document $collection * @param Document $attribute - * @param string $projectId + * @param Document $project */ - protected function deleteAttribute(Document $database, Document $collection, Document $attribute, string $projectId): void + protected function deleteAttribute(Document $database, Document $collection, Document $attribute, Document $project): void { + $projectId = $project->getId(); $dbForConsole = $this->getConsoleDB(); - $dbForProject = $this->getProjectDB($projectId); + $dbForProject = $this->getProjectDB($project); $events = Event::generateEvents('databases.[databaseId].collections.[collectionId].attributes.[attributeId].delete', [ 'databaseId' => $database->getId(), @@ -225,7 +227,7 @@ class DatabaseV1 extends Worker } if ($exists) { // Delete the duplicate if created, else update in db - $this->deleteIndex($database, $collection, $index, $projectId); + $this->deleteIndex($database, $collection, $index, $project); } else { $dbForProject->updateDocument('indexes', $index->getId(), $index); } @@ -241,12 +243,13 @@ class DatabaseV1 extends Worker * @param Document $database * @param Document $collection * @param Document $index - * @param string $projectId + * @param Document $project */ - protected function createIndex(Document $database, Document $collection, Document $index, string $projectId): void + protected function createIndex(Document $database, Document $collection, Document $index, Document $project): void { + $projectId = $project->getId(); $dbForConsole = $this->getConsoleDB(); - $dbForProject = $this->getProjectDB($projectId); + $dbForProject = $this->getProjectDB($project); $events = Event::generateEvents('databases.[databaseId].collections.[collectionId].indexes.[indexId].update', [ 'databaseId' => $database->getId(), @@ -298,12 +301,13 @@ class DatabaseV1 extends Worker * @param Document $database * @param Document $collection * @param Document $index - * @param string $projectId + * @param Document $project */ - protected function deleteIndex(Document $database, Document $collection, Document $index, string $projectId): void + protected function deleteIndex(Document $database, Document $collection, Document $index, Document $project): void { + $projectId = $project->getId(); $dbForConsole = $this->getConsoleDB(); - $dbForProject = $this->getProjectDB($projectId); + $dbForProject = $this->getProjectDB($project); $events = Event::generateEvents('databases.[databaseId].collections.[collectionId].indexes.[indexId].delete', [ 'databaseId' => $database->getId(), diff --git a/app/workers/deletes.php b/app/workers/deletes.php index 44ae1136a8..f898778386 100644 --- a/app/workers/deletes.php +++ b/app/workers/deletes.php @@ -40,28 +40,28 @@ class DeletesV1 extends Worker switch ($document->getCollection()) { case DELETE_TYPE_DATABASES: - $this->deleteDatabase($document, $project->getId()); + $this->deleteDatabase($document, $project); break; case DELETE_TYPE_COLLECTIONS: - $this->deleteCollection($document, $project->getId()); + $this->deleteCollection($document, $project); break; case DELETE_TYPE_PROJECTS: $this->deleteProject($document); break; case DELETE_TYPE_FUNCTIONS: - $this->deleteFunction($document, $project->getId()); + $this->deleteFunction($document, $project); break; case DELETE_TYPE_DEPLOYMENTS: - $this->deleteDeployment($document, $project->getId()); + $this->deleteDeployment($document, $project); break; case DELETE_TYPE_USERS: - $this->deleteUser($document, $project->getId()); + $this->deleteUser($document, $project); break; case DELETE_TYPE_TEAMS: - $this->deleteMemberships($document, $project->getId()); + $this->deleteMemberships($document, $project); break; case DELETE_TYPE_BUCKETS: - $this->deleteBucket($document, $project->getId()); + $this->deleteBucket($document, $project); break; default: Console::error('No lazy delete operation available for document of type: ' . $document->getCollection()); @@ -82,7 +82,7 @@ class DeletesV1 extends Worker $document = new Document($this->args['document'] ?? []); if (!$document->isEmpty()) { - $this->deleteAuditLogsByResource('document/' . $document->getId(), $project->getId()); + $this->deleteAuditLogsByResource('document/' . $document->getId(), $project); } break; @@ -109,7 +109,7 @@ class DeletesV1 extends Worker break; case DELETE_TYPE_CACHE_BY_RESOURCE: - $this->deleteCacheByResource($project->getId()); + $this->deleteCacheByResource($this->args['resource']); break; case DELETE_TYPE_CACHE_BY_TIMESTAMP: $this->deleteCacheByDate(); @@ -125,12 +125,12 @@ class DeletesV1 extends Worker } /** - * @param string $projectId + * @param string $resource */ - protected function deleteCacheByResource(string $projectId): void + protected function deleteCacheByResource(string $resource): void { $this->deleteCacheFiles([ - Query::equal('resource', [$this->args['resource']]), + Query::equal('resource', [$resource]), ]); } @@ -143,9 +143,10 @@ class DeletesV1 extends Worker protected function deleteCacheFiles($query): void { - $this->deleteForProjectIds(function (string $projectId) use ($query) { + $this->deleteForProjectIds(function (Document $project) use ($query) { - $dbForProject = $this->getProjectDB($projectId); + $projectId = $project->getId(); + $dbForProject = $this->getProjectDB($project); $cache = new Cache( new Filesystem(APP_STORAGE_CACHE . DIRECTORY_SEPARATOR . 'app-' . $projectId) ); @@ -169,34 +170,35 @@ class DeletesV1 extends Worker /** * @param Document $document database document - * @param string $projectId + * @param Document $projectId */ - protected function deleteDatabase(Document $document, string $projectId): void + protected function deleteDatabase(Document $document, Document $project): void { $databaseId = $document->getId(); + $projectId = $project->getId(); - $dbForProject = $this->getProjectDB($projectId); + $dbForProject = $this->getProjectDB($project); - $this->deleteByGroup('database_' . $document->getInternalId(), [], $dbForProject, function ($document) use ($projectId) { - $this->deleteCollection($document, $projectId); + $this->deleteByGroup('database_' . $document->getInternalId(), [], $dbForProject, function ($document) use ($project) { + $this->deleteCollection($document, $project); }); $dbForProject->deleteCollection('database_' . $document->getInternalId()); - $this->deleteAuditLogsByResource('database/' . $databaseId, $projectId); + $this->deleteAuditLogsByResource('database/' . $databaseId, $project); } /** * @param Document $document teams document - * @param string $projectId + * @param Document $project */ - protected function deleteCollection(Document $document, string $projectId): void + protected function deleteCollection(Document $document, Document $project): void { $collectionId = $document->getId(); $databaseId = $document->getAttribute('databaseId'); $databaseInternalId = $document->getAttribute('databaseInternalId'); - $dbForProject = $this->getProjectDB($projectId); + $dbForProject = $this->getProjectDB($project); $dbForProject->deleteCollection('database_' . $databaseInternalId . '_collection_' . $document->getInternalId()); @@ -210,7 +212,7 @@ class DeletesV1 extends Worker Query::equal('collectionId', [$collectionId]) ], $dbForProject); - $this->deleteAuditLogsByResource('database/' . $databaseId . '/collection/' . $collectionId, $projectId); + $this->deleteAuditLogsByResource('database/' . $databaseId . '/collection/' . $collectionId, $project); } /** @@ -219,8 +221,9 @@ class DeletesV1 extends Worker */ protected function deleteUsageStats(string $hourlyUsageRetentionDatetime) { - $this->deleteForProjectIds(function (string $projectId) use ($hourlyUsageRetentionDatetime) { - $dbForProject = $this->getProjectDB($projectId); + $this->deleteForProjectIds(function (Document $project) use ($hourlyUsageRetentionDatetime) { + $dbForProject = $this->getProjectDB($project); + // Delete Usage stats $this->deleteByGroup('stats', [ Query::lessThan('time', $hourlyUsageRetentionDatetime), Query::equal('period', ['1h']), @@ -230,16 +233,16 @@ class DeletesV1 extends Worker /** * @param Document $document teams document - * @param string $projectId + * @param Document $project */ - protected function deleteMemberships(Document $document, string $projectId): void + protected function deleteMemberships(Document $document, Document $project): void { $teamId = $document->getAttribute('teamId', ''); // Delete Memberships $this->deleteByGroup('memberships', [ Query::equal('teamId', [$teamId]) - ], $this->getProjectDB($projectId)); + ], $this->getProjectDB($project)); } /** @@ -250,7 +253,7 @@ class DeletesV1 extends Worker $projectId = $document->getId(); // Delete all DBs - $this->getProjectDB($projectId)->delete($projectId); + $this->getProjectDB($document)->delete($projectId); // Delete all storage directories $uploads = new Local(APP_STORAGE_UPLOADS . '/app-' . $document->getId()); @@ -262,30 +265,30 @@ class DeletesV1 extends Worker /** * @param Document $document user document - * @param string $projectId + * @param Document $project */ - protected function deleteUser(Document $document, string $projectId): void + protected function deleteUser(Document $document, Document $project): void { $userId = $document->getId(); // Delete all sessions of this user from the sessions table and update the sessions field of the user record $this->deleteByGroup('sessions', [ Query::equal('userId', [$userId]) - ], $this->getProjectDB($projectId)); + ], $this->getProjectDB($project)); - $this->getProjectDB($projectId)->deleteCachedDocument('users', $userId); + $this->getProjectDB($project)->deleteCachedDocument('users', $userId); // Delete Memberships and decrement team membership counts $this->deleteByGroup('memberships', [ Query::equal('userId', [$userId]) - ], $this->getProjectDB($projectId), function (Document $document) use ($projectId) { + ], $this->getProjectDB($project), function (Document $document) use ($project) { if ($document->getAttribute('confirm')) { // Count only confirmed members $teamId = $document->getAttribute('teamId'); - $team = $this->getProjectDB($projectId)->getDocument('teams', $teamId); + $team = $this->getProjectDB($project)->getDocument('teams', $teamId); if (!$team->isEmpty()) { $team = $this - ->getProjectDB($projectId) + ->getProjectDB($project) ->updateDocument( 'teams', $teamId, @@ -299,7 +302,7 @@ class DeletesV1 extends Worker // Delete tokens $this->deleteByGroup('tokens', [ Query::equal('userId', [$userId]) - ], $this->getProjectDB($projectId)); + ], $this->getProjectDB($project)); } /** @@ -307,8 +310,8 @@ class DeletesV1 extends Worker */ protected function deleteExecutionLogs(string $datetime): void { - $this->deleteForProjectIds(function (string $projectId) use ($datetime) { - $dbForProject = $this->getProjectDB($projectId); + $this->deleteForProjectIds(function (Document $project) use ($datetime) { + $dbForProject = $this->getProjectDB($project); // Delete Executions $this->deleteByGroup('executions', [ Query::lessThan('$createdAt', $datetime) @@ -321,8 +324,8 @@ class DeletesV1 extends Worker */ protected function deleteExpiredSessions(string $datetime): void { - $this->deleteForProjectIds(function (string $projectId) use ($datetime) { - $dbForProject = $this->getProjectDB($projectId); + $this->deleteForProjectIds(function (Document $project) use ($datetime) { + $dbForProject = $this->getProjectDB($project); // Delete Sessions $this->deleteByGroup('sessions', [ Query::lessThan('expire', $datetime) @@ -335,8 +338,8 @@ class DeletesV1 extends Worker */ protected function deleteRealtimeUsage(string $datetime): void { - $this->deleteForProjectIds(function (string $projectId) use ($datetime) { - $dbForProject = $this->getProjectDB($projectId); + $this->deleteForProjectIds(function (Document $project) use ($datetime) { + $dbForProject = $this->getProjectDB($project); // Delete Dead Realtime Logs $this->deleteByGroup('realtime', [ Query::lessThan('timestamp', $datetime) @@ -354,8 +357,9 @@ class DeletesV1 extends Worker throw new Exception('Failed to delete audit logs. No datetime provided'); } - $this->deleteForProjectIds(function (string $projectId) use ($datetime) { - $dbForProject = $this->getProjectDB($projectId); + $this->deleteForProjectIds(function (Document $project) use ($datetime) { + $projectId = $project->getId(); + $dbForProject = $this->getProjectDB($project); $timeLimit = new TimeLimit("", 0, 1, $dbForProject); $abuse = new Abuse($timeLimit); $status = $abuse->cleanup($datetime); @@ -375,8 +379,9 @@ class DeletesV1 extends Worker throw new Exception('Failed to delete audit logs. No datetime provided'); } - $this->deleteForProjectIds(function (string $projectId) use ($datetime) { - $dbForProject = $this->getProjectDB($projectId); + $this->deleteForProjectIds(function (Document $project) use ($datetime) { + $projectId = $project->getId(); + $dbForProject = $this->getProjectDB($project); $audit = new Audit($dbForProject); $status = $audit->cleanup($datetime); if (!$status) { @@ -387,11 +392,11 @@ class DeletesV1 extends Worker /** * @param string $resource - * @param string $projectId + * @param Document $project */ - protected function deleteAuditLogsByResource(string $resource, string $projectId): void + protected function deleteAuditLogsByResource(string $resource, Document $project): void { - $dbForProject = $this->getProjectDB($projectId); + $dbForProject = $this->getProjectDB($project); $this->deleteByGroup(Audit::COLLECTION, [ Query::equal('resource', [$resource]) @@ -400,11 +405,12 @@ class DeletesV1 extends Worker /** * @param Document $document function document - * @param string $projectId + * @param Document $project */ - protected function deleteFunction(Document $document, string $projectId): void + protected function deleteFunction(Document $document, Document $project): void { - $dbForProject = $this->getProjectDB($projectId); + $projectId = $project->getId(); + $dbForProject = $this->getProjectDB($project); $functionId = $document->getId(); /** @@ -473,11 +479,12 @@ class DeletesV1 extends Worker /** * @param Document $document deployment document - * @param string $projectId + * @param Document $project */ - protected function deleteDeployment(Document $document, string $projectId): void + protected function deleteDeployment(Document $document, Document $project): void { - $dbForProject = $this->getProjectDB($projectId); + $projectId = $project->getId(); + $dbForProject = $this->getProjectDB($project); $deploymentId = $document->getId(); $functionId = $document->getAttribute('resourceId'); @@ -562,13 +569,11 @@ class DeletesV1 extends Worker $chunk++; /** @var string[] $projectIds */ - $projectIds = array_map(fn (Document $project) => $project->getId(), $projects); - $sum = count($projects); Console::info('Executing delete function for chunk #' . $chunk . '. Found ' . $sum . ' projects'); - foreach ($projectIds as $projectId) { - $callback($projectId); + foreach ($projects as $project) { + $callback($project); $count++; } } @@ -660,9 +665,10 @@ class DeletesV1 extends Worker } } - protected function deleteBucket(Document $document, string $projectId) + protected function deleteBucket(Document $document, Document $project) { - $dbForProject = $this->getProjectDB($projectId); + $projectId = $project->getId(); + $dbForProject = $this->getProjectDB($project); $dbForProject->deleteCollection('bucket_' . $document->getInternalId()); $device = $this->getDevice(APP_STORAGE_UPLOADS . '/app-' . $projectId); diff --git a/app/workers/functions.php b/app/workers/functions.php index 1df776383f..1ba4b6575b 100644 --- a/app/workers/functions.php +++ b/app/workers/functions.php @@ -52,7 +52,7 @@ class FunctionsV1 extends Worker return; } - $database = $this->getProjectDB($project->getId()); + $database = $this->getProjectDB($project); /** * Handle Event execution. diff --git a/composer.json b/composer.json index 6c3cf1950f..1748f7cac6 100644 --- a/composer.json +++ b/composer.json @@ -48,10 +48,10 @@ "utopia-php/abuse": "0.15.*", "utopia-php/analytics": "0.2.*", "utopia-php/audit": "0.16.*", - "utopia-php/cache": "0.6.*", + "utopia-php/cache": "0.8.*", "utopia-php/cli": "0.13.*", "utopia-php/config": "0.2.*", - "utopia-php/database": "0.27.0", + "utopia-php/database": "dev-feat-update-create as 0.27.0", "utopia-php/locale": "0.4.*", "utopia-php/registry": "0.5.*", "utopia-php/preloader": "0.2.*", @@ -61,6 +61,7 @@ "utopia-php/websocket": "0.1.0", "utopia-php/image": "0.5.*", "utopia-php/orchestration": "0.6.*", + "utopia-php/pools": "0.1.*", "resque/php-resque": "1.3.6", "matomo/device-detector": "6.0.0", "dragonmantank/cron-expression": "3.3.1", diff --git a/composer.lock b/composer.lock index da90bebf25..86d47f4a19 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "fe23114c0eec940a582137c2ecac8e6a", + "content-hash": "c33b697a2f1422eb6fccfd860a7e0169", "packages": [ { "name": "adhocore/jwt", @@ -345,79 +345,6 @@ }, "time": "2022-06-20T22:56:59+00:00" }, - { - "name": "composer/package-versions-deprecated", - "version": "1.11.99.5", - "source": { - "type": "git", - "url": "https://github.com/composer/package-versions-deprecated.git", - "reference": "b4f54f74ef3453349c24a845d22392cd31e65f1d" - }, - "dist": { - "type": "zip", - "url": "https://api.github.com/repos/composer/package-versions-deprecated/zipball/b4f54f74ef3453349c24a845d22392cd31e65f1d", - "reference": "b4f54f74ef3453349c24a845d22392cd31e65f1d", - "shasum": "" - }, - "require": { - "composer-plugin-api": "^1.1.0 || ^2.0", - "php": "^7 || ^8" - }, - "replace": { - "ocramius/package-versions": "1.11.99" - }, - "require-dev": { - "composer/composer": "^1.9.3 || ^2.0@dev", - "ext-zip": "^1.13", - "phpunit/phpunit": "^6.5 || ^7" - }, - "type": "composer-plugin", - "extra": { - "class": "PackageVersions\\Installer", - "branch-alias": { - "dev-master": "1.x-dev" - } - }, - "autoload": { - "psr-4": { - "PackageVersions\\": "src/PackageVersions" - } - }, - "notification-url": "https://packagist.org/downloads/", - "license": [ - "MIT" - ], - "authors": [ - { - "name": "Marco Pivetta", - "email": "ocramius@gmail.com" - }, - { - "name": "Jordi Boggiano", - "email": "j.boggiano@seld.be" - } - ], - "description": "Composer plugin that provides efficient querying for installed package versions (no runtime IO)", - "support": { - "issues": "https://github.com/composer/package-versions-deprecated/issues", - "source": "https://github.com/composer/package-versions-deprecated/tree/1.11.99.5" - }, - "funding": [ - { - "url": "https://packagist.com", - "type": "custom" - }, - { - "url": "https://github.com/composer", - "type": "github" - }, - { - "url": "https://tidelift.com/funding/github/packagist/composer/composer", - "type": "tidelift" - } - ], - "time": "2022-01-17T14:14:24+00:00" - }, { "name": "dragonmantank/cron-expression", "version": "v3.3.1", @@ -876,61 +803,6 @@ }, "time": "2020-12-26T17:45:17+00:00" }, - { - "name": "jean85/pretty-package-versions", - "version": "1.6.0", - "source": { - "type": "git", - "url": "https://github.com/Jean85/pretty-package-versions.git", - "reference": "1e0104b46f045868f11942aea058cd7186d6c303" - }, - "dist": { - "type": "zip", - "url": "https://api.github.com/repos/Jean85/pretty-package-versions/zipball/1e0104b46f045868f11942aea058cd7186d6c303", - "reference": "1e0104b46f045868f11942aea058cd7186d6c303", - "shasum": "" - }, - "require": { - "composer/package-versions-deprecated": "^1.8.0", - "php": "^7.0|^8.0" - }, - "require-dev": { - "phpunit/phpunit": "^6.0|^8.5|^9.2" - }, - "type": "library", - "extra": { - "branch-alias": { - "dev-master": "1.x-dev" - } - }, - "autoload": { - "psr-4": { - "Jean85\\": "src/" - } - }, - "notification-url": "https://packagist.org/downloads/", - "license": [ - "MIT" - ], - "authors": [ - { - "name": "Alessandro Lai", - "email": "alessandro.lai85@gmail.com" - } - ], - "description": "A wrapper for ocramius/package-versions to get pretty versions strings", - "keywords": [ - "composer", - "package", - "release", - "versions" - ], - "support": { - "issues": "https://github.com/Jean85/pretty-package-versions/issues", - "source": "https://github.com/Jean85/pretty-package-versions/tree/1.6.0" - }, - "time": "2021-02-04T16:20:16+00:00" - }, { "name": "matomo/device-detector", "version": "6.0.0", @@ -1000,74 +872,6 @@ }, "time": "2022-04-11T09:58:17+00:00" }, - { - "name": "mongodb/mongodb", - "version": "1.8.0", - "source": { - "type": "git", - "url": "https://github.com/mongodb/mongo-php-library.git", - "reference": "953dbc19443aa9314c44b7217a16873347e6840d" - }, - "dist": { - "type": "zip", - "url": "https://api.github.com/repos/mongodb/mongo-php-library/zipball/953dbc19443aa9314c44b7217a16873347e6840d", - "reference": "953dbc19443aa9314c44b7217a16873347e6840d", - "shasum": "" - }, - "require": { - "ext-hash": "*", - "ext-json": "*", - "ext-mongodb": "^1.8.1", - "jean85/pretty-package-versions": "^1.2", - "php": "^7.0 || ^8.0", - "symfony/polyfill-php80": "^1.19" - }, - "require-dev": { - "squizlabs/php_codesniffer": "^3.5, <3.5.5", - "symfony/phpunit-bridge": "5.x-dev" - }, - "type": "library", - "extra": { - "branch-alias": { - "dev-master": "1.8.x-dev" - } - }, - "autoload": { - "files": [ - "src/functions.php" - ], - "psr-4": { - "MongoDB\\": "src/" - } - }, - "notification-url": "https://packagist.org/downloads/", - "license": [ - "Apache-2.0" - ], - "authors": [ - { - "name": "Andreas Braun", - "email": "andreas.braun@mongodb.com" - }, - { - "name": "Jeremy Mikola", - "email": "jmikola@gmail.com" - } - ], - "description": "MongoDB driver library", - "homepage": "https://jira.mongodb.org/browse/PHPLIB", - "keywords": [ - "database", - "driver", - "mongodb", - "persistence" - ], - "support": { - "issues": "https://github.com/mongodb/mongo-php-library/issues", - "source": "https://github.com/mongodb/mongo-php-library/tree/1.8.0" - }, - "time": "2020-11-25T12:26:02+00:00" - }, { "name": "mustangostang/spyc", "version": "0.6.3", @@ -1656,89 +1460,6 @@ ], "time": "2022-02-25T11:15:52+00:00" }, - { - "name": "symfony/polyfill-php80", - "version": "v1.26.0", - "source": { - "type": "git", - "url": "https://github.com/symfony/polyfill-php80.git", - "reference": "cfa0ae98841b9e461207c13ab093d76b0fa7bace" - }, - "dist": { - "type": "zip", - "url": "https://api.github.com/repos/symfony/polyfill-php80/zipball/cfa0ae98841b9e461207c13ab093d76b0fa7bace", - "reference": "cfa0ae98841b9e461207c13ab093d76b0fa7bace", - "shasum": "" - }, - "require": { - "php": ">=7.1" - }, - "type": "library", - "extra": { - "branch-alias": { - "dev-main": "1.26-dev" - }, - "thanks": { - "name": "symfony/polyfill", - "url": "https://github.com/symfony/polyfill" - } - }, - "autoload": { - "files": [ - "bootstrap.php" - ], - "psr-4": { - "Symfony\\Polyfill\\Php80\\": "" - }, - "classmap": [ - "Resources/stubs" - ] - }, - "notification-url": "https://packagist.org/downloads/", - "license": [ - "MIT" - ], - "authors": [ - { - "name": "Ion Bazan", - "email": "ion.bazan@gmail.com" - }, - { - "name": "Nicolas Grekas", - "email": "p@tchwork.com" - }, - { - "name": "Symfony Community", - "homepage": "https://symfony.com/contributors" - } - ], - "description": "Symfony polyfill backporting some PHP 8.0+ features to lower PHP versions", - "homepage": "https://symfony.com", - "keywords": [ - "compatibility", - "polyfill", - "portable", - "shim" - ], - "support": { - "source": "https://github.com/symfony/polyfill-php80/tree/v1.26.0" - }, - "funding": [ - { - "url": "https://symfony.com/sponsor", - "type": "custom" - }, - { - "url": "https://github.com/fabpot", - "type": "github" - }, - { - "url": "https://tidelift.com/funding/github/packagist/symfony/symfony", - "type": "tidelift" - } - ], - "time": "2022-05-10T07:21:04+00:00" - }, { "name": "utopia-php/abuse", "version": "0.15.0", @@ -1897,24 +1618,26 @@ }, { "name": "utopia-php/cache", - "version": "0.6.1", + "version": "0.8.0", "source": { "type": "git", "url": "https://github.com/utopia-php/cache.git", - "reference": "9889235a6d3da6cbb1f435201529da4d27c30e79" + "reference": "212e66100a1f32e674fca5d9bc317cc998303089" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/cache/zipball/9889235a6d3da6cbb1f435201529da4d27c30e79", - "reference": "9889235a6d3da6cbb1f435201529da4d27c30e79", + "url": "https://api.github.com/repos/utopia-php/cache/zipball/212e66100a1f32e674fca5d9bc317cc998303089", + "reference": "212e66100a1f32e674fca5d9bc317cc998303089", "shasum": "" }, "require": { "ext-json": "*", + "ext-memcached": "*", "ext-redis": "*", "php": ">=8.0" }, "require-dev": { + "laravel/pint": "1.2.*", "phpunit/phpunit": "^9.3", "vimeo/psalm": "4.13.1" }, @@ -1928,12 +1651,6 @@ "license": [ "MIT" ], - "authors": [ - { - "name": "Eldad Fux", - "email": "eldad@appwrite.io" - } - ], "description": "A simple cache library to manage application cache storing, loading and purging", "keywords": [ "cache", @@ -1944,9 +1661,9 @@ ], "support": { "issues": "https://github.com/utopia-php/cache/issues", - "source": "https://github.com/utopia-php/cache/tree/0.6.1" + "source": "https://github.com/utopia-php/cache/tree/0.8.0" }, - "time": "2022-08-10T08:12:46+00:00" + "time": "2022-10-16T16:48:09+00:00" }, { "name": "utopia-php/cli", @@ -2054,29 +1771,29 @@ }, { "name": "utopia-php/database", - "version": "0.27.0", + "version": "dev-feat-update-create", "source": { "type": "git", "url": "https://github.com/utopia-php/database.git", - "reference": "0644cd8a7efee2fc745ed70dbf2b73a78eeefa52" + "reference": "8ba52f4ce40674ed9f6e8f72e300d5d9a4900590" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/database/zipball/0644cd8a7efee2fc745ed70dbf2b73a78eeefa52", - "reference": "0644cd8a7efee2fc745ed70dbf2b73a78eeefa52", + "url": "https://api.github.com/repos/utopia-php/database/zipball/8ba52f4ce40674ed9f6e8f72e300d5d9a4900590", + "reference": "8ba52f4ce40674ed9f6e8f72e300d5d9a4900590", "shasum": "" }, "require": { - "ext-mongodb": "*", - "ext-pdo": "*", - "ext-redis": "*", - "mongodb/mongodb": "1.8.0", "php": ">=8.0", - "utopia-php/cache": "0.6.*", + "utopia-php/cache": "0.8.*", "utopia-php/framework": "0.*.*" }, "require-dev": { + "ext-mongodb": "*", + "ext-pdo": "*", + "ext-redis": "*", "fakerphp/faker": "^1.14", + "mongodb/mongodb": "1.8.0", "phpunit/phpunit": "^9.4", "swoole/ide-helper": "4.8.0", "utopia-php/cli": "^0.11.0", @@ -2102,9 +1819,9 @@ ], "support": { "issues": "https://github.com/utopia-php/database/issues", - "source": "https://github.com/utopia-php/database/tree/0.27.0" + "source": "https://github.com/utopia-php/database/tree/feat-update-create" }, - "time": "2022-10-16T07:39:26+00:00" + "time": "2022-10-28T12:01:54+00:00" }, { "name": "utopia-php/domains", @@ -2433,6 +2150,59 @@ }, "time": "2022-07-13T16:47:18+00:00" }, + { + "name": "utopia-php/pools", + "version": "0.1.0", + "source": { + "type": "git", + "url": "https://github.com/utopia-php/pools.git", + "reference": "5a467a569a80aefc846a97dc195b4adc2fd71805" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/utopia-php/pools/zipball/5a467a569a80aefc846a97dc195b4adc2fd71805", + "reference": "5a467a569a80aefc846a97dc195b4adc2fd71805", + "shasum": "" + }, + "require": { + "ext-mongodb": "*", + "ext-pdo": "*", + "ext-redis": "*", + "php": ">=8.0" + }, + "require-dev": { + "phpunit/phpunit": "^9.4", + "vimeo/psalm": "4.0.1" + }, + "type": "library", + "autoload": { + "psr-4": { + "Utopia\\Pools\\": "src/Pools" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Team Appwrite", + "email": "team@appwrite.io" + } + ], + "description": "A simple library to manage connection pools", + "keywords": [ + "framework", + "php", + "pools", + "utopia" + ], + "support": { + "issues": "https://github.com/utopia-php/pools/issues", + "source": "https://github.com/utopia-php/pools/tree/0.1.0" + }, + "time": "2022-10-11T19:31:07+00:00" + }, { "name": "utopia-php/preloader", "version": "0.2.4", @@ -5347,9 +5117,18 @@ "time": "2022-09-28T08:42:51+00:00" } ], - "aliases": [], + "aliases": [ + { + "package": "utopia-php/database", + "version": "dev-feat-update-create", + "alias": "0.27.0", + "alias_normalized": "0.27.0.0" + } + ], "minimum-stability": "stable", - "stability-flags": [], + "stability-flags": { + "utopia-php/database": 20 + }, "prefer-stable": false, "prefer-lowest": false, "platform": { diff --git a/docker-compose.yml b/docker-compose.yml index 18ea38f658..081d330f1a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -84,6 +84,7 @@ services: - ./public:/usr/src/code/public - ./src:/usr/src/code/src - ./dev:/usr/local/dev + - ./vendor/utopia-php/framework:/usr/src/code/vendor/utopia-php/framework depends_on: - mariadb - redis @@ -109,15 +110,20 @@ services: - _APP_OPENSSL_KEY_V1 - _APP_DOMAIN - _APP_DOMAIN_TARGET - - _APP_REDIS_HOST - - _APP_REDIS_PORT - - _APP_REDIS_USER - - _APP_REDIS_PASS - _APP_DB_HOST - _APP_DB_PORT - _APP_DB_SCHEMA - _APP_DB_USER - _APP_DB_PASS + - _APP_REDIS_HOST + - _APP_REDIS_PORT + - _APP_REDIS_USER + - _APP_REDIS_PASS + - _APP_CONNECTIONS_DB_PROJECT + - _APP_CONNECTIONS_DB_CONSOLE + - _APP_CONNECTIONS_CACHE + - _APP_CONNECTIONS_QUEUE + - _APP_CONNECTIONS_PUBSUB - _APP_SMTP_HOST - _APP_SMTP_PORT - _APP_SMTP_SECURE @@ -211,13 +217,19 @@ services: - _APP_WORKER_PER_CORE - _APP_OPTIONS_ABUSE - _APP_OPENSSL_KEY_V1 - - _APP_REDIS_HOST - - _APP_REDIS_PORT - _APP_DB_HOST - _APP_DB_PORT - _APP_DB_SCHEMA - _APP_DB_USER - _APP_DB_PASS + - _APP_REDIS_HOST + - _APP_REDIS_PORT + - _APP_REDIS_USER + - _APP_REDIS_PASS + - _APP_CONNECTIONS_DB_CONSOLE + - _APP_CONNECTIONS_DB_PROJECT + - _APP_CONNECTIONS_CACHE + - _APP_CONNECTIONS_PUBSUB - _APP_USAGE_STATS - _APP_LOGGING_PROVIDER - _APP_LOGGING_CONFIG @@ -238,15 +250,19 @@ services: environment: - _APP_ENV - _APP_OPENSSL_KEY_V1 - - _APP_REDIS_HOST - - _APP_REDIS_PORT - - _APP_REDIS_USER - - _APP_REDIS_PASS - _APP_DB_HOST - _APP_DB_PORT - _APP_DB_SCHEMA - _APP_DB_USER - _APP_DB_PASS + - _APP_REDIS_HOST + - _APP_REDIS_PORT + - _APP_REDIS_USER + - _APP_REDIS_PASS + - _APP_CONNECTIONS_DB_CONSOLE + - _APP_CONNECTIONS_DB_PROJECT + - _APP_CONNECTIONS_CACHE + - _APP_CONNECTIONS_QUEUE - _APP_LOGGING_PROVIDER - _APP_LOGGING_CONFIG @@ -272,6 +288,7 @@ services: - _APP_REDIS_PORT - _APP_REDIS_USER - _APP_REDIS_PASS + - _APP_CONNECTIONS_QUEUE - _APP_LOGGING_PROVIDER - _APP_LOGGING_CONFIG @@ -296,15 +313,19 @@ services: environment: - _APP_ENV - _APP_OPENSSL_KEY_V1 - - _APP_REDIS_HOST - - _APP_REDIS_PORT - - _APP_REDIS_USER - - _APP_REDIS_PASS - _APP_DB_HOST - _APP_DB_PORT - _APP_DB_SCHEMA - _APP_DB_USER - _APP_DB_PASS + - _APP_REDIS_HOST + - _APP_REDIS_PORT + - _APP_REDIS_USER + - _APP_REDIS_PASS + - _APP_CONNECTIONS_DB_CONSOLE + - _APP_CONNECTIONS_DB_PROJECT + - _APP_CONNECTIONS_CACHE + - _APP_CONNECTIONS_QUEUE - _APP_STORAGE_DEVICE - _APP_STORAGE_S3_ACCESS_KEY - _APP_STORAGE_S3_SECRET @@ -348,15 +369,19 @@ services: environment: - _APP_ENV - _APP_OPENSSL_KEY_V1 - - _APP_REDIS_HOST - - _APP_REDIS_PORT - - _APP_REDIS_USER - - _APP_REDIS_PASS - _APP_DB_HOST - _APP_DB_PORT - _APP_DB_SCHEMA - _APP_DB_USER - _APP_DB_PASS + - _APP_REDIS_HOST + - _APP_REDIS_PORT + - _APP_REDIS_USER + - _APP_REDIS_PASS + - _APP_CONNECTIONS_DB_CONSOLE + - _APP_CONNECTIONS_DB_PROJECT + - _APP_CONNECTIONS_CACHE + - _APP_CONNECTIONS_QUEUE - _APP_LOGGING_PROVIDER - _APP_LOGGING_CONFIG @@ -378,15 +403,19 @@ services: - _APP_OPENSSL_KEY_V1 - _APP_EXECUTOR_SECRET - _APP_EXECUTOR_HOST - - _APP_REDIS_HOST - - _APP_REDIS_PORT - - _APP_REDIS_USER - - _APP_REDIS_PASS - _APP_DB_HOST - _APP_DB_PORT - _APP_DB_SCHEMA - _APP_DB_USER - _APP_DB_PASS + - _APP_REDIS_HOST + - _APP_REDIS_PORT + - _APP_REDIS_USER + - _APP_REDIS_PASS + - _APP_CONNECTIONS_DB_CONSOLE + - _APP_CONNECTIONS_DB_PROJECT + - _APP_CONNECTIONS_CACHE + - _APP_CONNECTIONS_QUEUE - _APP_LOGGING_PROVIDER - _APP_LOGGING_CONFIG @@ -411,15 +440,19 @@ services: - _APP_DOMAIN - _APP_DOMAIN_TARGET - _APP_SYSTEM_SECURITY_EMAIL_ADDRESS - - _APP_REDIS_HOST - - _APP_REDIS_PORT - - _APP_REDIS_USER - - _APP_REDIS_PASS - _APP_DB_HOST - _APP_DB_PORT - _APP_DB_SCHEMA - _APP_DB_USER - _APP_DB_PASS + - _APP_REDIS_HOST + - _APP_REDIS_PORT + - _APP_REDIS_USER + - _APP_REDIS_PASS + - _APP_CONNECTIONS_DB_CONSOLE + - _APP_CONNECTIONS_DB_PROJECT + - _APP_CONNECTIONS_CACHE + - _APP_CONNECTIONS_QUEUE - _APP_LOGGING_PROVIDER - _APP_LOGGING_CONFIG @@ -440,15 +473,19 @@ services: environment: - _APP_ENV - _APP_OPENSSL_KEY_V1 - - _APP_REDIS_HOST - - _APP_REDIS_PORT - - _APP_REDIS_USER - - _APP_REDIS_PASS - _APP_DB_HOST - _APP_DB_PORT - _APP_DB_SCHEMA - _APP_DB_USER - _APP_DB_PASS + - _APP_REDIS_HOST + - _APP_REDIS_PORT + - _APP_REDIS_USER + - _APP_REDIS_PASS + - _APP_CONNECTIONS_DB_CONSOLE + - _APP_CONNECTIONS_DB_PROJECT + - _APP_CONNECTIONS_CACHE + - _APP_CONNECTIONS_QUEUE - _APP_FUNCTIONS_TIMEOUT - _APP_EXECUTOR_SECRET - _APP_EXECUTOR_HOST @@ -540,6 +577,7 @@ services: - _APP_REDIS_PORT - _APP_REDIS_USER - _APP_REDIS_PASS + - _APP_CONNECTIONS_QUEUE - _APP_SMTP_HOST - _APP_SMTP_PORT - _APP_SMTP_SECURE @@ -566,6 +604,7 @@ services: - _APP_REDIS_PORT - _APP_REDIS_USER - _APP_REDIS_PASS + - _APP_CONNECTIONS_QUEUE - _APP_SMS_PROVIDER - _APP_SMS_FROM - _APP_LOGGING_PROVIDER @@ -589,15 +628,18 @@ services: - _APP_DOMAIN - _APP_DOMAIN_TARGET - _APP_OPENSSL_KEY_V1 - - _APP_REDIS_HOST - - _APP_REDIS_PORT - - _APP_REDIS_USER - - _APP_REDIS_PASS - _APP_DB_HOST - _APP_DB_PORT - _APP_DB_SCHEMA - _APP_DB_USER - _APP_DB_PASS + - _APP_REDIS_HOST + - _APP_REDIS_PORT + - _APP_REDIS_USER + - _APP_REDIS_PASS + - _APP_CONNECTIONS_DB_CONSOLE + - _APP_CONNECTIONS_DB_PROJECT + - _APP_CONNECTIONS_CACHE - _APP_MAINTENANCE_INTERVAL - _APP_MAINTENANCE_RETENTION_EXECUTION - _APP_MAINTENANCE_RETENTION_CACHE @@ -634,6 +676,9 @@ services: - _APP_REDIS_PORT - _APP_REDIS_USER - _APP_REDIS_PASS + - _APP_CONNECTIONS_DB_CONSOLE + - _APP_CONNECTIONS_DB_PROJECT + - _APP_CONNECTIONS_CACHE - _APP_LOGGING_PROVIDER - _APP_LOGGING_CONFIG @@ -655,6 +700,7 @@ services: - _APP_REDIS_PORT - _APP_REDIS_USER - _APP_REDIS_PASS + - _APP_CONNECTIONS_QUEUE mariadb: image: mariadb:10.7 # fix issues when upgrading using: mysql_upgrade -u root -p @@ -677,7 +723,6 @@ services: # smtp: # image: appwrite/smtp:1.2.0 # container_name: appwrite-smtp - # restart: unless-stopped # networks: # - appwrite # environment: @@ -764,7 +809,6 @@ services: image: adminer container_name: appwrite-adminer <<: *x-logging - restart: always ports: - 9506:8080 networks: @@ -772,7 +816,6 @@ services: # redis-commander: # image: rediscommander/redis-commander:latest - # restart: unless-stopped # networks: # - appwrite # environment: @@ -782,7 +825,6 @@ services: # resque: # image: appwrite/resque-web:1.1.0 - # restart: unless-stopped # networks: # - appwrite # ports: @@ -796,7 +838,6 @@ services: # chronograf: # image: chronograf:1.6 # container_name: appwrite-chronograf - # restart: unless-stopped # networks: # - appwrite # volumes: diff --git a/docs/references/health/get-cache.md b/docs/references/health/get-cache.md index 91abcd6bc5..632c02208d 100644 --- a/docs/references/health/get-cache.md +++ b/docs/references/health/get-cache.md @@ -1 +1 @@ -Check the Appwrite in-memory cache server is up and connection is successful. \ No newline at end of file +Check the Appwrite in-memory cache servers are up and connection is successful. \ No newline at end of file diff --git a/docs/references/health/get-db.md b/docs/references/health/get-db.md index 9652d0d3e3..7381e51f70 100644 --- a/docs/references/health/get-db.md +++ b/docs/references/health/get-db.md @@ -1 +1 @@ -Check the Appwrite database server is up and connection is successful. \ No newline at end of file +Check the Appwrite database servers are up and connection is successful. \ No newline at end of file diff --git a/docs/references/health/get-pubsub.md b/docs/references/health/get-pubsub.md new file mode 100644 index 0000000000..8f86411e8f --- /dev/null +++ b/docs/references/health/get-pubsub.md @@ -0,0 +1 @@ +Check the Appwrite pub-sub servers are up and connection is successful. \ No newline at end of file diff --git a/docs/references/health/get-queue.md b/docs/references/health/get-queue.md new file mode 100644 index 0000000000..e4558f941f --- /dev/null +++ b/docs/references/health/get-queue.md @@ -0,0 +1 @@ +Check the Appwrite queue messaging servers are up and connection is successful. \ No newline at end of file diff --git a/phpunit.xml b/phpunit.xml index 4012c8c276..927e91567a 100644 --- a/phpunit.xml +++ b/phpunit.xml @@ -7,7 +7,7 @@ convertWarningsToExceptions="true" processIsolation="false" stopOnFailure="false" -> + > diff --git a/src/Appwrite/Extend/PDO.php b/src/Appwrite/Extend/PDO.php deleted file mode 100644 index bc3f42afb9..0000000000 --- a/src/Appwrite/Extend/PDO.php +++ /dev/null @@ -1,110 +0,0 @@ -dsn = $dsn; - $this->username = $username; - $this->passwd = $passwd; - $this->options = $options; - - $this->pdo = new PDONative($dsn, $username, $passwd, $options); - } - - public function setAttribute($attribute, $value) - { - return $this->pdo->setAttribute($attribute, $value); - } - - public function prepare($statement, $driver_options = null) - { - return new PDOStatement($this, $this->pdo->prepare($statement, [])); - } - - public function quote($string, $parameter_type = PDONative::PARAM_STR) - { - return $this->pdo->quote($string, $parameter_type); - } - - public function beginTransaction() - { - try { - $result = $this->pdo->beginTransaction(); - } catch (\Throwable $th) { - $this->pdo = $this->reconnect(); - $result = $this->pdo->beginTransaction(); - } - - return $result; - } - - public function rollBack() - { - try { - $result = $this->pdo->rollBack(); - } catch (\Throwable $th) { - $this->pdo = $this->reconnect(); - return false; - } - - return $result; - } - - public function commit() - { - try { - $result = $this->pdo->commit(); - } catch (\Throwable $th) { - $this->pdo = $this->reconnect(); - $result = $this->pdo->commit(); - } - - return $result; - } - - public function reconnect(): PDONative - { - $this->pdo = new PDONative($this->dsn, $this->username, $this->passwd, $this->options); - - echo '[PDO] MySQL connection restarted' . PHP_EOL; - - // Connection settings - $this->pdo->setAttribute(PDONative::ATTR_DEFAULT_FETCH_MODE, PDONative::FETCH_ASSOC); // Return arrays - $this->pdo->setAttribute(PDONative::ATTR_ERRMODE, PDONative::ERRMODE_EXCEPTION); // Handle all errors with exceptions - - return $this->pdo; - } -} diff --git a/src/Appwrite/Extend/PDOStatement.php b/src/Appwrite/Extend/PDOStatement.php deleted file mode 100644 index 9c5a83ec34..0000000000 --- a/src/Appwrite/Extend/PDOStatement.php +++ /dev/null @@ -1,115 +0,0 @@ -pdo = &$pdo; - $this->PDOStatement = $PDOStatement; - } - - public function bindValue($parameter, $value, $data_type = PDONative::PARAM_STR) - { - $this->values[$parameter] = ['value' => $value, 'data_type' => $data_type]; - - $result = $this->PDOStatement->bindValue($parameter, $value, $data_type); - - return $result; - } - - public function bindParam($parameter, &$variable, $data_type = PDONative::PARAM_STR, $length = null, $driver_options = null) - { - $this->params[$parameter] = ['value' => &$variable, 'data_type' => $data_type, 'length' => $length, 'driver_options' => $driver_options]; - - $result = $this->PDOStatement->bindParam($parameter, $variable, $data_type, $length, $driver_options); - - return $result; - } - - public function bindColumn($column, &$param, $type = null, $maxlen = null, $driverdata = null) - { - $this->columns[$column] = ['param' => &$param, 'type' => $type, 'maxlen' => $maxlen, 'driverdata' => $driverdata]; - - $result = $this->PDOStatement->bindColumn($column, $param, $type, $maxlen, $driverdata); - - return $result; - } - - public function execute($input_parameters = null) - { - try { - $result = $this->PDOStatement->execute($input_parameters); - } catch (\Throwable $th) { - $this->pdo = $this->pdo->reconnect(); - $this->PDOStatement = $this->pdo->prepare($this->PDOStatement->queryString, []); - - foreach ($this->values as $key => $set) { - $this->PDOStatement->bindValue($key, $set['value'], $set['data_type']); - } - - foreach ($this->params as $key => $set) { - $this->PDOStatement->bindParam($key, $set['variable'], $set['data_type'], $set['length'], $set['driver_options']); - } - - foreach ($this->columns as $key => $set) { - $this->PDOStatement->bindColumn($key, $set['param'], $set['type'], $set['maxlen'], $set['driverdata']); - } - - $result = $this->PDOStatement->execute($input_parameters); - } - - return $result; - } - - public function fetch($fetch_style = PDONative::FETCH_ASSOC, $cursor_orientation = PDONative::FETCH_ORI_NEXT, $cursor_offset = 0) - { - $result = $this->PDOStatement->fetch($fetch_style, $cursor_orientation, $cursor_offset); - - return $result; - } - - /** - * Fetch All - * - * @param int $fetch_style - * @param mixed $fetch_args - * - * @return array|false - */ - public function fetchAll(int $fetch_style = PDO::FETCH_BOTH, mixed ...$fetch_args) - { - $result = $this->PDOStatement->fetchAll(); - - return $result; - } -} diff --git a/src/Appwrite/Migration/Migration.php b/src/Appwrite/Migration/Migration.php index c87c907bf0..2ac2641b95 100644 --- a/src/Appwrite/Migration/Migration.php +++ b/src/Appwrite/Migration/Migration.php @@ -86,8 +86,6 @@ abstract class Migration { $this->project = $project; $this->projectDB = $projectDB; - $this->projectDB->setNamespace('_' . $this->project->getId()); - $this->consoleDB = $consoleDB; return $this; diff --git a/src/Appwrite/Resque/Worker.php b/src/Appwrite/Resque/Worker.php index dd7cebd084..5d05d77576 100644 --- a/src/Appwrite/Resque/Worker.php +++ b/src/Appwrite/Resque/Worker.php @@ -2,12 +2,12 @@ namespace Appwrite\Resque; +use Exception; use Utopia\App; use Utopia\Cache\Cache; -use Utopia\Cache\Adapter\Redis as RedisCache; -use Utopia\CLI\Console; +use Utopia\Config\Config; +use Utopia\Cache\Adapter\Sharding; use Utopia\Database\Database; -use Utopia\Database\Adapter\MariaDB; use Utopia\Storage\Device; use Utopia\Storage\Storage; use Utopia\Storage\Device\Local; @@ -16,7 +16,7 @@ use Utopia\Storage\Device\Linode; use Utopia\Storage\Device\Wasabi; use Utopia\Storage\Device\Backblaze; use Utopia\Storage\Device\S3; -use Exception; +use Utopia\Database\Document; use Utopia\Database\Validator\Authorization; abstract class Worker @@ -136,7 +136,12 @@ abstract class Worker */ public function tearDown(): void { + global $register; + try { + $pools = $register->get('pools'); /** @var \Utopia\Pools\Group $pools */ + $pools->reclaim(); + $this->shutdown(); } catch (\Throwable $error) { foreach (self::$errorCallbacks as $errorCallback) { @@ -158,23 +163,32 @@ abstract class Worker { \array_push(self::$errorCallbacks, $callback); } + /** * Get internal project database - * @param string $projectId + * @param Document $project * @return Database */ - protected function getProjectDB(string $projectId): Database + protected function getProjectDB(Document $project): Database { - $consoleDB = $this->getConsoleDB(); + global $register; - if ($projectId === 'console') { - return $consoleDB; + $pools = $register->get('pools'); /** @var \Utopia\Pools\Group $pools */ + + if ($project->isEmpty() || $project->getId() === 'console') { + return $this->getConsoleDB(); } - /** @var Document $project */ - $project = Authorization::skip(fn() => $consoleDB->getDocument('projects', $projectId)); + $dbAdapter = $pools + ->get($project->getAttribute('database')) + ->pop() + ->getResource() + ; - return $this->getDB(self::DATABASE_PROJECT, $projectId, $project->getInternalId()); + $database = new Database($dbAdapter, $this->getCache()); + $database->setNamespace('_' . $project->getInternalId()); + + return $database; } /** @@ -183,67 +197,46 @@ abstract class Worker */ protected function getConsoleDB(): Database { - return $this->getDB(self::DATABASE_CONSOLE); + global $register; + + $pools = $register->get('pools'); /** @var \Utopia\Pools\Group $pools */ + + $dbAdapter = $pools + ->get('console') + ->pop() + ->getResource() + ; + + $database = new Database($dbAdapter, $this->getCache()); + + $database->setNamespace('console'); + + return $database; } + /** - * Get console database - * @param string $type One of (internal, external, console) - * @param string $projectId of internal or external DB - * @return Database + * Get Cache + * @return Cache */ - private function getDB(string $type, string $projectId = '', string $projectInternalId = ''): Database + protected function getCache(): Cache { global $register; - $namespace = ''; - $sleep = DATABASE_RECONNECT_SLEEP; // overwritten when necessary + $pools = $register->get('pools'); /** @var \Utopia\Pools\Group $pools */ - switch ($type) { - case self::DATABASE_PROJECT: - if (!$projectId) { - throw new \Exception('ProjectID not provided - cannot get database'); - } - $namespace = "_{$projectInternalId}"; - break; - case self::DATABASE_CONSOLE: - $namespace = "_console"; - $sleep = 5; // ConsoleDB needs extra sleep time to ensure tables are created - break; - default: - throw new \Exception('Unknown database type: ' . $type); - break; + $list = Config::getParam('pools-cache', []); + $adapters = []; + + foreach ($list as $value) { + $adapters[] = $pools + ->get($value) + ->pop() + ->getResource() + ; } - $attempts = 0; - - do { - try { - $attempts++; - $cache = new Cache(new RedisCache($register->get('cache'))); - $database = new Database(new MariaDB($register->get('db')), $cache); - $database->setDefaultDatabase(App::getEnv('_APP_DB_SCHEMA', 'appwrite')); - $database->setNamespace($namespace); // Main DB - - if (!empty($projectId) && !$database->getDocument('projects', $projectId)->isEmpty()) { - throw new \Exception("Project does not exist: {$projectId}"); - } - - if ($type === self::DATABASE_CONSOLE && !$database->exists($database->getDefaultDatabase(), Database::METADATA)) { - throw new \Exception('Console project not ready'); - } - - break; // leave loop if successful - } catch (\Exception $e) { - Console::warning("Database not ready. Retrying connection ({$attempts})..."); - if ($attempts >= DATABASE_RECONNECT_MAX_ATTEMPTS) { - throw new \Exception('Failed to connect to database: ' . $e->getMessage()); - } - sleep($sleep); - } - } while ($attempts < DATABASE_RECONNECT_MAX_ATTEMPTS); - - return $database; + return new Cache(new Sharding($adapters)); } /** @@ -266,7 +259,6 @@ abstract class Worker return $this->getDevice(APP_STORAGE_UPLOADS . '/app-' . $projectId); } - /** * Get Builds Storage Device * @param string $projectId of the project diff --git a/src/Appwrite/Utopia/Response.php b/src/Appwrite/Utopia/Response.php index e23335365a..07134c6ea9 100644 --- a/src/Appwrite/Utopia/Response.php +++ b/src/Appwrite/Utopia/Response.php @@ -207,6 +207,7 @@ class Response extends SwooleResponse public const MODEL_HEALTH_QUEUE = 'healthQueue'; public const MODEL_HEALTH_TIME = 'healthTime'; public const MODEL_HEALTH_ANTIVIRUS = 'healthAntivirus'; + public const MODEL_HEALTH_STATUS_LIST = 'healthStatusList'; // Deprecated public const MODEL_PERMISSIONS = 'permissions'; @@ -268,6 +269,7 @@ class Response extends SwooleResponse ->setModel(new BaseList('Phones List', self::MODEL_PHONE_LIST, 'phones', self::MODEL_PHONE)) ->setModel(new BaseList('Metric List', self::MODEL_METRIC_LIST, 'metrics', self::MODEL_METRIC, true, false)) ->setModel(new BaseList('Variables List', self::MODEL_VARIABLE_LIST, 'variables', self::MODEL_VARIABLE)) + ->setModel(new BaseList('Status List', self::MODEL_HEALTH_STATUS_LIST, 'statuses', self::MODEL_HEALTH_STATUS)) // Entities ->setModel(new Database()) ->setModel(new Collection()) diff --git a/src/Appwrite/Utopia/Response/Model/HealthStatus.php b/src/Appwrite/Utopia/Response/Model/HealthStatus.php index 23756de131..ba340107ac 100644 --- a/src/Appwrite/Utopia/Response/Model/HealthStatus.php +++ b/src/Appwrite/Utopia/Response/Model/HealthStatus.php @@ -10,6 +10,12 @@ class HealthStatus extends Model public function __construct() { $this + ->addRule('name', [ + 'type' => self::TYPE_STRING, + 'description' => 'Name of the service.', + 'default' => '', + 'example' => 'database', + ]) ->addRule('ping', [ 'type' => self::TYPE_INTEGER, 'description' => 'Duration in milliseconds how long the health check took.', diff --git a/tests/e2e/Services/Health/HealthCustomServerTest.php b/tests/e2e/Services/Health/HealthCustomServerTest.php index 47a2268e21..96c9bde5c7 100644 --- a/tests/e2e/Services/Health/HealthCustomServerTest.php +++ b/tests/e2e/Services/Health/HealthCustomServerTest.php @@ -47,9 +47,9 @@ class HealthCustomServerTest extends Scope ], $this->getHeaders()), []); $this->assertEquals(200, $response['headers']['status-code']); - $this->assertEquals('pass', $response['body']['status']); - $this->assertIsInt($response['body']['ping']); - $this->assertLessThan(100, $response['body']['ping']); + $this->assertEquals('pass', $response['body']['statuses'][0]['status']); + $this->assertIsInt($response['body']['statuses'][0]['ping']); + $this->assertLessThan(100, $response['body']['statuses'][0]['ping']); /** * Test for FAILURE @@ -69,9 +69,53 @@ class HealthCustomServerTest extends Scope ], $this->getHeaders()), []); $this->assertEquals(200, $response['headers']['status-code']); - $this->assertEquals('pass', $response['body']['status']); - $this->assertIsInt($response['body']['ping']); - $this->assertLessThan(100, $response['body']['ping']); + $this->assertEquals('pass', $response['body']['statuses'][0]['status']); + $this->assertIsInt($response['body']['statuses'][0]['ping']); + $this->assertLessThan(100, $response['body']['statuses'][0]['ping']); + + /** + * Test for FAILURE + */ + + return []; + } + + public function testQueueSuccess(): array + { + /** + * Test for SUCCESS + */ + $response = $this->client->call(Client::METHOD_GET, '/health/queue', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), []); + + $this->assertEquals(200, $response['headers']['status-code']); + $this->assertEquals('pass', $response['body']['statuses'][0]['status']); + $this->assertIsInt($response['body']['statuses'][0]['ping']); + $this->assertLessThan(100, $response['body']['statuses'][0]['ping']); + + /** + * Test for FAILURE + */ + + return []; + } + + public function testPubSubSuccess(): array + { + /** + * Test for SUCCESS + */ + $response = $this->client->call(Client::METHOD_GET, '/health/pubsub', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), []); + + $this->assertEquals(200, $response['headers']['status-code']); + $this->assertEquals('pass', $response['body']['statuses'][0]['status']); + $this->assertIsInt($response['body']['statuses'][0]['ping']); + $this->assertLessThan(100, $response['body']['statuses'][0]['ping']); /** * Test for FAILURE