From ebac3f5547ea94d5fad16193d79002586fed0a1b Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Tue, 9 Aug 2022 05:52:53 +0000 Subject: [PATCH] usage containers updated --- app/tasks/usage.php | 71 ++++++++++++++++----------------- app/views/install/compose.phtml | 40 +++++++++++++++++-- docker-compose.yml | 46 +++++++++++++++++++-- 3 files changed, 115 insertions(+), 42 deletions(-) diff --git a/app/tasks/usage.php b/app/tasks/usage.php index dba356abb6..78a12fd406 100644 --- a/app/tasks/usage.php +++ b/app/tasks/usage.php @@ -4,13 +4,16 @@ global $cli, $register; use Appwrite\Stats\Usage; use Appwrite\Stats\UsageDB; +use Appwrite\Usage\Calculators\Aggregator; +use Appwrite\Usage\Calculators\Database; +use Appwrite\Usage\Calculators\TimeSeries; use InfluxDB\Database as InfluxDatabase; use Utopia\App; use Utopia\Cache\Adapter\Redis as RedisCache; use Utopia\Cache\Cache; use Utopia\CLI\Console; use Utopia\Database\Adapter\MariaDB; -use Utopia\Database\Database; +use Utopia\Database\Database as UtopiaDatabase; use Utopia\Database\Validator\Authorization; use Utopia\Registry\Registry; use Utopia\Logger\Log; @@ -19,7 +22,7 @@ use Utopia\Validator\WhiteList; Authorization::disable(); Authorization::setDefaultStatus(false); -function getDatabase(Registry &$register, string $namespace): Database +function getDatabase(Registry &$register, string $namespace): UtopiaDatabase { $attempts = 0; @@ -31,7 +34,7 @@ function getDatabase(Registry &$register, string $namespace): Database $redis = $register->get('cache'); $cache = new Cache(new RedisCache($redis)); - $database = new Database(new MariaDB($db), $cache); + $database = new UtopiaDatabase(new MariaDB($db), $cache); $database->setDefaultDatabase(App::getEnv('_APP_DB_SCHEMA', 'appwrite')); $database->setNamespace($namespace); @@ -39,7 +42,7 @@ function getDatabase(Registry &$register, string $namespace): Database throw new Exception('Projects collection not ready'); } break; // leave loop if successful - } catch (\Exception$e) { + } catch (\Exception $e) { Console::warning("Database not ready. Retrying connection ({$attempts})..."); if ($attempts >= DATABASE_RECONNECT_MAX_ATTEMPTS) { throw new \Exception('Failed to connect to database: ' . $e->getMessage()); @@ -66,7 +69,7 @@ function getInfluxDB(Registry &$register): InfluxDatabase if (in_array('telegraf', $client->listDatabases())) { break; // leave the do-while if successful } - } catch (\Throwable$th) { + } catch (\Throwable $th) { Console::warning("InfluxDB not ready. Retrying connection ({$attempts})..."); if ($attempts >= $max) { throw new \Exception('InfluxDB database not ready yet'); @@ -119,48 +122,44 @@ $cli Console::title('Usage Aggregation V1'); Console::success(APP_NAME . ' usage aggregation process v1 has started'); - $interval = (int) App::getEnv('_APP_USAGE_TIMESERIES_INTERVAL', '30'); // 30 seconds (by default) - $database = getDatabase($register, '_console'); $influxDB = getInfluxDB($register); - $usage = new Usage($database, $influxDB, $logError); - $usageDB = new UsageDB($database, $logError); + if ($type == 'timeseries') { - $iterations = 0; - Console::loop(function () use ($interval, $usage, $usageDB, &$iterations) { - $now = date('d-m-Y H:i:s', time()); - Console::info("[{$now}] Aggregating usage data every {$interval} seconds"); + $interval = (int) App::getEnv('_APP_USAGE_TIMESERIES_INTERVAL', '30'); // 30 seconds (by default) - $loopStart = microtime(true); - /** - * Aggregate InfluxDB every 30 seconds - */ - $usage->collect(); + $usage = new TimeSeries($database, $influxDB, $logError); + + Console::loop(function () use ($interval, $usage) { + $now = date('d-m-Y H:i:s', time()); + Console::info("[{$now}] Aggregating Timeseries Usage data every {$interval} seconds"); + $loopStart = microtime(true); + + $usage->collect(); - if ($iterations % 30 != 0 && App::getEnv('_APP_ENV', 'production') == 'production') { // return if 30 iterations has not passed - $iterations++; $loopTook = microtime(true) - $loopStart; $now = date('d-m-Y H:i:s', time()); Console::info("[{$now}] Aggregation took {$loopTook} seconds"); - return; - } + }, $interval); + } - $iterations = 0; // Reset iterations to prevent overflow when running for long time - /** - * Aggregate MariaDB every 15 minutes - * Some of the queries here might contain full-table scans. - */ - $now = date('d-m-Y H:i:s', time()); - Console::info("[{$now}] Aggregating database counters."); + if ($type == 'database') { + $interval = (int) App::getEnv('_APP_USAGE_DATABASE_INTERVAL', '900'); // 15 minutes (by default) + $usage = new Database($database, $logError); + $aggregrator = new Aggregator($database, $logError); - $usageDB->collect(); + Console::loop(function () use ($interval, $usage, $aggregrator) { + $now = date('d-m-Y H:i:s', time()); + Console::info("[{$now}] Aggregating database usage every {$interval} seconds."); + $loopStart = microtime(true); + $usage->collect(); + $aggregrator->collect(); + $loopTook = microtime(true) - $loopStart; + $now = date('d-m-Y H:i:s', time()); - $iterations++; - $loopTook = microtime(true) - $loopStart; - $now = date('d-m-Y H:i:s', time()); - - Console::info("[{$now}] Aggregation took {$loopTook} seconds"); - }, $interval); + Console::info("[{$now}] Aggregation took {$loopTook} seconds"); + }, $interval); + } }); diff --git a/app/views/install/compose.phtml b/app/views/install/compose.phtml index f504892d56..32ef53ff1b 100644 --- a/app/views/install/compose.phtml +++ b/app/views/install/compose.phtml @@ -548,10 +548,44 @@ services: - _APP_MAINTENANCE_RETENTION_ABUSE - _APP_MAINTENANCE_RETENTION_AUDIT - appwrite-usage: + appwrite-usage-timeseries: image: /: - entrypoint: usage - container_name: appwrite-usage + entrypoint: + - usage + - --type=timeseries + container_name: appwrite-usage-timeseries + <<: *x-logging + restart: unless-stopped + networks: + - appwrite + depends_on: + - influxdb + - mariadb + environment: + - _APP_ENV + - _APP_OPENSSL_KEY_V1 + - _APP_DB_HOST + - _APP_DB_PORT + - _APP_DB_SCHEMA + - _APP_DB_USER + - _APP_DB_PASS + - _APP_INFLUXDB_HOST + - _APP_INFLUXDB_PORT + - _APP_USAGE_TIMESERIES_INTERVAL + - _APP_USAGE_DATABASE_INTERVAL + - _APP_REDIS_HOST + - _APP_REDIS_PORT + - _APP_REDIS_USER + - _APP_REDIS_PASS + - _APP_LOGGING_PROVIDER + - _APP_LOGGING_CONFIG + + appwrite-usage-database: + image: /: + entrypoint: + - usage + - --type=database + container_name: appwrite-usage-database <<: *x-logging restart: unless-stopped networks: diff --git a/docker-compose.yml b/docker-compose.yml index f8b7f7947f..1daf5b3df1 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -582,10 +582,50 @@ services: - _APP_MAINTENANCE_RETENTION_ABUSE - _APP_MAINTENANCE_RETENTION_AUDIT - appwrite-usage: - entrypoint: usage + appwrite-usage-timeseries: + entrypoint: + - usage + - --type=timeseries <<: *x-logging - container_name: appwrite-usage + container_name: appwrite-usage-timeseries + build: + context: . + args: + - DEBUG=false + networks: + - appwrite + volumes: + - ./app:/usr/src/code/app + - ./src:/usr/src/code/src + - ./dev:/usr/local/dev + depends_on: + - influxdb + - mariadb + environment: + - _APP_ENV + - _APP_OPENSSL_KEY_V1 + - _APP_DB_HOST + - _APP_DB_PORT + - _APP_DB_SCHEMA + - _APP_DB_USER + - _APP_DB_PASS + - _APP_INFLUXDB_HOST + - _APP_INFLUXDB_PORT + - _APP_USAGE_TIMESERIES_INTERVAL + - _APP_USAGE_DATABASE_INTERVAL + - _APP_REDIS_HOST + - _APP_REDIS_PORT + - _APP_REDIS_USER + - _APP_REDIS_PASS + - _APP_LOGGING_PROVIDER + - _APP_LOGGING_CONFIG + + appwrite-usage-database: + entrypoint: + - usage + - --type=database + <<: *x-logging + container_name: appwrite-usage-database build: context: . args: