diff --git a/app/cli.php b/app/cli.php index 09b3dc9413..0539c98313 100644 --- a/app/cli.php +++ b/app/cli.php @@ -3,30 +3,11 @@ require_once __DIR__ . '/init.php'; require_once __DIR__ . '/controllers/general.php'; -use Utopia\App; -use Utopia\CLI\CLI; -use Utopia\CLI\Console; +use Appwrite\Task\Tasks; use Utopia\Database\Validator\Authorization; Authorization::disable(); -$cli = new CLI(); +Tasks::init(); +Tasks::getCli()->run(); -include 'tasks/doctor.php'; -include 'tasks/maintenance.php'; -include 'tasks/install.php'; -include 'tasks/migrate.php'; -include 'tasks/sdks.php'; -include 'tasks/specs.php'; -include 'tasks/ssl.php'; -include 'tasks/vars.php'; -include 'tasks/usage.php'; - -$cli - ->task('version') - ->desc('Get the server version') - ->action(function () { - Console::log(App::getEnv('_APP_VERSION', 'UNKNOWN')); - }); - -$cli->run(); diff --git a/composer.json b/composer.json index ded0834bb3..2cf3e44004 100644 --- a/composer.json +++ b/composer.json @@ -43,7 +43,7 @@ "utopia-php/analytics": "0.2.*", "utopia-php/audit": "0.8.*", "utopia-php/cache": "0.6.*", - "utopia-php/cli": "0.12.*", + "utopia-php/cli": "dev-feat-allow-adding-task as 0.12.0", "utopia-php/config": "0.2.*", "utopia-php/database": "0.18.*", "utopia-php/locale": "0.4.*", diff --git a/composer.lock b/composer.lock index 9d29bc8ac7..3460506d35 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "dedc6a6328b4fdc5dfbd556a08534403", + "content-hash": "4bd2cfb69f671c01e68b67221f8664fd", "packages": [ { "name": "adhocore/jwt", @@ -1947,16 +1947,16 @@ }, { "name": "utopia-php/cli", - "version": "0.12.0", + "version": "dev-feat-allow-adding-task", "source": { "type": "git", "url": "https://github.com/utopia-php/cli.git", - "reference": "6d164b752efeb1ca089e3a517bc274d8b383474b" + "reference": "1ecc371b018f6e65d7d70b65f1c8b7a74c9ebcb2" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/cli/zipball/6d164b752efeb1ca089e3a517bc274d8b383474b", - "reference": "6d164b752efeb1ca089e3a517bc274d8b383474b", + "url": "https://api.github.com/repos/utopia-php/cli/zipball/1ecc371b018f6e65d7d70b65f1c8b7a74c9ebcb2", + "reference": "1ecc371b018f6e65d7d70b65f1c8b7a74c9ebcb2", "shasum": "" }, "require": { @@ -1994,9 +1994,9 @@ ], "support": { "issues": "https://github.com/utopia-php/cli/issues", - "source": "https://github.com/utopia-php/cli/tree/0.12.0" + "source": "https://github.com/utopia-php/cli/tree/feat-allow-adding-task" }, - "time": "2022-02-18T22:10:41+00:00" + "time": "2022-07-08T01:44:31+00:00" }, { "name": "utopia-php/config", @@ -5346,9 +5346,18 @@ "time": "2022-05-17T05:48:52+00:00" } ], - "aliases": [], + "aliases": [ + { + "package": "utopia-php/cli", + "version": "dev-feat-allow-adding-task", + "alias": "0.12.0", + "alias_normalized": "0.12.0.0" + } + ], "minimum-stability": "stable", - "stability-flags": [], + "stability-flags": { + "utopia-php/cli": 20 + }, "prefer-stable": false, "prefer-lowest": false, "platform": { @@ -5370,5 +5379,5 @@ "platform-overrides": { "php": "8.0" }, - "plugin-api-version": "2.3.0" + "plugin-api-version": "2.2.0" } diff --git a/src/Appwrite/Task/Task.php b/src/Appwrite/Task/Task.php new file mode 100644 index 0000000000..a10049b407 --- /dev/null +++ b/src/Appwrite/Task/Task.php @@ -0,0 +1,9 @@ +addTask(Vars::getTask()); + self::$cli->addTask(Usage::getTask()); + self::$cli->addTask(Version::getTask()); + } + + public static function getCli(): CLI + { + return self::$cli; + } +} diff --git a/src/Appwrite/Task/Usage.php b/src/Appwrite/Task/Usage.php new file mode 100644 index 0000000000..777dc303f6 --- /dev/null +++ b/src/Appwrite/Task/Usage.php @@ -0,0 +1,178 @@ +get('db'); + $redis = $register->get('cache'); + + $cache = new Cache(new RedisCache($redis)); + $database = new Database(new MariaDB($db), $cache); + $database->setDefaultDatabase(App::getEnv('_APP_DB_SCHEMA', 'appwrite')); + $database->setNamespace($namespace); + + if (!$database->exists($database->getDefaultDatabase(), 'projects')) { + throw new Exception('Projects collection not ready'); + } + break; // leave loop if successful + } catch (\Exception$e) { + Console::warning("Database not ready. Retrying connection ({$attempts})..."); + if ($attempts >= DATABASE_RECONNECT_MAX_ATTEMPTS) { + throw new \Exception('Failed to connect to database: ' . $e->getMessage()); + } + sleep(DATABASE_RECONNECT_SLEEP); + } + } while ($attempts < DATABASE_RECONNECT_MAX_ATTEMPTS); + + return $database; + } + + protected static function getInfluxDB(Registry &$register): InfluxDatabase + { + /** @var InfluxDB\Client $client */ + $client = $register->get('influxdb'); + $attempts = 0; + $max = 10; + $sleep = 1; + + do { // check if telegraf database is ready + try { + $attempts++; + $database = $client->selectDB('telegraf'); + if (in_array('telegraf', $client->listDatabases())) { + break; // leave the do-while if successful + } + } catch (\Throwable$th) { + Console::warning("InfluxDB not ready. Retrying connection ({$attempts})..."); + if ($attempts >= $max) { + throw new \Exception('InfluxDB database not ready yet'); + } + sleep($sleep); + } + } while ($attempts < $max); + return $database; + } + + public static function getTask(): CLITask + { + global $register; + + Authorization::disable(); + Authorization::setDefaultStatus(false); + + $logError = function (Throwable $error, string $action = 'syncUsageStats') use ($register) { + $logger = $register->get('logger'); + + if ($logger) { + $version = App::getEnv('_APP_VERSION', 'UNKNOWN'); + + $log = new Log(); + $log->setNamespace("usage"); + $log->setServer(\gethostname()); + $log->setVersion($version); + $log->setType(Log::TYPE_ERROR); + $log->setMessage($error->getMessage()); + + $log->addTag('code', $error->getCode()); + $log->addTag('verboseType', get_class($error)); + + $log->addExtra('file', $error->getFile()); + $log->addExtra('line', $error->getLine()); + $log->addExtra('trace', $error->getTraceAsString()); + $log->addExtra('detailedTrace', $error->getTrace()); + + $log->setAction($action); + + $isProduction = App::getEnv('_APP_ENV', 'development') === 'production'; + $log->setEnvironment($isProduction ? Log::ENVIRONMENT_PRODUCTION : Log::ENVIRONMENT_STAGING); + + $responseCode = $logger->addLog($log); + Console::info('Usage stats log pushed with status code: ' . $responseCode); + } + + Console::warning("Failed: {$error->getMessage()}"); + Console::warning($error->getTraceAsString()); + }; + + $usage = new CLITask('usage'); + + $usage + ->desc('Schedules syncing data from influxdb to Appwrite console db') + ->action(function () use ($register, $logError) { + Console::title('Usage Aggregation V1'); + Console::success(APP_NAME . ' usage aggregation process v1 has started'); + + $interval = (int) App::getEnv('_APP_USAGE_AGGREGATION_INTERVAL', '30'); // 30 seconds (by default) + + $database = self::getDatabase($register, '_console'); + $influxDB = self::getInfluxDB($register); + + $usage = new InfluxUsage($database, $influxDB, $logError); + $usageDB = new UsageDB($database, $logError); + + $iterations = 0; + Console::loop(function () use ($interval, $usage, $usageDB, &$iterations) { + $now = date('d-m-Y H:i:s', time()); + Console::info("[{$now}] Aggregating usage data every {$interval} seconds"); + + $loopStart = microtime(true); + + /** + * Aggregate InfluxDB every 30 seconds + */ + $usage->collect(); + + if ($iterations % 30 != 0) { // return if 30 iterations has not passed + $iterations++; + $loopTook = microtime(true) - $loopStart; + $now = date('d-m-Y H:i:s', time()); + Console::info("[{$now}] Aggregation took {$loopTook} seconds"); + return; + } + + $iterations = 0; // Reset iterations to prevent overflow when running for long time + /** + * Aggregate MariaDB every 15 minutes + * Some of the queries here might contain full-table scans. + */ + $now = date('d-m-Y H:i:s', time()); + Console::info("[{$now}] Aggregating database counters."); + + $usageDB->collect(); + + $iterations++; + $loopTook = microtime(true) - $loopStart; + $now = date('d-m-Y H:i:s', time()); + + Console::info("[{$now}] Aggregation took {$loopTook} seconds"); + }, $interval); + }); + self::$task = $usage; + return self::$task; + } +} \ No newline at end of file diff --git a/src/Appwrite/Task/Vars.php b/src/Appwrite/Task/Vars.php new file mode 100644 index 0000000000..7cfb1904ac --- /dev/null +++ b/src/Appwrite/Task/Vars.php @@ -0,0 +1,34 @@ +desc('List all the server environment variables') + ->action(function () { + $config = Config::getParam('variables', []); + $vars = []; + + foreach ($config as $category) { + foreach ($category['variables'] ?? [] as $var) { + $vars[] = $var; + } + } + + foreach ($vars as $key => $value) { + Console::log('- ' . $value['name'] . '=' . App::getEnv($value['name'], '')); + } + }); + self::$task = $vars; + return self::$task; + } +} \ No newline at end of file diff --git a/src/Appwrite/Task/Version.php b/src/Appwrite/Task/Version.php new file mode 100644 index 0000000000..d63e1306c5 --- /dev/null +++ b/src/Appwrite/Task/Version.php @@ -0,0 +1,22 @@ +desc('Get the server version') + ->action(function () { + Console::log(App::getEnv('_APP_VERSION', 'UNKNOWN')); + }); + self::$task = $version; + return self::$task; + } +}