From af19787ed9a331b1f8dcfed2c38a6e8dbc602080 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Tue, 14 Jun 2022 23:40:56 +0000 Subject: [PATCH] logging updates --- app/tasks/usage.php | 36 +++++- app/views/install/compose.phtml | 2 + composer.lock | 206 ++++++++++++++++---------------- docker-compose.yml | 2 + src/Appwrite/Stats/Usage.php | 4 +- src/Appwrite/Stats/UsageDB.php | 8 +- 6 files changed, 146 insertions(+), 112 deletions(-) diff --git a/app/tasks/usage.php b/app/tasks/usage.php index a46ab739b2..106ace9107 100644 --- a/app/tasks/usage.php +++ b/app/tasks/usage.php @@ -13,6 +13,7 @@ use Utopia\Database\Adapter\MariaDB; use Utopia\Database\Database; use Utopia\Database\Validator\Authorization; use Utopia\Registry\Registry; +use Utopia\Logger\Log; function getDatabase(Registry &$register, string $namespace): Database { @@ -72,9 +73,38 @@ function getInfluxDB(Registry &$register): InfluxDatabase return $database; } -$logError = function ($message, $stackTrace) { - Console::warning("Failed: {$message}"); - Console::warning($stackTrace); +$logError = function (Throwable $error, string $action = 'syncUsageStats') use ($register) { + $logger = $register->get('logger'); + + if ($logger) { + $version = App::getEnv('_APP_VERSION', 'UNKNOWN'); + + $log = new Log(); + $log->setNamespace("realtime"); + $log->setServer(\gethostname()); + $log->setVersion($version); + $log->setType(Log::TYPE_ERROR); + $log->setMessage($error->getMessage()); + + $log->addTag('code', $error->getCode()); + $log->addTag('verboseType', get_class($error)); + + $log->addExtra('file', $error->getFile()); + $log->addExtra('line', $error->getLine()); + $log->addExtra('trace', $error->getTraceAsString()); + $log->addExtra('detailedTrace', $error->getTrace()); + + $log->setAction($action); + + $isProduction = App::getEnv('_APP_ENV', 'development') === 'production'; + $log->setEnvironment($isProduction ? Log::ENVIRONMENT_PRODUCTION : Log::ENVIRONMENT_STAGING); + + $responseCode = $logger->addLog($log); + Console::info('Usage stats log pushed with status code: ' . $responseCode); + } + + Console::warning("Failed: {$error->getMessage()}"); + Console::warning($error->getTraceAsString()); }; $cli diff --git a/app/views/install/compose.phtml b/app/views/install/compose.phtml index 6fa8335bfa..cf9a111391 100644 --- a/app/views/install/compose.phtml +++ b/app/views/install/compose.phtml @@ -552,6 +552,8 @@ services: - _APP_REDIS_PORT - _APP_REDIS_USER - _APP_REDIS_PASS + - _APP_LOGGING_PROVIDER + - _APP_LOGGING_CONFIG appwrite-schedule: image: /: diff --git a/composer.lock b/composer.lock index 77a95ecc29..2f77c1c23b 100644 --- a/composer.lock +++ b/composer.lock @@ -1583,16 +1583,16 @@ }, { "name": "squizlabs/php_codesniffer", - "version": "3.6.2", + "version": "3.7.0", "source": { "type": "git", "url": "https://github.com/squizlabs/PHP_CodeSniffer.git", - "reference": "5e4e71592f69da17871dba6e80dd51bce74a351a" + "reference": "a2cd51b45bcaef9c1f2a4bda48f2dd2fa2b95563" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/squizlabs/PHP_CodeSniffer/zipball/5e4e71592f69da17871dba6e80dd51bce74a351a", - "reference": "5e4e71592f69da17871dba6e80dd51bce74a351a", + "url": "https://api.github.com/repos/squizlabs/PHP_CodeSniffer/zipball/a2cd51b45bcaef9c1f2a4bda48f2dd2fa2b95563", + "reference": "a2cd51b45bcaef9c1f2a4bda48f2dd2fa2b95563", "shasum": "" }, "require": { @@ -1635,29 +1635,29 @@ "source": "https://github.com/squizlabs/PHP_CodeSniffer", "wiki": "https://github.com/squizlabs/PHP_CodeSniffer/wiki" }, - "time": "2021-12-12T21:44:58+00:00" + "time": "2022-06-13T06:31:38+00:00" }, { "name": "symfony/deprecation-contracts", - "version": "v2.5.1", + "version": "v3.1.0", "source": { "type": "git", "url": "https://github.com/symfony/deprecation-contracts.git", - "reference": "e8b495ea28c1d97b5e0c121748d6f9b53d075c66" + "reference": "07f1b9cc2ffee6aaafcf4b710fbc38ff736bd918" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/deprecation-contracts/zipball/e8b495ea28c1d97b5e0c121748d6f9b53d075c66", - "reference": "e8b495ea28c1d97b5e0c121748d6f9b53d075c66", + "url": "https://api.github.com/repos/symfony/deprecation-contracts/zipball/07f1b9cc2ffee6aaafcf4b710fbc38ff736bd918", + "reference": "07f1b9cc2ffee6aaafcf4b710fbc38ff736bd918", "shasum": "" }, "require": { - "php": ">=7.1" + "php": ">=8.1" }, "type": "library", "extra": { "branch-alias": { - "dev-main": "2.5-dev" + "dev-main": "3.1-dev" }, "thanks": { "name": "symfony/contracts", @@ -1686,7 +1686,7 @@ "description": "A generic function and convention to trigger deprecation notices", "homepage": "https://symfony.com", "support": { - "source": "https://github.com/symfony/deprecation-contracts/tree/v2.5.1" + "source": "https://github.com/symfony/deprecation-contracts/tree/v3.1.0" }, "funding": [ { @@ -1702,89 +1702,7 @@ "type": "tidelift" } ], - "time": "2022-01-02T09:53:40+00:00" - }, - { - "name": "symfony/polyfill-ctype", - "version": "v1.26.0", - "source": { - "type": "git", - "url": "https://github.com/symfony/polyfill-ctype.git", - "reference": "6fd1b9a79f6e3cf65f9e679b23af304cd9e010d4" - }, - "dist": { - "type": "zip", - "url": "https://api.github.com/repos/symfony/polyfill-ctype/zipball/6fd1b9a79f6e3cf65f9e679b23af304cd9e010d4", - "reference": "6fd1b9a79f6e3cf65f9e679b23af304cd9e010d4", - "shasum": "" - }, - "require": { - "php": ">=7.1" - }, - "provide": { - "ext-ctype": "*" - }, - "suggest": { - "ext-ctype": "For best performance" - }, - "type": "library", - "extra": { - "branch-alias": { - "dev-main": "1.26-dev" - }, - "thanks": { - "name": "symfony/polyfill", - "url": "https://github.com/symfony/polyfill" - } - }, - "autoload": { - "files": [ - "bootstrap.php" - ], - "psr-4": { - "Symfony\\Polyfill\\Ctype\\": "" - } - }, - "notification-url": "https://packagist.org/downloads/", - "license": [ - "MIT" - ], - "authors": [ - { - "name": "Gert de Pagter", - "email": "BackEndTea@gmail.com" - }, - { - "name": "Symfony Community", - "homepage": "https://symfony.com/contributors" - } - ], - "description": "Symfony polyfill for ctype functions", - "homepage": "https://symfony.com", - "keywords": [ - "compatibility", - "ctype", - "polyfill", - "portable" - ], - "support": { - "source": "https://github.com/symfony/polyfill-ctype/tree/v1.26.0" - }, - "funding": [ - { - "url": "https://symfony.com/sponsor", - "type": "custom" - }, - { - "url": "https://github.com/fabpot", - "type": "github" - }, - { - "url": "https://tidelift.com/funding/github/packagist/symfony/symfony", - "type": "tidelift" - } - ], - "time": "2022-05-24T11:49:31+00:00" + "time": "2022-02-25T11:15:52+00:00" }, { "name": "symfony/polyfill-php80", @@ -2905,21 +2823,21 @@ }, { "name": "webmozart/assert", - "version": "1.10.0", + "version": "1.11.0", "source": { "type": "git", "url": "https://github.com/webmozarts/assert.git", - "reference": "6964c76c7804814a842473e0c8fd15bab0f18e25" + "reference": "11cb2199493b2f8a3b53e7f19068fc6aac760991" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/webmozarts/assert/zipball/6964c76c7804814a842473e0c8fd15bab0f18e25", - "reference": "6964c76c7804814a842473e0c8fd15bab0f18e25", + "url": "https://api.github.com/repos/webmozarts/assert/zipball/11cb2199493b2f8a3b53e7f19068fc6aac760991", + "reference": "11cb2199493b2f8a3b53e7f19068fc6aac760991", "shasum": "" }, "require": { - "php": "^7.2 || ^8.0", - "symfony/polyfill-ctype": "^1.8" + "ext-ctype": "*", + "php": "^7.2 || ^8.0" }, "conflict": { "phpstan/phpstan": "<0.12.20", @@ -2957,9 +2875,9 @@ ], "support": { "issues": "https://github.com/webmozarts/assert/issues", - "source": "https://github.com/webmozarts/assert/tree/1.10.0" + "source": "https://github.com/webmozarts/assert/tree/1.11.0" }, - "time": "2021-03-09T10:59:23+00:00" + "time": "2022-06-03T18:03:27+00:00" } ], "packages-dev": [ @@ -5086,6 +5004,88 @@ ], "time": "2022-04-18T20:38:04+00:00" }, + { + "name": "symfony/polyfill-ctype", + "version": "v1.26.0", + "source": { + "type": "git", + "url": "https://github.com/symfony/polyfill-ctype.git", + "reference": "6fd1b9a79f6e3cf65f9e679b23af304cd9e010d4" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/symfony/polyfill-ctype/zipball/6fd1b9a79f6e3cf65f9e679b23af304cd9e010d4", + "reference": "6fd1b9a79f6e3cf65f9e679b23af304cd9e010d4", + "shasum": "" + }, + "require": { + "php": ">=7.1" + }, + "provide": { + "ext-ctype": "*" + }, + "suggest": { + "ext-ctype": "For best performance" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-main": "1.26-dev" + }, + "thanks": { + "name": "symfony/polyfill", + "url": "https://github.com/symfony/polyfill" + } + }, + "autoload": { + "files": [ + "bootstrap.php" + ], + "psr-4": { + "Symfony\\Polyfill\\Ctype\\": "" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Gert de Pagter", + "email": "BackEndTea@gmail.com" + }, + { + "name": "Symfony Community", + "homepage": "https://symfony.com/contributors" + } + ], + "description": "Symfony polyfill for ctype functions", + "homepage": "https://symfony.com", + "keywords": [ + "compatibility", + "ctype", + "polyfill", + "portable" + ], + "support": { + "source": "https://github.com/symfony/polyfill-ctype/tree/v1.26.0" + }, + "funding": [ + { + "url": "https://symfony.com/sponsor", + "type": "custom" + }, + { + "url": "https://github.com/fabpot", + "type": "github" + }, + { + "url": "https://tidelift.com/funding/github/packagist/symfony/symfony", + "type": "tidelift" + } + ], + "time": "2022-05-24T11:49:31+00:00" + }, { "name": "symfony/polyfill-mbstring", "version": "v1.26.0", diff --git a/docker-compose.yml b/docker-compose.yml index b6e1ed68b2..c5e11f1c51 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -588,6 +588,8 @@ services: - _APP_REDIS_PORT - _APP_REDIS_USER - _APP_REDIS_PASS + - _APP_LOGGING_PROVIDER + - _APP_LOGGING_CONFIG appwrite-schedule: entrypoint: schedule diff --git a/src/Appwrite/Stats/Usage.php b/src/Appwrite/Stats/Usage.php index c18f852cd6..9a885c011f 100644 --- a/src/Appwrite/Stats/Usage.php +++ b/src/Appwrite/Stats/Usage.php @@ -202,7 +202,7 @@ class Usage $this->latestTime[$metric][$period] = $time; } catch (\Exception $e) { // if projects are deleted this might fail if (is_callable($this->errorHandler)) { - call_user_func($this->errorHandler, "Unable to save data for project {$projectId} and metric {$metric}: {$e->getMessage()}", $e->getTraceAsString()); + call_user_func($this->errorHandler, $e, "sync_project_{$projectId}_metric_{$metric}"); } else { throw $e; } @@ -290,7 +290,7 @@ class Usage $this->syncFromInfluxDB($metric, $options, $period); } catch (\Exception $e) { if (is_callable($this->errorHandler)) { - call_user_func($this->errorHandler, $e->getMessage(), $e->getTraceAsString()); + call_user_func($this->errorHandler, $e); } else { throw $e; } diff --git a/src/Appwrite/Stats/UsageDB.php b/src/Appwrite/Stats/UsageDB.php index 1cdd34bb1e..cc312b69ac 100644 --- a/src/Appwrite/Stats/UsageDB.php +++ b/src/Appwrite/Stats/UsageDB.php @@ -49,7 +49,7 @@ class UsageDB extends Usage } } catch (\Exception$e) { // if projects are deleted this might fail if (is_callable($this->errorHandler)) { - call_user_func($this->errorHandler, "Unable to save data for project {$projectId} and metric {$metric}: {$e->getMessage()}", $e->getTraceAsString()); + call_user_func($this->errorHandler, $e, "sync_project_{$projectId}_metric_{$metric}"); } else { throw $e; } @@ -81,7 +81,7 @@ class UsageDB extends Usage $results = $this->database->find($collection, $queries, $limit, cursor:$latestDocument); } catch (\Exception $e) { if (is_callable($this->errorHandler)) { - call_user_func($this->errorHandler, "Unable to fetch documents for project {$projectId} and collection {$collection}: {$e->getMessage()}", $e->getTraceAsString()); + call_user_func($this->errorHandler, $e, "fetch_documents_project_{$projectId}_collection_{$collection}"); return; } else { throw $e; @@ -122,7 +122,7 @@ class UsageDB extends Usage return $sum; } catch (\Exception $e) { if (is_callable($this->errorHandler)) { - call_user_func($this->errorHandler, "Unable to fetch sum for project {$projectId} and metric {$metric}: {$e->getMessage()}", $e->getTraceAsString()); + call_user_func($this->errorHandler, $e, "fetch_sum_project_{$projectId}_collection_{$collection}"); } else { throw $e; } @@ -148,7 +148,7 @@ class UsageDB extends Usage return $count; } catch (\Exception $e) { if (is_callable($this->errorHandler)) { - call_user_func($this->errorHandler, "Unable to fetch count for project {$projectId} and metric {$metric}: {$e->getMessage()}", $e->getTraceAsString()); + call_user_func($this->errorHandler, $e, "fetch_count_project_{$projectId}_collection_{$collection}"); } else { throw $e; }