From 0d4b70633918eaecaf9f4ea9b4a1eeadf4840917 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Mon, 2 Aug 2021 11:56:08 +0545 Subject: [PATCH 01/12] save usage data directly to statsd without redis queue --- app/controllers/shared/api.php | 54 ++++++++++++++++++++++++++++++---- 1 file changed, 49 insertions(+), 5 deletions(-) diff --git a/app/controllers/shared/api.php b/app/controllers/shared/api.php index cdbf561e20..901178a94e 100644 --- a/app/controllers/shared/api.php +++ b/app/controllers/shared/api.php @@ -169,7 +169,7 @@ App::init(function ($utopia, $request, $response, $project, $user) { }, ['utopia', 'request', 'response', 'project', 'user'], 'auth'); -App::shutdown(function ($utopia, $request, $response, $project, $events, $audits, $usage, $deletes, $database, $mode) { +App::shutdown(function ($utopia, $request, $response, $project, $events, $audits, $statsd, $usage, $deletes, $database, $mode) { /** @var Utopia\App $utopia */ /** @var Utopia\Swoole\Request $request */ /** @var Appwrite\Utopia\Response $response */ @@ -221,9 +221,53 @@ App::shutdown(function ($utopia, $request, $response, $project, $events, $audits $usage ->setParam('networkRequestSize', $request->getSize() + $usage->getParam('storage')) - ->setParam('networkResponseSize', $response->getSize()) - ->trigger() - ; + ->setParam('networkResponseSize', $response->getSize()); + + statsdUpdate($statsd, $usage); + + } -}, ['utopia', 'request', 'response', 'project', 'events', 'audits', 'usage', 'deletes', 'database', 'mode'], 'api'); +}, ['utopia', 'request', 'response', 'project', 'events', 'audits', 'statsd', 'usage', 'deletes', 'database', 'mode'], 'api'); + +function statsdUpdate($statsd, $usage): void +{ + /** @var Appwrite\Event\Event $usage */ + + $projectId = $usage->getParam('projectId') ?? ''; + + $storage = $usage->getParam('storage') ?? 0; + + $networkRequestSize = $usage->getParam('networkRequestSize') ?? 0; + $networkResponseSize = $usage->getParam('networkResponseSize') ?? 0; + + $httpMethod = $usage->getParam('httpMethod') ?? ''; + $httpRequest = $usage->getParam('httpRequest') ?? 0; + + $functionId = $usage->getParam('functionId') ?? ''; + $functionExecution = $usage->getParam('functionExecution') ?? 0; + $functionExecutionTime = $usage->getParam('functionExecutionTime') ?? 0; + $functionStatus = $usage->getParam('functionStatus') ?? ''; + + $tags = ",project={$projectId},version=".App::getEnv('_APP_VERSION', 'UNKNOWN'); + + // the global namespace is prepended to every key (optional) + $statsd->setNamespace('appwrite.usage'); + + if($httpRequest >= 1) { + $statsd->increment('requests.all'.$tags.',method='.\strtolower($httpMethod)); + } + + if($functionExecution >= 1) { + $statsd->increment('executions.all'.$tags.',functionId='.$functionId.',functionStatus='.$functionStatus); + $statsd->count('executions.time'.$tags.',functionId='.$functionId, $functionExecutionTime); + } + + $statsd->count('network.inbound'.$tags, $networkRequestSize); + $statsd->count('network.outbound'.$tags, $networkResponseSize); + $statsd->count('network.all'.$tags, $networkRequestSize + $networkResponseSize); + + if($storage >= 1) { + $statsd->count('storage.all'.$tags, $storage); + } +} \ No newline at end of file From ab841925ff351c6eb29a4953c1a469334e415016 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Mon, 2 Aug 2021 11:58:42 +0545 Subject: [PATCH 02/12] drop usage worker --- Dockerfile | 1 - app/workers/usage.php | 71 ------------------------------------------- bin/worker-usage | 10 ------ docker-compose.yml | 22 -------------- 4 files changed, 104 deletions(-) delete mode 100644 app/workers/usage.php delete mode 100644 bin/worker-usage diff --git a/Dockerfile b/Dockerfile index f3a87058af..8cba654921 100755 --- a/Dockerfile +++ b/Dockerfile @@ -239,7 +239,6 @@ RUN chmod +x /usr/local/bin/doctor && \ chmod +x /usr/local/bin/worker-functions && \ chmod +x /usr/local/bin/worker-mails && \ chmod +x /usr/local/bin/worker-tasks && \ - chmod +x /usr/local/bin/worker-usage && \ chmod +x /usr/local/bin/worker-webhooks # Letsencrypt Permissions diff --git a/app/workers/usage.php b/app/workers/usage.php deleted file mode 100644 index b5a3f885af..0000000000 --- a/app/workers/usage.php +++ /dev/null @@ -1,71 +0,0 @@ -get('statsd', true); - - $projectId = $this->args['projectId'] ?? ''; - - $storage = $this->args['storage'] ?? 0; - - $networkRequestSize = $this->args['networkRequestSize'] ?? 0; - $networkResponseSize = $this->args['networkResponseSize'] ?? 0; - - $httpMethod = $this->args['httpMethod'] ?? ''; - $httpRequest = $this->args['httpRequest'] ?? 0; - - $functionId = $this->args['functionId'] ?? ''; - $functionExecution = $this->args['functionExecution'] ?? 0; - $functionExecutionTime = $this->args['functionExecutionTime'] ?? 0; - $functionStatus = $this->args['functionStatus'] ?? ''; - - $tags = ",project={$projectId},version=".App::getEnv('_APP_VERSION', 'UNKNOWN'); - - // the global namespace is prepended to every key (optional) - $statsd->setNamespace('appwrite.usage'); - - if($httpRequest >= 1) { - $statsd->increment('requests.all'.$tags.',method='.\strtolower($httpMethod)); - } - - if($functionExecution >= 1) { - $statsd->increment('executions.all'.$tags.',functionId='.$functionId.',functionStatus='.$functionStatus); - $statsd->count('executions.time'.$tags.',functionId='.$functionId, $functionExecutionTime); - } - - $statsd->count('network.inbound'.$tags, $networkRequestSize); - $statsd->count('network.outbound'.$tags, $networkResponseSize); - $statsd->count('network.all'.$tags, $networkRequestSize + $networkResponseSize); - - if($storage >= 1) { - $statsd->count('storage.all'.$tags, $storage); - } - } - - public function shutdown(): void - { - } -} \ No newline at end of file diff --git a/bin/worker-usage b/bin/worker-usage deleted file mode 100644 index 4174acce23..0000000000 --- a/bin/worker-usage +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/sh - -if [ -z "$_APP_REDIS_USER" ] && [ -z "$_APP_REDIS_PASS" ] -then - REDIS_BACKEND="${_APP_REDIS_HOST}:${_APP_REDIS_PORT}" -else - REDIS_BACKEND="redis://${_APP_REDIS_USER}:${_APP_REDIS_PASS}@${_APP_REDIS_HOST}:${_APP_REDIS_PORT}" -fi - -INTERVAL=1 QUEUE='v1-usage' APP_INCLUDE='/usr/src/code/app/workers/usage.php' php /usr/src/code/vendor/bin/resque -dopcache.preload=opcache.preload=/usr/src/code/app/preload.php \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index ed50016ed5..5f93426b4e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -122,28 +122,6 @@ services: - _APP_FUNCTIONS_MEMORY_SWAP - _APP_FUNCTIONS_RUNTIMES - appwrite-worker-usage: - entrypoint: worker-usage - container_name: appwrite-worker-usage - build: - context: . - networks: - - appwrite - volumes: - - ./app:/usr/src/code/app - - ./src:/usr/src/code/src - depends_on: - - redis - - telegraf - environment: - - _APP_ENV - - _APP_REDIS_HOST - - _APP_REDIS_PORT - - _APP_REDIS_USER - - _APP_REDIS_PASS - - _APP_STATSD_HOST - - _APP_STATSD_PORT - appwrite-worker-audits: entrypoint: worker-audits container_name: appwrite-worker-audits From 7d38b83abf5907983dc9b5b025fc2c713aa109f0 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Mon, 2 Aug 2021 12:20:27 +0545 Subject: [PATCH 03/12] statsd env to appwrite image --- docker-compose.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docker-compose.yml b/docker-compose.yml index 5f93426b4e..89c88c68d2 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -121,6 +121,8 @@ services: - _APP_FUNCTIONS_MEMORY - _APP_FUNCTIONS_MEMORY_SWAP - _APP_FUNCTIONS_RUNTIMES + - _APP_STATSD_HOST + - _APP_STATSD_PORT appwrite-worker-audits: entrypoint: worker-audits From bc4fede216cd0a14ecd654005b0b3afa3c825913 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Mon, 2 Aug 2021 12:41:12 +0545 Subject: [PATCH 04/12] refactor usage update and fix functions worker --- app/controllers/shared/api.php | 69 +++++++++++----------------------- app/workers/functions.php | 35 +++++++++-------- docker-compose.yml | 2 + 3 files changed, 43 insertions(+), 63 deletions(-) diff --git a/app/controllers/shared/api.php b/app/controllers/shared/api.php index 901178a94e..29333f6488 100644 --- a/app/controllers/shared/api.php +++ b/app/controllers/shared/api.php @@ -218,56 +218,31 @@ App::shutdown(function ($utopia, $request, $response, $project, $events, $audits && $project->getId() && $mode !== APP_MODE_ADMIN //TODO: add check to make sure user is admin && !empty($route->getLabel('sdk.namespace', null))) { // Don't calculate console usage on admin mode + + $storage = $usage->getParam('storage') ?? 0; + + $networkRequestSize = $request->getSize() + $usage->getParam('storage'); + $networkResponseSize = $response->getSize(); - $usage - ->setParam('networkRequestSize', $request->getSize() + $usage->getParam('storage')) - ->setParam('networkResponseSize', $response->getSize()); + $httpMethod = $usage->getParam('httpMethod') ?? ''; + $httpRequest = $usage->getParam('httpRequest') ?? 0; - statsdUpdate($statsd, $usage); + $tags = ",project={$project->getId()},version=".App::getEnv('_APP_VERSION', 'UNKNOWN'); + // the global namespace is prepended to every key (optional) + $statsd->setNamespace('appwrite.usage'); + if($httpRequest >= 1) { + $statsd->increment('requests.all'.$tags.',method='.\strtolower($httpMethod)); + } + + $statsd->count('network.inbound'.$tags, $networkRequestSize); + $statsd->count('network.outbound'.$tags, $networkResponseSize); + $statsd->count('network.all'.$tags, $networkRequestSize + $networkResponseSize); + + if($storage >= 1) { + $statsd->count('storage.all'.$tags, $storage); + } } -}, ['utopia', 'request', 'response', 'project', 'events', 'audits', 'statsd', 'usage', 'deletes', 'database', 'mode'], 'api'); - -function statsdUpdate($statsd, $usage): void -{ - /** @var Appwrite\Event\Event $usage */ - - $projectId = $usage->getParam('projectId') ?? ''; - - $storage = $usage->getParam('storage') ?? 0; - - $networkRequestSize = $usage->getParam('networkRequestSize') ?? 0; - $networkResponseSize = $usage->getParam('networkResponseSize') ?? 0; - - $httpMethod = $usage->getParam('httpMethod') ?? ''; - $httpRequest = $usage->getParam('httpRequest') ?? 0; - - $functionId = $usage->getParam('functionId') ?? ''; - $functionExecution = $usage->getParam('functionExecution') ?? 0; - $functionExecutionTime = $usage->getParam('functionExecutionTime') ?? 0; - $functionStatus = $usage->getParam('functionStatus') ?? ''; - - $tags = ",project={$projectId},version=".App::getEnv('_APP_VERSION', 'UNKNOWN'); - - // the global namespace is prepended to every key (optional) - $statsd->setNamespace('appwrite.usage'); - - if($httpRequest >= 1) { - $statsd->increment('requests.all'.$tags.',method='.\strtolower($httpMethod)); - } - - if($functionExecution >= 1) { - $statsd->increment('executions.all'.$tags.',functionId='.$functionId.',functionStatus='.$functionStatus); - $statsd->count('executions.time'.$tags.',functionId='.$functionId, $functionExecutionTime); - } - - $statsd->count('network.inbound'.$tags, $networkRequestSize); - $statsd->count('network.outbound'.$tags, $networkResponseSize); - $statsd->count('network.all'.$tags, $networkRequestSize + $networkResponseSize); - - if($storage >= 1) { - $statsd->count('storage.all'.$tags, $storage); - } -} \ No newline at end of file +}, ['utopia', 'request', 'response', 'project', 'events', 'audits', 'statsd', 'usage', 'deletes', 'database', 'mode'], 'api'); \ No newline at end of file diff --git a/app/workers/functions.php b/app/workers/functions.php index 510346fb48..6941c69765 100644 --- a/app/workers/functions.php +++ b/app/workers/functions.php @@ -134,8 +134,6 @@ class FunctionsV1 extends Worker public function run(): void { - global $register; - $projectId = $this->args['projectId'] ?? ''; $functionId = $this->args['functionId'] ?? ''; $webhooks = $this->args['webhooks'] ?? []; @@ -279,7 +277,7 @@ class FunctionsV1 extends Worker */ public function execute(string $trigger, string $projectId, string $executionId, Database $database, Document $function, string $event = '', string $eventData = '', string $data = '', array $webhooks = [], string $userId = '', string $jwt = ''): void { - global $list; + global $list, $register; $runtimes = Config::getParam('runtimes'); @@ -477,21 +475,26 @@ class FunctionsV1 extends Worker ->setParam('eventData', $execution->getArrayCopy(array_keys($executionModel->getRules()))); $executionUpdate->trigger(); - - $usage = new Event('v1-usage', 'UsageV1'); - - $usage - ->setParam('projectId', $projectId) - ->setParam('functionId', $function->getId()) - ->setParam('functionExecution', 1) - ->setParam('functionStatus', $functionStatus) - ->setParam('functionExecutionTime', $executionTime * 1000) // ms - ->setParam('networkRequestSize', 0) - ->setParam('networkResponseSize', 0) - ; if(App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') { - $usage->trigger(); + $statsd = $register->get('statsd'); + + $storage = 0; + + $functionExecutionTime = $executionTime * 1000; + + $tags = ",project={$projectId},version=".App::getEnv('_APP_VERSION', 'UNKNOWN'); + + // the global namespace is prepended to every key (optional) + $statsd->setNamespace('appwrite.usage'); + + $statsd->increment('executions.all'.$tags.',functionId='.$function->getId().',functionStatus='.$functionStatus); + $statsd->count('executions.time'.$tags.',functionId='.$function->getId(), $functionExecutionTime); + + if($storage >= 1) { + $statsd->count('storage.all'.$tags, $storage); + } + } $this->cleanup(); diff --git a/docker-compose.yml b/docker-compose.yml index 89c88c68d2..51ecde502e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -315,6 +315,8 @@ services: - _APP_FUNCTIONS_MEMORY - _APP_FUNCTIONS_MEMORY_SWAP - _APP_USAGE_STATS + - _APP_STATSD_HOST + - _APP_STATSD_PORT - DOCKERHUB_PULL_USERNAME - DOCKERHUB_PULL_PASSWORD From afe772f8fc02d7ead00a75f74de0cfe2b2cbe8f3 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Mon, 2 Aug 2021 14:51:56 +0545 Subject: [PATCH 05/12] fix worker statsd not found issue --- app/controllers/shared/api.php | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/app/controllers/shared/api.php b/app/controllers/shared/api.php index 29333f6488..c1a1f8c827 100644 --- a/app/controllers/shared/api.php +++ b/app/controllers/shared/api.php @@ -169,7 +169,7 @@ App::init(function ($utopia, $request, $response, $project, $user) { }, ['utopia', 'request', 'response', 'project', 'user'], 'auth'); -App::shutdown(function ($utopia, $request, $response, $project, $events, $audits, $statsd, $usage, $deletes, $database, $mode) { +App::shutdown(function ($utopia, $request, $response, $project, $register, $events, $audits, $usage, $deletes, $database, $mode) { /** @var Utopia\App $utopia */ /** @var Utopia\Swoole\Request $request */ /** @var Appwrite\Utopia\Response $response */ @@ -228,7 +228,8 @@ App::shutdown(function ($utopia, $request, $response, $project, $events, $audits $httpRequest = $usage->getParam('httpRequest') ?? 0; $tags = ",project={$project->getId()},version=".App::getEnv('_APP_VERSION', 'UNKNOWN'); - + + $statsd = $register->get('statsd'); // the global namespace is prepended to every key (optional) $statsd->setNamespace('appwrite.usage'); @@ -245,4 +246,4 @@ App::shutdown(function ($utopia, $request, $response, $project, $events, $audits } } -}, ['utopia', 'request', 'response', 'project', 'events', 'audits', 'statsd', 'usage', 'deletes', 'database', 'mode'], 'api'); \ No newline at end of file +}, ['utopia', 'request', 'response', 'project', 'register', 'events', 'audits', 'usage', 'deletes', 'database', 'mode'], 'api'); \ No newline at end of file From 23caa625505706deda9f2da3094f0deebd0edbed Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Mon, 2 Aug 2021 15:14:53 +0545 Subject: [PATCH 06/12] remove unused params --- app/workers/functions.php | 7 ------- 1 file changed, 7 deletions(-) diff --git a/app/workers/functions.php b/app/workers/functions.php index 6941c69765..676c784e5a 100644 --- a/app/workers/functions.php +++ b/app/workers/functions.php @@ -479,8 +479,6 @@ class FunctionsV1 extends Worker if(App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') { $statsd = $register->get('statsd'); - $storage = 0; - $functionExecutionTime = $executionTime * 1000; $tags = ",project={$projectId},version=".App::getEnv('_APP_VERSION', 'UNKNOWN'); @@ -490,11 +488,6 @@ class FunctionsV1 extends Worker $statsd->increment('executions.all'.$tags.',functionId='.$function->getId().',functionStatus='.$functionStatus); $statsd->count('executions.time'.$tags.',functionId='.$function->getId(), $functionExecutionTime); - - if($storage >= 1) { - $statsd->count('storage.all'.$tags, $storage); - } - } $this->cleanup(); From ffe9c99157ada92750f048f9205831be42717f1f Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Sun, 8 Aug 2021 11:21:08 +0545 Subject: [PATCH 07/12] new statsd class and implementation --- app/controllers/shared/api.php | 34 +++---------- app/init.php | 3 +- app/workers/functions.php | 20 +++++--- src/Appwrite/Statsd/Statsd.php | 93 ++++++++++++++++++++++++++++++++++ 4 files changed, 114 insertions(+), 36 deletions(-) create mode 100644 src/Appwrite/Statsd/Statsd.php diff --git a/app/controllers/shared/api.php b/app/controllers/shared/api.php index c1a1f8c827..9655b768c1 100644 --- a/app/controllers/shared/api.php +++ b/app/controllers/shared/api.php @@ -18,7 +18,7 @@ App::init(function ($utopia, $request, $response, $project, $user, $register, $e /** @var Utopia\Registry\Registry $register */ /** @var Appwrite\Event\Event $events */ /** @var Appwrite\Event\Event $audits */ - /** @var Appwrite\Event\Event $usage */ + /** @var Appwrite\Statsd\Statsd $usage */ /** @var Appwrite\Event\Event $deletes */ /** @var Appwrite\Event\Event $database */ /** @var Appwrite\Event\Event $functions */ @@ -176,7 +176,7 @@ App::shutdown(function ($utopia, $request, $response, $project, $register, $even /** @var Utopia\Database\Document $project */ /** @var Appwrite\Event\Event $events */ /** @var Appwrite\Event\Event $audits */ - /** @var Appwrite\Event\Event $usage */ + /** @var Appwrite\Statsd\Statsd $usage */ /** @var Appwrite\Event\Event $deletes */ /** @var Appwrite\Event\Event $database */ /** @var Appwrite\Event\Event $functions */ @@ -218,32 +218,12 @@ App::shutdown(function ($utopia, $request, $response, $project, $register, $even && $project->getId() && $mode !== APP_MODE_ADMIN //TODO: add check to make sure user is admin && !empty($route->getLabel('sdk.namespace', null))) { // Don't calculate console usage on admin mode - - $storage = $usage->getParam('storage') ?? 0; - - $networkRequestSize = $request->getSize() + $usage->getParam('storage'); - $networkResponseSize = $response->getSize(); - $httpMethod = $usage->getParam('httpMethod') ?? ''; - $httpRequest = $usage->getParam('httpRequest') ?? 0; - - $tags = ",project={$project->getId()},version=".App::getEnv('_APP_VERSION', 'UNKNOWN'); - - $statsd = $register->get('statsd'); - // the global namespace is prepended to every key (optional) - $statsd->setNamespace('appwrite.usage'); - - if($httpRequest >= 1) { - $statsd->increment('requests.all'.$tags.',method='.\strtolower($httpMethod)); - } - - $statsd->count('network.inbound'.$tags, $networkRequestSize); - $statsd->count('network.outbound'.$tags, $networkResponseSize); - $statsd->count('network.all'.$tags, $networkRequestSize + $networkResponseSize); - - if($storage >= 1) { - $statsd->count('storage.all'.$tags, $storage); - } + $usage + ->setParam('networkRequestSize', $request->getSize() + $usage->getParam('storage')) + ->setParam('networkResponseSize', $response->getSize()) + ->save() + ; } }, ['utopia', 'request', 'response', 'project', 'register', 'events', 'audits', 'usage', 'deletes', 'database', 'mode'], 'api'); \ No newline at end of file diff --git a/app/init.php b/app/init.php index 9d8413e64d..bb5278445d 100644 --- a/app/init.php +++ b/app/init.php @@ -26,6 +26,7 @@ use Appwrite\Database\Adapter\Redis as RedisAdapter; use Appwrite\Database\Document; use Appwrite\Event\Event; use Appwrite\OpenSSL\OpenSSL; +use Appwrite\Statsd\Statsd; use Utopia\App; use Utopia\View; use Utopia\Config\Config; @@ -378,7 +379,7 @@ App::setResource('audits', function($register) { }, ['register']); App::setResource('usage', function($register) { - return new Event(Event::USAGE_QUEUE_NAME, Event::USAGE_CLASS_NAME); + return new Statsd($register->get('statsd')); }, ['register']); App::setResource('mails', function($register) { diff --git a/app/workers/functions.php b/app/workers/functions.php index 676c784e5a..75dc3ca5e9 100644 --- a/app/workers/functions.php +++ b/app/workers/functions.php @@ -2,6 +2,7 @@ use Appwrite\Event\Event; use Appwrite\Resque\Worker; +use Appwrite\Statsd\Statsd; use Appwrite\Utopia\Response\Model\Execution; use Cron\CronExpression; use Swoole\Runtime; @@ -479,15 +480,18 @@ class FunctionsV1 extends Worker if(App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') { $statsd = $register->get('statsd'); - $functionExecutionTime = $executionTime * 1000; + $usage = new Statsd($statsd); - $tags = ",project={$projectId},version=".App::getEnv('_APP_VERSION', 'UNKNOWN'); - - // the global namespace is prepended to every key (optional) - $statsd->setNamespace('appwrite.usage'); - - $statsd->increment('executions.all'.$tags.',functionId='.$function->getId().',functionStatus='.$functionStatus); - $statsd->count('executions.time'.$tags.',functionId='.$function->getId(), $functionExecutionTime); + $usage + ->setParam('projectId', $projectId) + ->setParam('functionId', $function->getId()) + ->setParam('functionExecution', 1) + ->setParam('functionStatus', $functionStatus) + ->setParam('functionExecutionTime', $executionTime * 1000) // ms + ->setParam('networkRequestSize', 0) + ->setParam('networkResponseSize', 0) + ->save() + ; } $this->cleanup(); diff --git a/src/Appwrite/Statsd/Statsd.php b/src/Appwrite/Statsd/Statsd.php new file mode 100644 index 0000000000..134b890565 --- /dev/null +++ b/src/Appwrite/Statsd/Statsd.php @@ -0,0 +1,93 @@ +statsd = $statsd; + } + + /** + * @param string $key + * @param mixed $value + * + * @return $this + */ + public function setParam(string $key, $value): self + { + $this->params[$key] = $value; + + return $this; + } + + /** + * Save to statsd. + */ + public function save(): void + { + $projectId = $this->params['projectId'] ?? ''; + + $storage = $this->params['storage'] ?? 0; + + $networkRequestSize = $this->params['networkRequestSize'] ?? 0; + $networkResponseSize = $this->params['networkResponseSize'] ?? 0; + + $httpMethod = $this->params['httpMethod'] ?? ''; + $httpRequest = $this->params['httpRequest'] ?? 0; + + $functionId = $this->params['functionId'] ?? ''; + $functionExecution = $this->params['functionExecution'] ?? 0; + $functionExecutionTime = $this->params['functionExecutionTime'] ?? 0; + $functionStatus = $this->params['functionStatus'] ?? ''; + + $tags = ",project={$projectId},version=" . App::getEnv('_APP_VERSION', 'UNKNOWN'); + + // the global namespace is prepended to every key (optional) + $this->statsd->setNamespace('appwrite.usage'); + + if ($httpRequest >= 1) { + $this->statsd->increment('requests.all' . $tags . ',method=' . \strtolower($httpMethod)); + } + + if ($functionExecution >= 1) { + $this->statsd->increment('executions.all' . $tags . ',functionId=' . $functionId . ',functionStatus=' . $functionStatus); + $this->statsd->count('executions.time' . $tags . ',functionId=' . $functionId, $functionExecutionTime); + } + + $this->statsd->count('network.inbound' . $tags, $networkRequestSize); + $this->statsd->count('network.outbound' . $tags, $networkResponseSize); + $this->statsd->count('network.all' . $tags, $networkRequestSize + $networkResponseSize); + + if ($storage >= 1) { + $this->statsd->count('storage.all' . $tags, $storage); + } + + $this->reset(); + } + + public function reset(): self + { + $this->params = []; + + return $this; + } +} From a8f864a07ddc9b5bcf3ce0e7f2c4a320e98f946e Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Sun, 8 Aug 2021 12:13:50 +0545 Subject: [PATCH 08/12] Update src/Appwrite/Statsd/Statsd.php Co-authored-by: Eldad A. Fux --- src/Appwrite/Statsd/Statsd.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Appwrite/Statsd/Statsd.php b/src/Appwrite/Statsd/Statsd.php index 134b890565..e1fdbc46e9 100644 --- a/src/Appwrite/Statsd/Statsd.php +++ b/src/Appwrite/Statsd/Statsd.php @@ -40,7 +40,7 @@ class Statsd } /** - * Save to statsd. + * Submit data to StatsD. */ public function save(): void { From fa46f14092b1d097d5b7fa8fa4cda2fa04327e16 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Sun, 8 Aug 2021 12:16:20 +0545 Subject: [PATCH 09/12] refactor the class --- app/controllers/shared/api.php | 6 +++--- app/init.php | 1 + app/workers/functions.php | 6 +++--- .../{Statsd/Statsd.php => Stats/Stats.php} | 16 +++++++++++++--- 4 files changed, 20 insertions(+), 9 deletions(-) rename src/Appwrite/{Statsd/Statsd.php => Stats/Stats.php} (89%) diff --git a/app/controllers/shared/api.php b/app/controllers/shared/api.php index 9655b768c1..a8d5f20cee 100644 --- a/app/controllers/shared/api.php +++ b/app/controllers/shared/api.php @@ -18,7 +18,7 @@ App::init(function ($utopia, $request, $response, $project, $user, $register, $e /** @var Utopia\Registry\Registry $register */ /** @var Appwrite\Event\Event $events */ /** @var Appwrite\Event\Event $audits */ - /** @var Appwrite\Statsd\Statsd $usage */ + /** @var Appwrite\Stats\Stats $usage */ /** @var Appwrite\Event\Event $deletes */ /** @var Appwrite\Event\Event $database */ /** @var Appwrite\Event\Event $functions */ @@ -176,7 +176,7 @@ App::shutdown(function ($utopia, $request, $response, $project, $register, $even /** @var Utopia\Database\Document $project */ /** @var Appwrite\Event\Event $events */ /** @var Appwrite\Event\Event $audits */ - /** @var Appwrite\Statsd\Statsd $usage */ + /** @var Appwrite\Stats\Stats $usage */ /** @var Appwrite\Event\Event $deletes */ /** @var Appwrite\Event\Event $database */ /** @var Appwrite\Event\Event $functions */ @@ -222,7 +222,7 @@ App::shutdown(function ($utopia, $request, $response, $project, $register, $even $usage ->setParam('networkRequestSize', $request->getSize() + $usage->getParam('storage')) ->setParam('networkResponseSize', $response->getSize()) - ->save() + ->submit() ; } diff --git a/app/init.php b/app/init.php index bb5278445d..5c20816e43 100644 --- a/app/init.php +++ b/app/init.php @@ -249,6 +249,7 @@ $register->set('statsd', function () { // Register DB connection return $statsd; }); + $register->set('smtp', function () { $mail = new PHPMailer(true); diff --git a/app/workers/functions.php b/app/workers/functions.php index 75dc3ca5e9..1b36b7cdc4 100644 --- a/app/workers/functions.php +++ b/app/workers/functions.php @@ -2,7 +2,7 @@ use Appwrite\Event\Event; use Appwrite\Resque\Worker; -use Appwrite\Statsd\Statsd; +use Appwrite\Stats\Stats; use Appwrite\Utopia\Response\Model\Execution; use Cron\CronExpression; use Swoole\Runtime; @@ -480,7 +480,7 @@ class FunctionsV1 extends Worker if(App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') { $statsd = $register->get('statsd'); - $usage = new Statsd($statsd); + $usage = new Stats($statsd); $usage ->setParam('projectId', $projectId) @@ -490,7 +490,7 @@ class FunctionsV1 extends Worker ->setParam('functionExecutionTime', $executionTime * 1000) // ms ->setParam('networkRequestSize', 0) ->setParam('networkResponseSize', 0) - ->save() + ->submit() ; } diff --git a/src/Appwrite/Statsd/Statsd.php b/src/Appwrite/Stats/Stats.php similarity index 89% rename from src/Appwrite/Statsd/Statsd.php rename to src/Appwrite/Stats/Stats.php index e1fdbc46e9..4b9eddf85f 100644 --- a/src/Appwrite/Statsd/Statsd.php +++ b/src/Appwrite/Stats/Stats.php @@ -1,10 +1,10 @@ params[$key])) ? $this->params[$key] : null; + } + /** * Submit data to StatsD. */ - public function save(): void + public function submit(): void { $projectId = $this->params['projectId'] ?? ''; From 8255bdb06a5e9b7a1631646acbb8f644759f299f Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Sun, 8 Aug 2021 12:24:16 +0545 Subject: [PATCH 10/12] fix issue --- app/init.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/init.php b/app/init.php index 53272c0276..2394d778c0 100644 --- a/app/init.php +++ b/app/init.php @@ -29,7 +29,7 @@ use Appwrite\Network\Validator\Email; use Appwrite\Network\Validator\IP; use Appwrite\Network\Validator\URL; use Appwrite\OpenSSL\OpenSSL; -use Appwrite\Statsd\Statsd; +use Appwrite\Stats\Stats; use Utopia\App; use Utopia\View; use Utopia\Config\Config; @@ -423,7 +423,7 @@ App::setResource('audits', function($register) { }, ['register']); App::setResource('usage', function($register) { - return new Statsd($register->get('statsd')); + return new Stats($register->get('statsd')); }, ['register']); App::setResource('mails', function($register) { From 72948624ee6cc416d27524c97031557ca8d49cbc Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Sun, 8 Aug 2021 14:09:23 +0545 Subject: [PATCH 11/12] simple unit test for stats class --- tests/unit/Stats/StatsTest.php | 61 ++++++++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) create mode 100644 tests/unit/Stats/StatsTest.php diff --git a/tests/unit/Stats/StatsTest.php b/tests/unit/Stats/StatsTest.php new file mode 100644 index 0000000000..ad660330ab --- /dev/null +++ b/tests/unit/Stats/StatsTest.php @@ -0,0 +1,61 @@ +object = new Stats($statsd); + } + + public function tearDown(): void + { + } + + public function testParams() + { + $this->object + ->setParam('statsKey1', 'statsValue1') + ->setParam('statsKey2', 'statsValue2') + ; + + $this->object->submit(); + + $this->assertEquals(null, $this->object->getParam('statsKey1')); + $this->assertEquals(null, $this->object->getParam('statsKey2')); + $this->assertEquals(null, $this->object->getParam('statsKey3')); + } + + public function testReset() + { + $this->object + ->setParam('statsKey1', 'statsValue1') + ->setParam('statsKey2', 'statsValue2') + ; + + $this->assertEquals('statsValue1', $this->object->getParam('statsKey1')); + $this->assertEquals('statsValue2', $this->object->getParam('statsKey2')); + + $this->object->reset(); + + $this->assertEquals(null, $this->object->getParam('statsKey1')); + $this->assertEquals(null, $this->object->getParam('statsKey2')); + $this->assertEquals(null, $this->object->getParam('statsKey3')); + } +} From d73ddb581c1fd5f759442dec0108fcd2f9de11c2 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Sun, 8 Aug 2021 15:08:44 +0545 Subject: [PATCH 12/12] added namespace and modified tests --- src/Appwrite/Stats/Stats.php | 28 +++++++++++++++++++++++++++- tests/unit/Stats/StatsTest.php | 34 +++++++++++++++++++++------------- 2 files changed, 48 insertions(+), 14 deletions(-) diff --git a/src/Appwrite/Stats/Stats.php b/src/Appwrite/Stats/Stats.php index 4b9eddf85f..b07c4c85c5 100644 --- a/src/Appwrite/Stats/Stats.php +++ b/src/Appwrite/Stats/Stats.php @@ -16,6 +16,11 @@ class Stats */ protected $statsd; + /** + * @var string + */ + protected $namespace = 'appwrite.usage'; + /** * Event constructor. * @@ -49,6 +54,26 @@ class Stats return (isset($this->params[$key])) ? $this->params[$key] : null; } + /** + * @param string $namespace + * + * @return $this + */ + public function setNamespace(string $namespace): self + { + $this->namespace = $namespace; + + return $this; + } + + /** + * @return string + */ + public function getNamespace() + { + return $this->namespace; + } + /** * Submit data to StatsD. */ @@ -72,7 +97,7 @@ class Stats $tags = ",project={$projectId},version=" . App::getEnv('_APP_VERSION', 'UNKNOWN'); // the global namespace is prepended to every key (optional) - $this->statsd->setNamespace('appwrite.usage'); + $this->statsd->setNamespace($this->namespace); if ($httpRequest >= 1) { $this->statsd->increment('requests.all' . $tags . ',method=' . \strtolower($httpMethod)); @@ -97,6 +122,7 @@ class Stats public function reset(): self { $this->params = []; + $this->namespace = 'appwrite.usage'; return $this; } diff --git a/tests/unit/Stats/StatsTest.php b/tests/unit/Stats/StatsTest.php index ad660330ab..784ac77fb4 100644 --- a/tests/unit/Stats/StatsTest.php +++ b/tests/unit/Stats/StatsTest.php @@ -20,7 +20,7 @@ class StatsTest extends TestCase $connection = new \Domnikl\Statsd\Connection\UdpSocket($host, $port); $statsd = new \Domnikl\Statsd\Client($connection); - + $this->object = new Stats($statsd); } @@ -28,34 +28,42 @@ class StatsTest extends TestCase { } + public function testNamespace() + { + $this->object->setNamespace('appwritetest.usage'); + $this->assertEquals('appwritetest.usage', $this->object->getNamespace()); + } + public function testParams() { $this->object - ->setParam('statsKey1', 'statsValue1') - ->setParam('statsKey2', 'statsValue2') + ->setParam('projectId', 'appwrite_test') + ->setParam('networkRequestSize', 100) ; + $this->assertEquals('appwrite_test', $this->object->getParam('projectId')); + $this->assertEquals(100, $this->object->getParam('networkRequestSize')); + $this->object->submit(); - $this->assertEquals(null, $this->object->getParam('statsKey1')); - $this->assertEquals(null, $this->object->getParam('statsKey2')); - $this->assertEquals(null, $this->object->getParam('statsKey3')); + $this->assertEquals(null, $this->object->getParam('projectId')); + $this->assertEquals(null, $this->object->getParam('networkRequestSize')); } public function testReset() { $this->object - ->setParam('statsKey1', 'statsValue1') - ->setParam('statsKey2', 'statsValue2') + ->setParam('projectId', 'appwrite_test') + ->setParam('networkRequestSize', 100) ; - $this->assertEquals('statsValue1', $this->object->getParam('statsKey1')); - $this->assertEquals('statsValue2', $this->object->getParam('statsKey2')); + $this->assertEquals('appwrite_test', $this->object->getParam('projectId')); + $this->assertEquals(100, $this->object->getParam('networkRequestSize')); $this->object->reset(); - $this->assertEquals(null, $this->object->getParam('statsKey1')); - $this->assertEquals(null, $this->object->getParam('statsKey2')); - $this->assertEquals(null, $this->object->getParam('statsKey3')); + $this->assertEquals(null, $this->object->getParam('projectId')); + $this->assertEquals(null, $this->object->getParam('networkRequestSize')); + $this->assertEquals('appwrite.usage', $this->object->getNamespace()); } }