diff --git a/Dockerfile b/Dockerfile index 12ac410b73..68b53747be 100755 --- a/Dockerfile +++ b/Dockerfile @@ -238,7 +238,6 @@ RUN chmod +x /usr/local/bin/doctor && \ chmod +x /usr/local/bin/worker-deletes && \ chmod +x /usr/local/bin/worker-functions && \ chmod +x /usr/local/bin/worker-mails && \ - chmod +x /usr/local/bin/worker-usage && \ chmod +x /usr/local/bin/worker-webhooks # Letsencrypt Permissions diff --git a/app/controllers/shared/api.php b/app/controllers/shared/api.php index 8a9256720f..674a5004b4 100644 --- a/app/controllers/shared/api.php +++ b/app/controllers/shared/api.php @@ -18,7 +18,7 @@ App::init(function ($utopia, $request, $response, $project, $user, $register, $e /** @var Utopia\Registry\Registry $register */ /** @var Appwrite\Event\Event $events */ /** @var Appwrite\Event\Event $audits */ - /** @var Appwrite\Event\Event $usage */ + /** @var Appwrite\Stats\Stats $usage */ /** @var Appwrite\Event\Event $deletes */ /** @var Appwrite\Event\Event $database */ /** @var Appwrite\Event\Event $functions */ @@ -162,14 +162,14 @@ App::init(function ($utopia, $request, $project) { }, ['utopia', 'request', 'project'], 'auth'); -App::shutdown(function ($utopia, $request, $response, $project, $events, $audits, $usage, $deletes, $database, $mode) { +App::shutdown(function ($utopia, $request, $response, $project, $register, $events, $audits, $usage, $deletes, $database, $mode) { /** @var Utopia\App $utopia */ /** @var Utopia\Swoole\Request $request */ /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Document $project */ /** @var Appwrite\Event\Event $events */ /** @var Appwrite\Event\Event $audits */ - /** @var Appwrite\Event\Event $usage */ + /** @var Appwrite\Stats\Stats $usage */ /** @var Appwrite\Event\Event $deletes */ /** @var Appwrite\Event\Event $database */ /** @var Appwrite\Event\Event $functions */ @@ -215,8 +215,8 @@ App::shutdown(function ($utopia, $request, $response, $project, $events, $audits $usage ->setParam('networkRequestSize', $request->getSize() + $usage->getParam('storage')) ->setParam('networkResponseSize', $response->getSize()) - ->trigger() + ->submit() ; } -}, ['utopia', 'request', 'response', 'project', 'events', 'audits', 'usage', 'deletes', 'database', 'mode'], 'api'); +}, ['utopia', 'request', 'response', 'project', 'register', 'events', 'audits', 'usage', 'deletes', 'database', 'mode'], 'api'); \ No newline at end of file diff --git a/app/init.php b/app/init.php index 72b3dedc7e..2394d778c0 100644 --- a/app/init.php +++ b/app/init.php @@ -29,6 +29,7 @@ use Appwrite\Network\Validator\Email; use Appwrite\Network\Validator\IP; use Appwrite\Network\Validator\URL; use Appwrite\OpenSSL\OpenSSL; +use Appwrite\Stats\Stats; use Utopia\App; use Utopia\View; use Utopia\Config\Config; @@ -291,6 +292,7 @@ $register->set('statsd', function () { // Register DB connection return $statsd; }); + $register->set('smtp', function () { $mail = new PHPMailer(true); @@ -421,7 +423,7 @@ App::setResource('audits', function($register) { }, ['register']); App::setResource('usage', function($register) { - return new Event(Event::USAGE_QUEUE_NAME, Event::USAGE_CLASS_NAME); + return new Stats($register->get('statsd')); }, ['register']); App::setResource('mails', function($register) { diff --git a/app/workers/functions.php b/app/workers/functions.php index 9e002dd70f..bed0bda138 100644 --- a/app/workers/functions.php +++ b/app/workers/functions.php @@ -2,6 +2,7 @@ use Appwrite\Event\Event; use Appwrite\Resque\Worker; +use Appwrite\Stats\Stats; use Appwrite\Utopia\Response\Model\Execution; use Cron\CronExpression; use Swoole\Runtime; @@ -134,8 +135,6 @@ class FunctionsV1 extends Worker public function run(): void { - global $register; - $projectId = $this->args['projectId'] ?? ''; $functionId = $this->args['functionId'] ?? ''; $webhooks = $this->args['webhooks'] ?? []; @@ -279,7 +278,7 @@ class FunctionsV1 extends Worker */ public function execute(string $trigger, string $projectId, string $executionId, Database $database, Document $function, string $event = '', string $eventData = '', string $data = '', array $webhooks = [], string $userId = '', string $jwt = ''): void { - global $list; + global $list, $register; $runtimes = Config::getParam('runtimes'); @@ -477,21 +476,22 @@ class FunctionsV1 extends Worker ->setParam('eventData', $execution->getArrayCopy(array_keys($executionModel->getRules()))); $executionUpdate->trigger(); - - $usage = new Event('v1-usage', 'UsageV1'); - - $usage - ->setParam('projectId', $projectId) - ->setParam('functionId', $function->getId()) - ->setParam('functionExecution', 1) - ->setParam('functionStatus', $functionStatus) - ->setParam('functionExecutionTime', $executionTime * 1000) // ms - ->setParam('networkRequestSize', 0) - ->setParam('networkResponseSize', 0) - ; if(App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') { - $usage->trigger(); + $statsd = $register->get('statsd'); + + $usage = new Stats($statsd); + + $usage + ->setParam('projectId', $projectId) + ->setParam('functionId', $function->getId()) + ->setParam('functionExecution', 1) + ->setParam('functionStatus', $functionStatus) + ->setParam('functionExecutionTime', $executionTime * 1000) // ms + ->setParam('networkRequestSize', 0) + ->setParam('networkResponseSize', 0) + ->submit() + ; } $this->cleanup(); diff --git a/app/workers/usage.php b/app/workers/usage.php deleted file mode 100644 index b5a3f885af..0000000000 --- a/app/workers/usage.php +++ /dev/null @@ -1,71 +0,0 @@ -get('statsd', true); - - $projectId = $this->args['projectId'] ?? ''; - - $storage = $this->args['storage'] ?? 0; - - $networkRequestSize = $this->args['networkRequestSize'] ?? 0; - $networkResponseSize = $this->args['networkResponseSize'] ?? 0; - - $httpMethod = $this->args['httpMethod'] ?? ''; - $httpRequest = $this->args['httpRequest'] ?? 0; - - $functionId = $this->args['functionId'] ?? ''; - $functionExecution = $this->args['functionExecution'] ?? 0; - $functionExecutionTime = $this->args['functionExecutionTime'] ?? 0; - $functionStatus = $this->args['functionStatus'] ?? ''; - - $tags = ",project={$projectId},version=".App::getEnv('_APP_VERSION', 'UNKNOWN'); - - // the global namespace is prepended to every key (optional) - $statsd->setNamespace('appwrite.usage'); - - if($httpRequest >= 1) { - $statsd->increment('requests.all'.$tags.',method='.\strtolower($httpMethod)); - } - - if($functionExecution >= 1) { - $statsd->increment('executions.all'.$tags.',functionId='.$functionId.',functionStatus='.$functionStatus); - $statsd->count('executions.time'.$tags.',functionId='.$functionId, $functionExecutionTime); - } - - $statsd->count('network.inbound'.$tags, $networkRequestSize); - $statsd->count('network.outbound'.$tags, $networkResponseSize); - $statsd->count('network.all'.$tags, $networkRequestSize + $networkResponseSize); - - if($storage >= 1) { - $statsd->count('storage.all'.$tags, $storage); - } - } - - public function shutdown(): void - { - } -} \ No newline at end of file diff --git a/bin/worker-usage b/bin/worker-usage deleted file mode 100644 index 4174acce23..0000000000 --- a/bin/worker-usage +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/sh - -if [ -z "$_APP_REDIS_USER" ] && [ -z "$_APP_REDIS_PASS" ] -then - REDIS_BACKEND="${_APP_REDIS_HOST}:${_APP_REDIS_PORT}" -else - REDIS_BACKEND="redis://${_APP_REDIS_USER}:${_APP_REDIS_PASS}@${_APP_REDIS_HOST}:${_APP_REDIS_PORT}" -fi - -INTERVAL=1 QUEUE='v1-usage' APP_INCLUDE='/usr/src/code/app/workers/usage.php' php /usr/src/code/vendor/bin/resque -dopcache.preload=opcache.preload=/usr/src/code/app/preload.php \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 38377f99b7..42c909ced5 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -121,26 +121,6 @@ services: - _APP_FUNCTIONS_MEMORY - _APP_FUNCTIONS_MEMORY_SWAP - _APP_FUNCTIONS_RUNTIMES - - appwrite-worker-usage: - entrypoint: worker-usage - container_name: appwrite-worker-usage - build: - context: . - networks: - - appwrite - volumes: - - ./app:/usr/src/code/app - - ./src:/usr/src/code/src - depends_on: - - redis - - telegraf - environment: - - _APP_ENV - - _APP_REDIS_HOST - - _APP_REDIS_PORT - - _APP_REDIS_USER - - _APP_REDIS_PASS - _APP_STATSD_HOST - _APP_STATSD_PORT @@ -308,6 +288,8 @@ services: - _APP_FUNCTIONS_MEMORY - _APP_FUNCTIONS_MEMORY_SWAP - _APP_USAGE_STATS + - _APP_STATSD_HOST + - _APP_STATSD_PORT - DOCKERHUB_PULL_USERNAME - DOCKERHUB_PULL_PASSWORD diff --git a/src/Appwrite/Stats/Stats.php b/src/Appwrite/Stats/Stats.php new file mode 100644 index 0000000000..b07c4c85c5 --- /dev/null +++ b/src/Appwrite/Stats/Stats.php @@ -0,0 +1,129 @@ +statsd = $statsd; + } + + /** + * @param string $key + * @param mixed $value + * + * @return $this + */ + public function setParam(string $key, $value): self + { + $this->params[$key] = $value; + + return $this; + } + + /** + * @param string $key + * + * @return mixed|null + */ + public function getParam(string $key) + { + return (isset($this->params[$key])) ? $this->params[$key] : null; + } + + /** + * @param string $namespace + * + * @return $this + */ + public function setNamespace(string $namespace): self + { + $this->namespace = $namespace; + + return $this; + } + + /** + * @return string + */ + public function getNamespace() + { + return $this->namespace; + } + + /** + * Submit data to StatsD. + */ + public function submit(): void + { + $projectId = $this->params['projectId'] ?? ''; + + $storage = $this->params['storage'] ?? 0; + + $networkRequestSize = $this->params['networkRequestSize'] ?? 0; + $networkResponseSize = $this->params['networkResponseSize'] ?? 0; + + $httpMethod = $this->params['httpMethod'] ?? ''; + $httpRequest = $this->params['httpRequest'] ?? 0; + + $functionId = $this->params['functionId'] ?? ''; + $functionExecution = $this->params['functionExecution'] ?? 0; + $functionExecutionTime = $this->params['functionExecutionTime'] ?? 0; + $functionStatus = $this->params['functionStatus'] ?? ''; + + $tags = ",project={$projectId},version=" . App::getEnv('_APP_VERSION', 'UNKNOWN'); + + // the global namespace is prepended to every key (optional) + $this->statsd->setNamespace($this->namespace); + + if ($httpRequest >= 1) { + $this->statsd->increment('requests.all' . $tags . ',method=' . \strtolower($httpMethod)); + } + + if ($functionExecution >= 1) { + $this->statsd->increment('executions.all' . $tags . ',functionId=' . $functionId . ',functionStatus=' . $functionStatus); + $this->statsd->count('executions.time' . $tags . ',functionId=' . $functionId, $functionExecutionTime); + } + + $this->statsd->count('network.inbound' . $tags, $networkRequestSize); + $this->statsd->count('network.outbound' . $tags, $networkResponseSize); + $this->statsd->count('network.all' . $tags, $networkRequestSize + $networkResponseSize); + + if ($storage >= 1) { + $this->statsd->count('storage.all' . $tags, $storage); + } + + $this->reset(); + } + + public function reset(): self + { + $this->params = []; + $this->namespace = 'appwrite.usage'; + + return $this; + } +} diff --git a/tests/unit/Stats/StatsTest.php b/tests/unit/Stats/StatsTest.php new file mode 100644 index 0000000000..784ac77fb4 --- /dev/null +++ b/tests/unit/Stats/StatsTest.php @@ -0,0 +1,69 @@ +object = new Stats($statsd); + } + + public function tearDown(): void + { + } + + public function testNamespace() + { + $this->object->setNamespace('appwritetest.usage'); + $this->assertEquals('appwritetest.usage', $this->object->getNamespace()); + } + + public function testParams() + { + $this->object + ->setParam('projectId', 'appwrite_test') + ->setParam('networkRequestSize', 100) + ; + + $this->assertEquals('appwrite_test', $this->object->getParam('projectId')); + $this->assertEquals(100, $this->object->getParam('networkRequestSize')); + + $this->object->submit(); + + $this->assertEquals(null, $this->object->getParam('projectId')); + $this->assertEquals(null, $this->object->getParam('networkRequestSize')); + } + + public function testReset() + { + $this->object + ->setParam('projectId', 'appwrite_test') + ->setParam('networkRequestSize', 100) + ; + + $this->assertEquals('appwrite_test', $this->object->getParam('projectId')); + $this->assertEquals(100, $this->object->getParam('networkRequestSize')); + + $this->object->reset(); + + $this->assertEquals(null, $this->object->getParam('projectId')); + $this->assertEquals(null, $this->object->getParam('networkRequestSize')); + $this->assertEquals('appwrite.usage', $this->object->getNamespace()); + } +}