From 63371ce76eb0dff66896fe691ac347675924ad50 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Thu, 11 Aug 2022 05:25:39 +0000 Subject: [PATCH] functions usage stats using route labels --- app/controllers/api/functions.php | 70 ++++++++++--------- app/controllers/shared/api.php | 5 +- app/workers/builds.php | 6 +- app/workers/functions.php | 6 +- src/Appwrite/Usage/Calculators/TimeSeries.php | 8 +-- src/Appwrite/Usage/Stats.php | 27 +++---- .../Utopia/Response/Model/UsageFunction.php | 43 ++++++++++-- .../Utopia/Response/Model/UsageFunctions.php | 37 +++++++--- 8 files changed, 133 insertions(+), 69 deletions(-) diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index 62484cb2ed..7e0e85f8a4 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -223,9 +223,14 @@ App::get('/v1/functions/:functionId/usage') ]; $metrics = [ - "functions.$functionId.executions", - "functions.$functionId.failures", - "functions.$functionId.compute" + "executions.$functionId.compute.total", + "executions.$functionId.compute.success", + "executions.$functionId.compute.failure", + "executions.$functionId.compute.time", + "builds.$functionId.compute.total", + "builds.$functionId.compute.success", + "builds.$functionId.compute.failure", + "builds.$functionId.compute.time", ]; $stats = []; @@ -268,9 +273,14 @@ App::get('/v1/functions/:functionId/usage') $usage = new Document([ 'range' => $range, - 'functionsExecutions' => $stats["functions.$functionId.executions"], - 'functionsFailures' => $stats["functions.$functionId.failures"], - 'functionsCompute' => $stats["functions.$functionId.compute"] + 'executionsTotal' => $stats["executions.$functionId.compute.total"], + 'executionsFailure' => $stats["executions.$functionId.compute.failure"], + 'executionsSuccesse' => $stats["executions.$functionId.compute.success"], + 'executionsTime' => $stats["executions.$functionId.compute.time"], + 'buildsTotal' => $stats["builds.$functionId.compute.total"], + 'buildsFailure' => $stats["builds.$functionId.compute.failure"], + 'buildsSuccess' => $stats["builds.$functionId.compute.success"], + 'buildsTime' => $stats["builds.$functionId.compute.time"] ]); } @@ -314,11 +324,14 @@ App::get('/v1/functions/usage') ]; $metrics = [ - "functions.executions", - "functions.failures", - "functions.executionTime", - "functions.buildTime", - "functions.compute" + 'executions.$all.compute.total', + 'executions.$all.compute.failure', + 'executions.$all.compute.success', + 'executions.$all.compute.time', + 'builds.$all.compute.total', + 'builds.$all.compute.failure', + 'builds.$all.compute.success', + 'builds.$all.compute.time', ]; $stats = []; @@ -361,11 +374,14 @@ App::get('/v1/functions/usage') $usage = new Document([ 'range' => $range, - 'functionsExecutions' => $stats["functions.executions"], - 'functionsFailures' => $stats["functions.failures"], - 'functionsCompute' => $stats["functions.compute"], - 'functionsExecutionTime' => $stats["functions.executionTime"], - 'functionsBuildTime' => $stats["functions.buildTime"] + 'executionsTotal' => $stats[$metrics[0]], + 'executionsFailure' => $stats[$metrics[1]], + 'executionsSuccess' => $stats[$metrics[2]], + 'executionsTime' => $stats[$metrics[3]], + 'buildsTotal' => $stats[$metrics[4]], + 'buildsFailure' => $stats[$metrics[5]], + 'buildsSuccess' => $stats[$metrics[6]], + 'buildsTime' => $stats[$metrics[7]], ]); } @@ -563,12 +579,11 @@ App::post('/v1/functions/:functionId/deployments') ->inject('request') ->inject('response') ->inject('dbForProject') - ->inject('usage') ->inject('events') ->inject('project') ->inject('deviceFunctions') ->inject('deviceLocal') - ->action(function (string $functionId, string $entrypoint, mixed $code, bool $activate, Request $request, Response $response, Database $dbForProject, Stats $usage, Event $events, Document $project, Device $deviceFunctions, Device $deviceLocal) { + ->action(function (string $functionId, string $entrypoint, mixed $code, bool $activate, Request $request, Response $response, Database $dbForProject, Event $events, Document $project, Device $deviceFunctions, Device $deviceLocal) { $function = $dbForProject->getDocument('functions', $functionId); @@ -692,8 +707,6 @@ App::post('/v1/functions/:functionId/deployments') ->setDeployment($deployment) ->setProject($project) ->trigger(); - - $usage->setParam('storage', $deployment->getAttribute('size', 0)); } else { if ($deployment->isEmpty()) { $deployment = $dbForProject->createDocument('deployments', new Document([ @@ -839,11 +852,10 @@ App::delete('/v1/functions/:functionId/deployments/:deploymentId') ->param('deploymentId', '', new UID(), 'Deployment ID.') ->inject('response') ->inject('dbForProject') - ->inject('usage') ->inject('deletes') ->inject('events') ->inject('deviceFunctions') - ->action(function (string $functionId, string $deploymentId, Response $response, Database $dbForProject, Stats $usage, Delete $deletes, Event $events, Device $deviceFunctions) { + ->action(function (string $functionId, string $deploymentId, Response $response, Database $dbForProject, Delete $deletes, Event $events, Device $deviceFunctions) { $function = $dbForProject->getDocument('functions', $functionId); if ($function->isEmpty()) { @@ -871,9 +883,6 @@ App::delete('/v1/functions/:functionId/deployments/:deploymentId') ]))); } - $usage - ->setParam('storage', $deployment->getAttribute('size', 0) * -1); - $events ->setParam('functionId', $function->getId()) ->setParam('deploymentId', $deployment->getId()); @@ -890,6 +899,8 @@ App::post('/v1/functions/:functionId/executions') ->desc('Create Execution') ->label('scope', 'execution.write') ->label('event', 'functions.[functionId].executions.[executionId].create') + ->label('usage.metric', 'executions.{scope}.compute') + ->label('usage.params', ['functionId'=>'request.functionId','executionStatus' => 'response.status', 'executionTime', 'response.time']) ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_KEY, APP_AUTH_TYPE_JWT]) ->label('sdk.namespace', 'functions') ->label('sdk.method', 'createExecution') @@ -907,8 +918,7 @@ App::post('/v1/functions/:functionId/executions') ->inject('dbForProject') ->inject('user') ->inject('events') - ->inject('usage') - ->action(function (string $functionId, string $data, bool $async, Response $response, Document $project, Database $dbForProject, Document $user, Event $events, Stats $usage) { + ->action(function (string $functionId, string $data, bool $async, Response $response, Document $project, Database $dbForProject, Document $user, Event $events) { $function = Authorization::skip(fn () => $dbForProject->getDocument('functions', $functionId)); @@ -1060,12 +1070,6 @@ App::post('/v1/functions/:functionId/executions') Authorization::skip(fn () => $dbForProject->updateDocument('executions', $executionId, $execution)); - $usage - ->setParam('functionId', $function->getId()) - ->setParam('functionExecution', 1) - ->setParam('functionStatus', $execution->getAttribute('status', '')) - ->setParam('functionExecutionTime', $execution->getAttribute('time') * 1000); // ms - $response ->setStatusCode(Response::STATUS_CODE_CREATED) ->dynamic($execution, Response::MODEL_EXECUTION); diff --git a/app/controllers/shared/api.php b/app/controllers/shared/api.php index 8652853f99..be77705d9b 100644 --- a/app/controllers/shared/api.php +++ b/app/controllers/shared/api.php @@ -348,15 +348,16 @@ App::shutdown() ) { // Don't calculate console usage on admin mode $metric = $route->getLabel('usage.metric', ''); - $params = $route->getLabel('usage.params', []); + $usageParams = $route->getLabel('usage.params', []); if(!empty($metric)) { $usage->setParam($metric, 1); - foreach ($params as $param => $value) { + foreach ($usageParams as $param => $value) { $parts = explode('.',$value); $namespace = $parts[0]; $key = $parts[1]; + $params = []; switch ($namespace) { case 'response': $params = $responsePayload; diff --git a/app/workers/builds.php b/app/workers/builds.php index 722510b6cf..55fd9e4826 100644 --- a/app/workers/builds.php +++ b/app/workers/builds.php @@ -222,9 +222,9 @@ class BuildsV1 extends Worker $usage ->setParam('projectId', $project->getId()) ->setParam('functionId', $function->getId()) - ->setParam('functionBuild', 1) - ->setParam('functionBuildStatus', $build->getAttribute('status', '')) - ->setParam('functionBuildTime', $build->getAttribute('duration') * 1000) // ms + ->setParam('builds.{scope}.compute', 1) + ->setParam('buildStatus', $build->getAttribute('status', '')) + ->setParam('buildTime', $build->getAttribute('duration') * 1000) // ms ->setParam('networkRequestSize', 0) ->setParam('networkResponseSize', 0) ->submit(); diff --git a/app/workers/functions.php b/app/workers/functions.php index 125d1a1022..0fe8fb3fa6 100644 --- a/app/workers/functions.php +++ b/app/workers/functions.php @@ -359,9 +359,9 @@ class FunctionsV1 extends Worker $usage ->setParam('projectId', $project->getId()) ->setParam('functionId', $function->getId()) - ->setParam('functionExecution', 1) - ->setParam('functionStatus', $execution->getAttribute('status', '')) - ->setParam('functionExecutionTime', $execution->getAttribute('time') * 1000) // ms + ->setParam('executions.{scope}.compute', 1) + ->setParam('executionStatus', $execution->getAttribute('status', '')) + ->setParam('executionTime', $execution->getAttribute('time') * 1000) // ms ->setParam('networkRequestSize', 0) ->setParam('networkResponseSize', 0) ->submit(); diff --git a/src/Appwrite/Usage/Calculators/TimeSeries.php b/src/Appwrite/Usage/Calculators/TimeSeries.php index 1f09086f19..b1f59ec1f5 100644 --- a/src/Appwrite/Usage/Calculators/TimeSeries.php +++ b/src/Appwrite/Usage/Calculators/TimeSeries.php @@ -192,13 +192,13 @@ class TimeSeries extends Calculator 'builds.$all.compute.total' => [ 'table' => 'appwrite_usage_builds_{scope}_compute', ], - 'executions.$all.compute.failures' => [ + 'executions.$all.compute.failure' => [ 'table' => 'appwrite_usage_executions_{scope}_compute', 'filters' => [ 'functionStatus' => 'failed', ], ], - 'builds.$all.compute.failures' => [ + 'builds.$all.compute.failure' => [ 'table' => 'appwrite_usage_builds_{scope}_compute', 'filters' => [ 'functionStatus' => 'failed', @@ -232,14 +232,14 @@ class TimeSeries extends Calculator 'table' => 'appwrite_usage_builds_{scope}_compute_time', 'groupBy' => ['functionId'], ], - 'executions.functionId.compute.failures' => [ + 'executions.functionId.compute.failure' => [ 'table' => 'appwrite_usage_executions_{scope}_compute', 'groupBy' => ['functionId'], 'filters' => [ 'functionStatus' => 'failed', ], ], - 'builds.functionId.compute.failures' => [ + 'builds.functionId.compute.failure' => [ 'table' => 'appwrite_usage_builds_{scope}_compute', 'groupBy' => ['functionId'], 'filters' => [ diff --git a/src/Appwrite/Usage/Stats.php b/src/Appwrite/Usage/Stats.php index 5d63494215..7071142cf8 100644 --- a/src/Appwrite/Usage/Stats.php +++ b/src/Appwrite/Usage/Stats.php @@ -89,15 +89,6 @@ class Stats $httpMethod = $this->params['httpMethod'] ?? ''; $httpRequest = $this->params['httpRequest'] ?? 0; - $functionId = $this->params['functionId'] ?? ''; - $functionExecution = $this->params['functionExecution'] ?? 0; - $functionExecutionTime = $this->params['functionExecutionTime'] ?? 0; - $functionStatus = $this->params['functionStatus'] ?? ''; - $functionBuildTime = $this->params['functionBuildTime'] ?? 0; - $functionBuild = $this->params['functionBuild'] ?? 0; - $functionBuildStatus = $this->params['functionBuildStatus'] ?? ''; - $functionCompute = $functionExecutionTime + $functionBuildTime; - $tags = ",projectId={$projectId},version=" . App::getEnv('_APP_VERSION', 'UNKNOWN'); // the global namespace is prepended to every key (optional) @@ -185,13 +176,25 @@ class Stats $this->statsd->count('storage.all' . $storageTags, $storage); } + $functionId = $this->params['functionId'] ?? ''; + $functionExecution = $this->params['executions.{scope}.compute'] ?? 0; + $functionExecutionTime = ($this->params['executionTime'] ?? 0) * 1000; // ms + $functionExecutionStatus = $this->params['executionStatus'] ?? ''; + + $functionBuild = $this->params['builds.{scope}.compute'] ?? 0; + $functionBuildTime = ($this->params['buildTime'] ?? 0) * 1000; // ms + $functionBuildStatus = $this->params['buildStatus'] ?? ''; + $functionCompute = $functionExecutionTime + $functionBuildTime; + if ($functionExecution >= 1) { - $this->statsd->increment('executions.{scope}.compute' . $tags . ',functionId=' . $functionId . ',functionStatus=' . $functionStatus); - $this->statsd->count('executions.{scope}.compute.time' . $tags . ',functionId=' . $functionId, $functionExecutionTime); + $this->statsd->increment('executions.{scope}.compute' . $tags . ',functionId=' . $functionId . ',functionStatus=' . $functionExecutionStatus); + if($functionExecutionTime > 0) { + $this->statsd->count('executions.{scope}.compute.time' . $tags . ',functionId=' . $functionId, $functionExecutionTime); + } } if ($functionBuild >= 1) { $this->statsd->increment('builds.{scope}.compute' . $tags . ',functionId=' . $functionId . ',functionBuildStatus=' . $functionBuildStatus); - $this->statsd->count('builds.{scope}.compute.time' . $tags . ',functionId=' . $functionId, $functionExecutionTime); + $this->statsd->count('builds.{scope}.compute.time' . $tags . ',functionId=' . $functionId, $functionBuildTime); } if ($functionBuild + $functionExecution >= 1) { $this->statsd->count('project.{scope}.compute.time' . $tags . ',functionId=' . $functionId, $functionCompute); diff --git a/src/Appwrite/Utopia/Response/Model/UsageFunction.php b/src/Appwrite/Utopia/Response/Model/UsageFunction.php index 8467bc25f4..676e6c5578 100644 --- a/src/Appwrite/Utopia/Response/Model/UsageFunction.php +++ b/src/Appwrite/Utopia/Response/Model/UsageFunction.php @@ -16,27 +16,62 @@ class UsageFunction extends Model 'default' => '', 'example' => '30d', ]) - ->addRule('functionsExecutions', [ + ->addRule('executionsTotal', [ 'type' => Response::MODEL_METRIC_LIST, - 'description' => 'Aggregated stats for function executions.', + 'description' => 'Aggregated stats for number of function executions.', 'default' => [], 'example' => new \stdClass(), 'array' => true ]) - ->addRule('functionsFailures', [ + ->addRule('executionsFailure', [ 'type' => Response::MODEL_METRIC_LIST, 'description' => 'Aggregated stats for function execution failures.', 'default' => [], 'example' => new \stdClass(), 'array' => true ]) - ->addRule('functionsCompute', [ + ->addRule('executionsSuccess', [ + 'type' => Response::MODEL_METRIC_LIST, + 'description' => 'Aggregated stats for function execution successes.', + 'default' => [], + 'example' => new \stdClass(), + 'array' => true + ]) + ->addRule('executionsTime', [ 'type' => Response::MODEL_METRIC_LIST, 'description' => 'Aggregated stats for function execution duration.', 'default' => [], 'example' => new \stdClass(), 'array' => true ]) + ->addRule('buildsTotal', [ + 'type' => Response::MODEL_METRIC_LIST, + 'description' => 'Aggregated stats for number of function builds.', + 'default' => [], + 'example' => new \stdClass(), + 'array' => true + ]) + ->addRule('buildsFailure', [ + 'type' => Response::MODEL_METRIC_LIST, + 'description' => 'Aggregated stats for function build failures.', + 'default' => [], + 'example' => new \stdClass(), + 'array' => true + ]) + ->addRule('buildsSuccess', [ + 'type' => Response::MODEL_METRIC_LIST, + 'description' => 'Aggregated stats for function build successes.', + 'default' => [], + 'example' => new \stdClass(), + 'array' => true + ]) + ->addRule('buildsTime', [ + 'type' => Response::MODEL_METRIC_LIST, + 'description' => 'Aggregated stats for function build duration.', + 'default' => [], + 'example' => new \stdClass(), + 'array' => true + ]) ; } diff --git a/src/Appwrite/Utopia/Response/Model/UsageFunctions.php b/src/Appwrite/Utopia/Response/Model/UsageFunctions.php index e5cad4e489..9fb9fbabcc 100644 --- a/src/Appwrite/Utopia/Response/Model/UsageFunctions.php +++ b/src/Appwrite/Utopia/Response/Model/UsageFunctions.php @@ -16,37 +16,58 @@ class UsageFunctions extends Model 'default' => '', 'example' => '30d', ]) - ->addRule('functionsExecutions', [ + ->addRule('executionsTotal', [ 'type' => Response::MODEL_METRIC_LIST, - 'description' => 'Aggregated stats for function executions.', + 'description' => 'Aggregated stats for number of function executions.', 'default' => [], 'example' => new \stdClass(), 'array' => true ]) - ->addRule('functionsFailures', [ + ->addRule('executionsFailure', [ 'type' => Response::MODEL_METRIC_LIST, 'description' => 'Aggregated stats for function execution failures.', 'default' => [], 'example' => new \stdClass(), 'array' => true ]) - ->addRule('functionsCompute', [ + ->addRule('executionsSuccess', [ + 'type' => Response::MODEL_METRIC_LIST, + 'description' => 'Aggregated stats for function execution successes.', + 'default' => [], + 'example' => new \stdClass(), + 'array' => true + ]) + ->addRule('executionsTime', [ 'type' => Response::MODEL_METRIC_LIST, 'description' => 'Aggregated stats for function execution duration.', 'default' => [], 'example' => new \stdClass(), 'array' => true ]) - ->addRule('functionsExecutionTime', [ + ->addRule('buildsTotal', [ 'type' => Response::MODEL_METRIC_LIST, - 'description' => 'Aggregated stats for function execution duration.', + 'description' => 'Aggregated stats for number of function builds.', 'default' => [], 'example' => new \stdClass(), 'array' => true ]) - ->addRule('functionsBuildTime', [ + ->addRule('buildsFailure', [ 'type' => Response::MODEL_METRIC_LIST, - 'description' => 'Aggregated stats for function execution duration.', + 'description' => 'Aggregated stats for function build failures.', + 'default' => [], + 'example' => new \stdClass(), + 'array' => true + ]) + ->addRule('buildsSuccess', [ + 'type' => Response::MODEL_METRIC_LIST, + 'description' => 'Aggregated stats for function build successes.', + 'default' => [], + 'example' => new \stdClass(), + 'array' => true + ]) + ->addRule('buildsTime', [ + 'type' => Response::MODEL_METRIC_LIST, + 'description' => 'Aggregated stats for function build duration.', 'default' => [], 'example' => new \stdClass(), 'array' => true