Add usage endpoints

This commit is contained in:
Khushboo Verma
2024-10-24 15:21:40 +02:00
parent 8bda7f1e1e
commit d876085f6d
10 changed files with 535 additions and 10 deletions
+2 -2
View File
@@ -655,7 +655,7 @@ App::get('/v1/functions/:functionId/usage')
App::get('/v1/functions/usage')
->desc('Get functions usage')
->groups(['api', 'functions'])
->groups(['api', 'functions', 'usage'])
->label('scope', 'functions.read')
->label('sdk.auth', [APP_AUTH_TYPE_ADMIN])
->label('sdk.namespace', 'functions')
@@ -2565,7 +2565,7 @@ App::get('/v1/functions/templates')
->desc('List function templates')
->label('scope', 'public')
->label('sdk.namespace', 'functions')
->label('sdk.method', 'listTemplates') // TODO: Change to listFunctionTemplates later
->label('sdk.method', 'listTemplates')
->label('sdk.auth', [APP_AUTH_TYPE_ADMIN])
->label('sdk.description', '/docs/references/functions/list-templates.md')
->label('sdk.response.code', Response::STATUS_CODE_OK)
+9
View File
@@ -133,6 +133,15 @@ $databaseListener = function (string $event, Document $document, Document $proje
$queueForUsage
->addMetric(METRIC_FUNCTIONS, $value); // per project
if ($event === Database::EVENT_DOCUMENT_DELETE) {
$queueForUsage
->addReduce($document);
}
break;
case $document->getCollection() === 'sites':
$queueForUsage
->addMetric(METRIC_SITES, $value); // per project
if ($event === Database::EVENT_DOCUMENT_DELETE) {
$queueForUsage
->addReduce($document);
+7
View File
@@ -259,6 +259,7 @@ const METRIC_FILES = 'files';
const METRIC_FILES_STORAGE = 'files.storage';
const METRIC_BUCKET_ID_FILES = '{bucketInternalId}.files';
const METRIC_BUCKET_ID_FILES_STORAGE = '{bucketInternalId}.files.storage';
const METRIC_SITES = 'sites';
const METRIC_FUNCTIONS = 'functions';
const METRIC_DEPLOYMENTS = 'deployments';
const METRIC_DEPLOYMENTS_STORAGE = 'deployments.storage';
@@ -286,6 +287,12 @@ const METRIC_EXECUTIONS_MB_SECONDS = 'executions.mbSeconds';
const METRIC_FUNCTION_ID_EXECUTIONS = '{functionInternalId}.executions';
const METRIC_FUNCTION_ID_EXECUTIONS_COMPUTE = '{functionInternalId}.executions.compute';
const METRIC_FUNCTION_ID_EXECUTIONS_MB_SECONDS = '{functionInternalId}.executions.mbSeconds';
const METRIC_SITE_ID_DEPLOYMENTS = '{resourceType}.{resourceInternalId}.deployments';
const METRIC_SITE_ID_DEPLOYMENTS_STORAGE = '{resourceType}.{resourceInternalId}.deployments.storage';
const METRIC_SITE_ID_BUILDS = '{siteInternalId}.builds';
const METRIC_SITE_ID_BUILDS_STORAGE = '{siteInternalId}.builds.storage';
const METRIC_SITE_ID_BUILDS_COMPUTE = '{siteInternalId}.builds.compute';
const METRIC_SITE_ID_BUILDS_MB_SECONDS = '{siteInternalId}.builds.mbSeconds';
const METRIC_NETWORK_REQUESTS = 'network.requests';
const METRIC_NETWORK_INBOUND = 'network.inbound';
const METRIC_NETWORK_OUTBOUND = 'network.outbound';
@@ -0,0 +1,129 @@
<?php
namespace Appwrite\Platform\Modules\Sites\Http\Sites;
use Appwrite\Extend\Exception;
use Appwrite\Platform\Modules\Compute\Base;
use Appwrite\Utopia\Response;
use Utopia\Config\Config;
use Utopia\Database\Database;
use Utopia\Database\Document;
use Utopia\Database\Query;
use Utopia\Database\Validator\Authorization;
use Utopia\Database\Validator\UID;
use Utopia\Platform\Action;
use Utopia\Platform\Scope\HTTP;
use Utopia\Validator\WhiteList;
class GetSiteUsage extends Base
{
use HTTP;
public static function getName()
{
return 'getSiteUsage';
}
public function __construct()
{
$this
->setHttpMethod(Action::HTTP_REQUEST_METHOD_GET)
->setHttpPath('/v1/sites/:siteId/usage')
->desc('Get site usage')
->groups(['api', 'sites', 'usage'])
->label('scope', 'functions.read') // TODO: Update scope to sites.read
->label('sdk.auth', [APP_AUTH_TYPE_ADMIN])
->label('sdk.namespace', 'sites')
->label('sdk.method', 'getSiteUsage')
->label('sdk.response.code', Response::STATUS_CODE_OK)
->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
->label('sdk.response.model', Response::MODEL_USAGE_SITE)
->param('siteId', '', new UID(), 'Site ID.')
->param('range', '30d', new WhiteList(['24h', '30d', '90d']), 'Date range.', true)
->inject('response')
->inject('dbForProject')
->callback([$this, 'action']);
}
public function action(string $siteId, string $range, Response $response, Database $dbForProject)
{
$site = $dbForProject->getDocument('sites', $siteId);
if ($site->isEmpty()) {
throw new Exception(Exception::SITE_NOT_FOUND);
}
$periods = Config::getParam('usage', []);
$stats = $usage = [];
$days = $periods[$range];
$metrics = [
str_replace(['{resourceType}', '{resourceInternalId}'], ['sites', $site->getInternalId()], METRIC_SITE_ID_DEPLOYMENTS),
str_replace(['{resourceType}', '{resourceInternalId}'], ['sites', $site->getInternalId()], METRIC_SITE_ID_DEPLOYMENTS_STORAGE),
str_replace('{siteInternalId}', $site->getInternalId(), METRIC_SITE_ID_BUILDS),
str_replace('{siteInternalId}', $site->getInternalId(), METRIC_SITE_ID_BUILDS_STORAGE),
str_replace('{siteInternalId}', $site->getInternalId(), METRIC_SITE_ID_BUILDS_COMPUTE),
str_replace('{siteInternalId}', $site->getInternalId(), METRIC_SITE_ID_BUILDS_MB_SECONDS)
];
Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats) {
foreach ($metrics as $metric) {
$result = $dbForProject->findOne('stats', [
Query::equal('metric', [$metric]),
Query::equal('period', ['inf'])
]);
$stats[$metric]['total'] = $result['value'] ?? 0;
$limit = $days['limit'];
$period = $days['period'];
$results = $dbForProject->find('stats', [
Query::equal('metric', [$metric]),
Query::equal('period', [$period]),
Query::limit($limit),
Query::orderDesc('time'),
]);
$stats[$metric]['data'] = [];
foreach ($results as $result) {
$stats[$metric]['data'][$result->getAttribute('time')] = [
'value' => $result->getAttribute('value'),
];
}
}
});
$format = match ($days['period']) {
'1h' => 'Y-m-d\TH:00:00.000P',
'1d' => 'Y-m-d\T00:00:00.000P',
};
foreach ($metrics as $metric) {
$usage[$metric]['total'] = $stats[$metric]['total'];
$usage[$metric]['data'] = [];
$leap = time() - ($days['limit'] * $days['factor']);
while ($leap < time()) {
$leap += $days['factor'];
$formatDate = date($format, $leap);
$usage[$metric]['data'][] = [
'value' => $stats[$metric]['data'][$formatDate]['value'] ?? 0,
'date' => $formatDate,
];
}
}
$response->dynamic(new Document([
'range' => $range,
'deploymentsTotal' => $usage[$metrics[0]]['total'],
'deploymentsStorageTotal' => $usage[$metrics[1]]['total'],
'buildsTotal' => $usage[$metrics[2]]['total'],
'buildsStorageTotal' => $usage[$metrics[3]]['total'],
'buildsTimeTotal' => $usage[$metrics[4]]['total'],
'deployments' => $usage[$metrics[0]]['data'],
'deploymentsStorage' => $usage[$metrics[1]]['data'],
'builds' => $usage[$metrics[2]]['data'],
'buildsStorage' => $usage[$metrics[3]]['data'],
'buildsTime' => $usage[$metrics[4]]['data'],
'buildsMbSecondsTotal' => $usage[$metrics[7]]['total'],
'buildsMbSeconds' => $usage[$metrics[7]]['data']
// TODO: Add more metrics for requests, bandwidth, etc.
]), Response::MODEL_USAGE_SITE);
}
}
@@ -0,0 +1,121 @@
<?php
namespace Appwrite\Platform\Modules\Sites\Http\Sites;
use Appwrite\Platform\Modules\Compute\Base;
use Appwrite\Utopia\Response;
use Utopia\Config\Config;
use Utopia\Database\Database;
use Utopia\Database\Document;
use Utopia\Database\Query;
use Utopia\Database\Validator\Authorization;
use Utopia\Platform\Action;
use Utopia\Platform\Scope\HTTP;
use Utopia\Validator\WhiteList;
class GetSitesUsage extends Base
{
use HTTP;
public static function getName()
{
return 'getSitesUsage';
}
public function __construct()
{
$this
->setHttpMethod(Action::HTTP_REQUEST_METHOD_GET)
->setHttpPath('/v1/sites/usage')
->desc('Get sites usage')
->groups(['api', 'sites', 'usage'])
->label('scope', 'functions.read') // TODO: Update scope to sites.read
->label('sdk.auth', [APP_AUTH_TYPE_ADMIN])
->label('sdk.namespace', 'sites')
->label('sdk.method', 'getUsage')
->label('sdk.response.code', Response::STATUS_CODE_OK)
->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
->label('sdk.response.model', Response::MODEL_USAGE_SITES)
->param('range', '30d', new WhiteList(['24h', '30d', '90d']), 'Date range.', true)
->inject('response')
->inject('dbForProject')
->callback([$this, 'action']);
}
public function action(string $range, Response $response, Database $dbForProject)
{
$periods = Config::getParam('usage', []);
$stats = $usage = [];
$days = $periods[$range];
$metrics = [
METRIC_SITES,
METRIC_DEPLOYMENTS,
METRIC_DEPLOYMENTS_STORAGE,
METRIC_BUILDS,
METRIC_BUILDS_STORAGE,
METRIC_BUILDS_COMPUTE,
METRIC_BUILDS_MB_SECONDS,
];
Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats) {
foreach ($metrics as $metric) {
$result = $dbForProject->findOne('stats', [
Query::equal('metric', [$metric]),
Query::equal('period', ['inf'])
]);
$stats[$metric]['total'] = $result['value'] ?? 0;
$limit = $days['limit'];
$period = $days['period'];
$results = $dbForProject->find('stats', [
Query::equal('metric', [$metric]),
Query::equal('period', [$period]),
Query::limit($limit),
Query::orderDesc('time'),
]);
$stats[$metric]['data'] = [];
foreach ($results as $result) {
$stats[$metric]['data'][$result->getAttribute('time')] = [
'value' => $result->getAttribute('value'),
];
}
}
});
$format = match ($days['period']) {
'1h' => 'Y-m-d\TH:00:00.000P',
'1d' => 'Y-m-d\T00:00:00.000P',
};
foreach ($metrics as $metric) {
$usage[$metric]['total'] = $stats[$metric]['total'];
$usage[$metric]['data'] = [];
$leap = time() - ($days['limit'] * $days['factor']);
while ($leap < time()) {
$leap += $days['factor'];
$formatDate = date($format, $leap);
$usage[$metric]['data'][] = [
'value' => $stats[$metric]['data'][$formatDate]['value'] ?? 0,
'date' => $formatDate,
];
}
}
$response->dynamic(new Document([
'range' => $range,
'sitesTotal' => $usage[$metrics[0]]['total'],
'deploymentsTotal' => $usage[$metrics[1]]['total'],
'deploymentsStorageTotal' => $usage[$metrics[2]]['total'],
'buildsTotal' => $usage[$metrics[3]]['total'],
'buildsStorageTotal' => $usage[$metrics[4]]['total'],
'buildsTimeTotal' => $usage[$metrics[5]]['total'],
'sites' => $usage[$metrics[0]]['data'],
'deployments' => $usage[$metrics[1]]['data'],
'deploymentsStorage' => $usage[$metrics[2]]['data'],
'builds' => $usage[$metrics[3]]['data'],
'buildsStorage' => $usage[$metrics[4]]['data'],
'buildsTime' => $usage[$metrics[5]]['data'],
'buildsMbSecondsTotal' => $usage[$metrics[8]]['total'],
'buildsMbSeconds' => $usage[$metrics[8]]['data']
]), Response::MODEL_USAGE_SITES);
}
}
@@ -12,13 +12,13 @@ use Utopia\Validator\ArrayList;
use Utopia\Validator\Range;
use Utopia\Validator\WhiteList;
class ListSiteTemplates extends Base
class ListTemplates extends Base
{
use HTTP;
public static function getName()
{
return 'listSiteTemplates';
return 'listTemplates';
}
public function __construct()
@@ -26,11 +26,11 @@ class ListSiteTemplates extends Base
$this
->setHttpMethod(Action::HTTP_REQUEST_METHOD_GET)
->setHttpPath('/v1/sites/templates')
->desc('List site templates')
->desc('List templates')
->groups(['api'])
->label('scope', 'public')
->label('sdk.namespace', 'sites')
->label('sdk.method', 'listSiteTemplates')
->label('sdk.method', 'listTemplates')
->label('sdk.auth', [APP_AUTH_TYPE_ADMIN])
->label('sdk.description', '/docs/references/sites/list-templates.md')
->label('sdk.response.code', Response::STATUS_CODE_OK)
@@ -48,8 +48,6 @@ class ListSiteTemplates extends Base
{
$templates = Config::getParam('site-templates', []);
var_dump($templates);
if (!empty($frameworks)) {
$templates = \array_filter($templates, function ($template) use ($frameworks) {
return \count(\array_intersect($frameworks, \array_column($template['frameworks'], 'name'))) > 0;
@@ -13,9 +13,11 @@ use Appwrite\Platform\Modules\Sites\Http\Deployments\UpdateDeployment;
use Appwrite\Platform\Modules\Sites\Http\Sites\CreateSite;
use Appwrite\Platform\Modules\Sites\Http\Sites\DeleteSite;
use Appwrite\Platform\Modules\Sites\Http\Sites\GetSite;
use Appwrite\Platform\Modules\Sites\Http\Sites\GetSitesUsage;
use Appwrite\Platform\Modules\Sites\Http\Sites\GetSiteUsage;
use Appwrite\Platform\Modules\Sites\Http\Sites\ListFrameworks;
use Appwrite\Platform\Modules\Sites\Http\Sites\ListSites;
use Appwrite\Platform\Modules\Sites\Http\Sites\ListSiteTemplates;
use Appwrite\Platform\Modules\Sites\Http\Sites\ListTemplates;
use Appwrite\Platform\Modules\Sites\Http\Sites\UpdateSite;
use Appwrite\Platform\Modules\Sites\Http\Variables\CreateVariable;
use Appwrite\Platform\Modules\Sites\Http\Variables\DeleteVariable;
@@ -48,6 +50,8 @@ class Http extends Service
$this->addAction(ListVariables::getName(), new ListVariables());
$this->addAction(UpdateVariable::getName(), new UpdateVariable());
$this->addAction(DeleteVariable::getName(), new DeleteVariable());
$this->addAction(ListSiteTemplates::getName(), new ListSiteTemplates());
$this->addAction(ListTemplates::getName(), new ListTemplates());
$this->addAction(GetSiteUsage::getName(), new GetSiteUsage());
$this->addAction(GetSitesUsage::getName(), new GetSitesUsage());
}
}
+6
View File
@@ -106,6 +106,8 @@ use Appwrite\Utopia\Response\Model\UsageDatabases;
use Appwrite\Utopia\Response\Model\UsageFunction;
use Appwrite\Utopia\Response\Model\UsageFunctions;
use Appwrite\Utopia\Response\Model\UsageProject;
use Appwrite\Utopia\Response\Model\UsageSite;
use Appwrite\Utopia\Response\Model\UsageSites;
use Appwrite\Utopia\Response\Model\UsageStorage;
use Appwrite\Utopia\Response\Model\UsageUsers;
use Appwrite\Utopia\Response\Model\User;
@@ -144,6 +146,8 @@ class Response extends SwooleResponse
public const MODEL_USAGE_STORAGE = 'usageStorage';
public const MODEL_USAGE_FUNCTIONS = 'usageFunctions';
public const MODEL_USAGE_FUNCTION = 'usageFunction';
public const MODEL_USAGE_SITES = 'usageSites';
public const MODEL_USAGE_SITE = 'usageSite';
public const MODEL_USAGE_PROJECT = 'usageProject';
// Database
@@ -483,6 +487,8 @@ class Response extends SwooleResponse
->setModel(new UsageBuckets())
->setModel(new UsageFunctions())
->setModel(new UsageFunction())
->setModel(new UsageSites())
->setModel(new UsageSite())
->setModel(new UsageProject())
->setModel(new Headers())
->setModel(new Specification())
@@ -0,0 +1,119 @@
<?php
namespace Appwrite\Utopia\Response\Model;
use Appwrite\Utopia\Response;
use Appwrite\Utopia\Response\Model;
class UsageSite extends Model
{
public function __construct()
{
$this
->addRule('range', [
'type' => self::TYPE_STRING,
'description' => 'The time range of the usage stats.',
'default' => '',
'example' => '30d',
])
->addRule('deploymentsTotal', [
'type' => self::TYPE_INTEGER,
'description' => 'Total aggregated number of site deployments.',
'default' => 0,
'example' => 0,
])
->addRule('deploymentsStorageTotal', [
'type' => self::TYPE_INTEGER,
'description' => 'Total aggregated sum of site deployments storage.',
'default' => 0,
'example' => 0,
])
->addRule('buildsTotal', [
'type' => self::TYPE_INTEGER,
'description' => 'Total aggregated number of site builds.',
'default' => 0,
'example' => 0,
])
->addRule('buildsStorageTotal', [
'type' => self::TYPE_INTEGER,
'description' => 'total aggregated sum of site builds storage.',
'default' => 0,
'example' => 0,
])
->addRule('buildsTimeTotal', [
'type' => self::TYPE_INTEGER,
'description' => 'Total aggregated sum of site builds compute time.',
'default' => 0,
'example' => 0,
])
->addRule('buildsMbSecondsTotal', [
'type' => self::TYPE_INTEGER,
'description' => 'Total aggregated sum of site builds mbSeconds.',
'default' => 0,
'example' => 0,
])
->addRule('deployments', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated number of site deployments per period.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('deploymentsStorage', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated number of site deployments storage per period.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('builds', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated number of site builds per period.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('buildsStorage', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated sum of site builds storage per period.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('buildsTime', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated sum of site builds compute time per period.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('buildsMbSeconds', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated number of site builds mbSeconds per period.',
'default' => [],
'example' => [],
'array' => true
])
;
}
/**
* Get Name
*
* @return string
*/
public function getName(): string
{
return 'UsageSite';
}
/**
* Get Type
*
* @return string
*/
public function getType(): string
{
return Response::MODEL_USAGE_SITE;
}
}
@@ -0,0 +1,132 @@
<?php
namespace Appwrite\Utopia\Response\Model;
use Appwrite\Utopia\Response;
use Appwrite\Utopia\Response\Model;
class UsageSites extends Model
{
public function __construct()
{
$this
->addRule('range', [
'type' => self::TYPE_STRING,
'description' => 'Time range of the usage stats.',
'default' => '',
'example' => '30d',
])
->addRule('sitesTotal', [
'type' => self::TYPE_INTEGER,
'description' => 'Total aggregated number of sites.',
'default' => 0,
'example' => 0,
])
->addRule('deploymentsTotal', [
'type' => self::TYPE_INTEGER,
'description' => 'Total aggregated number of sites deployments.',
'default' => 0,
'example' => 0,
])
->addRule('deploymentsStorageTotal', [
'type' => self::TYPE_INTEGER,
'description' => 'Total aggregated sum of sites deployment storage.',
'default' => 0,
'example' => 0,
])
->addRule('buildsTotal', [
'type' => self::TYPE_INTEGER,
'description' => 'Total aggregated number of sites build.',
'default' => 0,
'example' => 0,
])
->addRule('buildsStorageTotal', [
'type' => self::TYPE_INTEGER,
'description' => 'total aggregated sum of sites build storage.',
'default' => 0,
'example' => 0,
])
->addRule('buildsTimeTotal', [
'type' => self::TYPE_INTEGER,
'description' => 'Total aggregated sum of sites build compute time.',
'default' => 0,
'example' => 0,
])
->addRule('buildsMbSecondsTotal', [
'type' => self::TYPE_INTEGER,
'description' => 'Total aggregated sum of sites build mbSeconds.',
'default' => 0,
'example' => 0,
])
->addRule('sites', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated number of sites per period.',
'default' => 0,
'example' => 0,
'array' => true
])
->addRule('deployments', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated number of sites deployment per period.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('deploymentsStorage', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated number of sites deployment storage per period.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('builds', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated number of sites build per period.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('buildsStorage', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated sum of sites build storage per period.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('buildsTime', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated sum of sites build compute time per period.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('buildsMbSeconds', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated sum of sites build mbSeconds per period.',
'default' => [],
'example' => [],
'array' => true
])
;
}
/**
* Get Name
*
* @return string
*/
public function getName(): string
{
return 'UsageSites';
}
/**
* Get Type
*
* @return string
*/
public function getType(): string
{
return Response::MODEL_USAGE_SITES;
}
}