Appwrite/appwrite
1
0
Fork 0
mirror of https://github.com/appwrite/appwrite.git synced 2026-05-26 13:51:13 +00:00
Code Issues Packages Projects Releases Wiki Activity

reset usage

Browse Source
This commit is contained in:
Damodar Lohani
2025-01-30 04:33:42 +00:00
parent 3fde465dfa
commit 469dbf0a31
2 changed files with 26 additions and 293 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/Appwrite/Event/Usage.php
+8 -31
View File
@@ -3,18 +3,14 @@
namespace Appwrite\Event;
use Utopia\Database\Document;
use Utopia\Queue\Client;
use Utopia\Queue\Connection;
class Usage extends Event
{
public const TYPE_USAGE_DUMP = 'usage-dump';
public const TYPE_USAGE_COUNT = 'stats-resources';
protected array $metrics = [];
protected array $reduce = [];
protected string $type = self::TYPE_USAGE_DUMP;
public function __construct(protected Connection $connection)
{
parent::__construct($connection);
@@ -24,12 +20,6 @@ class Usage extends Event
->setClass(Event::USAGE_CLASS_NAME);
}
public function setType(string $type): self
{
$this->type = $type;
return $this;
}
/**
* Add reduce.
*
@@ -52,7 +42,6 @@ class Usage extends Event
*/
public function addMetric(string $key, int $value): self
{
$this->metrics[] = [
'key' => $key,
'value' => $value,
@@ -61,21 +50,6 @@ class Usage extends Event
return $this;
}
/**
* Prepare the payload for the usage event.
*
* @return array
*/
protected function preparePayload(): array
{
return [
'project' => $this->project,
'reduce' => $this->reduce,
'metrics' => $this->metrics,
'type' => $this->type,
];
}
/**
* Sends metrics to the usage worker.
*
@@ -83,8 +57,11 @@ class Usage extends Event
*/
public function trigger(): string|bool
{
parent::trigger();
$this->metrics = [];
return true;
$client = new Client($this->queue, $this->connection);
return $client->enqueue([
'project' => $this->getProject(),
'reduce' => $this->reduce,
'metrics' => $this->metrics,
]);
}
}
}
src/Appwrite/Platform/Workers/Usage.php
+18 -262
View File
@@ -2,41 +2,22 @@
namespace Appwrite\Platform\Workers;
use Appwrite\Event\Usage as UsageEvent;
use Appwrite\Event\UsageDump;
use Appwrite\Platform\Action;
use Exception;
use Throwable;
use Utopia\CLI\Console;
use Utopia\Database\Database;
use Utopia\Database\DateTime;
use Utopia\Database\Document;
use Utopia\Database\Query;
use Utopia\Platform\Action;
use Utopia\Queue\Message;
use Utopia\System\System;
class Usage extends Action
{
/**
* Date format for different periods
*/
protected array $periods = [
'1h' => 'Y-m-d H:00',
'1d' => 'Y-m-d 00:00',
'inf' => '0000-00-00 00:00'
];
/**
* Log Error Callback
*
* @var callable
*/
protected mixed $logError;
private array $stats = [];
private int $lastTriggeredTime = 0;
private int $aggregationInterval = 20;
private int $keys = 0;
private const INFINITY_PERIOD = '_inf_';
private const KEYS_THRESHOLD = 20000;
private const KEYS_THRESHOLD = 10000;
public static function getName(): string
{
@@ -48,50 +29,37 @@ class Usage extends Action
*/
public function __construct()
{
$this
->desc('Usage worker')
->inject('message')
->inject('project')
->inject('getProjectDB')
->inject('queueForUsageDump')
->inject('getLogsDB')
->inject('dbForPlatform')
->inject('logError')
->callback([$this, 'action']);
$this->aggregationInterval = (int) System::getEnv('_APP_USAGE_AGGREGATION_INTERVAL', '20');
$this
->desc('Usage worker')
->inject('message')
->inject('getProjectDB')
->inject('queueForUsageDump')
->callback(function (Message $message, callable $getProjectDB, UsageDump $queueForUsageDump) {
$this->action($message, $getProjectDB, $queueForUsageDump);
});
$this->lastTriggeredTime = time();
}
/**
* @param Message $message
* @param Document $project
* @param callable $getProjectDB
* @param UsageDump $queueForUsageDump
* @return void
* @throws \Utopia\Database\Exception
* @throws Exception
*/
public function action(Message $message, Document $project, callable $getProjectDB, UsageDump $queueForUsageDump, callable $getLogsDB, Database $dbForPlatform, callable $logError): void
public function action(Message $message, callable $getProjectDB, UsageDump $queueForUsageDump): void
{
$this->logError = $logError;
$payload = $message->getPayload() ?? [];
if (empty($payload)) {
throw new Exception('Missing payload');
}
//Todo Figure out way to preserve keys when the container is being recreated @shimonewman
if (empty($project->getAttribute('database'))) {
var_dump($payload);
return;
}
$type = $payload['type'] ?? UsageEvent::TYPE_USAGE_DUMP;
if ($type === UsageEvent::TYPE_USAGE_COUNT) {
$this->countForProject($dbForPlatform, $getLogsDB, $getProjectDB, $project);
return;
}
$aggregationInterval = (int) System::getEnv('_APP_USAGE_AGGREGATION_INTERVAL', '20');
$project = new Document($payload['project'] ?? []);
$projectId = $project->getInternalId();
foreach ($payload['reduce'] ?? [] as $document) {
if (empty($document)) {
@@ -106,12 +74,7 @@ class Usage extends Action
);
}
$this->stats[$projectId]['project'] = [
'$id' => $project->getId(),
'$internalId' => $project->getInternalId(),
'database' => $project->getAttribute('database'),
];
$this->stats[$projectId]['project'] = $project;
$this->stats[$projectId]['receivedAt'] = DateTime::now();
foreach ($payload['metrics'] ?? [] as $metric) {
$this->keys++;
@@ -126,7 +89,7 @@ class Usage extends Action
// If keys crossed threshold or X time passed since the last send and there are some keys in the array ($this->stats)
if (
$this->keys >= self::KEYS_THRESHOLD ||
(time() - $this->lastTriggeredTime > $this->aggregationInterval && $this->keys > 0)
(time() - $this->lastTriggeredTime > $aggregationInterval && $this->keys > 0)
) {
Console::warning('[' . DateTime::now() . '] Aggregated ' . $this->keys . ' keys');
@@ -314,211 +277,4 @@ class Usage extends Action
console::error("[reducer] " . " {DateTime::now()} " . " {$project->getInternalId()} " . " {$e->getMessage()}");
}
}
protected function countForProject(Database $dbForPlatform, callable $getLogsDB, callable $getProjectDB, Document $project): void
{
Console::info('Begining count for: ' . $project->getId());
try {
/** @var \Utopia\Database\Database $dbForLogs */
$dbForLogs = call_user_func($getLogsDB, $project);
/** @var \Utopia\Database\Database $dbForProject */
$dbForProject = call_user_func($getProjectDB, $project);
$region = $project->getAttribute('region');
$platforms = $dbForPlatform->count('platforms', [
Query::equal('projectInternalId', [$project->getInternalId()])
]);
$webhooks = $dbForPlatform->count('webhooks', [
Query::equal('projectInternalId', [$project->getInternalId()])
]);
$keys = $dbForPlatform->count('keys', [
Query::equal('projectInternalId', [$project->getInternalId()])
]);
$databases = $dbForProject->count('databases');
$buckets = $dbForProject->count('buckets');
$users = $dbForProject->count('users');
$last30Days = (new \DateTime())->sub(\DateInterval::createFromDateString('30 days'))->format('Y-m-d 00:00:00');
$usersActive = $dbForProject->count('users', [
Query::greaterThanEqual('accessedAt', $last30Days)
]);
$teams = $dbForProject->count('teams');
$functions = $dbForProject->count('functions');
$messages = $dbForProject->count('messages');
$providers = $dbForProject->count('providers');
$topics = $dbForProject->count('topics');
$metrics = [
METRIC_DATABASES => $databases,
METRIC_BUCKETS => $buckets,
METRIC_USERS => $users,
METRIC_FUNCTIONS => $functions,
METRIC_TEAMS => $teams,
METRIC_MESSAGES => $messages,
METRIC_USERS_ACTIVE => $usersActive,
METRIC_WEBHOOKS => $webhooks,
METRIC_PLATFORMS => $platforms,
METRIC_PROVIDERS => $providers,
METRIC_TOPICS => $topics,
METRIC_KEYS => $keys,
];
foreach ($metrics as $metric => $value) {
$this->createOrUpdateMetric($dbForLogs, $region, $metric, $value);
}
try {
$this->countForBuckets($dbForProject, $dbForLogs, $region);
} catch (Throwable $th) {
call_user_func_array($this->logError, [$th, "StatsResources", "count_for_buckets_{$project->getId()}"]);
}
try {
$this->countForDatabase($dbForProject, $dbForLogs, $region);
} catch (Throwable $th) {
call_user_func_array($this->logError, [$th, "StatsResources", "count_for_database_{$project->getId()}"]);
}
try {
$this->countForFunctions($dbForProject, $dbForLogs, $region);
} catch (Throwable $th) {
call_user_func_array($this->logError, [$th, "StatsResources", "count_for_functions_{$project->getId()}"]);
}
} catch (Throwable $th) {
call_user_func_array($this->logError, [$th, "StatsResources", "count_for_project_{$project->getId()}"]);
}
Console::info('End of count for: ' . $project->getId());
}
protected function countForBuckets(Database $dbForProject, Database $dbForLogs, string $region)
{
$totalFiles = 0;
$totalStorage = 0;
$this->foreachDocument($dbForProject, 'buckets', [], function ($bucket) use ($dbForProject, $dbForLogs, $region, &$totalFiles, &$totalStorage) {
$files = $dbForProject->count('bucket_' . $bucket->getInternalId());
$metric = str_replace('{bucketInternalId}', $bucket->getInternalId(), METRIC_BUCKET_ID_FILES);
$this->createOrUpdateMetric($dbForLogs, $region, $metric, $files);
$storage = $dbForProject->sum('bucket_' . $bucket->getInternalId(), 'sizeActual');
$metric = str_replace('{bucketInternalId}', $bucket->getInternalId(), METRIC_BUCKET_ID_FILES_STORAGE);
$this->createOrUpdateMetric($dbForLogs, $region, $metric, $storage);
$totalStorage += $storage;
$totalFiles += $files;
});
$this->createOrUpdateMetric($dbForLogs, $region, METRIC_FILES, $totalFiles);
$this->createOrUpdateMetric($dbForLogs, $region, METRIC_FILES_STORAGE, $totalStorage);
}
protected function countForDatabase(Database $dbForProject, Database $dbForLogs, string $region)
{
$totalCollections = 0;
$totalDocuments = 0;
$this->foreachDocument($dbForProject, 'databases', [], function ($database) use ($dbForProject, $dbForLogs, $region, &$totalCollections, &$totalDocuments) {
$collections = $dbForProject->count('database_' . $database->getInternalId());
$metric = str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_COLLECTIONS);
$this->createOrUpdateMetric($dbForLogs, $region, $metric, $collections);
$documents = $this->countForCollections($dbForProject, $dbForLogs, $database, $region);
$totalDocuments += $documents;
$totalCollections += $collections;
});
$this->createOrUpdateMetric($dbForLogs, $region, METRIC_COLLECTIONS, $totalCollections);
$this->createOrUpdateMetric($dbForLogs, $region, METRIC_DOCUMENTS, $totalDocuments);
}
protected function countForCollections(Database $dbForProject, Database $dbForLogs, Document $database, string $region): int
{
$databaseDocuments = 0;
$this->foreachDocument($dbForProject, 'database_' . $database->getInternalId(), [], function ($collection) use ($dbForProject, $dbForLogs, $database, $region, &$totalCollections, &$databaseDocuments) {
$documents = $dbForProject->count('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId());
$metric = str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$database->getInternalId(), $collection->getInternalId()], METRIC_DATABASE_ID_COLLECTION_ID_DOCUMENTS);
$this->createOrUpdateMetric($dbForLogs, $region, $metric, $documents);
$databaseDocuments += $documents;
});
$metric = str_replace(['{databaseInternalId}'], [$database->getInternalId()], METRIC_DATABASE_ID_DOCUMENTS);
$this->createOrUpdateMetric($dbForLogs, $region, $metric, $databaseDocuments);
return $databaseDocuments;
}
protected function countForFunctions(Database $dbForProject, Database $dbForLogs, string $region)
{
$deploymentsStorage = $dbForProject->sum('deployments', 'size');
$buildsStorage = $dbForProject->sum('builds', 'size');
$this->createOrUpdateMetric($dbForLogs, $region, METRIC_DEPLOYMENTS_STORAGE, $deploymentsStorage);
$this->createOrUpdateMetric($dbForLogs, $region, METRIC_BUILDS_STORAGE, $buildsStorage);
$deployments = $dbForProject->count('deployments');
$builds = $dbForProject->count('builds');
$this->createOrUpdateMetric($dbForLogs, $region, METRIC_DEPLOYMENTS, $deployments);
$this->createOrUpdateMetric($dbForLogs, $region, METRIC_BUILDS, $builds);
$this->foreachDocument($dbForProject, 'functions', [], function (Document $function) use ($dbForProject, $dbForLogs, $region) {
$functionDeploymentsStorage = $dbForProject->sum('deployments', 'size', [
Query::equal('resourceInternalId', [$function->getInternalId()]),
Query::equal('resourceType', [RESOURCE_TYPE_FUNCTIONS]),
]);
$this->createOrUpdateMetric($dbForLogs, $region, str_replace(['{resourceType}','{resourceInternalId}'], [RESOURCE_TYPE_FUNCTIONS,$function->getInternalId()], METRIC_RESOURCE_TYPE_ID_DEPLOYMENTS_STORAGE), $functionDeploymentsStorage);
$functionDeployments = $dbForProject->count('deployments', [
Query::equal('resourceInternalId', [$function->getInternalId()]),
Query::equal('resourceType', [RESOURCE_TYPE_FUNCTIONS]),
]);
$this->createOrUpdateMetric($dbForLogs, $region, str_replace(['{resourceType}','{resourceInternalId}'], [RESOURCE_TYPE_FUNCTIONS,$function->getInternalId()], METRIC_RESOURCE_TYPE_ID_DEPLOYMENTS), $functionDeployments);
/**
* As deployments and builds have 1-1 relationship,
* the count for one should match the other
*/
$this->createOrUpdateMetric($dbForLogs, $region, str_replace(['{resourceType}','{resourceInternalId}'], [RESOURCE_TYPE_FUNCTIONS,$function->getInternalId()], METRIC_RESOURCE_TYPE_ID_BUILDS), $functionDeployments);
$functionBuildsStorage = 0;
$this->foreachDocument($dbForProject, 'deployments', [
Query::equal('resourceInternalId', [$function->getInternalId()]),
Query::equal('resourceType', [RESOURCE_TYPE_FUNCTIONS]),
], function (Document $deployment) use ($dbForProject, &$functionBuildsStorage): void {
$build = $dbForProject->getDocument('builds', $deployment->getAttribute('buildId', ''));
$functionBuildsStorage += $build->getAttribute('size', 0);
});
$this->createOrUpdateMetric($dbForLogs, $region, str_replace(['{resourceType}','{resourceInternalId}'], [RESOURCE_TYPE_FUNCTIONS,$function->getInternalId()], METRIC_RESOURCE_TYPE_ID_BUILDS_STORAGE), $functionBuildsStorage);
});
}
protected function createOrUpdateMetric(Database $dbForLogs, string $region, string $metric, int $value)
{
foreach ($this->periods as $period => $format) {
$time = 'inf' === $period ? null : \date($format, \time());
$id = \md5("{$time}_{$period}_{$metric}");
$current = $dbForLogs->getDocument('usage', $id);
if ($current->isEmpty()) {
$dbForLogs->createDocument('usage', new Document([
'$id' => $id,
'metric' => $metric,
'period' => $period,
'region' => $region,
'value' => $value,
]));
Console::success('Usage logs created for metric: ' . $metric . ' period:'. $period);
} else {
$dbForLogs->updateDocument('usage', $id, $current->setAttribute('value', $value));
Console::success('Usage logs updated for metric: ' . $metric . ' period:'. $period);
}
}
}
}
}