From 22ab23d009cfb0af9edc397baaf396734a0b37d2 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Thu, 30 Jan 2025 04:05:51 +0000 Subject: [PATCH] feat: Stats resources worker --- Dockerfile | 3 +- bin/worker-stats-resources | 3 + docker-compose.yml | 31 ++ src/Appwrite/Platform/Services/Tasks.php | 2 +- src/Appwrite/Platform/Services/Workers.php | 3 +- .../Platform/Workers/StatsResources.php | 278 ++++++++++++++++++ 6 files changed, 317 insertions(+), 3 deletions(-) create mode 100644 bin/worker-stats-resources create mode 100644 src/Appwrite/Platform/Workers/StatsResources.php diff --git a/Dockerfile b/Dockerfile index c38e0493fe..1d50684535 100755 --- a/Dockerfile +++ b/Dockerfile @@ -87,7 +87,8 @@ RUN chmod +x /usr/local/bin/doctor && \ chmod +x /usr/local/bin/worker-webhooks && \ chmod +x /usr/local/bin/worker-usage && \ chmod +x /usr/local/bin/worker-usage-dump && \ - chmod +x /usr/local/bin/stats-resources + chmod +x /usr/local/bin/stats-resources && \ + chmod +x /usr/local/bin/worker-stats-resources # Letsencrypt Permissions RUN mkdir -p /etc/letsencrypt/live/ && chmod -Rf 755 /etc/letsencrypt/live/ diff --git a/bin/worker-stats-resources b/bin/worker-stats-resources new file mode 100644 index 0000000000..9c5d2bebff --- /dev/null +++ b/bin/worker-stats-resources @@ -0,0 +1,3 @@ +#!/bin/sh + +php /usr/src/code/app/worker.php stats-resources $@ \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index d294896adc..d6345abddb 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -754,6 +754,37 @@ services: - _APP_DATABASE_SHARED_TABLES - _APP_USAGE_COUNT_INTERVAL + appwrite-worker-stats-resources: + entrypoint: worker-stats-resources + <<: *x-logging + container_name: appwrite-worker-stats-resources + image: appwrite-dev + networks: + - appwrite + volumes: + - ./app:/usr/src/code/app + - ./src:/usr/src/code/src + depends_on: + - redis + - mariadb + environment: + - _APP_ENV + - _APP_WORKER_PER_CORE + - _APP_OPENSSL_KEY_V1 + - _APP_DB_HOST + - _APP_DB_PORT + - _APP_DB_SCHEMA + - _APP_DB_USER + - _APP_DB_PASS + - _APP_REDIS_HOST + - _APP_REDIS_PORT + - _APP_REDIS_USER + - _APP_REDIS_PASS + - _APP_USAGE_STATS + - _APP_LOGGING_CONFIG + - _APP_USAGE_AGGREGATION_INTERVAL + - _APP_DATABASE_SHARED_TABLES + appwrite-worker-usage: entrypoint: worker-usage <<: *x-logging diff --git a/src/Appwrite/Platform/Services/Tasks.php b/src/Appwrite/Platform/Services/Tasks.php index 2622bd18a2..eb6d775da9 100644 --- a/src/Appwrite/Platform/Services/Tasks.php +++ b/src/Appwrite/Platform/Services/Tasks.php @@ -14,8 +14,8 @@ use Appwrite\Platform\Tasks\ScheduleMessages; use Appwrite\Platform\Tasks\SDKs; use Appwrite\Platform\Tasks\Specs; use Appwrite\Platform\Tasks\SSL; -use Appwrite\Platform\Tasks\Upgrade; use Appwrite\Platform\Tasks\StatsResources; +use Appwrite\Platform\Tasks\Upgrade; use Appwrite\Platform\Tasks\Vars; use Appwrite\Platform\Tasks\Version; use Utopia\Platform\Service; diff --git a/src/Appwrite/Platform/Services/Workers.php b/src/Appwrite/Platform/Services/Workers.php index 0e79f4257c..150c8cfd23 100644 --- a/src/Appwrite/Platform/Services/Workers.php +++ b/src/Appwrite/Platform/Services/Workers.php @@ -11,6 +11,7 @@ use Appwrite\Platform\Workers\Functions; use Appwrite\Platform\Workers\Mails; use Appwrite\Platform\Workers\Messaging; use Appwrite\Platform\Workers\Migrations; +use Appwrite\Platform\Workers\StatsResources; use Appwrite\Platform\Workers\Usage; use Appwrite\Platform\Workers\UsageDump; use Appwrite\Platform\Workers\Webhooks; @@ -34,7 +35,7 @@ class Workers extends Service ->addAction(UsageDump::getName(), new UsageDump()) ->addAction(Usage::getName(), new Usage()) ->addAction(Migrations::getName(), new Migrations()) - + ->addAction(StatsResources::getName(), new StatsResources()) ; } } diff --git a/src/Appwrite/Platform/Workers/StatsResources.php b/src/Appwrite/Platform/Workers/StatsResources.php new file mode 100644 index 0000000000..e4ba59c464 --- /dev/null +++ b/src/Appwrite/Platform/Workers/StatsResources.php @@ -0,0 +1,278 @@ + 'Y-m-d H:00', + '1d' => 'Y-m-d 00:00', + 'inf' => '0000-00-00 00:00' + ]; + + public static function getName(): string + { + return 'stats-resources'; + } + + /** + * @throws Exception + */ + public function __construct() + { + $this + ->desc('Stats resources worker') + ->inject('message') + ->inject('project') + ->inject('getProjectDB') + ->inject('getLogsDB') + ->inject('dbForPlatform') + ->inject('logError') + ->callback([$this, 'action']); + } + + /** + * @param Message $message + * @param Document $project + * @param callable $getProjectDB + * @param UsageDump $queueForUsageDump + * @return void + * @throws \Utopia\Database\Exception + * @throws Exception + */ + public function action(Message $message, Document $project, callable $getProjectDB, callable $getLogsDB, Database $dbForPlatform, callable $logError): void + { + $this->logError = $logError; + + $payload = $message->getPayload() ?? []; + if (empty($payload)) { + throw new Exception('Missing payload'); + } + + if (empty($project->getAttribute('database'))) { + var_dump($payload); + return; + } + + $this->countForProject($dbForPlatform, $getLogsDB, $getProjectDB, $project); + } + + + protected function countForProject(Database $dbForPlatform, callable $getLogsDB, callable $getProjectDB, Document $project): void + { + Console::info('Begining count for: ' . $project->getId()); + + try { + /** @var \Utopia\Database\Database $dbForLogs */ + $dbForLogs = call_user_func($getLogsDB, $project); + /** @var \Utopia\Database\Database $dbForProject */ + $dbForProject = call_user_func($getProjectDB, $project); + + $region = $project->getAttribute('region'); + + $platforms = $dbForPlatform->count('platforms', [ + Query::equal('projectInternalId', [$project->getInternalId()]) + ]); + $webhooks = $dbForPlatform->count('webhooks', [ + Query::equal('projectInternalId', [$project->getInternalId()]) + ]); + $keys = $dbForPlatform->count('keys', [ + Query::equal('projectInternalId', [$project->getInternalId()]) + ]); + $databases = $dbForProject->count('databases'); + $buckets = $dbForProject->count('buckets'); + $users = $dbForProject->count('users'); + + $last30Days = (new \DateTime())->sub(\DateInterval::createFromDateString('30 days'))->format('Y-m-d 00:00:00'); + $usersActive = $dbForProject->count('users', [ + Query::greaterThanEqual('accessedAt', $last30Days) + ]); + $teams = $dbForProject->count('teams'); + $functions = $dbForProject->count('functions'); + $messages = $dbForProject->count('messages'); + $providers = $dbForProject->count('providers'); + $topics = $dbForProject->count('topics'); + + $metrics = [ + METRIC_DATABASES => $databases, + METRIC_BUCKETS => $buckets, + METRIC_USERS => $users, + METRIC_FUNCTIONS => $functions, + METRIC_TEAMS => $teams, + METRIC_MESSAGES => $messages, + METRIC_USERS_ACTIVE => $usersActive, + METRIC_WEBHOOKS => $webhooks, + METRIC_PLATFORMS => $platforms, + METRIC_PROVIDERS => $providers, + METRIC_TOPICS => $topics, + METRIC_KEYS => $keys, + ]; + + foreach ($metrics as $metric => $value) { + $this->createOrUpdateMetric($dbForLogs, $region, $metric, $value); + } + + try { + $this->countForBuckets($dbForProject, $dbForLogs, $region); + } catch (Throwable $th) { + call_user_func_array($this->logError, [$th, "StatsResources", "count_for_buckets_{$project->getId()}"]); + } + + try { + $this->countForDatabase($dbForProject, $dbForLogs, $region); + } catch (Throwable $th) { + call_user_func_array($this->logError, [$th, "StatsResources", "count_for_database_{$project->getId()}"]); + } + + try { + $this->countForFunctions($dbForProject, $dbForLogs, $region); + } catch (Throwable $th) { + call_user_func_array($this->logError, [$th, "StatsResources", "count_for_functions_{$project->getId()}"]); + } + } catch (Throwable $th) { + call_user_func_array($this->logError, [$th, "StatsResources", "count_for_project_{$project->getId()}"]); + } + + Console::info('End of count for: ' . $project->getId()); + } + + protected function countForBuckets(Database $dbForProject, Database $dbForLogs, string $region) + { + $totalFiles = 0; + $totalStorage = 0; + $this->foreachDocument($dbForProject, 'buckets', [], function ($bucket) use ($dbForProject, $dbForLogs, $region, &$totalFiles, &$totalStorage) { + $files = $dbForProject->count('bucket_' . $bucket->getInternalId()); + + $metric = str_replace('{bucketInternalId}', $bucket->getInternalId(), METRIC_BUCKET_ID_FILES); + $this->createOrUpdateMetric($dbForLogs, $region, $metric, $files); + + $storage = $dbForProject->sum('bucket_' . $bucket->getInternalId(), 'sizeActual'); + $metric = str_replace('{bucketInternalId}', $bucket->getInternalId(), METRIC_BUCKET_ID_FILES_STORAGE); + $this->createOrUpdateMetric($dbForLogs, $region, $metric, $storage); + + $totalStorage += $storage; + $totalFiles += $files; + }); + + $this->createOrUpdateMetric($dbForLogs, $region, METRIC_FILES, $totalFiles); + $this->createOrUpdateMetric($dbForLogs, $region, METRIC_FILES_STORAGE, $totalStorage); + } + + protected function countForDatabase(Database $dbForProject, Database $dbForLogs, string $region) + { + $totalCollections = 0; + $totalDocuments = 0; + + $this->foreachDocument($dbForProject, 'databases', [], function ($database) use ($dbForProject, $dbForLogs, $region, &$totalCollections, &$totalDocuments) { + $collections = $dbForProject->count('database_' . $database->getInternalId()); + + $metric = str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_COLLECTIONS); + $this->createOrUpdateMetric($dbForLogs, $region, $metric, $collections); + + $documents = $this->countForCollections($dbForProject, $dbForLogs, $database, $region); + + $totalDocuments += $documents; + $totalCollections += $collections; + }); + + $this->createOrUpdateMetric($dbForLogs, $region, METRIC_COLLECTIONS, $totalCollections); + $this->createOrUpdateMetric($dbForLogs, $region, METRIC_DOCUMENTS, $totalDocuments); + } + protected function countForCollections(Database $dbForProject, Database $dbForLogs, Document $database, string $region): int + { + $databaseDocuments = 0; + $this->foreachDocument($dbForProject, 'database_' . $database->getInternalId(), [], function ($collection) use ($dbForProject, $dbForLogs, $database, $region, &$totalCollections, &$databaseDocuments) { + $documents = $dbForProject->count('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId()); + + $metric = str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$database->getInternalId(), $collection->getInternalId()], METRIC_DATABASE_ID_COLLECTION_ID_DOCUMENTS); + $this->createOrUpdateMetric($dbForLogs, $region, $metric, $documents); + + $databaseDocuments += $documents; + }); + + $metric = str_replace(['{databaseInternalId}'], [$database->getInternalId()], METRIC_DATABASE_ID_DOCUMENTS); + $this->createOrUpdateMetric($dbForLogs, $region, $metric, $databaseDocuments); + + return $databaseDocuments; + } + + protected function countForFunctions(Database $dbForProject, Database $dbForLogs, string $region) + { + $deploymentsStorage = $dbForProject->sum('deployments', 'size'); + $buildsStorage = $dbForProject->sum('builds', 'size'); + $this->createOrUpdateMetric($dbForLogs, $region, METRIC_DEPLOYMENTS_STORAGE, $deploymentsStorage); + $this->createOrUpdateMetric($dbForLogs, $region, METRIC_BUILDS_STORAGE, $buildsStorage); + + $deployments = $dbForProject->count('deployments'); + $builds = $dbForProject->count('builds'); + $this->createOrUpdateMetric($dbForLogs, $region, METRIC_DEPLOYMENTS, $deployments); + $this->createOrUpdateMetric($dbForLogs, $region, METRIC_BUILDS, $builds); + + + $this->foreachDocument($dbForProject, 'functions', [], function (Document $function) use ($dbForProject, $dbForLogs, $region) { + $functionDeploymentsStorage = $dbForProject->sum('deployments', 'size', [ + Query::equal('resourceInternalId', [$function->getInternalId()]), + Query::equal('resourceType', [RESOURCE_TYPE_FUNCTIONS]), + ]); + $this->createOrUpdateMetric($dbForLogs, $region, str_replace(['{resourceType}','{resourceInternalId}'], [RESOURCE_TYPE_FUNCTIONS,$function->getInternalId()], METRIC_RESOURCE_TYPE_ID_DEPLOYMENTS_STORAGE), $functionDeploymentsStorage); + + $functionDeployments = $dbForProject->count('deployments', [ + Query::equal('resourceInternalId', [$function->getInternalId()]), + Query::equal('resourceType', [RESOURCE_TYPE_FUNCTIONS]), + ]); + $this->createOrUpdateMetric($dbForLogs, $region, str_replace(['{resourceType}','{resourceInternalId}'], [RESOURCE_TYPE_FUNCTIONS,$function->getInternalId()], METRIC_RESOURCE_TYPE_ID_DEPLOYMENTS), $functionDeployments); + + /** + * As deployments and builds have 1-1 relationship, + * the count for one should match the other + */ + $this->createOrUpdateMetric($dbForLogs, $region, str_replace(['{resourceType}','{resourceInternalId}'], [RESOURCE_TYPE_FUNCTIONS,$function->getInternalId()], METRIC_RESOURCE_TYPE_ID_BUILDS), $functionDeployments); + + $functionBuildsStorage = 0; + + $this->foreachDocument($dbForProject, 'deployments', [ + Query::equal('resourceInternalId', [$function->getInternalId()]), + Query::equal('resourceType', [RESOURCE_TYPE_FUNCTIONS]), + ], function (Document $deployment) use ($dbForProject, &$functionBuildsStorage): void { + $build = $dbForProject->getDocument('builds', $deployment->getAttribute('buildId', '')); + $functionBuildsStorage += $build->getAttribute('size', 0); + }); + + $this->createOrUpdateMetric($dbForLogs, $region, str_replace(['{resourceType}','{resourceInternalId}'], [RESOURCE_TYPE_FUNCTIONS,$function->getInternalId()], METRIC_RESOURCE_TYPE_ID_BUILDS_STORAGE), $functionBuildsStorage); + }); + } + + protected function createOrUpdateMetric(Database $dbForLogs, string $region, string $metric, int $value) + { + foreach ($this->periods as $period => $format) { + $time = 'inf' === $period ? null : \date($format, \time()); + $id = \md5("{$time}_{$period}_{$metric}"); + $current = $dbForLogs->getDocument('usage', $id); + if ($current->isEmpty()) { + $dbForLogs->createDocument('usage', new Document([ + '$id' => $id, + 'metric' => $metric, + 'period' => $period, + 'region' => $region, + 'value' => $value, + ])); + Console::success('Usage logs created for metric: ' . $metric . ' period:'. $period); + } else { + $dbForLogs->updateDocument('usage', $id, $current->setAttribute('value', $value)); + Console::success('Usage logs updated for metric: ' . $metric . ' period:'. $period); + } + } + } +}