diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e0db158628..11966d14f3 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -319,12 +319,16 @@ These are the current metrics we collect usage stats for: | users | Total number of users per project| | executions | Total number of executions per project | | databases | Total number of databases per project | +| databases.storage | Total amount of storage used by all databases per project (in bytes) | +| databases.storage_disk | Total amount of storage used by all database per project on disk (in bytes) | | collections | Total number of collections per project | | {databaseInternalId}.collections | Total number of collections per database| | {databaseInternalId}.storage | Sum of database storage (in bytes) | +| {databaseInternalId}.storage_disk | Sum of database storage on disk (in bytes) | | documents | Total number of documents per project | | {databaseInternalId}.{collectionInternalId}.documents | Total number of documents per collection | | {databaseInternalId}.{collectionInternalId}.storage | Sum of database storage used by the collection (in bytes) | +| {databsaeInternalId}.{collectionInternalId}.storage_disk | Sum of database storage used by the collection on disk (in bytes) | | buckets | Total number of buckets per project | | files | Total number of files per project | | {bucketInternalId}.files.storage | Sum of files.storage per bucket (in bytes) | diff --git a/app/config/specs/swagger2-1.6.x-console.json b/app/config/specs/swagger2-1.6.x-console.json index 51935a5e01..5d0f39186b 100644 --- a/app/config/specs/swagger2-1.6.x-console.json +++ b/app/config/specs/swagger2-1.6.x-console.json @@ -37270,6 +37270,12 @@ "x-example": 0, "format": "int32" }, + "storageTotal": { + "type": "integer", + "description": "Total aggregated number of total databases storage in bytes.", + "x-example": 0, + "format": "int32" + }, "databases": { "type": "array", "description": "Aggregated number of databases per period.", @@ -37296,6 +37302,15 @@ "$ref": "#\/definitions\/metric" }, "x-example": [] + }, + "storage": { + "type": "array", + "description": "Aggregated number of databases storage in bytes per period.", + "items": { + "type": "object", + "$ref": "#\/definitions\/metric" + }, + "x-example": [] } }, "required": [ @@ -37303,9 +37318,11 @@ "databasesTotal", "collectionsTotal", "documentsTotal", + "storageTotal", "databases", "collections", - "documents" + "documents", + "storage" ] }, "usageDatabase": { @@ -37329,6 +37346,12 @@ "x-example": 0, "format": "int32" }, + "storageTotal": { + "type": "integer", + "description": "Total aggregated number of total storage used in bytes.", + "x-example": 0, + "format": "int32" + }, "collections": { "type": "array", "description": "Aggregated number of collections per period.", @@ -37346,14 +37369,25 @@ "$ref": "#\/definitions\/metric" }, "x-example": [] + }, + "storage": { + "type": "array", + "description": "Aggregated storage used in bytes per period.", + "items": { + "type": "object", + "$ref": "#\/definitions\/metric" + }, + "x-example": [] } }, "required": [ "range", "collectionsTotal", "documentsTotal", + "storageTotal", "collections", - "documents" + "documents", + "storage" ] }, "usageCollection": { @@ -37921,6 +37955,12 @@ "x-example": 0, "format": "int32" }, + "databasesStorageTotal": { + "type": "integer", + "description": "Total aggregated sum of databases storage size (in bytes).", + "x-example": 0, + "format": "int32" + }, "usersTotal": { "type": "integer", "description": "Total aggregated number of users.", @@ -38023,6 +38063,15 @@ }, "x-example": [] }, + "databasesStorageBreakdown": { + "type": "array", + "description": "Aggregated breakdown in totals of usage by databases.", + "items": { + "type": "object", + "$ref": "#\/definitions\/metricBreakdown" + }, + "x-example": [] + }, "executionsMbSecondsBreakdown": { "type": "array", "description": "Aggregated breakdown in totals of execution mbSeconds by functions.", @@ -38055,6 +38104,7 @@ "executionsTotal", "documentsTotal", "databasesTotal", + "databasesStorageTotal", "usersTotal", "filesStorageTotal", "functionsStorageTotal", @@ -38069,6 +38119,7 @@ "executions", "executionsBreakdown", "bucketsBreakdown", + "databasesStorageBreakdown", "executionsMbSecondsBreakdown", "buildsMbSecondsBreakdown", "functionsStorageBreakdown" diff --git a/src/Appwrite/Platform/Workers/UsageDump.php b/src/Appwrite/Platform/Workers/UsageDump.php index 1b35b0ffd1..7bd848d586 100644 --- a/src/Appwrite/Platform/Workers/UsageDump.php +++ b/src/Appwrite/Platform/Workers/UsageDump.php @@ -12,6 +12,10 @@ use Utopia\Platform\Action; use Utopia\Queue\Message; use Utopia\System\System; +const METRIC_COLLECTION_LEVEL_STORAGE = 4; +const METRIC_DATABASE_LEVEL_STORAGE = 3; +const METRIC_PROJECT_LEVEL_STORAGE = 2; + class UsageDump extends Action { protected array $stats = []; @@ -71,8 +75,8 @@ class UsageDump extends Action continue; } - if (str_contains($key, 'databases.storage') && $value === 1) { - $this->handleDBStorageCalculation($key, $dbForProject); + if (str_contains($key, METRIC_DATABASES_STORAGE)) { + $this->handleDatabaseStorage($key, $dbForProject); return; } @@ -114,7 +118,7 @@ class UsageDump extends Action } } - private function handleDBStorageCalculation(string $key, Database $dbForProject): void + private function handleDatabaseStorage(string $key, Database $dbForProject): void { $data = explode('.', $key); $start = microtime(true); @@ -165,7 +169,7 @@ class UsageDump extends Action switch (count($data)) { // Collection Level - case 3: + case METRIC_COLLECTION_LEVEL_STORAGE: $databaseInternalId = $data[0]; $collectionInternalId = $data[1]; @@ -185,17 +189,17 @@ class UsageDump extends Action $updateMetric($dbForProject, $diskDiff, $key . '_disk', $period, $time); // Update Database - $databaseKey = $data[0] . '.databases.storage'; + $databaseKey = str_replace(['{databaseInternalId}'], [$data[0]], METRIC_DATABASE_ID_STORAGE); $updateMetric($dbForProject, $diff, $databaseKey, $period, $time); $updateMetric($dbForProject, $diskDiff, $databaseKey . '_disk', $period, $time); // Update Project - $projectKey = 'databases.storage'; + $projectKey = METRIC_DATABASES_STORAGE; $updateMetric($dbForProject, $diff, $projectKey, $period, $time); $updateMetric($dbForProject, $diskDiff, $projectKey . '_disk', $period, $time); break; // Database Level - case 2: + case METRIC_DATABASE_LEVEL_STORAGE: $databaseInternalId = $data[0]; $collections = $dbForProject->find('database_' . $databaseInternalId); @@ -212,17 +216,17 @@ class UsageDump extends Action } // Update Database - $databaseKey = $data[0] . '.databases.storage'; + $databaseKey = str_replace(['{databaseInternalId}'], [$data[0]], METRIC_DATABASE_ID_STORAGE); $updateMetric($dbForProject, $diff, $databaseKey, $period, $time); $updateMetric($dbForProject, $diskDiff, $databaseKey . '_disk', $period, $time); // Update Project - $projectKey = 'databases.storage'; + $projectKey = METRIC_DATABASES_STORAGE; $updateMetric($dbForProject, $diff, $projectKey, $period, $time); $updateMetric($dbForProject, $diskDiff, $projectKey . '_disk', $period, $time); break; // Project Level - case 1: + case METRIC_PROJECT_LEVEL_STORAGE: // Get all project databases $databases = $dbForProject->find('database'); @@ -240,7 +244,7 @@ class UsageDump extends Action $diskDiff = $diskValue - $previousValue; // Update Project - $projectKey = 'databases.storage'; + $projectKey = METRIC_DATABASES_STORAGE; $updateMetric($dbForProject, $diff, $projectKey, $period, $time); $updateMetric($dbForProject, $diskDiff, $projectKey . '_disk', $period, $time); break;