Add DB Storage Metrics into usage APIs

CONTRIBUTING.md

@@ -321,8 +321,10 @@ These are the current metrics we collect usage stats for:
 | databases | Total number of databases per project             | 
 | collections | Total number of collections per project | 
 | {databaseInternalId}.collections | Total number of collections per database| 
+| {databaseInternalId}.storage | Sum of database storage (in bytes) |
 | documents | Total number of documents per project             | 
 | {databaseInternalId}.{collectionInternalId}.documents | Total number of documents per collection | 
+| {databaseInternalId}.{collectionInternalId}.storage | Sum of database storage used by the collection (in bytes) |
 | buckets | Total number of buckets per project               | 
 | files | Total number of files per project                 |
 | {bucketInternalId}.files.storage | Sum of files.storage per bucket (in bytes)                  |

app/controllers/api/databases.php

@@ -3665,6 +3665,7 @@ App::get('/v1/databases/usage')
             METRIC_DATABASES,
             METRIC_COLLECTIONS,
             METRIC_DOCUMENTS,
+            METRIC_DATABASES_STORAGE,
         ];
 
         Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats) {
@@ -3715,9 +3716,11 @@ App::get('/v1/databases/usage')
             'databasesTotal'   => $usage[$metrics[0]]['total'],
             'collectionsTotal' => $usage[$metrics[1]]['total'],
             'documentsTotal'   => $usage[$metrics[2]]['total'],
+            'databasesStorageTotal'   => $usage[$metrics[3]]['total'],
             'databases'   => $usage[$metrics[0]]['data'],
             'collections' => $usage[$metrics[1]]['data'],
             'documents'   => $usage[$metrics[2]]['data'],
+            'databasesStorage'   => $usage[$metrics[3]]['data'],
         ]), Response::MODEL_USAGE_DATABASES);
     });
 
@@ -3749,6 +3752,7 @@ App::get('/v1/databases/:databaseId/usage')
         $metrics = [
             str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_COLLECTIONS),
             str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_DOCUMENTS),
+            str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_STORAGE)
         ];
 
         Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats) {
@@ -3799,8 +3803,10 @@ App::get('/v1/databases/:databaseId/usage')
             'range' => $range,
             'collectionsTotal'   => $usage[$metrics[0]]['total'],
             'documentsTotal'   => $usage[$metrics[1]]['total'],
+            'storageTotal'   => $usage[$metrics[2]]['total'],
             'collections'   => $usage[$metrics[0]]['data'],
             'documents'   => $usage[$metrics[1]]['data'],
+            'storage'   => $usage[$metrics[2]]['data'],
         ]), Response::MODEL_USAGE_DATABASE);
     });
 

app/controllers/api/project.php

@@ -42,6 +42,7 @@ App::get('/v1/project/usage')
                 METRIC_EXECUTIONS,
                 METRIC_DOCUMENTS,
                 METRIC_DATABASES,
+                METRIC_DATABASES_STORAGE,
                 METRIC_USERS,
                 METRIC_BUCKETS,
                 METRIC_FILES_STORAGE
@@ -144,6 +145,22 @@ App::get('/v1/project/usage')
             ];
         }, $dbForProject->find('buckets'));
 
+        $databasesStorageBreakdown = array_map(function ($database) use ($dbForProject) {
+            $id = $database->getId();
+            $name = $database->getAttribute('name');
+            $metric = str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_STORAGE);
+            $value = $dbForProject->findOne('stats', [
+                Query::equal('metric', [$metric]),
+                Query::equal('period', ['inf'])
+            ]);
+
+            return [
+                'resourceId' => $id,
+                'name' => $name,
+                'value' => $value['value'] ?? 0,
+            ];
+        }, $dbForProject->find('databases'));
+
         // merge network inbound + outbound
         $projectBandwidth = [];
         foreach ($usage[METRIC_NETWORK_INBOUND] as $item) {
@@ -173,11 +190,13 @@ App::get('/v1/project/usage')
             'executionsTotal' => $total[METRIC_EXECUTIONS],
             'documentsTotal' => $total[METRIC_DOCUMENTS],
             'databasesTotal' => $total[METRIC_DATABASES],
+            'databasesStorageTotal' => $total[METRIC_DATABASES_STORAGE],
             'usersTotal' => $total[METRIC_USERS],
             'bucketsTotal' => $total[METRIC_BUCKETS],
             'filesStorageTotal' => $total[METRIC_FILES_STORAGE],
             'executionsBreakdown' => $executionsBreakdown,
-            'bucketsBreakdown' => $bucketsBreakdown
+            'bucketsBreakdown' => $bucketsBreakdown,
+            'databasesStorageBreakdown' => $databasesStorageBreakdown,
         ]), Response::MODEL_USAGE_PROJECT);
     });
 

app/controllers/shared/api.php

@@ -55,7 +55,7 @@ $parseLabel = function (string $label, array $responsePayload, array $requestPar
     return $label;
 };
 
-$databaseListener = function (string $event, Document $document, Document $project, Usage $queueForUsage, Database $dbForProject) {
+$databaseListener = function (string $event, Document $document, Document $project, Usage $queueForUsage, EventDatabase $queueForDatabase, Database $dbForProject) {
 
     $value = 1;
     if ($event === Database::EVENT_DOCUMENT_DELETE) {
@@ -109,6 +109,16 @@ $databaseListener = function (string $event, Document $document, Document $proje
                 ->addMetric(METRIC_DOCUMENTS, $value)  // per project
                 ->addMetric(str_replace('{databaseInternalId}', $databaseInternalId, METRIC_DATABASE_ID_DOCUMENTS), $value) // per database
                 ->addMetric(str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$databaseInternalId, $collectionInternalId], METRIC_DATABASE_ID_COLLECTION_ID_DOCUMENTS), $value);  // per collection
+
+            $queueForDatabase
+            ->setType(DATABASE_TYPE_CALCULATE_STORAGE_USAGE)
+            ->setPayload([
+                'collectionInternalId' => $collectionInternalId,
+                'databaseInternalId' => $databaseInternalId,
+            ]);
+
+            $queueForDatabase->trigger();
+            
             break;
         case $document->getCollection() === 'buckets': //buckets
             $queueForUsage
@@ -406,8 +416,8 @@ App::init()
         $queueForMessaging->setProject($project);
 
         $dbForProject
-            ->on(Database::EVENT_DOCUMENT_CREATE, 'calculate-usage', fn ($event, $document) => $databaseListener($event, $document, $project, $queueForUsage, $dbForProject))
-            ->on(Database::EVENT_DOCUMENT_DELETE, 'calculate-usage', fn ($event, $document) => $databaseListener($event, $document, $project, $queueForUsage, $dbForProject));
+            ->on(Database::EVENT_DOCUMENT_CREATE, 'calculate-usage', fn ($event, $document) => $databaseListener($event, $document, $project, $queueForUsage, $queueForDatabase, $dbForProject))
+            ->on(Database::EVENT_DOCUMENT_DELETE, 'calculate-usage', fn ($event, $document) => $databaseListener($event, $document, $project, $queueForUsage, $queueForDatabase, $dbForProject));
 
         $useCache = $route->getLabel('cache', false);
         if ($useCache) {

app/init.php

@@ -154,6 +154,7 @@ const DATABASE_TYPE_DELETE_ATTRIBUTE = 'deleteAttribute';
 const DATABASE_TYPE_DELETE_INDEX = 'deleteIndex';
 const DATABASE_TYPE_DELETE_COLLECTION = 'deleteCollection';
 const DATABASE_TYPE_DELETE_DATABASE = 'deleteDatabase';
+const DATABASE_TYPE_CALCULATE_STORAGE_USAGE = 'calculateStorageUsage';
 
 // Build Worker Types
 const BUILD_TYPE_DEPLOYMENT = 'deployment';
@@ -216,10 +217,13 @@ const METRIC_MESSAGES_COUNTRY_CODE  = '{countryCode}.messages';
 const METRIC_SESSIONS  = 'sessions';
 const METRIC_DATABASES = 'databases';
 const METRIC_COLLECTIONS = 'collections';
+const METRIC_DATABASES_STORAGE = 'databases.storage';
 const METRIC_DATABASE_ID_COLLECTIONS = '{databaseInternalId}.collections';
+const METRIC_DATABASE_ID_STORAGE = '{databaseInternalId}.storage';
 const METRIC_DOCUMENTS = 'documents';
 const METRIC_DATABASE_ID_DOCUMENTS = '{databaseInternalId}.documents';
 const METRIC_DATABASE_ID_COLLECTION_ID_DOCUMENTS = '{databaseInternalId}.{collectionInternalId}.documents';
+const METRIC_DATABASE_ID_COLLECTION_ID_STORAGE = '{databaseInternalId}.{collectionInternalId}.storage';
 const METRIC_BUCKETS = 'buckets';
 const METRIC_FILES  = 'files';
 const METRIC_FILES_STORAGE  = 'files.storage';

src/Appwrite/Event/Database.php

@@ -119,6 +119,15 @@ class Database extends Event
 
         $client = new Client($this->queue, $this->connection);
 
+        if ($this->type === DATABASE_TYPE_CALCULATE_STORAGE_USAGE) {
+            $result = $client->enqueue(array_merge([
+                'project' => $this->project,
+                'type' => $this->type,
+                'user' => $this->user
+            ], $this->payload));
+            return $result;
+        }
+
         try {
             $result = $client->enqueue([
                 'project' => $this->project,

src/Appwrite/Platform/Workers/Databases.php

@@ -3,6 +3,7 @@
 namespace Appwrite\Platform\Workers;
 
 use Appwrite\Event\Event;
+use Appwrite\Event\Usage;
 use Appwrite\Messaging\Adapter\Realtime;
 use Exception;
 use Utopia\Audit\Audit;
@@ -36,19 +37,21 @@ class Databases extends Action
             ->inject('message')
             ->inject('dbForConsole')
             ->inject('dbForProject')
+            ->inject('queueForUsage')
             ->inject('log')
-            ->callback(fn (Message $message, Database $dbForConsole, Database $dbForProject, Log $log) => $this->action($message, $dbForConsole, $dbForProject, $log));
+            ->callback(fn (Message $message, Database $dbForConsole, Database $dbForProject, Usage $queueForUsage, Log $log) => $this->action($message, $dbForConsole, $dbForProject, $queueForUsage, $log));
     }
 
     /**
      * @param Message $message
      * @param Database $dbForConsole
      * @param Database $dbForProject
+     * @param Usage $queueForUsage
      * @param Log $log
      * @return void
      * @throws \Exception
      */
-    public function action(Message $message, Database $dbForConsole, Database $dbForProject, Log $log): void
+    public function action(Message $message, Database $dbForConsole, Database $dbForProject, Usage $queueForUsage, Log $log): void
     {
         $payload = $message->getPayload() ?? [];
 
@@ -65,11 +68,13 @@ class Databases extends Action
         $log->addTag('projectId', $project->getId());
         $log->addTag('type', $type);
 
-        if ($database->isEmpty()) {
+        if ($database->isEmpty() && $type !== DATABASE_TYPE_CALCULATE_STORAGE_USAGE) {
             throw new Exception('Missing database');
         }
 
-        $log->addTag('databaseId', $database->getId());
+        if (!$database->isEmpty()) {
+            $log->addTag('databaseId', $database->getId());
+        }
 
         match (\strval($type)) {
             DATABASE_TYPE_DELETE_DATABASE => $this->deleteDatabase($database, $project, $dbForProject),
@@ -78,6 +83,8 @@ class Databases extends Action
             DATABASE_TYPE_DELETE_ATTRIBUTE => $this->deleteAttribute($database, $collection, $document, $project, $dbForConsole, $dbForProject),
             DATABASE_TYPE_CREATE_INDEX => $this->createIndex($database, $collection, $document, $project, $dbForConsole, $dbForProject),
             DATABASE_TYPE_DELETE_INDEX => $this->deleteIndex($database, $collection, $document, $project, $dbForConsole, $dbForProject),
+            DATABASE_TYPE_CALCULATE_STORAGE_USAGE => $this->calculateStorageUsage($payload, $project, $dbForProject, $queueForUsage),
+            
             default => throw new \Exception('No database operation for type: ' . \strval($type)),
         };
     }
@@ -612,6 +619,56 @@ class Databases extends Action
         Console::info("Deleted {$count} document by group in " . ($executionEnd - $executionStart) . " seconds");
     }
 
+   /**
+     * @param Document $database
+     * @param Document $collection
+     * @param Database $dbForConsole
+     * @param Usage $queueForUsage
+     * @return void
+     * @throws Exception
+     * @throws Authorization
+     * @throws DatabaseException
+     */
+    private function calculateStorageUsage(array $payload, Document $project, Database $dbForProject, Usage $queueForUsage): void
+    {
+        if (!isset($payload['databaseInternalId'])) {
+            throw new Exception('Missing Database');
+        }
+
+        if (!isset($payload['collectionInternalId'])) {
+            throw new Exception('Missing Collection');
+        }
+
+        $databaseInternalId = $payload['databaseInternalId'];
+        $collectionInternalId = $payload['collectionInternalId'];
+
+        // Calculate storage usage for collection
+        $collectionStorageUsage = $dbForProject->getSizeOfCollection('database_'. $databaseInternalId . '_collection_' . $collectionInternalId);
+
+        //TODO: Optimize using reduce
+        // Calculate storage usage for database
+        $databsaeStorageUsage = 0;
+
+        $collections = $dbForProject->find('database_' . $databaseInternalId);
+        foreach ($collections as $collection) {
+            $databsaeStorageUsage += $dbForProject->getSizeOfCollection('database_' . $databaseInternalId . '_collection_' . $collection->getInternalId());
+        }
+
+        $queueForUsage
+            ->addMetric(str_replace('{databaseInternalId}', $databaseInternalId, METRIC_DATABASE_ID_STORAGE), $databsaeStorageUsage)
+            ->addMetric(str_replace([
+                '{databaseInternalId}',
+                '{collectionInternalId}'
+            ], [
+                $databaseInternalId,
+                $collectionInternalId
+            ], METRIC_DATABASE_ID_COLLECTION_ID_STORAGE), $collectionStorageUsage);
+
+        Console::info('Calculated storage usage for database: ' . $databaseInternalId . ' and collection: ' . $collectionInternalId);
+        $queueForUsage->setProject($project);
+        $queueForUsage->trigger();
+    }
+
     protected function trigger(
         Document $database,
         Document $collection,

src/Appwrite/Utopia/Response/Model/UsageDatabase.php

@@ -28,6 +28,12 @@ class UsageDatabase extends Model
                 'default' => 0,
                 'example' => 0,
             ])
+            ->addRule('storageTotal', [
+                'type' => self::TYPE_INTEGER,
+                'description' => 'Total aggregated sum of databases storage size (in bytes)',
+                'default' => 0,
+                'example' => 0,
+            ])
             ->addRule('collections', [
                 'type' => Response::MODEL_METRIC,
                 'description' => 'Aggregated  number of collections per period.',
@@ -42,6 +48,13 @@ class UsageDatabase extends Model
                 'example' => [],
                 'array' => true
             ])
+            ->addRule('storage', [
+                'type' => Response::MODEL_METRIC,
+                'description' => 'Aggregated  storage size (in bytes) per period.',
+                'default' => [],
+                'example' => [],
+                'array' => true
+            ]);
         ;
     }
 

src/Appwrite/Utopia/Response/Model/UsageDatabases.php

@@ -34,6 +34,12 @@ class UsageDatabases extends Model
                 'default' => 0,
                 'example' => 0,
             ])
+            ->addRule('databasesStorageTotal', [
+                'type' => self::TYPE_INTEGER,
+                'description' => 'Total aggregated sum of databases storage size (in bytes)',
+                'default' => 0,
+                'example' => 0,
+            ])
             ->addRule('databases', [
                 'type' => Response::MODEL_METRIC,
                 'description' => 'Aggregated number of databases per period.',
@@ -55,6 +61,13 @@ class UsageDatabases extends Model
                 'example' => [],
                 'array' => true
             ])
+            ->addRule('databasesStorage', [
+                'type' => Response::MODEL_METRIC,
+                'description' => 'Aggregated sum of databases storage size (in bytes) per period.',
+                'default' => [],
+                'example' => [],
+                'array' => true
+            ])
         ;
     }
 

src/Appwrite/Utopia/Response/Model/UsageProject.php

@@ -28,6 +28,12 @@ class UsageProject extends Model
                 'default' => 0,
                 'example' => 0,
             ])
+            ->addRule('databasesStorageTotal', [
+                'type' => self::TYPE_INTEGER,
+                'description' => 'Total aggregated sum of databases storage size (in bytes)',
+                'default' => 0,
+                'example' => 0,
+            ])
             ->addRule('usersTotal', [
                 'type' => self::TYPE_INTEGER,
                 'description' => 'Total aggregated number of users.',
@@ -88,6 +94,13 @@ class UsageProject extends Model
                 'example' => [],
                 'array' => true
             ])
+            ->addRule('databasesStorageBreakdown', [
+                'type' => Response::MODEL_METRIC_BREAKDOWN,
+                'description' => 'Aggregated breakdown in totals of usage by databases storage.',
+                'default' => [],
+                'example' => [],
+                'array' => true
+            ])
         ;
     }