diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index d3a7d060b0..11966d14f3 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -319,10 +319,16 @@ These are the current metrics we collect usage stats for: | users | Total number of users per project| | executions | Total number of executions per project | | databases | Total number of databases per project | +| databases.storage | Total amount of storage used by all databases per project (in bytes) | +| databases.storage_disk | Total amount of storage used by all database per project on disk (in bytes) | | collections | Total number of collections per project | | {databaseInternalId}.collections | Total number of collections per database| +| {databaseInternalId}.storage | Sum of database storage (in bytes) | +| {databaseInternalId}.storage_disk | Sum of database storage on disk (in bytes) | | documents | Total number of documents per project | | {databaseInternalId}.{collectionInternalId}.documents | Total number of documents per collection | +| {databaseInternalId}.{collectionInternalId}.storage | Sum of database storage used by the collection (in bytes) | +| {databsaeInternalId}.{collectionInternalId}.storage_disk | Sum of database storage used by the collection on disk (in bytes) | | buckets | Total number of buckets per project | | files | Total number of files per project | | {bucketInternalId}.files.storage | Sum of files.storage per bucket (in bytes) | diff --git a/app/config/specs/open-api3-1.6.x-console.json b/app/config/specs/open-api3-1.6.x-console.json index 86a6ee7e53..a3727dcf72 100644 --- a/app/config/specs/open-api3-1.6.x-console.json +++ b/app/config/specs/open-api3-1.6.x-console.json @@ -36879,6 +36879,12 @@ "x-example": 0, "format": "int32" }, + "storageTotal": { + "type": "integer", + "description": "Total aggregated number of total databases storage in bytes.", + "x-example": 0, + "format": "int32" + }, "databases": { "type": "array", "description": "Aggregated number of databases per period.", @@ -36902,6 +36908,14 @@ "$ref": "#\/components\/schemas\/metric" }, "x-example": [] + }, + "storage": { + "type": "array", + "description": "An array of the aggregated number of databases storage in bytes per period.", + "items": { + "$ref": "#\/components\/schemas\/metric" + }, + "x-example": [] } }, "required": [ @@ -36909,9 +36923,11 @@ "databasesTotal", "collectionsTotal", "documentsTotal", + "storageTotal", "databases", "collections", - "documents" + "documents", + "storage" ] }, "usageDatabase": { @@ -36935,6 +36951,12 @@ "x-example": 0, "format": "int32" }, + "storageTotal": { + "type": "integer", + "description": "Total aggregated number of total storage used in bytes.", + "x-example": 0, + "format": "int32" + }, "collections": { "type": "array", "description": "Aggregated number of collections per period.", @@ -36950,14 +36972,24 @@ "$ref": "#\/components\/schemas\/metric" }, "x-example": [] + }, + "storage": { + "type": "array", + "description": "Aggregated storage used in bytes per period.", + "items": { + "$ref": "#\/components\/schemas\/metric" + }, + "x-example": [] } }, "required": [ "range", "collectionsTotal", "documentsTotal", + "storageTotal", "collections", - "documents" + "documents", + "storage" ] }, "usageCollection": { @@ -37498,6 +37530,12 @@ "x-example": 0, "format": "int32" }, + "databasesStorageTotal": { + "type": "integer", + "description": "Total aggregated sum of databases storage size (in bytes).", + "x-example": 0, + "format": "int32" + }, "usersTotal": { "type": "integer", "description": "Total aggregated number of users.", @@ -37594,6 +37632,14 @@ }, "x-example": [] }, + "databasesStorageBreakdown": { + "type": "array", + "description": "An array of the aggregated breakdown of storage usage by databases.", + "items": { + "$ref": "#\/components\/schemas\/metricBreakdown" + }, + "x-example": [] + }, "executionsMbSecondsBreakdown": { "type": "array", "description": "Aggregated breakdown in totals of execution mbSeconds by functions.", @@ -37623,6 +37669,7 @@ "executionsTotal", "documentsTotal", "databasesTotal", + "databasesStorageTotal", "usersTotal", "filesStorageTotal", "functionsStorageTotal", @@ -37637,6 +37684,7 @@ "executions", "executionsBreakdown", "bucketsBreakdown", + "databasesStorageBreakdown", "executionsMbSecondsBreakdown", "buildsMbSecondsBreakdown", "functionsStorageBreakdown" diff --git a/app/config/specs/open-api3-latest-console.json b/app/config/specs/open-api3-latest-console.json index 86a6ee7e53..a3727dcf72 100644 --- a/app/config/specs/open-api3-latest-console.json +++ b/app/config/specs/open-api3-latest-console.json @@ -36879,6 +36879,12 @@ "x-example": 0, "format": "int32" }, + "storageTotal": { + "type": "integer", + "description": "Total aggregated number of total databases storage in bytes.", + "x-example": 0, + "format": "int32" + }, "databases": { "type": "array", "description": "Aggregated number of databases per period.", @@ -36902,6 +36908,14 @@ "$ref": "#\/components\/schemas\/metric" }, "x-example": [] + }, + "storage": { + "type": "array", + "description": "An array of the aggregated number of databases storage in bytes per period.", + "items": { + "$ref": "#\/components\/schemas\/metric" + }, + "x-example": [] } }, "required": [ @@ -36909,9 +36923,11 @@ "databasesTotal", "collectionsTotal", "documentsTotal", + "storageTotal", "databases", "collections", - "documents" + "documents", + "storage" ] }, "usageDatabase": { @@ -36935,6 +36951,12 @@ "x-example": 0, "format": "int32" }, + "storageTotal": { + "type": "integer", + "description": "Total aggregated number of total storage used in bytes.", + "x-example": 0, + "format": "int32" + }, "collections": { "type": "array", "description": "Aggregated number of collections per period.", @@ -36950,14 +36972,24 @@ "$ref": "#\/components\/schemas\/metric" }, "x-example": [] + }, + "storage": { + "type": "array", + "description": "Aggregated storage used in bytes per period.", + "items": { + "$ref": "#\/components\/schemas\/metric" + }, + "x-example": [] } }, "required": [ "range", "collectionsTotal", "documentsTotal", + "storageTotal", "collections", - "documents" + "documents", + "storage" ] }, "usageCollection": { @@ -37498,6 +37530,12 @@ "x-example": 0, "format": "int32" }, + "databasesStorageTotal": { + "type": "integer", + "description": "Total aggregated sum of databases storage size (in bytes).", + "x-example": 0, + "format": "int32" + }, "usersTotal": { "type": "integer", "description": "Total aggregated number of users.", @@ -37594,6 +37632,14 @@ }, "x-example": [] }, + "databasesStorageBreakdown": { + "type": "array", + "description": "An array of the aggregated breakdown of storage usage by databases.", + "items": { + "$ref": "#\/components\/schemas\/metricBreakdown" + }, + "x-example": [] + }, "executionsMbSecondsBreakdown": { "type": "array", "description": "Aggregated breakdown in totals of execution mbSeconds by functions.", @@ -37623,6 +37669,7 @@ "executionsTotal", "documentsTotal", "databasesTotal", + "databasesStorageTotal", "usersTotal", "filesStorageTotal", "functionsStorageTotal", @@ -37637,6 +37684,7 @@ "executions", "executionsBreakdown", "bucketsBreakdown", + "databasesStorageBreakdown", "executionsMbSecondsBreakdown", "buildsMbSecondsBreakdown", "functionsStorageBreakdown" diff --git a/app/config/specs/swagger2-1.6.x-console.json b/app/config/specs/swagger2-1.6.x-console.json index 49d60e980b..731f57fdd6 100644 --- a/app/config/specs/swagger2-1.6.x-console.json +++ b/app/config/specs/swagger2-1.6.x-console.json @@ -37402,6 +37402,12 @@ "x-example": 0, "format": "int32" }, + "storageTotal": { + "type": "integer", + "description": "Total aggregated number of total databases storage in bytes.", + "x-example": 0, + "format": "int32" + }, "databases": { "type": "array", "description": "Aggregated number of databases per period.", @@ -37428,6 +37434,15 @@ "$ref": "#\/definitions\/metric" }, "x-example": [] + }, + "storage": { + "type": "array", + "description": "An array of the aggregated number of databases storage in bytes per period.", + "items": { + "type": "object", + "$ref": "#\/definitions\/metric" + }, + "x-example": [] } }, "required": [ @@ -37435,9 +37450,11 @@ "databasesTotal", "collectionsTotal", "documentsTotal", + "storageTotal", "databases", "collections", - "documents" + "documents", + "storage" ] }, "usageDatabase": { @@ -37461,6 +37478,12 @@ "x-example": 0, "format": "int32" }, + "storageTotal": { + "type": "integer", + "description": "Total aggregated number of total storage used in bytes.", + "x-example": 0, + "format": "int32" + }, "collections": { "type": "array", "description": "Aggregated number of collections per period.", @@ -37478,14 +37501,25 @@ "$ref": "#\/definitions\/metric" }, "x-example": [] + }, + "storage": { + "type": "array", + "description": "Aggregated storage used in bytes per period.", + "items": { + "type": "object", + "$ref": "#\/definitions\/metric" + }, + "x-example": [] } }, "required": [ "range", "collectionsTotal", "documentsTotal", + "storageTotal", "collections", - "documents" + "documents", + "storage" ] }, "usageCollection": { @@ -38053,6 +38087,12 @@ "x-example": 0, "format": "int32" }, + "databasesStorageTotal": { + "type": "integer", + "description": "Total aggregated sum of databases storage size (in bytes).", + "x-example": 0, + "format": "int32" + }, "usersTotal": { "type": "integer", "description": "Total aggregated number of users.", @@ -38155,6 +38195,15 @@ }, "x-example": [] }, + "databasesStorageBreakdown": { + "type": "array", + "description": "An array of the aggregated breakdown of storage usage by databases.", + "items": { + "type": "object", + "$ref": "#\/definitions\/metricBreakdown" + }, + "x-example": [] + }, "executionsMbSecondsBreakdown": { "type": "array", "description": "Aggregated breakdown in totals of execution mbSeconds by functions.", @@ -38187,6 +38236,7 @@ "executionsTotal", "documentsTotal", "databasesTotal", + "databasesStorageTotal", "usersTotal", "filesStorageTotal", "functionsStorageTotal", @@ -38201,6 +38251,7 @@ "executions", "executionsBreakdown", "bucketsBreakdown", + "databasesStorageBreakdown", "executionsMbSecondsBreakdown", "buildsMbSecondsBreakdown", "functionsStorageBreakdown" diff --git a/app/config/specs/swagger2-latest-console.json b/app/config/specs/swagger2-latest-console.json index 49d60e980b..731f57fdd6 100644 --- a/app/config/specs/swagger2-latest-console.json +++ b/app/config/specs/swagger2-latest-console.json @@ -37402,6 +37402,12 @@ "x-example": 0, "format": "int32" }, + "storageTotal": { + "type": "integer", + "description": "Total aggregated number of total databases storage in bytes.", + "x-example": 0, + "format": "int32" + }, "databases": { "type": "array", "description": "Aggregated number of databases per period.", @@ -37428,6 +37434,15 @@ "$ref": "#\/definitions\/metric" }, "x-example": [] + }, + "storage": { + "type": "array", + "description": "An array of the aggregated number of databases storage in bytes per period.", + "items": { + "type": "object", + "$ref": "#\/definitions\/metric" + }, + "x-example": [] } }, "required": [ @@ -37435,9 +37450,11 @@ "databasesTotal", "collectionsTotal", "documentsTotal", + "storageTotal", "databases", "collections", - "documents" + "documents", + "storage" ] }, "usageDatabase": { @@ -37461,6 +37478,12 @@ "x-example": 0, "format": "int32" }, + "storageTotal": { + "type": "integer", + "description": "Total aggregated number of total storage used in bytes.", + "x-example": 0, + "format": "int32" + }, "collections": { "type": "array", "description": "Aggregated number of collections per period.", @@ -37478,14 +37501,25 @@ "$ref": "#\/definitions\/metric" }, "x-example": [] + }, + "storage": { + "type": "array", + "description": "Aggregated storage used in bytes per period.", + "items": { + "type": "object", + "$ref": "#\/definitions\/metric" + }, + "x-example": [] } }, "required": [ "range", "collectionsTotal", "documentsTotal", + "storageTotal", "collections", - "documents" + "documents", + "storage" ] }, "usageCollection": { @@ -38053,6 +38087,12 @@ "x-example": 0, "format": "int32" }, + "databasesStorageTotal": { + "type": "integer", + "description": "Total aggregated sum of databases storage size (in bytes).", + "x-example": 0, + "format": "int32" + }, "usersTotal": { "type": "integer", "description": "Total aggregated number of users.", @@ -38155,6 +38195,15 @@ }, "x-example": [] }, + "databasesStorageBreakdown": { + "type": "array", + "description": "An array of the aggregated breakdown of storage usage by databases.", + "items": { + "type": "object", + "$ref": "#\/definitions\/metricBreakdown" + }, + "x-example": [] + }, "executionsMbSecondsBreakdown": { "type": "array", "description": "Aggregated breakdown in totals of execution mbSeconds by functions.", @@ -38187,6 +38236,7 @@ "executionsTotal", "documentsTotal", "databasesTotal", + "databasesStorageTotal", "usersTotal", "filesStorageTotal", "functionsStorageTotal", @@ -38201,6 +38251,7 @@ "executions", "executionsBreakdown", "bucketsBreakdown", + "databasesStorageBreakdown", "executionsMbSecondsBreakdown", "buildsMbSecondsBreakdown", "functionsStorageBreakdown" diff --git a/app/controllers/api/databases.php b/app/controllers/api/databases.php index 2b468bab8f..9c110647af 100644 --- a/app/controllers/api/databases.php +++ b/app/controllers/api/databases.php @@ -5,6 +5,7 @@ use Appwrite\Detector\Detector; use Appwrite\Event\Database as EventDatabase; use Appwrite\Event\Delete; use Appwrite\Event\Event; +use Appwrite\Event\Usage; use Appwrite\Extend\Exception; use Appwrite\Network\Validator\Email; use Appwrite\Utopia\Database\Validator\CustomId; @@ -452,7 +453,8 @@ App::post('/v1/databases') ->inject('response') ->inject('dbForProject') ->inject('queueForEvents') - ->action(function (string $databaseId, string $name, bool $enabled, Response $response, Database $dbForProject, Event $queueForEvents) { + ->inject('queueForUsage') + ->action(function (string $databaseId, string $name, bool $enabled, Response $response, Database $dbForProject, Event $queueForEvents, Usage $queueForUsage) { $databaseId = $databaseId == 'unique()' ? ID::unique() : $databaseId; @@ -502,6 +504,7 @@ App::post('/v1/databases') } $queueForEvents->setParam('databaseId', $database->getId()); + $queueForUsage->addMetric(str_replace(['{databaseInternalId}'], [$database->getInternalId()], METRIC_DATABASE_ID_STORAGE), 1); // per database $response ->setStatusCode(Response::STATUS_CODE_CREATED) @@ -733,7 +736,8 @@ App::delete('/v1/databases/:databaseId') ->inject('dbForProject') ->inject('queueForDatabase') ->inject('queueForEvents') - ->action(function (string $databaseId, Response $response, Database $dbForProject, EventDatabase $queueForDatabase, Event $queueForEvents) { + ->inject('queueForUsage') + ->action(function (string $databaseId, Response $response, Database $dbForProject, EventDatabase $queueForDatabase, Event $queueForEvents, Usage $queueForUsage) { $database = $dbForProject->getDocument('databases', $databaseId); @@ -756,6 +760,9 @@ App::delete('/v1/databases/:databaseId') ->setParam('databaseId', $database->getId()) ->setPayload($response->output($database, Response::MODEL_DATABASE)); + $queueForUsage + ->addMetric(METRIC_DATABASES_STORAGE, 1); // Global, deletion forces full recalculation + $response->noContent(); }); @@ -2350,7 +2357,8 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/attributes/:key ->inject('dbForProject') ->inject('queueForDatabase') ->inject('queueForEvents') - ->action(function (string $databaseId, string $collectionId, string $key, Response $response, Database $dbForProject, EventDatabase $queueForDatabase, Event $queueForEvents) { + ->inject('queueForUsage') + ->action(function (string $databaseId, string $collectionId, string $key, Response $response, Database $dbForProject, EventDatabase $queueForDatabase, Event $queueForEvents, Usage $queueForUsage) { $db = Authorization::skip(fn () => $dbForProject->getDocument('databases', $databaseId)); @@ -2435,6 +2443,9 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/attributes/:key ->setContext('database', $db) ->setPayload($response->output($attribute, $model)); + $queueForUsage + ->addMetric(str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$db->getInternalId(), $collection->getInternalId()], METRIC_DATABASE_ID_COLLECTION_ID_STORAGE), 1); // per collection + $response->noContent(); }); @@ -2810,8 +2821,10 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents') ->inject('dbForProject') ->inject('user') ->inject('queueForEvents') + ->inject('queueForUsage') ->inject('mode') - ->action(function (string $databaseId, string $documentId, string $collectionId, string|array $data, ?array $permissions, Response $response, Database $dbForProject, Document $user, Event $queueForEvents, string $mode) { + ->action(function (string $databaseId, string $documentId, string $collectionId, string|array $data, ?array $permissions, Response $response, Database $dbForProject, Document $user, Event $queueForEvents, Usage $queueForUsage, string $mode) { + $data = (\is_string($data)) ? \json_decode($data, true) : $data; // Cast to JSON array if (empty($data)) { @@ -3026,6 +3039,9 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents') ->setContext('database', $database) ->setPayload($response->getPayload(), sensitive: $relationships); + + $queueForUsage + ->addMetric(str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$database->getInternalId(), $collection->getInternalId()], METRIC_DATABASE_ID_COLLECTION_ID_STORAGE), 1); // per collection }); App::get('/v1/databases/:databaseId/collections/:collectionId/documents') @@ -3642,8 +3658,9 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/documents/:docu ->inject('dbForProject') ->inject('queueForDeletes') ->inject('queueForEvents') + ->inject('queueForUsage') ->inject('mode') - ->action(function (string $databaseId, string $collectionId, string $documentId, ?\DateTime $requestTimestamp, Response $response, Database $dbForProject, Delete $queueForDeletes, Event $queueForEvents, string $mode) { + ->action(function (string $databaseId, string $collectionId, string $documentId, ?\DateTime $requestTimestamp, Response $response, Database $dbForProject, Delete $queueForDeletes, Event $queueForEvents, Usage $queueForUsage, string $mode) { $database = Authorization::skip(fn () => $dbForProject->getDocument('databases', $databaseId)); $isAPIKey = Auth::isAppUser(Authorization::getRoles()); @@ -3728,6 +3745,9 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/documents/:docu ->setContext('database', $database) ->setPayload($response->output($document, Response::MODEL_DOCUMENT), sensitive: $relationships); + $queueForUsage + ->addMetric(str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$database->getInternalId(), $collection->getInternalId()], METRIC_DATABASE_ID_COLLECTION_ID_STORAGE), 1); // per collection + $response->noContent(); }); @@ -3753,6 +3773,7 @@ App::get('/v1/databases/usage') METRIC_DATABASES, METRIC_COLLECTIONS, METRIC_DOCUMENTS, + METRIC_DATABASES_STORAGE ]; Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats) { @@ -3803,9 +3824,11 @@ App::get('/v1/databases/usage') 'databasesTotal' => $usage[$metrics[0]]['total'], 'collectionsTotal' => $usage[$metrics[1]]['total'], 'documentsTotal' => $usage[$metrics[2]]['total'], + 'storageTotal' => $usage[$metrics[3]]['total'], 'databases' => $usage[$metrics[0]]['data'], 'collections' => $usage[$metrics[1]]['data'], 'documents' => $usage[$metrics[2]]['data'], + 'storage' => $usage[$metrics[3]]['data'], ]), Response::MODEL_USAGE_DATABASES); }); @@ -3837,6 +3860,7 @@ App::get('/v1/databases/:databaseId/usage') $metrics = [ str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_COLLECTIONS), str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_DOCUMENTS), + str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_STORAGE) ]; Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats) { @@ -3887,8 +3911,10 @@ App::get('/v1/databases/:databaseId/usage') 'range' => $range, 'collectionsTotal' => $usage[$metrics[0]]['total'], 'documentsTotal' => $usage[$metrics[1]]['total'], + 'storageTotal' => $usage[$metrics[2]]['total'], 'collections' => $usage[$metrics[0]]['data'], 'documents' => $usage[$metrics[1]]['data'], + 'storage' => $usage[$metrics[2]]['data'], ]), Response::MODEL_USAGE_DATABASE); }); diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index 6e429fd8cf..9c3e6782b4 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -1718,7 +1718,7 @@ App::post('/v1/functions/:functionId/executions') ->label('sdk.request.type', Response::CONTENT_TYPE_JSON) ->param('functionId', '', new UID(), 'Function ID.') ->param('body', '', new Payload(10485760, 0), 'HTTP body of execution. Default value is empty string.', true) - ->param('async', false, new Boolean(), 'Execute code in the background. Default value is false.', true) + ->param('async', false, new Boolean(true), 'Execute code in the background. Default value is false.', true) ->param('path', '/', new Text(2048), 'HTTP path of execution. Path can include query params. Default value is /', true) ->param('method', 'POST', new Whitelist(['GET', 'POST', 'PUT', 'PATCH', 'DELETE', 'OPTIONS'], true), 'HTTP method of execution. Default value is GET.', true) ->param('headers', [], new AnyOf([new Assoc(), new Text(65535)], AnyOf::TYPE_MIXED), 'HTTP headers of execution. Defaults to empty.', true) @@ -1733,7 +1733,8 @@ App::post('/v1/functions/:functionId/executions') ->inject('queueForUsage') ->inject('queueForFunctions') ->inject('geodb') - ->action(function (string $functionId, string $body, bool $async, string $path, string $method, mixed $headers, ?string $scheduledAt, Response $response, Request $request, Document $project, Database $dbForProject, Database $dbForConsole, Document $user, Event $queueForEvents, Usage $queueForUsage, Func $queueForFunctions, Reader $geodb) { + ->action(function (string $functionId, string $body, mixed $async, string $path, string $method, mixed $headers, ?string $scheduledAt, Response $response, Request $request, Document $project, Database $dbForProject, Database $dbForConsole, Document $user, Event $queueForEvents, Usage $queueForUsage, Func $queueForFunctions, Reader $geodb) { + $async = \strval($async) === 'true' || \strval($async) === '1'; if (!$async && !is_null($scheduledAt)) { throw new Exception(Exception::GENERAL_BAD_REQUEST, 'Scheduled executions must run asynchronously. Set scheduledAt to a future date, or set async to true.'); diff --git a/app/controllers/api/project.php b/app/controllers/api/project.php index d885e980df..816886590a 100644 --- a/app/controllers/api/project.php +++ b/app/controllers/api/project.php @@ -47,6 +47,7 @@ App::get('/v1/project/usage') METRIC_USERS, METRIC_BUCKETS, METRIC_FILES_STORAGE, + METRIC_DATABASES_STORAGE, METRIC_DEPLOYMENTS_STORAGE, METRIC_BUILDS_STORAGE ], @@ -56,6 +57,7 @@ App::get('/v1/project/usage') METRIC_NETWORK_OUTBOUND, METRIC_USERS, METRIC_EXECUTIONS, + METRIC_DATABASES_STORAGE, METRIC_EXECUTIONS_MB_SECONDS, METRIC_BUILDS_MB_SECONDS ] @@ -182,6 +184,23 @@ App::get('/v1/project/usage') ]; }, $dbForProject->find('buckets')); + $databasesStorageBreakdown = array_map(function ($database) use ($dbForProject) { + $id = $database->getId(); + $name = $database->getAttribute('name'); + $metric = str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_STORAGE); + + $value = $dbForProject->findOne('stats', [ + Query::equal('metric', [$metric]), + Query::equal('period', ['inf']) + ]); + + return [ + 'resourceId' => $id, + 'name' => $name, + 'value' => $value['value'] ?? 0, + ]; + }, $dbForProject->find('databases')); + $functionsStorageBreakdown = array_map(function ($function) use ($dbForProject) { $id = $function->getId(); $name = $function->getAttribute('name'); @@ -269,6 +288,7 @@ App::get('/v1/project/usage') 'buildsMbSecondsTotal' => $total[METRIC_BUILDS_MB_SECONDS], 'documentsTotal' => $total[METRIC_DOCUMENTS], 'databasesTotal' => $total[METRIC_DATABASES], + 'databasesStorageTotal' => $total[METRIC_DATABASES_STORAGE], 'usersTotal' => $total[METRIC_USERS], 'bucketsTotal' => $total[METRIC_BUCKETS], 'filesStorageTotal' => $total[METRIC_FILES_STORAGE], @@ -279,6 +299,7 @@ App::get('/v1/project/usage') 'executionsMbSecondsBreakdown' => $executionsMbSecondsBreakdown, 'buildsMbSecondsBreakdown' => $buildsMbSecondsBreakdown, 'bucketsBreakdown' => $bucketsBreakdown, + 'databasesStorageBreakdown' => $databasesStorageBreakdown, 'executionsMbSecondsBreakdown' => $executionsMbSecondsBreakdown, 'buildsMbSecondsBreakdown' => $buildsMbSecondsBreakdown, 'functionsStorageBreakdown' => $functionsStorageBreakdown, diff --git a/app/controllers/shared/api.php b/app/controllers/shared/api.php index 6d87940ff7..357d73adc8 100644 --- a/app/controllers/shared/api.php +++ b/app/controllers/shared/api.php @@ -95,7 +95,7 @@ $databaseListener = function (string $event, Document $document, Document $proje $databaseInternalId = $parts[1] ?? 0; $queueForUsage ->addMetric(METRIC_COLLECTIONS, $value) // per project - ->addMetric(str_replace('{databaseInternalId}', $databaseInternalId, METRIC_DATABASE_ID_COLLECTIONS), $value) // per database + ->addMetric(str_replace('{databaseInternalId}', $databaseInternalId, METRIC_DATABASE_ID_COLLECTIONS), $value) ; if ($event === Database::EVENT_DOCUMENT_DELETE) { diff --git a/app/init.php b/app/init.php index 468274e464..6c2bf369f5 100644 --- a/app/init.php +++ b/app/init.php @@ -241,10 +241,16 @@ const METRIC_MESSAGES_TYPE_PROVIDER_FAILED = METRIC_MESSAGES . '.{type}.{provid const METRIC_SESSIONS = 'sessions'; const METRIC_DATABASES = 'databases'; const METRIC_COLLECTIONS = 'collections'; +const METRIC_DATABASES_STORAGE = 'databases.storage'; +const METRIC_DATABASES_STORAGE_DISK = 'databases.storage_disk'; const METRIC_DATABASE_ID_COLLECTIONS = '{databaseInternalId}.collections'; +const METRIC_DATABASE_ID_STORAGE = '{databaseInternalId}.databases.storage'; +const METRIC_DATABASE_ID_STORAGE_DISK = '{databaseInternalId}.databases.storage_disk'; const METRIC_DOCUMENTS = 'documents'; const METRIC_DATABASE_ID_DOCUMENTS = '{databaseInternalId}.documents'; const METRIC_DATABASE_ID_COLLECTION_ID_DOCUMENTS = '{databaseInternalId}.{collectionInternalId}.documents'; +const METRIC_DATABASE_ID_COLLECTION_ID_STORAGE = '{databaseInternalId}.{collectionInternalId}.databases.storage'; +const METRIC_DATABASE_ID_COLLECTION_ID_STORAGE_DISK = '{databaseInternalId}.{collectionInternalId}.databases.storage_disk'; const METRIC_BUCKETS = 'buckets'; const METRIC_FILES = 'files'; const METRIC_FILES_STORAGE = 'files.storage'; diff --git a/app/views/install/compose.phtml b/app/views/install/compose.phtml index 8f83fed544..ad35135a6f 100644 --- a/app/views/install/compose.phtml +++ b/app/views/install/compose.phtml @@ -531,6 +531,8 @@ $image = $this->getParam('image', ''); - _APP_SMTP_USERNAME - _APP_SMTP_PASSWORD - _APP_LOGGING_CONFIG + - _APP_DOMAIN + - _APP_OPTIONS_FORCE_HTTPS appwrite-worker-messaging: image: /: diff --git a/composer.json b/composer.json index 7d261fbab7..d3bf4e1055 100644 --- a/composer.json +++ b/composer.json @@ -51,7 +51,7 @@ "utopia-php/cache": "0.10.*", "utopia-php/cli": "0.15.*", "utopia-php/config": "0.2.*", - "utopia-php/database": "0.53.*", + "utopia-php/database": "0.53.5-rc1", "utopia-php/domains": "0.5.*", "utopia-php/dsn": "0.2.1", "utopia-php/framework": "0.33.*", @@ -97,5 +97,11 @@ "platform": { "php": "8.3" } + }, + "repositories": { + "utopia-php/database": { + "type": "vcs", + "url": "https://github.com/utopia-php/database" + } } } diff --git a/composer.lock b/composer.lock index 4cd5853fd0..b3a94545b4 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "3a25a9050c489b1a2cb4e320af1586d5", + "content-hash": "7066d9ca32e7a1a60614effdc4701970", "packages": [ { "name": "adhocore/jwt", @@ -65,16 +65,16 @@ }, { "name": "appwrite/appwrite", - "version": "11.1.0", + "version": "10.1.0", "source": { "type": "git", "url": "https://github.com/appwrite/sdk-for-php.git", - "reference": "1d043f543acdb17b9fdb440b1b2dd208e400bad3" + "reference": "da579af70723cfc117b5af84375bdef117e27312" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/appwrite/sdk-for-php/zipball/1d043f543acdb17b9fdb440b1b2dd208e400bad3", - "reference": "1d043f543acdb17b9fdb440b1b2dd208e400bad3", + "url": "https://api.github.com/repos/appwrite/sdk-for-php/zipball/da579af70723cfc117b5af84375bdef117e27312", + "reference": "da579af70723cfc117b5af84375bdef117e27312", "shasum": "" }, "require": { @@ -83,8 +83,7 @@ "php": ">=7.1.0" }, "require-dev": { - "mockery/mockery": "^1.6.6", - "phpunit/phpunit": "^10" + "phpunit/phpunit": "3.7.35" }, "type": "library", "autoload": { @@ -100,10 +99,10 @@ "support": { "email": "team@appwrite.io", "issues": "https://github.com/appwrite/sdk-for-php/issues", - "source": "https://github.com/appwrite/sdk-for-php/tree/11.1.0", + "source": "https://github.com/appwrite/sdk-for-php/tree/10.1.0", "url": "https://appwrite.io/support" }, - "time": "2024-06-26T07:03:23+00:00" + "time": "2023-11-20T09:56:12+00:00" }, { "name": "appwrite/php-clamav", @@ -1724,16 +1723,16 @@ }, { "name": "utopia-php/database", - "version": "0.53.4", + "version": "0.53.5-rc1", "source": { "type": "git", "url": "https://github.com/utopia-php/database.git", - "reference": "36a0e89d983afc1368635282e04fa762220a1d2a" + "reference": "689ba22063bf46def385da8695ba7a921e81e38d" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/database/zipball/36a0e89d983afc1368635282e04fa762220a1d2a", - "reference": "36a0e89d983afc1368635282e04fa762220a1d2a", + "url": "https://api.github.com/repos/utopia-php/database/zipball/689ba22063bf46def385da8695ba7a921e81e38d", + "reference": "689ba22063bf46def385da8695ba7a921e81e38d", "shasum": "" }, "require": { @@ -1760,7 +1759,38 @@ "Utopia\\Database\\": "src/Database" } }, - "notification-url": "https://packagist.org/downloads/", + "autoload-dev": { + "psr-4": { + "Tests\\E2E\\": "tests/e2e", + "Tests\\Unit\\": "tests/unit" + } + }, + "scripts": { + "build": [ + "Composer\\Config::disableProcessTimeout", + "docker compose build" + ], + "start": [ + "Composer\\Config::disableProcessTimeout", + "docker compose up -d" + ], + "test": [ + "Composer\\Config::disableProcessTimeout", + "docker compose exec tests vendor/bin/phpunit --configuration phpunit.xml" + ], + "lint": [ + "./vendor/bin/pint --test" + ], + "format": [ + "./vendor/bin/pint" + ], + "check": [ + "./vendor/bin/phpstan analyse --level 7 src tests --memory-limit 512M" + ], + "coverage": [ + "./vendor/bin/coverage-check ./tmp/clover.xml 90" + ] + }, "license": [ "MIT" ], @@ -1773,10 +1803,10 @@ "utopia" ], "support": { - "issues": "https://github.com/utopia-php/database/issues", - "source": "https://github.com/utopia-php/database/tree/0.53.4" + "source": "https://github.com/utopia-php/database/tree/0.53.5-rc1", + "issues": "https://github.com/utopia-php/database/issues" }, - "time": "2024-09-10T10:19:57+00:00" + "time": "2024-09-24T08:43:10+00:00" }, { "name": "utopia-php/domains", @@ -2175,35 +2205,27 @@ }, { "name": "utopia-php/migration", - "version": "0.6.1", + "version": "0.5.2", "source": { "type": "git", "url": "https://github.com/utopia-php/migration.git", - "reference": "6a064179dd0a7278bfbaf65d9abdef19d6d2bbe0" + "reference": "f18d44d4459f78c292dac9edde856fd156fe497a" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/migration/zipball/6a064179dd0a7278bfbaf65d9abdef19d6d2bbe0", - "reference": "6a064179dd0a7278bfbaf65d9abdef19d6d2bbe0", + "url": "https://api.github.com/repos/utopia-php/migration/zipball/f18d44d4459f78c292dac9edde856fd156fe497a", + "reference": "f18d44d4459f78c292dac9edde856fd156fe497a", "shasum": "" }, "require": { - "appwrite/appwrite": "11.1.*", - "ext-curl": "*", - "ext-openssl": "*", - "php": "8.3.*", - "utopia-php/database": "0.53.*", - "utopia-php/dsn": "0.2.*", - "utopia-php/framework": "0.33.*", - "utopia-php/storage": "0.18.*" + "appwrite/appwrite": "10.1.0", + "php": "8.*" }, "require-dev": { - "ext-pdo": "*", - "laravel/pint": "1.17.*", - "phpstan/phpstan": "1.11.*", - "phpunit/phpunit": "11.2.*", - "utopia-php/cli": "0.16.*", - "vlucas/phpdotenv": "5.6.*" + "laravel/pint": "1.*", + "phpunit/phpunit": "9.*", + "utopia-php/cli": "^0.18.0", + "vlucas/phpdotenv": "5.*" }, "type": "library", "autoload": { @@ -2225,9 +2247,9 @@ ], "support": { "issues": "https://github.com/utopia-php/migration/issues", - "source": "https://github.com/utopia-php/migration/tree/0.6.1" + "source": "https://github.com/utopia-php/migration/tree/0.5.2" }, - "time": "2024-09-24T09:54:09+00:00" + "time": "2024-07-22T09:27:07+00:00" }, { "name": "utopia-php/mongo", @@ -3323,16 +3345,16 @@ }, { "name": "laravel/pint", - "version": "v1.18.0", + "version": "v1.17.3", "source": { "type": "git", "url": "https://github.com/laravel/pint.git", - "reference": "eb2bcbd85034c1a508cf409b7ee318eb3137f090" + "reference": "9d77be916e145864f10788bb94531d03e1f7b482" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/laravel/pint/zipball/eb2bcbd85034c1a508cf409b7ee318eb3137f090", - "reference": "eb2bcbd85034c1a508cf409b7ee318eb3137f090", + "url": "https://api.github.com/repos/laravel/pint/zipball/9d77be916e145864f10788bb94531d03e1f7b482", + "reference": "9d77be916e145864f10788bb94531d03e1f7b482", "shasum": "" }, "require": { @@ -3385,7 +3407,7 @@ "issues": "https://github.com/laravel/pint/issues", "source": "https://github.com/laravel/pint" }, - "time": "2024-09-24T15:48:42+00:00" + "time": "2024-09-03T15:00:28+00:00" }, { "name": "matthiasmullie/minify", @@ -7004,7 +7026,9 @@ ], "aliases": [], "minimum-stability": "stable", - "stability-flags": [], + "stability-flags": { + "utopia-php/database": 5 + }, "prefer-stable": false, "prefer-lowest": false, "platform": { diff --git a/src/Appwrite/Platform/Workers/UsageDump.php b/src/Appwrite/Platform/Workers/UsageDump.php index b7097dbb04..e5d05bb2fb 100644 --- a/src/Appwrite/Platform/Workers/UsageDump.php +++ b/src/Appwrite/Platform/Workers/UsageDump.php @@ -4,6 +4,7 @@ namespace Appwrite\Platform\Workers; use Appwrite\Extend\Exception; use Utopia\CLI\Console; +use Utopia\Database\Database; use Utopia\Database\DateTime; use Utopia\Database\Document; use Utopia\Database\Exception\Duplicate; @@ -11,6 +12,10 @@ use Utopia\Platform\Action; use Utopia\Queue\Message; use Utopia\System\System; +const METRIC_COLLECTION_LEVEL_STORAGE = 4; +const METRIC_DATABASE_LEVEL_STORAGE = 3; +const METRIC_PROJECT_LEVEL_STORAGE = 2; + class UsageDump extends Action { protected array $stats = []; @@ -70,6 +75,15 @@ class UsageDump extends Action continue; } + if (str_contains($key, METRIC_DATABASES_STORAGE)) { + try { + $this->handleDatabaseStorage($key, $dbForProject); + } catch (\Exception $e) { + console::error('[' . DateTime::now() . '] failed to calculate database storage for key [' . $key . '] ' . $e->getMessage()); + } + continue; + } + foreach ($this->periods as $period => $format) { $time = 'inf' === $period ? null : date($format, time()); $id = \md5("{$time}_{$period}_{$key}"); @@ -107,4 +121,175 @@ class UsageDump extends Action } } } + + private function handleDatabaseStorage(string $key, Database $dbForProject): void + { + $data = explode('.', $key); + $start = microtime(true); + + $updateMetric = function (Database $dbForProject, int $value, string $key, string $period, string|null $time) { + $id = \md5("{$time}_{$period}_{$key}"); + + try { + $dbForProject->createDocument('stats', new Document([ + '$id' => $id, + 'period' => $period, + 'time' => $time, + 'metric' => $key, + 'value' => $value, + 'region' => System::getEnv('_APP_REGION', 'default'), + ])); + } catch (Duplicate $th) { + if ($value < 0) { + $dbForProject->decreaseDocumentAttribute( + 'stats', + $id, + 'value', + abs($value) + ); + } else { + $dbForProject->increaseDocumentAttribute( + 'stats', + $id, + 'value', + $value + ); + } + } + }; + + foreach ($this->periods as $period => $format) { + $time = 'inf' === $period ? null : date($format, time()); + $id = \md5("{$time}_{$period}_{$key}"); + + $value = 0; + $diskValue = 0; + $previousValue = 0; + try { + $previousValue = ($dbForProject->getDocument('stats', $id))->getAttribute('value', 0); + } catch (\Exception $e) { + // No previous value + } + + switch (count($data)) { + // Collection Level + case METRIC_COLLECTION_LEVEL_STORAGE: + Console::log('[' . DateTime::now() . '] Collection Level Storage Calculation [' . $key . ']'); + $databaseInternalId = $data[0]; + $collectionInternalId = $data[1]; + + try { + $value = $dbForProject->getSizeOfCollection('database_'.$databaseInternalId.'_collection_'.$collectionInternalId); + $diskValue = $dbForProject->getSizeOfCollectionOnDisk('database_'.$databaseInternalId.'_collection_'.$collectionInternalId); + } catch (\Exception $e) { + // Collection not found + if ($e->getMessage() !== 'Collection not found') { + throw $e; + } + } + + // Compare with previous value + $diff = $value - $previousValue; + $diskDiff = $diskValue - $previousValue; + + if ($diff === 0 && $diskDiff === 0) { + break; + } + + // Update Collection + $updateMetric($dbForProject, $diff, $key, $period, $time); + $updateMetric($dbForProject, $diskDiff, $key . '_disk', $period, $time); + + // Update Database + $databaseKey = str_replace(['{databaseInternalId}'], [$data[0]], METRIC_DATABASE_ID_STORAGE); + $updateMetric($dbForProject, $diff, $databaseKey, $period, $time); + $updateMetric($dbForProject, $diskDiff, $databaseKey . '_disk', $period, $time); + + // Update Project + $projectKey = METRIC_DATABASES_STORAGE; + $updateMetric($dbForProject, $diff, $projectKey, $period, $time); + $updateMetric($dbForProject, $diskDiff, $projectKey . '_disk', $period, $time); + break; + // Database Level + case METRIC_DATABASE_LEVEL_STORAGE: + Console::log('[' . DateTime::now() . '] Database Level Storage Calculation [' . $key . ']'); + $databaseInternalId = $data[0]; + + $collections = []; + try { + $collections = $dbForProject->find('database_' . $databaseInternalId); + } catch (\Exception $e) { + // Database not found + if ($e->getMessage() !== 'Collection not found') { + throw $e; + } + } + + foreach ($collections as $collection) { + try { + $value += $dbForProject->getSizeOfCollection('database_'.$databaseInternalId.'_collection_'.$collection->getInternalId()); + $diskValue += $dbForProject->getSizeOfCollectionOnDisk('database_'.$databaseInternalId.'_collection_'.$collection->getInternalId()); + } catch (\Exception $e) { + // Collection not found + if ($e->getMessage() !== 'Collection not found') { + throw $e; + } + } + } + + $diff = $value - $previousValue; + $diskDiff = $diskValue - $previousValue; + + if ($diff === 0 && $diskDiff === 0) { + break; + } + + // Update Database + $databaseKey = str_replace(['{databaseInternalId}'], [$data[0]], METRIC_DATABASE_ID_STORAGE); + $updateMetric($dbForProject, $diff, $databaseKey, $period, $time); + $updateMetric($dbForProject, $diskDiff, $databaseKey . '_disk', $period, $time); + + // Update Project + $projectKey = METRIC_DATABASES_STORAGE; + $updateMetric($dbForProject, $diff, $projectKey, $period, $time); + $updateMetric($dbForProject, $diskDiff, $projectKey . '_disk', $period, $time); + break; + // Project Level + case METRIC_PROJECT_LEVEL_STORAGE: + Console::log('[' . DateTime::now() . '] Project Level Storage Calculation [' . $key . ']'); + // Get all project databases + $databases = $dbForProject->find('database'); + + // Recalculate all databases + foreach ($databases as $database) { + $collections = $dbForProject->find('database_' . $database->getInternalId()); + + foreach ($collections as $collection) { + try { + $value += $dbForProject->getSizeOfCollection('database_'.$database->getInternalId().'_collection_'.$collection->getInternalId()); + $diskValue += $dbForProject->getSizeOfCollectionOnDisk('database_'.$database->getInternalId().'_collection_'.$collection->getInternalId()); + } catch (\Exception $e) { + // Collection not found + if ($e->getMessage() !== 'Collection not found') { + throw $e; + } + } + } + } + + $diff = $value - $previousValue; + $diskDiff = $diskValue - $previousValue; + + // Update Project + $projectKey = METRIC_DATABASES_STORAGE; + $updateMetric($dbForProject, $diff, $projectKey, $period, $time); + $updateMetric($dbForProject, $diskDiff, $projectKey . '_disk', $period, $time); + break; + } + } + + $end = microtime(true); + + console::log('[' . DateTime::now() . '] DB Storage Calculation [' . $key . '] took ' . (($end - $start) * 1000) . ' milliseconds'); + } } diff --git a/src/Appwrite/Utopia/Response/Model/UsageDatabase.php b/src/Appwrite/Utopia/Response/Model/UsageDatabase.php index d4733f2568..eb985baabb 100644 --- a/src/Appwrite/Utopia/Response/Model/UsageDatabase.php +++ b/src/Appwrite/Utopia/Response/Model/UsageDatabase.php @@ -28,6 +28,12 @@ class UsageDatabase extends Model 'default' => 0, 'example' => 0, ]) + ->addRule('storageTotal', [ + 'type' => self::TYPE_INTEGER, + 'description' => 'Total aggregated number of total storage used in bytes.', + 'default' => 0, + 'example' => 0, + ]) ->addRule('collections', [ 'type' => Response::MODEL_METRIC, 'description' => 'Aggregated number of collections per period.', @@ -42,6 +48,13 @@ class UsageDatabase extends Model 'example' => [], 'array' => true ]) + ->addRule('storage', [ + 'type' => Response::MODEL_METRIC, + 'description' => 'Aggregated storage used in bytes per period.', + 'default' => [], + 'example' => [], + 'array' => true + ]) ; } diff --git a/src/Appwrite/Utopia/Response/Model/UsageDatabases.php b/src/Appwrite/Utopia/Response/Model/UsageDatabases.php index f775f9489d..e0abba8ab8 100644 --- a/src/Appwrite/Utopia/Response/Model/UsageDatabases.php +++ b/src/Appwrite/Utopia/Response/Model/UsageDatabases.php @@ -34,6 +34,12 @@ class UsageDatabases extends Model 'default' => 0, 'example' => 0, ]) + ->addRule('storageTotal', [ + 'type' => self::TYPE_INTEGER, + 'description' => 'Total aggregated number of total databases storage in bytes.', + 'default' => 0, + 'example' => 0, + ]) ->addRule('databases', [ 'type' => Response::MODEL_METRIC, 'description' => 'Aggregated number of databases per period.', @@ -55,6 +61,13 @@ class UsageDatabases extends Model 'example' => [], 'array' => true ]) + ->addRule('storage', [ + 'type' => Response::MODEL_METRIC, + 'description' => 'An array of the aggregated number of databases storage in bytes per period.', + 'default' => [], + 'example' => [], + 'array' => true + ]) ; } diff --git a/src/Appwrite/Utopia/Response/Model/UsageProject.php b/src/Appwrite/Utopia/Response/Model/UsageProject.php index 2703691238..17d9271f04 100644 --- a/src/Appwrite/Utopia/Response/Model/UsageProject.php +++ b/src/Appwrite/Utopia/Response/Model/UsageProject.php @@ -28,6 +28,12 @@ class UsageProject extends Model 'default' => 0, 'example' => 0, ]) + ->addRule('databasesStorageTotal', [ + 'type' => self::TYPE_INTEGER, + 'description' => 'Total aggregated sum of databases storage size (in bytes).', + 'default' => 0, + 'example' => 0, + ]) ->addRule('usersTotal', [ 'type' => self::TYPE_INTEGER, 'description' => 'Total aggregated number of users.', @@ -118,6 +124,13 @@ class UsageProject extends Model 'example' => [], 'array' => true ]) + ->addRule('databasesStorageBreakdown', [ + 'type' => Response::MODEL_METRIC_BREAKDOWN, + 'description' => 'An array of the aggregated breakdown of storage usage by databases.', + 'default' => [], + 'example' => [], + 'array' => true + ]) ->addRule('executionsMbSecondsBreakdown', [ 'type' => Response::MODEL_METRIC_BREAKDOWN, 'description' => 'Aggregated breakdown in totals of execution mbSeconds by functions.', diff --git a/tests/e2e/General/UsageTest.php b/tests/e2e/General/UsageTest.php index d3623acffc..e5b9f0f819 100644 --- a/tests/e2e/General/UsageTest.php +++ b/tests/e2e/General/UsageTest.php @@ -143,7 +143,7 @@ class UsageTest extends Scope ); $this->assertEquals(200, $response['headers']['status-code']); - $this->assertEquals(20, count($response['body'])); + $this->assertEquals(22, count($response['body'])); $this->validateDates($response['body']['network']); $this->validateDates($response['body']['requests']); $this->validateDates($response['body']['users']); @@ -324,7 +324,7 @@ class UsageTest extends Scope ] ); - $this->assertEquals(20, count($response['body'])); + $this->assertEquals(22, count($response['body'])); $this->assertEquals(1, count($response['body']['requests'])); $this->assertEquals($requestsTotal, $response['body']['requests'][array_key_last($response['body']['requests'])]['value']); $this->validateDates($response['body']['requests']); @@ -545,7 +545,7 @@ class UsageTest extends Scope ] ); - $this->assertEquals(20, count($response['body'])); + $this->assertEquals(22, count($response['body'])); $this->assertEquals(1, count($response['body']['requests'])); $this->assertEquals(1, count($response['body']['network'])); $this->assertEquals($requestsTotal, $response['body']['requests'][array_key_last($response['body']['requests'])]['value']); @@ -590,6 +590,260 @@ class UsageTest extends Scope return $data; } + public function testDatabaseStoragePrepare(): array + { + $response = $this->client->call( + Client::METHOD_POST, + '/databases', + array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'] + ], $this->getHeaders()), + [ + 'databaseId' => 'unique()', + 'name' => 'dbStorageStats', + ] + ); + + $this->assertNotEmpty($response['body']['$id']); + $databaseId = $response['body']['$id']; + + $response = $this->client->call( + Client::METHOD_POST, + '/databases/' . $databaseId . '/collections', + array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'] + ], $this->getHeaders()), + [ + 'collectionId' => 'unique()', + 'name' => 'collectionStorageStats', + 'documentSecurity' => false, + 'permissions' => [ + Permission::read(Role::any()), + Permission::create(Role::any()), + Permission::update(Role::any()), + Permission::delete(Role::any()), + ], + ] + ); + + $this->assertNotEmpty($response['body']['$id']); + $collectionId = $response['body']['$id']; + + $response = $this->client->call( + Client::METHOD_POST, + '/databases/' . $databaseId . '/collections/' . $collectionId . '/attributes' . '/string', + array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'] + ], $this->getHeaders()), + [ + 'key' => 'data', + 'size' => 100000, + 'required' => true, + ] + ); + + return [ + 'databaseId' => $databaseId, + 'collectionId' => $collectionId, + ]; + } + + // /** @depends testDatabaseStoragePrepare */ + // #[Retry(count: 1)] + // public function testDatabaseStorageStatsCreateDocument(array $data): array + // { + // $databaseId = $data['databaseId']; + // $collectionId = $data['collectionId']; + + // $originalProjectMetrics = $this->client->call( + // Client::METHOD_GET, + // '/project/usage', + // $this->getConsoleHeaders(), + // [ + // 'period' => '1d', + // 'startDate' => self::getToday(), + // 'endDate' => self::getTomorrow(), + // ] + // ); + + // $this->assertEquals(200, $originalProjectMetrics['headers']['status-code']); + // $this->assertArrayHasKey('databasesStorageTotal', $originalProjectMetrics['body']); + + // $originalProjectMetrics = $originalProjectMetrics['body']; + + // $originalDatabaseMetrics = $this->client->call( + // Client::METHOD_GET, + // '/databases/' . $databaseId . '/usage?range=30d', + // $this->getConsoleHeaders() + // ); + + // $this->assertEquals(200, $originalDatabaseMetrics['headers']['status-code']); + // $this->assertArrayHasKey('storageTotal', $originalDatabaseMetrics['body']); + // $originalDatabaseMetrics = $originalDatabaseMetrics['body']; + + // // Create documents + // for ($i = 0; $i < 100; $i++) { + // $response = $this->client->call( + // Client::METHOD_POST, + // '/databases/' . $databaseId . '/collections/' . $collectionId . '/documents', + // array_merge([ + // 'content-type' => 'application/json', + // 'x-appwrite-project' => $this->getProject()['$id'] + // ], $this->getHeaders()), + // [ + // 'documentId' => 'unique()', + // 'data' => ['data' => str_repeat('a', 10000)], + // ] + // ); + + // $this->assertEquals(201, $response['headers']['status-code']); + // } + + // sleep(self::WAIT); + + // for ($i = 0; $i < 3; $i++) { + // try { + // $newProjectMetrics = $this->client->call( + // Client::METHOD_GET, + // '/project/usage', + // $this->getConsoleHeaders(), + // [ + // 'period' => '1d', + // 'startDate' => self::getToday(), + // 'endDate' => self::getTomorrow(), + // ] + // ); + + // $this->assertEquals(200, $newProjectMetrics['headers']['status-code']); + // $this->assertArrayHasKey('databasesStorageTotal', $newProjectMetrics['body']); + // $this->assertGreaterThan($originalProjectMetrics['databasesStorageTotal'], $newProjectMetrics['body']['databasesStorageTotal']); + + // $newProjectMetrics = $newProjectMetrics['body']; + + // $newDatabaseMetrics = $this->client->call( + // Client::METHOD_GET, + // '/databases/' . $databaseId . '/usage?range=30d', + // $this->getConsoleHeaders() + // ); + + // $this->assertEquals(200, $newDatabaseMetrics['headers']['status-code']); + // $this->assertArrayHasKey('storageTotal', $newDatabaseMetrics['body']); + // $this->assertGreaterThan($originalDatabaseMetrics['storageTotal'], $newDatabaseMetrics['body']['storageTotal']); + + // $newDatabaseMetrics = $newDatabaseMetrics['body']; + + // return [ + // 'databaseId' => $databaseId, + // 'collectionId' => $collectionId, + // 'currentProjectMetrics' => $newProjectMetrics, + // 'currentDatabaseMetrics' => $newDatabaseMetrics, + // ]; + // } catch (ExpectationFailedException $e) { + // if ($i === 2) { + // throw $e; + // } + // sleep(self::WAIT); + // continue; + // } + // } + // } + + // /** @depends testDatabaseStorageStatsCreateDocument */ + // #[Retry(count: 1)] + // public function testDatabaseStorageStatsDeleteDocument(array $data): array + // { + // $databaseId = $data['databaseId']; + // $collectionId = $data['collectionId']; + // $currentProjectMetrics = $data['currentProjectMetrics']; + // $currentDatabaseMetrics = $data['currentDatabaseMetrics']; + + // $documents = $this->client->call( + // Client::METHOD_GET, + // '/databases/' . $databaseId . '/collections/' . $collectionId . '/documents', + // array_merge([ + // 'x-appwrite-project' => $this->getProject()['$id'] + // ], $this->getHeaders()), + // [ + // 'queries' => [ + // Query::limit(50)->toString() + // ] + // ] + // ); + + // foreach ($documents['body']['documents'] as $document) { + // $response = $this->client->call( + // Client::METHOD_DELETE, + // '/databases/' . $databaseId . '/collections/' . $collectionId . '/documents/' . $document['$id'], + // array_merge([ + // 'x-appwrite-project' => $this->getProject()['$id'] + // ], $this->getHeaders()) + // ); + + // $this->assertEquals(204, $response['headers']['status-code']); + // } + + // sleep(self::WAIT); + + // for ($i = 0; $i < 3; $i++) { + // try { + // $newProjectMetrics = $this->client->call( + // Client::METHOD_GET, + // '/project/usage', + // $this->getConsoleHeaders(), + // [ + // 'period' => '1d', + // 'startDate' => self::getToday(), + // 'endDate' => self::getTomorrow(), + // ] + // ); + + // $this->assertEquals(200, $newProjectMetrics['headers']['status-code']); + // $this->assertArrayHasKey('databasesStorageTotal', $newProjectMetrics['body']); + // $this->assertLessThan($currentProjectMetrics['databasesStorageTotal'], $newProjectMetrics['body']['databasesStorageTotal']); + + // $newProjectMetrics = $newProjectMetrics['body']; + + // $newDatabaseMetrics = $this->client->call( + // Client::METHOD_GET, + // '/databases/' . $databaseId . '/usage?range=30d', + // $this->getConsoleHeaders() + // ); + + // $this->assertEquals(200, $newDatabaseMetrics['headers']['status-code']); + // $this->assertArrayHasKey('storageTotal', $newDatabaseMetrics['body']); + // $this->assertLessThan($currentDatabaseMetrics['storageTotal'], $newDatabaseMetrics['body']['storageTotal']); + + // $newDatabaseMetrics = $newDatabaseMetrics['body']; + + // return [ + // 'databaseId' => $databaseId, + // 'collectionId' => $collectionId, + // 'currentProjectMetrics' => $newProjectMetrics, + // 'currentDatabaseMetrics' => $newDatabaseMetrics, + // ]; + // } catch (ExpectationFailedException $e) { + // if ($i === 2) { + // throw $e; + // } + // sleep(self::WAIT); + // continue; + // } + // } + + // $newProjectMetrics = $this->client->call( + // Client::METHOD_GET, + // '/project/usage', + // $this->getConsoleHeaders(), + // [ + // 'period' => '1d', + // 'startDate' => self::getToday(), + // 'endDate' => self::getTomorrow(), + // ] + // ); + // } /** @depends testDatabaseStats */ public function testPrepareFunctionsStats(array $data): array diff --git a/tests/e2e/Services/Databases/DatabasesConsoleClientTest.php b/tests/e2e/Services/Databases/DatabasesConsoleClientTest.php index ca77cf2581..96bb0b5609 100644 --- a/tests/e2e/Services/Databases/DatabasesConsoleClientTest.php +++ b/tests/e2e/Services/Databases/DatabasesConsoleClientTest.php @@ -224,7 +224,7 @@ class DatabasesConsoleClientTest extends Scope ]); $this->assertEquals(200, $response['headers']['status-code']); - $this->assertEquals(5, count($response['body'])); + $this->assertEquals(7, count($response['body'])); $this->assertEquals('24h', $response['body']['range']); $this->assertIsNumeric($response['body']['documentsTotal']); $this->assertIsNumeric($response['body']['collectionsTotal']);