Merge remote-tracking branch 'origin/refactor-usage-sn' into feat-isolation-modes

# Conflicts:
#	composer.json
#	composer.lock
#	src/Appwrite/Platform/Tasks/CalcTierStats.php
#	tests/e2e/Services/Projects/ProjectsConsoleClientTest.php

src/Appwrite/Auth/Validator/Phone.php

@@ -32,7 +32,7 @@ class Phone extends Validator
      */
     public function isValid($value): bool
     {
-        return is_string($value) && !!\preg_match('/^\+[1-9]\d{1,14}$/', $value);
+        return is_string($value) && !!\preg_match('/^\+[1-9]\d{6,14}$/', $value);
     }
 
     /**

src/Appwrite/Event/Event.php

@@ -27,6 +27,9 @@ class Event
     public const USAGE_QUEUE_NAME = 'v1-usage';
     public const USAGE_CLASS_NAME = 'UsageV1';
 
+    public const USAGE_DUMP_QUEUE_NAME = 'v1-usage-dump';
+    public const USAGE_DUMP_CLASS_NAME = 'UsageDumpV1';
+
     public const WEBHOOK_QUEUE_NAME = 'v1-webhooks';
     public const WEBHOOK_CLASS_NAME = 'WebhooksV1';
 

src/Appwrite/Event/Mail.php

@@ -13,6 +13,8 @@ class Mail extends Event
     protected string $body = '';
     protected array $smtp = [];
     protected array $variables = [];
+    protected string $bodyTemplate = '';
+    protected array $attachment = [];
 
     public function __construct(protected Connection $connection)
     {
@@ -115,6 +117,29 @@ class Mail extends Event
         return $this->name;
     }
 
+    /**
+     * Sets bodyTemplate for the mail event.
+     *
+     * @param string $bodyTemplate
+     * @return self
+     */
+    public function setbodyTemplate(string $bodyTemplate): self
+    {
+        $this->bodyTemplate = $bodyTemplate;
+
+        return $this;
+    }
+
+    /**
+     * Returns subject for the mail event.
+     *
+     * @return string
+     */
+    public function getbodyTemplate(): string
+    {
+        return $this->bodyTemplate;
+    }
+
     /**
      * Set SMTP Host
      *
@@ -313,6 +338,64 @@ class Mail extends Event
         return $this;
     }
 
+    /**
+     * Set attachment
+     * @param string $content
+     * @param string $filename
+     * @param string $encoding
+     * @param string $type
+     * @return self
+     */
+    public function setAttachment(string $content, string $filename, string $encoding = 'base64', string $type = 'plain/text')
+    {
+        $this->attachment = [
+            'content' => base64_encode($content),
+            'filename' => $filename,
+            'encoding' => $encoding,
+            'type' => $type,
+        ];
+        return $this;
+    }
+
+    /**
+     * Get attachment
+     *
+     * @return array
+     */
+    public function getAttachment(): array
+    {
+        return $this->attachment;
+    }
+
+    /**
+     * Reset attachment
+     *
+     * @return self
+     */
+    public function resetAttachment(): self
+    {
+        $this->attachment = [];
+        return $this;
+    }
+
+    /**
+     * Reset
+     *
+     * @return self
+     */
+    public function reset(): self
+    {
+        $this->project = null;
+        $this->recipient = '';
+        $this->name = '';
+        $this->subject = '';
+        $this->body = '';
+        $this->variables = [];
+        $this->bodyTemplate = '';
+        $this->attachment = [];
+        return $this;
+    }
+
     /**
      * Executes the event and sends it to the mails worker.
      *
@@ -324,12 +407,15 @@ class Mail extends Event
         $client = new Client($this->queue, $this->connection);
 
         return $client->enqueue([
+            'project' => $this->project,
             'recipient' => $this->recipient,
             'name' => $this->name,
             'subject' => $this->subject,
+            'bodyTemplate' => $this->bodyTemplate,
             'body' => $this->body,
             'smtp' => $this->smtp,
             'variables' => $this->variables,
+            'attachment' => $this->attachment,
             'events' => Event::generateEvents($this->getEvent(), $this->getParams())
         ]);
     }

src/Appwrite/Event/Usage.php

@@ -2,6 +2,7 @@
 
 namespace Appwrite\Event;
 
+use Utopia\CLI\Console;
 use Utopia\Queue\Client;
 use Utopia\Queue\Connection;
 use Utopia\Database\Document;

src/Appwrite/Event/UsageDump.php

@@ -0,0 +1,47 @@
+<?php
+
+namespace Appwrite\Event;
+
+use Utopia\Queue\Client;
+use Utopia\Queue\Connection;
+
+class UsageDump extends Event
+{
+    protected array $stats;
+
+    public function __construct(protected Connection $connection)
+    {
+        parent::__construct($connection);
+
+        $this
+            ->setQueue(Event::USAGE_DUMP_QUEUE_NAME)
+            ->setClass(Event::USAGE_DUMP_CLASS_NAME);
+    }
+
+    /**
+     * Add Stats.
+     *
+     * @param array $stats
+     * @return self
+     */
+    public function setStats(array $stats): self
+    {
+        $this->stats = $stats;
+
+        return $this;
+    }
+
+    /**
+     * Sends metrics to the usage worker.
+     *
+     * @return string|bool
+     */
+    public function trigger(): string|bool
+    {
+        $client = new Client($this->queue, $this->connection);
+
+        return $client->enqueue([
+            'stats' => $this->stats,
+        ]);
+    }
+}

src/Appwrite/Extend/Exception.php

@@ -55,6 +55,7 @@ class Exception extends \Exception
     public const GENERAL_CODES_DISABLED            = 'general_codes_disabled';
     public const GENERAL_USAGE_DISABLED            = 'general_usage_disabled';
     public const GENERAL_NOT_IMPLEMENTED           = 'general_not_implemented';
+    public const GENERAL_BAD_REQUEST               = 'general_bad_request';
 
     /** Users */
     public const USER_COUNT_EXCEEDED               = 'user_count_exceeded';
@@ -84,8 +85,9 @@ class Exception extends \Exception
     public const USER_OAUTH2_BAD_REQUEST           = 'user_oauth2_bad_request';
     public const USER_OAUTH2_UNAUTHORIZED          = 'user_oauth2_unauthorized';
     public const USER_OAUTH2_PROVIDER_ERROR        = 'user_oauth2_provider_error';
-    public const USER_EMAIL_ALREADY_VERIFIED        = 'user_email_alread_verified';
-    public const USER_PHONE_ALREADY_VERIFIED        = 'user_phone_already_verified';
+    public const USER_EMAIL_ALREADY_VERIFIED       = 'user_email_alread_verified';
+    public const USER_PHONE_ALREADY_VERIFIED       = 'user_phone_already_verified';
+    public const USER_DELETION_PROHIBITED          = 'user_deletion_prohibited';
 
     /** Teams */
     public const TEAM_NOT_FOUND                    = 'team_not_found';
@@ -234,25 +236,22 @@ class Exception extends \Exception
     public const REALTIME_POLICY_VIOLATION          = 'realtime_policy_violation';
 
     /** Health */
-    public const QUEUE_SIZE_EXCEEDED                 = 'queue_size_exceeded';
+    public const HEALTH_QUEUE_SIZE_EXCEEDED          = 'health_queue_size_exceeded';
+    public const HEALTH_CERTIFICATE_EXPIRED          = 'health_certificate_expired';
+    public const HEALTH_INVALID_HOST                 = 'health_invalid_host';
 
     protected string $type = '';
     protected array $errors = [];
-    protected bool $publish = true;
+    protected bool $publish;
 
     public function __construct(string $type = Exception::GENERAL_UNKNOWN, string $message = null, int $code = null, \Throwable $previous = null)
     {
         $this->errors = Config::getParam('errors');
         $this->type = $type;
+        $this->code = $code ?? $this->errors[$type]['code'];
+        $this->message = $message ?? $this->errors[$type]['description'];
 
-        if (isset($this->errors[$type])) {
-            $this->code = $this->errors[$type]['code'];
-            $this->message = $this->errors[$type]['description'];
-            $this->publish = $this->errors[$type]['publish'] ?? true;
-        }
-
-        $this->message = $message ?? $this->message;
-        $this->code = $code ?? $this->code;
+        $this->publish = $this->errors[$type]['publish'] ?? ($this->code >= 500);
 
         parent::__construct($this->message, $this->code, $previous);
     }

src/Appwrite/Hooks/Hooks.php

@@ -0,0 +1,26 @@
+<?php
+
+namespace Appwrite\Hooks;
+
+class Hooks
+{
+    /**
+     * @var callable[] $hooks
+     */
+    private static array $hooks = [];
+
+    public static function add(string $name, callable $action)
+    {
+        self::$hooks[$name] = $action;
+    }
+
+    /**
+     * @param mixed[] $params
+     */
+    public function trigger(string $name, array $params = [])
+    {
+        if (isset(self::$hooks[$name])) {
+            call_user_func_array(self::$hooks[$name], $params);
+        }
+    }
+}

src/Appwrite/Migration/Migration.php

@@ -77,7 +77,8 @@ abstract class Migration
         '1.4.10' => 'V19',
         '1.4.11' => 'V19',
         '1.4.12' => 'V19',
-        '1.4.13' => 'V19'
+        '1.4.13' => 'V19',
+        '1.4.14' => 'V20'
     ];
 
     /**

src/Appwrite/Migration/Version/V20.php

@@ -0,0 +1,323 @@
+<?php
+
+namespace Appwrite\Migration\Version;
+
+use Appwrite\Migration\Migration;
+use PDOException;
+use Throwable;
+use Utopia\CLI\Console;
+use Utopia\Database\Database;
+use Utopia\Database\Document;
+use Utopia\Database\Exception;
+use Utopia\Database\Exception\Authorization;
+use Utopia\Database\Exception\Duplicate;
+use Utopia\Database\Exception\Structure;
+use Utopia\Database\Query;
+
+class V20 extends Migration
+{
+    /**
+     * @throws Throwable
+     */
+    public function execute(): void
+    {
+        if ($this->project->getInternalId() == 'console') {
+            return;
+        }
+
+        /**
+         * Disable SubQueries for Performance.
+         */
+        foreach (['subQueryIndexes', 'subQueryPlatforms', 'subQueryDomains', 'subQueryKeys', 'subQueryWebhooks', 'subQuerySessions', 'subQueryTokens', 'subQueryMemberships', 'subQueryVariables'] as $name) {
+            Database::addFilter(
+                $name,
+                fn () => null,
+                fn () => []
+            );
+        }
+
+        $this->migrateUsageMetrics('project.$all.network.requests', 'network.requests');
+        $this->migrateUsageMetrics('project.$all.network.outbound', 'network.outbound');
+        $this->migrateUsageMetrics('project.$all.network.inbound', 'network.inbound');
+        $this->migrateUsageMetrics('users.$all.count.total', 'users');
+        $this->migrateSessionsMetric();
+
+        Console::log('Migrating Project: ' . $this->project->getAttribute('name') . ' (' . $this->project->getId() . ')');
+        $this->projectDB->setNamespace("_{$this->project->getInternalId()}");
+
+        Console::info('Migrating Functions');
+        $this->migrateFunctions();
+
+        Console::info('Migrating Databases');
+        $this->migrateDatabases();
+
+        Console::info('Migrating Collections');
+        $this->migrateCollections();
+
+        Console::info('Migrating Buckets');
+        $this->migrateBuckets();
+    }
+
+    /**
+     * @return void
+     * @throws Authorization
+     * @throws Exception
+     * @throws Structure
+     */
+    protected function migrateSessionsMetric(): void
+    {
+        /**
+         * Creating inf metric
+         */
+
+        Console::info('Migrating Sessions metric');
+
+        $sessionsCreated = $this->projectDB->sum('stats', 'value', [
+            Query::equal('metric', [
+                'sessions.email-password.requests.create',
+                'sessions.magic-url.requests.create',
+                'sessions.anonymous.requests.create',
+                'sessions.invites.requests.create',
+                'sessions.jwt.requests.create',
+                'sessions.phone.requests.create'
+            ]),
+            Query::equal('period', ['1d']),
+        ]);
+
+        $query = $this->projectDB->findOne('stats', [
+            Query::equal('metric', ['sessions.$all.requests.delete']),
+            Query::equal('period', ['1d']),
+        ]);
+
+        $sessionsDeleted =  $query['value'] ?? 0;
+        $value = $sessionsCreated - $sessionsDeleted;
+        $this->createInfMetric('sessions', $value);
+    }
+
+    /**
+     * @param string $metric
+     * @param int $value
+     * @return void
+     * @throws Exception
+     * @throws Authorization
+     * @throws Structure
+     */
+    protected function createInfMetric(string $metric, int $value): void
+    {
+
+        try {
+            /**
+             * Creating inf metric
+             */
+            console::log("Creating inf metric to {$metric}");
+            $id = \md5("_inf_{$metric}");
+            $this->projectDB->createDocument('stats', new Document([
+                '$id' => $id,
+                'metric' => $metric,
+                'period' => 'inf',
+                'value'  => $value,
+                'time'   => null,
+                'region' => 'default',
+            ]));
+        } catch (Duplicate $th) {
+            console::log("Error while creating inf metric: duplicate id {$metric}  {$id}");
+        }
+    }
+
+    /**
+     * @param string $from
+     * @param string $to
+     * @return void
+     * @throws Exception
+     */
+    protected function migrateUsageMetrics(string $from, string $to): void
+    {
+        /**
+         * inf metric
+         */
+        if (
+            str_contains($from, '$all') ||
+            str_contains($from, '.total')
+        ) {
+            $query = $this->projectDB->sum('stats', 'value', [
+                Query::equal('metric', [$from]),
+                Query::equal('period', ['1d']),
+            ]);
+
+            $value = $query ?? 0;
+            $this->createInfMetric($to, $value);
+        }
+
+        try {
+            /**
+             * Update old metric format to new
+             */
+            $limit = 1000;
+            $sum = $limit;
+            $total = 0;
+            $latestDocument = null;
+            while ($sum === $limit) {
+                $paginationQueries = [Query::limit($limit)];
+                if ($latestDocument !== null) {
+                    $paginationQueries[] =  Query::cursorAfter($latestDocument);
+                }
+                $stats = $this->projectDB->find('stats', \array_merge($paginationQueries, [
+                    Query::equal('metric', [$from]),
+                ]));
+
+                $sum = count($stats);
+                $total = $total + $sum;
+                foreach ($stats as $stat) {
+                    $format = $stat['period'] === '1d' ? 'Y-m-d 00:00' : 'Y-m-d H:00';
+                    $time = date($format, strtotime($stat['time']));
+                    $this->projectDB->deleteDocument('stats', $stat->getId());
+                    $stat->setAttribute('$id', \md5("{$time}_{$stat['period']}_{$to}"));
+                    $stat->setAttribute('metric', $to);
+                    $this->projectDB->createDocument('stats', $stat);
+                    console::log("deleting metric {$from} and creating {$to}");
+                }
+                $latestDocument = !empty(array_key_last($stats)) ? $stats[array_key_last($stats)] : null;
+            }
+        } catch (Throwable $th) {
+            Console::warning("Error while updating metric  {$from}  " . $th->getMessage());
+        }
+    }
+    /**
+     * Migrate functions.
+     *
+     * @return void
+     * @throws \Exception
+     */
+    private function migrateFunctions(): void
+    {
+
+        $this->migrateUsageMetrics('deployment.$all.storage.size', 'deployments.storage');
+        $this->migrateUsageMetrics('builds.$all.compute.total', 'builds');
+        $this->migrateUsageMetrics('builds.$all.compute.time', 'builds.compute');
+        $this->migrateUsageMetrics('executions.$all.compute.total', 'executions');
+        $this->migrateUsageMetrics('executions.$all.compute.time', 'executions.compute');
+
+        foreach ($this->documentsIterator('functions') as $function) {
+            Console::log("Migrating Functions usage stats of {$function->getId()} ({$function->getAttribute('name')})");
+
+            $functionId = $function->getId();
+            $functionInternalId = $function->getInternalId();
+
+            $this->migrateUsageMetrics("deployment.$functionId.storage.size", "function.$functionInternalId.deployments.storage");
+            $this->migrateUsageMetrics("builds.$functionId.compute.total", "$functionInternalId.builds");
+            $this->migrateUsageMetrics("builds.$functionId.compute.time", "$functionInternalId.builds.compute");
+            $this->migrateUsageMetrics("executions.$functionId.compute.total", "$functionInternalId.executions");
+            $this->migrateUsageMetrics("executions.$functionId.compute.time", "$functionInternalId.executions.compute");
+        }
+    }
+
+    /**
+     * Migrate  Databases.
+     *
+     * @return void
+     * @throws \Exception
+     */
+    private function migrateDatabases(): void
+    {
+        // Project level
+        $this->migrateUsageMetrics('databases.$all.count.total', 'databases');
+        $this->migrateUsageMetrics('collections.$all.count.total', 'collections');
+        $this->migrateUsageMetrics('documents.$all.count.total', 'documents');
+
+        foreach ($this->documentsIterator('databases') as $database) {
+            Console::log("Migrating Collections of {$database->getId()} ({$database->getAttribute('name')})");
+
+            $databaseTable = "database_{$database->getInternalId()}";
+
+            // Database level
+            $databaseId = $database->getId();
+            $databaseInternalId = $database->getInternalId();
+
+            $this->migrateUsageMetrics("collections.$databaseId.count.total", "$databaseInternalId.collections");
+            $this->migrateUsageMetrics("documents.$databaseId.count.total", "$databaseInternalId.documents");
+
+            foreach ($this->documentsIterator($databaseTable) as $collection) {
+                $collectionTable = "{$databaseTable}_collection_{$collection->getInternalId()}";
+                Console::log("Migrating Collections of {$collectionTable} {$collection->getId()} ({$collection->getAttribute('name')})");
+
+                // Collection level
+                $collectionId =  $collection->getId() ;
+                $collectionInternalId =  $collection->getInternalId();
+
+                $this->migrateUsageMetrics("documents.$databaseId/$collectionId.count.total", "$databaseInternalId.$collectionInternalId.documents");
+            }
+        }
+    }
+
+    /**
+     * Migrate Collections.
+     *
+     * @return void
+     * @throws \Exception
+     */
+    private function migrateCollections(): void
+    {
+        $internalProjectId = $this->project->getInternalId();
+        $collectionType = match ($internalProjectId) {
+            'console' => 'console',
+            default => 'projects',
+        };
+
+        $collections = $this->collections[$collectionType];
+
+        foreach ($collections as $collection) {
+            $id = $collection['$id'];
+
+            Console::log("Migrating Collection \"{$id}\"");
+
+            $this->projectDB->setNamespace("_$internalProjectId");
+
+            switch ($id) {
+                case 'stats':
+                    try {
+                        /**
+                         * Delete 'type' attribute
+                         */
+                        $this->projectDB->deleteAttribute($id, 'type');
+                        /**
+                         * Alter `signed`  internal type on `value` attr
+                         */
+                        $this->projectDB->updateAttribute($id, 'value', null, null, null, null, true);
+                        $this->projectDB->deleteCachedCollection($id);
+                    } catch (Throwable $th) {
+                        Console::warning("'type' from {$id}: {$th->getMessage()}");
+                    }
+                    break;
+            }
+        }
+    }
+
+    /**
+     * Migrating all Bucket tables.
+     *
+     * @return void
+     * @throws \Exception
+     * @throws PDOException
+     */
+    protected function migrateBuckets(): void
+    {
+        // Project level
+        $this->migrateUsageMetrics('buckets.$all.count.total', 'buckets');
+        $this->migrateUsageMetrics('files.$all.count.total', 'files');
+        $this->migrateUsageMetrics('files.$all.storage.size', 'files.storage');
+        // There is also project.$all.storage.size which is the same as  files.$all.storage.size
+
+        foreach ($this->documentsIterator('buckets') as $bucket) {
+            $id = "bucket_{$bucket->getInternalId()}";
+            Console::log("Migrating Bucket {$id} {$bucket->getId()} ({$bucket->getAttribute('name')})");
+
+            // Bucket level
+            $bucketId = $bucket->getId();
+            $bucketInternalId = $bucket->getInternalId();
+
+             $this->migrateUsageMetrics("files.$bucketId.count.total", "$bucketInternalId.files");
+            $this->migrateUsageMetrics("files.$bucketId.storage.size", "$bucketInternalId.files.storage");
+            // some stats come with $ prefix in front of the id -> files.$650c3fda307b7fec4934.storage.size;
+        }
+    }
+}

src/Appwrite/Network/Validator/CNAME.php

@@ -6,6 +6,11 @@ use Utopia\Validator;
 
 class CNAME extends Validator
 {
+    /**
+     * @var mixed
+     */
+    protected mixed $logs;
+
     /**
      * @var string
      */
@@ -27,6 +32,14 @@ class CNAME extends Validator
         return 'Invalid CNAME record';
     }
 
+    /**
+     * @return mixed
+     */
+    public function getLogs(): mixed
+    {
+        return $this->logs;
+    }
+
     /**
      * Check if CNAME record target value matches selected target
      *
@@ -42,6 +55,7 @@ class CNAME extends Validator
 
         try {
             $records = \dns_get_record($domain, DNS_CNAME);
+            $this->logs = $records;
         } catch (\Throwable $th) {
             return false;
         }

src/Appwrite/Platform/Services/Tasks.php

@@ -12,7 +12,6 @@ use Appwrite\Platform\Tasks\SDKs;
 use Appwrite\Platform\Tasks\Specs;
 use Appwrite\Platform\Tasks\SSL;
 use Appwrite\Platform\Tasks\Hamster;
-use Appwrite\Platform\Tasks\Usage;
 use Appwrite\Platform\Tasks\Vars;
 use Appwrite\Platform\Tasks\Version;
 use Appwrite\Platform\Tasks\VolumeSync;
@@ -21,6 +20,9 @@ use Appwrite\Platform\Tasks\Upgrade;
 use Appwrite\Platform\Tasks\DeleteOrphanedProjects;
 use Appwrite\Platform\Tasks\GetMigrationStats;
 use Appwrite\Platform\Tasks\PatchRecreateRepositoriesDocuments;
+use Appwrite\Platform\Tasks\QueueRetry;
+use Appwrite\Platform\Tasks\CreateInfMetric;
+use Appwrite\Platform\Tasks\QueueCount;
 
 class Tasks extends Service
 {
@@ -29,7 +31,6 @@ class Tasks extends Service
         $this->type = self::TYPE_CLI;
         $this
             ->addAction(Version::getName(), new Version())
-            ->addAction(Usage::getName(), new Usage())
             ->addAction(Vars::getName(), new Vars())
             ->addAction(SSL::getName(), new SSL())
             ->addAction(Hamster::getName(), new Hamster())
@@ -46,7 +47,9 @@ class Tasks extends Service
             ->addAction(DeleteOrphanedProjects::getName(), new DeleteOrphanedProjects())
             ->addAction(PatchRecreateRepositoriesDocuments::getName(), new PatchRecreateRepositoriesDocuments())
             ->addAction(GetMigrationStats::getName(), new GetMigrationStats())
-
+            ->addAction(QueueRetry::getName(), new QueueRetry())
+            ->addAction(QueueCount::getName(), new QueueCount())
+            ->addAction(CreateInfMetric::getName(), new CreateInfMetric())
         ;
     }
 }

src/Appwrite/Platform/Services/Workers.php

@@ -13,6 +13,8 @@ use Appwrite\Platform\Workers\Functions;
 use Appwrite\Platform\Workers\Builds;
 use Appwrite\Platform\Workers\Deletes;
 use Appwrite\Platform\Workers\Hamster;
+use Appwrite\Platform\Workers\Usage;
+use Appwrite\Platform\Workers\UsageDump;
 use Appwrite\Platform\Workers\Migrations;
 
 class Workers extends Service
@@ -30,8 +32,10 @@ class Workers extends Service
             ->addAction(Functions::getName(), new Functions())
             ->addAction(Builds::getName(), new Builds())
             ->addAction(Deletes::getName(), new Deletes())
-            ->addAction(Migrations::getName(), new Migrations())
             ->addAction(Hamster::getName(), new Hamster())
+            ->addAction(UsageDump::getName(), new UsageDump())
+            ->addAction(Usage::getName(), new Usage())
+            ->addAction(Migrations::getName(), new Migrations())
 
         ;
     }

src/Appwrite/Platform/Tasks/CalcTierStats.php

@@ -5,6 +5,7 @@ namespace Appwrite\Platform\Tasks;
 use Exception;
 use League\Csv\CannotInsertRecord;
 use Utopia\App;
+use Utopia\Database\Document;
 use Utopia\Database\Validator\Authorization;
 use Utopia\Platform\Action;
 use Utopia\Cache\Cache;
@@ -15,6 +16,7 @@ use League\Csv\Writer;
 use PHPMailer\PHPMailer\PHPMailer;
 use Utopia\Pools\Group;
 use Utopia\Registry\Registry;
+use Utopia\Validator\Text;
 
 class CalcTierStats extends Action
 {
@@ -24,6 +26,7 @@ class CalcTierStats extends Action
     private array $columns = [
         'Project ID',
         'Organization ID',
+        'Organization Email',
         'Organization Members',
         'Teams',
         'Users',
@@ -49,8 +52,9 @@ class CalcTierStats extends Action
     protected string $date;
 
     private array $usageStats = [
-        'project.$all.network.requests'  => 'Requests',
-        'project.$all.network.bandwidth' => 'Bandwidth',
+        'network.requests'  => 'Requests',
+        'network.inbound'   => 'Inbound',
+        'network.outbound'  => 'Outbound',
 
     ];
 
@@ -64,270 +68,115 @@ class CalcTierStats extends Action
 
         $this
             ->desc('Get stats for projects')
+            ->param('after', '', new Text(36), 'After cursor', true)
+            ->param('projectId', '', new Text(36), 'Select project to validate', true)
             ->inject('pools')
             ->inject('cache')
             ->inject('dbForConsole')
+            ->inject('getProjectDB')
             ->inject('register')
-            ->callback(function (Group $pools, Cache $cache, Database $dbForConsole, Registry $register) {
-                $this->action($pools, $cache, $dbForConsole, $register);
+            ->callback(function ($after, $projectId, Group $pools, Cache $cache, Database $dbForConsole, callable $getProjectDB, Registry $register) {
+                $this->action($after, $projectId, $pools, $cache, $dbForConsole, $getProjectDB, $register);
             });
     }
 
-    /**
-     * @throws \Utopia\Exception
-     * @throws CannotInsertRecord
-     */
-    public function action(Group $pools, Cache $cache, Database $dbForConsole, Registry $register): void
+
+    public function action(string $after, string $projectId, Group $pools, Cache $cache, Database $dbForConsole, callable $getProjectDB, Registry $register): void
     {
         //docker compose exec -t appwrite calc-tier-stats
 
         Console::title('Cloud free tier  stats calculation V1');
         Console::success(APP_NAME . ' cloud free tier  stats calculation has started');
 
-        /* Initialise new Utopia app */
-        $app = new App('UTC');
-        $console = $app->getResource('console');
-
         /** CSV stuff */
         $this->date = date('Y-m-d');
         $this->path = "{$this->directory}/tier_stats_{$this->date}.csv";
         $csv = Writer::createFromPath($this->path, 'w');
         $csv->insertOne($this->columns);
 
-        /** Database connections */
-        $totalProjects = $dbForConsole->count('projects');
-        Console::success("Found a total of: {$totalProjects} projects");
+        if (!empty($projectId)) {
+            try {
+                console::log("Project " . $projectId);
+                $project = $dbForConsole->getDocument('projects', $projectId);
+                $dbForProject = call_user_func($getProjectDB, $project);
+                $data = $this->getData($project, $dbForConsole, $dbForProject);
+                $csv->insertOne($data);
+                $this->sendMail($register);
 
-        $projects = [$console];
-        $count = 0;
-        $limit = 100;
-        $sum = 100;
-        $offset = 0;
-        while (!empty($projects)) {
-            foreach ($projects as $project) {
-
-                /**
-                 * Skip user projects with id 'console'
-                 */
-                if ($project->getId() === 'console') {
-                    continue;
-                }
-
-                Console::info("Getting stats for {$project->getId()}");
-
-                try {
-                    $db = $project->getAttribute('database');
-                    $adapter = $pools
-                        ->get($db)
-                        ->pop()
-                        ->getResource();
-
-                    $dbForProject = new Database($adapter, $cache);
-                    $dbForProject->setDatabase('appwrite');
-                    $dbForProject->setNamespace('_' . $project->getInternalId());
-
-                    /** Get Project ID */
-                    $stats['Project ID'] = $project->getId();
-
-                    $stats['Organization ID']   = $project->getAttribute('teamId', null);
-
-                    /** Get Total Members */
-                    $teamInternalId = $project->getAttribute('teamInternalId', null);
-                    if ($teamInternalId) {
-                        $stats['Organization Members'] = $dbForConsole->count('memberships', [
-                            Query::equal('teamInternalId', [$teamInternalId])
-                        ]);
-                    } else {
-                        $stats['Organization Members'] = 0;
-                    }
-
-                    /** Get Total internal Teams */
-                    try {
-                        $stats['Teams'] = $dbForProject->count('teams', []);
-                    } catch (\Throwable) {
-                        $stats['Teams'] = 0;
-                    }
-
-                    /** Get Total users */
-                    try {
-                        $stats['Users'] = $dbForProject->count('users', []);
-                    } catch (\Throwable) {
-                        $stats['Users'] = 0;
-                    }
-
-                    /** Get Usage stats */
-                    $range = '30d';
-                    $periods = [
-                        '30d' => [
-                            'period' => '1d',
-                            'limit' => 30,
-                        ]
-                    ];
-
-                    $tmp = [];
-                    $metrics = $this->usageStats;
-                    Authorization::skip(function () use ($dbForProject, $periods, $range, $metrics, &$tmp) {
-                        foreach ($metrics as $metric => $name) {
-                            $limit = $periods[$range]['limit'];
-                            $period = $periods[$range]['period'];
-
-                            $requestDocs = $dbForProject->find('stats', [
-                                Query::equal('period', [$period]),
-                                Query::equal('metric', [$metric]),
-                                Query::limit($limit),
-                                Query::orderDesc('time'),
-                            ]);
-
-                            $tmp[$metric] = [];
-                            foreach ($requestDocs as $requestDoc) {
-                                if (empty($requestDoc)) {
-                                    continue;
-                                }
-
-                                $tmp[$metric][] = [
-                                    'value' => $requestDoc->getAttribute('value'),
-                                    'date' => $requestDoc->getAttribute('time'),
-                                ];
-                            }
-
-                            $tmp[$metric] = array_reverse($tmp[$metric]);
-                            $tmp[$metric] = array_sum(array_column($tmp[$metric], 'value'));
-                        }
-                    });
-
-                    foreach ($tmp as $key => $value) {
-                        $stats[$metrics[$key]]  = $value;
-                    }
-
-                    try {
-                        /** Get Domains */
-                        $stats['Domains'] = $dbForConsole->count('rules', [
-                            Query::equal('projectInternalId', [$project->getInternalId()]),
-                        ]);
-                    } catch (\Throwable) {
-                        $stats['Domains'] = 0;
-                    }
-
-                    try {
-                    /** Get Api keys */
-                        $stats['Api keys'] = $dbForConsole->count('keys', [
-                        Query::equal('projectInternalId', [$project->getInternalId()]),
-                        ]);
-                    } catch (\Throwable) {
-                        $stats['Api keys'] = 0;
-                    }
-
-                    try {
-                    /** Get Webhooks */
-                        $stats['Webhooks'] = $dbForConsole->count('webhooks', [
-                        Query::equal('projectInternalId', [$project->getInternalId()]),
-                        ]);
-                    } catch (\Throwable) {
-                        $stats['Webhooks'] = 0;
-                    }
-
-                    try {
-                    /** Get Platforms */
-                        $stats['Platforms'] = $dbForConsole->count('platforms', [
-                        Query::equal('projectInternalId', [$project->getInternalId()]),
-                        ]);
-                    } catch (\Throwable) {
-                        $stats['Platforms'] = 0;
-                    }
-
-                    /** Get Files & Buckets */
-                    $filesCount = 0;
-                    $filesSum = 0;
-                    $maxFileSize = 0;
-                    $counter = 0;
-                    try {
-                        $buckets = $dbForProject->find('buckets', []);
-                        foreach ($buckets as $bucket) {
-                            $file = $dbForProject->findOne('bucket_' . $bucket->getInternalId(), [Query::orderDesc('sizeOriginal'),]);
-                            if (empty($file)) {
-                                continue;
-                            }
-                            $filesSum   += $dbForProject->sum('bucket_' . $bucket->getInternalId(), 'sizeOriginal', []);
-                            $filesCount += $dbForProject->count('bucket_' . $bucket->getInternalId(), []);
-                            if ($file->getAttribute('sizeOriginal') > $maxFileSize) {
-                                $maxFileSize = $file->getAttribute('sizeOriginal');
-                            }
-                            $counter++;
-                        }
-                    } catch (\Throwable) {
-                        ;
-                    }
-                    $stats['Buckets'] = $counter;
-                    $stats['Files'] = $filesCount;
-                    $stats['Storage (bytes)'] = $filesSum;
-                    $stats['Max File Size (bytes)'] = $maxFileSize;
-
-
-                    try {
-                    /** Get Total Functions */
-                        $stats['Databases'] = $dbForProject->count('databases', []);
-                    } catch (\Throwable) {
-                        $stats['Databases'] = 0;
-                    }
-
-                    /** Get Total Functions */
-                    try {
-                        $stats['Functions'] = $dbForProject->count('functions', []);
-                    } catch (\Throwable) {
-                        $stats['Functions'] = 0;
-                    }
-
-                    /** Get Total Deployments */
-                    try {
-                        $stats['Deployments'] = $dbForProject->count('deployments', []);
-                    } catch (\Throwable) {
-                        $stats['Deployments'] = 0;
-                    }
-
-                    /** Get Total Executions */
-                    try {
-                        $stats['Executions'] = $dbForProject->count('executions', []);
-                    } catch (\Throwable) {
-                        $stats['Executions'] = 0;
-                    }
-
-                    /** Get Total Migrations */
-                    try {
-                        $stats['Migrations'] = $dbForProject->count('migrations', []);
-                    } catch (\Throwable) {
-                        $stats['Migrations'] = 0;
-                    }
-
-                    $csv->insertOne(array_values($stats));
-                } catch (\Throwable $th) {
-                    Console::error('Failed on project ("' . $project->getId() . '") version with error on File: ' . $th->getFile() . '  line no: ' . $th->getline() . ' with message: ' . $th->getMessage());
-                } finally {
-                    $pools
-                        ->get($db)
-                        ->reclaim();
-                }
+                return;
+            } catch (\Throwable $th) {
+                Console::error("Unexpected error occured with Project ID {$projectId}");
+                Console::error('[Error] Type: ' . get_class($th));
+                Console::error('[Error] Message: ' . $th->getMessage());
+                Console::error('[Error] File: ' . $th->getFile());
+                Console::error('[Error] Line: ' . $th->getLine());
             }
-
-            $sum = \count($projects);
-
-            $projects = $dbForConsole->find('projects', [
-                Query::limit($limit),
-                Query::offset($offset),
-            ]);
-
-            $offset = $offset + $limit;
-            $count = $count + $sum;
         }
 
-        Console::log('Iterated through ' . $count - 1 . '/' . $totalProjects . ' projects...');
+        $queries = [];
 
-        $pools
-            ->get('console')
-            ->reclaim();
+        if (!empty($after)) {
+            Console::info("Iterating remaining projects after project with ID {$after}");
+            $project = $dbForConsole->getDocument('projects', $after);
+            $queries = [Query::cursorAfter($project)];
+        } else {
+            Console::info("Iterating all projects");
+        }
 
+        $this->foreachDocument($dbForConsole, 'projects', $queries, function (Document $project) use ($getProjectDB, $dbForConsole, $csv) {
+            $projectId = $project->getId();
+            console::log("Project " . $projectId);
+            try {
+                $dbForProject = call_user_func($getProjectDB, $project);
+                $data = $this->getData($project, $dbForConsole, $dbForProject);
+                $csv->insertOne($data);
+            } catch (\Throwable $th) {
+                Console::error("Unexpected error occured with Project ID {$projectId}");
+                Console::error('[Error] Type: ' . get_class($th));
+                Console::error('[Error] Message: ' . $th->getMessage());
+                Console::error('[Error] File: ' . $th->getFile());
+                Console::error('[Error] Line: ' . $th->getLine());
+            }
+        });
+
+        $this->sendMail($register);
+    }
+
+    private function foreachDocument(Database $database, string $collection, array $queries = [], callable $callback = null): void
+    {
+        $limit = 1000;
+        $results = [];
+        $sum = $limit;
+        $latestDocument = null;
+
+        while ($sum === $limit) {
+            $newQueries = $queries;
+
+            if ($latestDocument != null) {
+                array_unshift($newQueries, Query::cursorAfter($latestDocument));
+            }
+            $newQueries[] = Query::limit($limit);
+            $results = $database->find('projects', $newQueries);
+
+            if (empty($results)) {
+                return;
+            }
+
+            $sum = count($results);
+
+            foreach ($results as $document) {
+                if (is_callable($callback)) {
+                    $callback($document);
+                }
+            }
+            $latestDocument = $results[array_key_last($results)];
+        }
+    }
+
+    private function sendMail(Registry $register): void
+    {
         /** @var PHPMailer $mail */
         $mail = $register->get('smtp');
-
         $mail->clearAddresses();
         $mail->clearAllRecipients();
         $mail->clearReplyTos();
@@ -339,7 +188,6 @@ class CalcTierStats extends Action
             /** Addresses */
             $mail->setFrom(App::getEnv('_APP_SYSTEM_EMAIL_ADDRESS', APP_EMAIL_TEAM), 'Appwrite Cloud Hamster');
             $recipients = explode(',', App::getEnv('_APP_USERS_STATS_RECIPIENTS', ''));
-
             foreach ($recipients as $recipient) {
                 $mail->addAddress($recipient);
             }
@@ -356,4 +204,206 @@ class CalcTierStats extends Action
             Console::error("Message could not be sent. Mailer Error: {$mail->ErrorInfo}");
         }
     }
+
+
+    private function getData(Document $project, Database $dbForConsole, Database $dbForProject): array
+    {
+        $stats['Project ID'] = $project->getId();
+        $stats['Organization ID']   = $project->getAttribute('teamId', null);
+
+        $teamInternalId = $project->getAttribute('teamInternalId', null);
+
+        if ($teamInternalId) {
+            $membership = $dbForConsole->findOne('memberships', [
+                Query::equal('teamInternalId', [$teamInternalId]),
+            ]);
+
+            if (!$membership || $membership->isEmpty()) {
+                Console::error('Membership not found. Skipping project : ' . $project->getId());
+            }
+
+            $userId = $membership->getAttribute('userId', null);
+            if ($userId) {
+                $user = $dbForConsole->getDocument('users', $userId);
+                $stats['Organization Email'] = $user->getAttribute('email', null);
+            }
+        } else {
+            Console::error("Email was not found for this Organization ID :{$teamInternalId}");
+        }
+
+        /** Get Total Members */
+        if ($teamInternalId) {
+            $stats['Organization Members'] = $dbForConsole->count('memberships', [
+                Query::equal('teamInternalId', [$teamInternalId])
+            ]);
+        } else {
+            $stats['Organization Members'] = 0;
+        }
+
+        /** Get Total internal Teams */
+        try {
+            $stats['Teams'] = $dbForProject->count('teams', []);
+        } catch (\Throwable) {
+            $stats['Teams'] = 0;
+        }
+
+        /** Get Total users */
+        try {
+            $stats['Users'] = $dbForProject->count('users', []);
+        } catch (\Throwable) {
+            $stats['Users'] = 0;
+        }
+
+        /** Get Usage stats */
+        $range = '30d';
+        $periods = [
+            '30d' => [
+                'period' => '1d',
+                'limit' => 30,
+            ]
+        ];
+
+        $tmp = [];
+        $metrics = $this->usageStats;
+        Authorization::skip(function () use ($dbForProject, $periods, $range, $metrics, &$tmp) {
+            foreach ($metrics as $metric => $name) {
+                $limit = $periods[$range]['limit'];
+                $period = $periods[$range]['period'];
+
+                $requestDocs = $dbForProject->find('stats_v2', [
+                    Query::equal('metric', [$metric]),
+                    Query::equal('period', [$period]),
+                    Query::limit($limit),
+                    Query::orderDesc('time'),
+                ]);
+
+                $tmp[$metric] = [];
+                foreach ($requestDocs as $requestDoc) {
+                    if (empty($requestDoc)) {
+                        continue;
+                    }
+
+                    $tmp[$metric][] = [
+                        'value' => $requestDoc->getAttribute('value'),
+                        'date' => $requestDoc->getAttribute('time'),
+                    ];
+                }
+
+                $tmp[$metric] = array_reverse($tmp[$metric]);
+                $tmp[$metric] = array_sum(array_column($tmp[$metric], 'value'));
+            }
+        });
+
+        foreach ($tmp as $key => $value) {
+            $stats[$metrics[$key]]  = $value;
+        }
+
+        /**
+         * Workaround to combine network.inbound+network.outbound as network.
+         */
+        $stats['Bandwidth'] = ($stats['Inbound'] ?? 0) + ($stats['Outbound'] ?? 0);
+        unset($stats['Inbound']);
+        unset($stats['Outbound']);
+
+        try {
+            /** Get Domains */
+            $stats['Domains'] = $dbForConsole->count('rules', [
+                Query::equal('projectInternalId', [$project->getInternalId()]),
+            ]);
+        } catch (\Throwable) {
+            $stats['Domains'] = 0;
+        }
+
+        try {
+            /** Get Api keys */
+            $stats['Api keys'] = $dbForConsole->count('keys', [
+                Query::equal('projectInternalId', [$project->getInternalId()]),
+            ]);
+        } catch (\Throwable) {
+            $stats['Api keys'] = 0;
+        }
+
+        try {
+            /** Get Webhooks */
+            $stats['Webhooks'] = $dbForConsole->count('webhooks', [
+                Query::equal('projectInternalId', [$project->getInternalId()]),
+            ]);
+        } catch (\Throwable) {
+            $stats['Webhooks'] = 0;
+        }
+
+        try {
+            /** Get Platforms */
+            $stats['Platforms'] = $dbForConsole->count('platforms', [
+                Query::equal('projectInternalId', [$project->getInternalId()]),
+            ]);
+        } catch (\Throwable) {
+            $stats['Platforms'] = 0;
+        }
+
+        /** Get Files & Buckets */
+        $filesCount = 0;
+        $filesSum = 0;
+        $maxFileSize = 0;
+        $counter = 0;
+        try {
+            $buckets = $dbForProject->find('buckets', []);
+            foreach ($buckets as $bucket) {
+                $file = $dbForProject->findOne('bucket_' . $bucket->getInternalId(), [Query::orderDesc('sizeOriginal'),]);
+                if (empty($file)) {
+                    continue;
+                }
+                $filesSum   += $dbForProject->sum('bucket_' . $bucket->getInternalId(), 'sizeOriginal', []);
+                $filesCount += $dbForProject->count('bucket_' . $bucket->getInternalId(), []);
+                if ($file->getAttribute('sizeOriginal') > $maxFileSize) {
+                    $maxFileSize = $file->getAttribute('sizeOriginal');
+                }
+                $counter++;
+            }
+        } catch (\Throwable $t) {
+            Console::error("Error while counting buckets: {$project->getId()}");
+        }
+        $stats['Buckets'] = $counter;
+        $stats['Files'] = $filesCount;
+        $stats['Storage (bytes)'] = $filesSum;
+        $stats['Max File Size (bytes)'] = $maxFileSize;
+
+
+        try {
+            /** Get Total Functions */
+            $stats['Databases'] = $dbForProject->count('databases', []);
+        } catch (\Throwable) {
+            $stats['Databases'] = 0;
+        }
+
+        /** Get Total Functions */
+        try {
+            $stats['Functions'] = $dbForProject->count('functions', []);
+        } catch (\Throwable) {
+            $stats['Functions'] = 0;
+        }
+
+        /** Get Total Deployments */
+        try {
+            $stats['Deployments'] = $dbForProject->count('deployments', []);
+        } catch (\Throwable) {
+            $stats['Deployments'] = 0;
+        }
+
+        /** Get Total Executions */
+        try {
+            $stats['Executions'] = $dbForProject->count('executions', []);
+        } catch (\Throwable) {
+            $stats['Executions'] = 0;
+        }
+
+        /** Get Total Migrations */
+        try {
+            $stats['Migrations'] = $dbForProject->count('migrations', []);
+        } catch (\Throwable) {
+            $stats['Migrations'] = 0;
+        }
+
+        return array_values($stats);
+    }
 }

src/Appwrite/Platform/Tasks/CreateInfMetric.php

@@ -0,0 +1,413 @@
+<?php
+
+namespace Appwrite\Platform\Tasks;
+
+use Utopia\App;
+use Utopia\Database\Document;
+use Utopia\Database\Exception;
+use Utopia\Database\Exception\Duplicate;
+use Utopia\Database\Query;
+use Utopia\Platform\Action;
+use Utopia\Cache\Cache;
+use Utopia\CLI\Console;
+use Utopia\Database\Database;
+use Utopia\Pools\Group;
+use Utopia\Registry\Registry;
+use Utopia\Validator\Text;
+
+class CreateInfMetric extends Action
+{
+    public static function getName(): string
+    {
+        return 'create-inf-metric';
+    }
+
+    public function __construct()
+    {
+
+        $this
+            ->desc('Create infinity stats metric')
+            ->param('after', '', new Text(36), 'After cursor', true)
+            ->param('projectId', '', new Text(36), 'Select project to validate', true)
+            ->inject('getProjectDB')
+            ->inject('dbForConsole')
+            ->callback(function (string $after, string $projectId, callable $getProjectDB, Database $dbForConsole) {
+                $this->action($after, $projectId, $getProjectDB, $dbForConsole);
+            });
+    }
+
+
+    /**
+     * @throws Exception
+     * @throws Exception\Timeout
+     * @throws Exception\Query
+     */
+    public function action(string $after, string $projectId, callable $getProjectDB, Database $dbForConsole): void
+    {
+
+        Console::title('Create infinity metric V1');
+        Console::success(APP_NAME . ' Create infinity metric started');
+
+        if (!empty($projectId)) {
+            try {
+                $project = $dbForConsole->getDocument('projects', $projectId);
+                $dbForProject = call_user_func($getProjectDB, $project);
+                $this->getUsageData($dbForProject, $project);
+            } catch (\Throwable $th) {
+                Console::error("Unexpected error occured with Project ID {$projectId}");
+                Console::error('[Error] Type: ' . get_class($th));
+                Console::error('[Error] Message: ' . $th->getMessage());
+                Console::error('[Error] File: ' . $th->getFile());
+                Console::error('[Error] Line: ' . $th->getLine());
+            }
+        } else {
+            $queries = [];
+            if (!empty($after)) {
+                Console::info("Iterating remaining projects after project with ID {$after}");
+                $project = $dbForConsole->getDocument('projects', $after);
+                $queries = [Query::cursorAfter($project)];
+            } else {
+                Console::info("Iterating all projects");
+            }
+            $this->foreachDocument($dbForConsole, 'projects', $queries, function (Document $project) use ($getProjectDB) {
+                $projectId = $project->getId();
+
+                try {
+                    $dbForProject = call_user_func($getProjectDB, $project);
+                    $this->getUsageData($dbForProject, $project);
+                } catch (\Throwable $th) {
+                    Console::error("Unexpected error occured with Project ID {$projectId}");
+                    Console::error('[Error] Type: ' . get_class($th));
+                    Console::error('[Error] Message: ' . $th->getMessage());
+                    Console::error('[Error] File: ' . $th->getFile());
+                    Console::error('[Error] Line: ' . $th->getLine());
+                }
+            });
+        }
+    }
+
+    /**
+     * @param Database $database
+     * @param string $collection
+     * @param array $queries
+     * @param callable|null $callback
+     * @return void
+     * @throws Exception
+     * @throws Exception\Query
+     * @throws Exception\Timeout
+     */
+    private function foreachDocument(Database $database, string $collection, array $queries = [], callable $callback = null): void
+    {
+        $limit = 1000;
+        $results = [];
+        $sum = $limit;
+        $latestDocument = null;
+
+        while ($sum === $limit) {
+            $newQueries = $queries;
+
+            if ($latestDocument != null) {
+                array_unshift($newQueries, Query::cursorAfter($latestDocument));
+            }
+            $newQueries[] = Query::limit($limit);
+            $results = $database->find($collection, $newQueries);
+
+            if (empty($results)) {
+                return;
+            }
+
+            $sum = count($results);
+
+            foreach ($results as $document) {
+                if (is_callable($callback)) {
+                    $callback($document);
+                }
+            }
+            $latestDocument = $results[array_key_last($results)];
+        }
+    }
+
+
+    /**
+     * @param Database $dbForProject
+     * @param Document $project
+     * @return void
+     */
+    private function getUsageData(Database $dbForProject, Document $project): void
+    {
+        try {
+            $this->network($dbForProject);
+            $this->sessions($dbForProject);
+            $this->users($dbForProject);
+            $this->teams($dbForProject);
+            $this->databases($dbForProject);
+            $this->functions($dbForProject);
+            $this->storage($dbForProject);
+        } catch (\Throwable $th) {
+            var_dump($th->getMessage());
+        }
+
+        Console::log('Finished project ' . $project->getId() . ' ' . $project->getInternalId());
+    }
+
+
+    /**
+     * @param Database $dbForProject
+     * @param string $metric
+     * @param int|float $value
+     * @return void
+     * @throws Exception
+     * @throws Exception\Authorization
+     * @throws Exception\Conflict
+     * @throws Exception\Restricted
+     * @throws Exception\Structure
+     */
+    private function createInfMetric(database $dbForProject, string $metric, int|float $value): void
+    {
+
+        try {
+            $id = \md5("_inf_{$metric}");
+            $dbForProject->deleteDocument('stats_v2', $id);
+            $dbForProject->createDocument('stats_v2', new Document([
+                '$id' => $id,
+                'metric' => $metric,
+                'period' => 'inf',
+                'value'  => (int)$value,
+                'time'   => null,
+                'region' => 'default',
+            ]));
+        } catch (Duplicate $th) {
+            console::log("Error while creating inf metric: duplicate id {$metric}  {$id}");
+        }
+    }
+
+    /**
+     * @param Database $dbForProject
+     * @param string $metric
+     * @return int|float
+     * @throws Exception
+     */
+    protected function getFromMetric(database $dbForProject, string $metric): int|float
+    {
+
+        return  $dbForProject->sum('stats_v2', 'value', [
+            Query::equal('metric', [
+                $metric,
+            ]),
+            Query::equal('period', ['1d']),
+        ]);
+    }
+
+    /**
+     * @param Database $dbForProject
+     * @throws Exception
+     * @throws Exception\Authorization
+     * @throws Exception\Conflict
+     * @throws Exception\Restricted
+     * @throws Exception\Structure
+     */
+    private function network(database $dbForProject)
+    {
+        $this->createInfMetric($dbForProject, 'network.inbound', $this->getFromMetric($dbForProject, 'network.inbound'));
+        $this->createInfMetric($dbForProject, 'network.outbound', $this->getFromMetric($dbForProject, 'network.outbound'));
+        $this->createInfMetric($dbForProject, 'network.requests', $this->getFromMetric($dbForProject, 'network.requests'));
+    }
+
+
+    /**
+     * @throws Exception\Authorization
+     * @throws Exception\Restricted
+     * @throws Exception\Conflict
+     * @throws Exception\Timeout
+     * @throws Exception\Structure
+     * @throws Exception
+     * @throws Exception\Query
+     */
+    private function storage(database $dbForProject)
+    {
+        $bucketsCount = 0;
+        $filesCount = 0;
+        $filesStorageSum = 0;
+
+        $buckets = $dbForProject->find('buckets');
+        foreach ($buckets as $bucket) {
+            $files = $dbForProject->count('bucket_' . $bucket->getInternalId());
+            $this->createInfMetric($dbForProject, $bucket->getInternalId() . '.files', $files);
+
+            $filesStorage = $dbForProject->sum('bucket_' . $bucket->getInternalId(), 'sizeOriginal');
+            $this->createInfMetric($dbForProject, $bucket->getInternalId() . '.files.storage', $filesStorage);
+
+            $bucketsCount++;
+            $filesCount += $files;
+            $filesStorageSum += $filesStorage;
+        }
+
+        $this->createInfMetric($dbForProject, 'buckets', $bucketsCount);
+        $this->createInfMetric($dbForProject, 'files', $filesCount);
+        $this->createInfMetric($dbForProject, 'files.storage', $filesStorageSum);
+    }
+
+
+    /**
+     * @throws Exception\Authorization
+     * @throws Exception\Timeout
+     * @throws Exception\Restricted
+     * @throws Exception\Structure
+     * @throws Exception\Conflict
+     * @throws Exception
+     * @throws Exception\Query
+     */
+    private function functions(Database $dbForProject)
+    {
+        $functionsCount = 0;
+        $deploymentsCount = 0;
+        $buildsCount = 0;
+        $buildsStorageSum = 0;
+        $buildsComputeSum = 0;
+        $executionsCount = 0;
+        $executionsComputeSum = 0;
+        $deploymentsStorageSum = 0;
+
+        //functions
+        $functions = $dbForProject->find('functions');
+        foreach ($functions as $function) {
+            //deployments
+            $deployments = $dbForProject->find('deployments', [
+                Query::equal('resourceType', ['functions']),
+                Query::equal('resourceInternalId', [$function->getInternalId()]),
+            ]);
+
+            $deploymentCount = 0;
+            $deploymentStorageSum = 0;
+            foreach ($deployments as $deployment) {
+                //builds
+                $builds = $dbForProject->count('builds', [
+                    Query::equal('deploymentInternalId', [$deployment->getInternalId()]),
+                ]);
+
+                $buildsCompute = $dbForProject->sum('builds', 'duration', [
+                    Query::equal('deploymentInternalId', [$deployment->getInternalId()]),
+                ]);
+
+                $buildsStorage = $dbForProject->sum('builds', 'size', [
+                    Query::equal('deploymentInternalId', [$deployment->getInternalId()]),
+                ]);
+
+                $this->createInfMetric($dbForProject, $function->getInternalId() . '.builds', $builds);
+                $this->createInfMetric($dbForProject, $function->getInternalId() . '.builds.storage', $buildsCompute * 1000);
+                $this->createInfMetric($dbForProject, $function->getInternalId() . '.builds.compute', $buildsStorage);
+
+                $buildsCount += $builds;
+                $buildsComputeSum += $buildsCompute;
+                $buildsStorageSum += $buildsStorage;
+
+
+                $deploymentCount++;
+                $deploymentsCount++;
+                $deploymentsStorageSum += $deployment['size'];
+                $deploymentStorageSum += $deployment['size'];
+            }
+            $this->createInfMetric($dbForProject, 'functions.' . $function->getInternalId() . '.deployments', $deploymentCount);
+            $this->createInfMetric($dbForProject, 'functions.' . $function->getInternalId() . '.deployments.storage', $deploymentStorageSum);
+
+            //executions
+            $executions = $dbForProject->count('executions', [
+                Query::equal('functionInternalId', [$function->getInternalId()]),
+            ]);
+
+            $executionsCompute = $dbForProject->sum('executions', 'duration', [
+                Query::equal('functionInternalId', [$function->getInternalId()]),
+            ]);
+
+            $this->createInfMetric($dbForProject, $function->getInternalId() . '.executions', $executions);
+            $this->createInfMetric($dbForProject, $function->getInternalId() . '.executions.compute', $executionsCompute * 1000);
+            $executionsCount += $executions;
+            $executionsComputeSum += $executionsCompute;
+
+            $functionsCount++;
+        }
+
+        $this->createInfMetric($dbForProject, 'functions', $functionsCount);
+        $this->createInfMetric($dbForProject, 'deployments', $deploymentsCount);
+        $this->createInfMetric($dbForProject, 'deployments.storage', $deploymentsStorageSum);
+        $this->createInfMetric($dbForProject, 'builds', $buildsCount);
+        $this->createInfMetric($dbForProject, 'builds.compute', $buildsComputeSum * 1000);
+        $this->createInfMetric($dbForProject, 'builds.storage', $buildsStorageSum);
+        $this->createInfMetric($dbForProject, 'executions', $executionsCount);
+        $this->createInfMetric($dbForProject, 'executions.compute', $executionsComputeSum * 1000);
+    }
+
+    /**
+     * @throws Exception\Authorization
+     * @throws Exception\Timeout
+     * @throws Exception\Structure
+     * @throws Exception\Restricted
+     * @throws Exception\Conflict
+     * @throws Exception
+     * @throws Exception\Query
+     */
+    private function databases(Database $dbForProject)
+    {
+        $databasesCount = 0;
+        $collectionsCount = 0;
+        $documentsCount = 0;
+        $databases = $dbForProject->find('databases');
+        foreach ($databases as $database) {
+            $collectionCount = 0;
+            $collections = $dbForProject->find('database_' . $database->getInternalId());
+            foreach ($collections as $collection) {
+                $documents = $dbForProject->count('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId());
+                $this->createInfMetric($dbForProject, $database->getInternalId() . '.' . $collection->getInternalId() . '.documents', $documents);
+                $documentsCount += $documents;
+                $collectionCount++;
+                $collectionsCount++;
+            }
+            $this->createInfMetric($dbForProject, $database->getInternalId() . '.collections', $collectionCount);
+            $this->createInfMetric($dbForProject, $database->getInternalId() . '.documents', $documentsCount);
+            $databasesCount++;
+        }
+        $this->createInfMetric($dbForProject, 'collections', $collectionsCount);
+        $this->createInfMetric($dbForProject, 'databases', $databasesCount);
+        $this->createInfMetric($dbForProject, 'documents', $documentsCount);
+    }
+
+
+    /**
+     * @throws Exception\Authorization
+     * @throws Exception\Structure
+     * @throws Exception\Restricted
+     * @throws Exception\Conflict
+     * @throws Exception
+     */
+    private function users(Database $dbForProject)
+    {
+        $users = $dbForProject->count('users');
+        $this->createInfMetric($dbForProject, 'users', $users);
+    }
+
+    /**
+     * @throws Exception\Authorization
+     * @throws Exception\Structure
+     * @throws Exception\Restricted
+     * @throws Exception\Conflict
+     * @throws Exception
+     */
+    private function sessions(Database $dbForProject)
+    {
+        $users = $dbForProject->count('sessions');
+        $this->createInfMetric($dbForProject, 'sessions', $users);
+    }
+
+    /**
+     * @throws Exception\Authorization
+     * @throws Exception\Structure
+     * @throws Exception\Restricted
+     * @throws Exception\Conflict
+     * @throws Exception
+     */
+    private function teams(Database $dbForProject)
+    {
+        $teams = $dbForProject->count('teams');
+        $this->createInfMetric($dbForProject, 'teams', $teams);
+    }
+}

src/Appwrite/Platform/Tasks/Install.php

@@ -8,6 +8,7 @@ use Appwrite\Docker\Env;
 use Appwrite\Utopia\View;
 use Utopia\CLI\Console;
 use Utopia\Config\Config;
+use Utopia\Validator\Boolean;
 use Utopia\Validator\Text;
 use Utopia\Platform\Action;
 
@@ -24,15 +25,16 @@ class Install extends Action
     {
         $this
             ->desc('Install Appwrite')
-            ->param('httpPort', '', new Text(4), 'Server HTTP port', true)
-            ->param('httpsPort', '', new Text(4), 'Server HTTPS port', true)
+            ->param('http-port', '', new Text(4), 'Server HTTP port', true)
+            ->param('https-port', '', new Text(4), 'Server HTTPS port', true)
             ->param('organization', 'appwrite', new Text(0), 'Docker Registry organization', true)
             ->param('image', 'appwrite', new Text(0), 'Main appwrite docker image', true)
             ->param('interactive', 'Y', new Text(1), 'Run an interactive session', true)
-            ->callback(fn ($httpPort, $httpsPort, $organization, $image, $interactive) => $this->action($httpPort, $httpsPort, $organization, $image, $interactive));
+            ->param('no-start', false, new Boolean(true), 'Run an interactive session', true)
+            ->callback(fn ($httpPort, $httpsPort, $organization, $image, $interactive, $noStart) => $this->action($httpPort, $httpsPort, $organization, $image, $interactive, $noStart));
     }
 
-    public function action(string $httpPort, string $httpsPort, string $organization, string $image, string $interactive): void
+    public function action(string $httpPort, string $httpsPort, string $organization, string $image, string $interactive, bool $noStart): void
     {
         $config = Config::getParam('variables');
         $defaultHTTPPort = '80';
@@ -220,9 +222,11 @@ class Install extends Action
             }
         }
 
-        Console::log("Running \"docker compose up -d --remove-orphans --renew-anon-volumes\"");
-
-        $exit = Console::execute("$env docker compose --project-directory $this->path up -d --remove-orphans --renew-anon-volumes", '', $stdout, $stderr);
+        $exit = 0;
+        if (!$noStart) {
+            Console::log("Running \"docker compose up -d --remove-orphans --renew-anon-volumes\"");
+            $exit = Console::execute("$env docker compose --project-directory $this->path up -d --remove-orphans --renew-anon-volumes", '', $stdout, $stderr);
+        }
 
         if ($exit !== 0) {
             $message = 'Failed to install Appwrite dockers';

src/Appwrite/Platform/Tasks/Maintenance.php

@@ -36,50 +36,78 @@ class Maintenance extends Action
 
         // # of days in seconds (1 day = 86400s)
         $interval = (int) App::getEnv('_APP_MAINTENANCE_INTERVAL', '86400');
-        $executionLogsRetention = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_EXECUTION', '1209600');
-        $auditLogRetention = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_AUDIT', '1209600');
-        $abuseLogsRetention = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_ABUSE', '86400');
+        $delay = (int) App::getEnv('_APP_MAINTENANCE_DELAY', '0');
         $usageStatsRetentionHourly = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_USAGE_HOURLY', '8640000'); //100 days
         $cacheRetention = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_CACHE', '2592000'); // 30 days
         $schedulesDeletionRetention = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_SCHEDULES', '86400'); // 1 Day
 
-        Console::loop(function () use ($interval, $executionLogsRetention, $abuseLogsRetention, $auditLogRetention, $cacheRetention, $schedulesDeletionRetention, $usageStatsRetentionHourly, $dbForConsole, $queueForDeletes, $queueForCertificates) {
+        Console::loop(function () use ($interval, $cacheRetention, $schedulesDeletionRetention, $usageStatsRetentionHourly, $dbForConsole, $queueForDeletes, $queueForCertificates) {
             $time = DateTime::now();
 
             Console::info("[{$time}] Notifying workers with maintenance tasks every {$interval} seconds");
-            $this->notifyDeleteExecutionLogs($executionLogsRetention, $queueForDeletes);
-            $this->notifyDeleteAbuseLogs($abuseLogsRetention, $queueForDeletes);
-            $this->notifyDeleteAuditLogs($auditLogRetention, $queueForDeletes);
-            $this->notifyDeleteUsageStats($usageStatsRetentionHourly, $queueForDeletes);
+
+            $this->foreachProject($dbForConsole, function (Document $project) use ($queueForDeletes, $usageStatsRetentionHourly) {
+                $queueForDeletes->setProject($project);
+
+                $this->notifyDeleteExecutionLogs($queueForDeletes);
+                $this->notifyDeleteAbuseLogs($queueForDeletes);
+                $this->notifyDeleteAuditLogs($queueForDeletes);
+                $this->notifyDeleteUsageStats($usageStatsRetentionHourly, $queueForDeletes);
+                $this->notifyDeleteExpiredSessions($queueForDeletes);
+            });
+
             $this->notifyDeleteConnections($queueForDeletes);
-            $this->notifyDeleteExpiredSessions($queueForDeletes);
             $this->renewCertificates($dbForConsole, $queueForCertificates);
             $this->notifyDeleteCache($cacheRetention, $queueForDeletes);
             $this->notifyDeleteSchedules($schedulesDeletionRetention, $queueForDeletes);
-        }, $interval);
+        }, $interval, $delay);
     }
 
-    private function notifyDeleteExecutionLogs(int $interval, Delete $queueForDeletes): void
+    protected function foreachProject(Database $dbForConsole, callable $callback): void
+    {
+        // TODO: @Meldiron name of this method no longer matches. It does not delete, and it gives whole document
+        $count = 0;
+        $chunk = 0;
+        $limit = 50;
+        $sum = $limit;
+        $executionStart = \microtime(true);
+
+        while ($sum === $limit) {
+            $projects = $dbForConsole->find('projects', [Query::limit($limit), Query::offset($chunk * $limit)]);
+
+            $chunk++;
+
+            /** @var string[] $projectIds */
+            $sum = count($projects);
+
+            foreach ($projects as $project) {
+                $callback($project);
+                $count++;
+            }
+        }
+
+        $executionEnd = \microtime(true);
+        Console::info("Found {$count} projects " . ($executionEnd - $executionStart) . " seconds");
+    }
+
+    private function notifyDeleteExecutionLogs(Delete $queueForDeletes): void
     {
         ($queueForDeletes)
             ->setType(DELETE_TYPE_EXECUTIONS)
-            ->setDatetime(DateTime::addSeconds(new \DateTime(), -1 * $interval))
             ->trigger();
     }
 
-    private function notifyDeleteAbuseLogs(int $interval, Delete $queueForDeletes): void
+    private function notifyDeleteAbuseLogs(Delete $queueForDeletes): void
     {
         ($queueForDeletes)
             ->setType(DELETE_TYPE_ABUSE)
-            ->setDatetime(DateTime::addSeconds(new \DateTime(), -1 * $interval))
             ->trigger();
     }
 
-    private function notifyDeleteAuditLogs(int $interval, Delete $queueForDeletes): void
+    private function notifyDeleteAuditLogs(Delete $queueForDeletes): void
     {
         ($queueForDeletes)
             ->setType(DELETE_TYPE_AUDIT)
-            ->setDatetime(DateTime::addSeconds(new \DateTime(), -1 * $interval))
             ->trigger();
     }
 

src/Appwrite/Platform/Tasks/QueueCount.php

@@ -0,0 +1,70 @@
+<?php
+
+namespace Appwrite\Platform\Tasks;
+
+use Appwrite\Event\Event;
+use Utopia\CLI\Console;
+use Utopia\Platform\Action;
+use Utopia\Queue\Client;
+use Utopia\Queue\Connection;
+use Utopia\Validator\WhiteList;
+
+class QueueCount extends Action
+{
+    public static function getName(): string
+    {
+        return 'queue-count';
+    }
+
+
+    public function __construct()
+    {
+        $this
+            ->desc('Return the number of from a specific queue identified by the name parameter with a specific type')
+            ->param('name', '', new WhiteList([
+                Event::DATABASE_QUEUE_NAME,
+                Event::DELETE_QUEUE_NAME,
+                Event::AUDITS_QUEUE_NAME,
+                Event::MAILS_QUEUE_NAME,
+                Event::FUNCTIONS_QUEUE_NAME,
+                Event::USAGE_QUEUE_NAME,
+                Event::WEBHOOK_QUEUE_NAME,
+                Event::CERTIFICATES_QUEUE_NAME,
+                Event::BUILDS_QUEUE_NAME,
+                Event::MESSAGING_QUEUE_NAME,
+                Event::MIGRATIONS_QUEUE_NAME,
+                Event::HAMSTER_QUEUE_NAME
+            ]), 'Queue name')
+            ->param('type', '', new WhiteList([
+                'success',
+                'failed',
+                'processing',
+            ]), 'Queue type')
+            ->inject('queue')
+            ->callback(fn ($name, $type, $queue) => $this->action($name, $type, $queue));
+    }
+
+    /**
+     * @param string $name The name of the queue to count the jobs from
+     * @param string $type The type of jobs to count
+     * @param Connection $queue
+     */
+    public function action(string $name, string $type, Connection $queue): void
+    {
+        if (!$name) {
+            Console::error('Missing required parameter $name');
+            return;
+        }
+
+        $queueClient = new Client($name, $queue);
+
+        $count = match ($type) {
+            'success' => $queueClient->countSuccessfulJobs(),
+            'failed' => $queueClient->countFailedJobs(),
+            'processing' => $queueClient->countProcessingJobs(),
+            default => 0
+        };
+
+        Console::log("Queue: '{$name}' has {$count} {$type} jobs.");
+    }
+}

src/Appwrite/Platform/Tasks/QueueRetry.php

@@ -0,0 +1,70 @@
+<?php
+
+namespace Appwrite\Platform\Tasks;
+
+use Appwrite\Event\Event;
+use Utopia\CLI\Console;
+use Utopia\Platform\Action;
+use Utopia\Queue\Client;
+use Utopia\Queue\Connection;
+use Utopia\Validator\Integer;
+use Utopia\Validator\WhiteList;
+use Utopia\Validator\Wildcard;
+
+class QueueRetry extends Action
+{
+    public static function getName(): string
+    {
+        return 'queue-retry';
+    }
+
+
+    public function __construct()
+    {
+        $this
+            ->desc('Retry failed jobs from a specific queue identified by the name parameter')
+            ->param('name', '', new WhiteList([
+                Event::DATABASE_QUEUE_NAME,
+                Event::DELETE_QUEUE_NAME,
+                Event::AUDITS_QUEUE_NAME,
+                Event::MAILS_QUEUE_NAME,
+                Event::FUNCTIONS_QUEUE_NAME,
+                Event::USAGE_QUEUE_NAME,
+                Event::WEBHOOK_CLASS_NAME,
+                Event::CERTIFICATES_QUEUE_NAME,
+                Event::BUILDS_QUEUE_NAME,
+                Event::MESSAGING_QUEUE_NAME,
+                Event::MIGRATIONS_QUEUE_NAME,
+                Event::HAMSTER_CLASS_NAME
+            ]), 'Queue name')
+            ->param('limit', 0, new Wildcard(), 'jobs limit', true)
+            ->inject('queue')
+            ->callback(fn ($name, $limit, $queue) => $this->action($name, $limit, $queue));
+    }
+
+    /**
+     * @param string $name The name of the queue to retry jobs from
+     * @param  mixed $limit
+     * @param Connection $queue
+     */
+    public function action(string $name, mixed $limit, Connection $queue): void
+    {
+
+        if (!$name) {
+            Console::error('Missing required parameter $name');
+            return;
+        }
+
+        $limit = (int)$limit;
+        $queueClient = new Client($name, $queue);
+
+        if ($queueClient->countFailedJobs() === 0) {
+            Console::error('No failed jobs found.');
+            return;
+        }
+
+        Console::log('Retrying failed jobs...');
+
+        $queueClient->retry($limit);
+    }
+}

src/Appwrite/Platform/Tasks/SSL.php

@@ -7,6 +7,7 @@ use Appwrite\Event\Certificate;
 use Utopia\App;
 use Utopia\CLI\Console;
 use Utopia\Database\Document;
+use Utopia\Validator\Boolean;
 use Utopia\Validator\Hostname;
 
 class SSL extends Action
@@ -21,19 +22,22 @@ class SSL extends Action
         $this
             ->desc('Validate server certificates')
             ->param('domain', App::getEnv('_APP_DOMAIN', ''), new Hostname(), 'Domain to generate certificate for. If empty, main domain will be used.', true)
+            ->param('skip-check', true, new Boolean(true), 'If DNS and renew check should be skipped. Defaults to true, and when true, all jobs will result in certificate generation attempt.', true)
             ->inject('queueForCertificates')
-            ->callback(fn (string $domain, Certificate $queueForCertificates) => $this->action($domain, $queueForCertificates));
+            ->callback(fn (string $domain, bool|string $skipCheck, Certificate $queueForCertificates) => $this->action($domain, $skipCheck, $queueForCertificates));
     }
 
-    public function action(string $domain, Certificate $queueForCertificates): void
+    public function action(string $domain, bool|string $skipCheck, Certificate $queueForCertificates): void
     {
+        $skipCheck = \strval($skipCheck) === 'true';
+
         Console::success('Scheduling a job to issue a TLS certificate for domain: ' . $domain);
 
         $queueForCertificates
             ->setDomain(new Document([
                 'domain' => $domain
             ]))
-            ->setSkipRenewCheck(true)
+            ->setSkipRenewCheck($skipCheck)
             ->trigger();
     }
 }

src/Appwrite/Platform/Tasks/Upgrade.php

@@ -3,6 +3,7 @@
 namespace Appwrite\Platform\Tasks;
 
 use Utopia\CLI\Console;
+use Utopia\Validator\Boolean;
 use Utopia\Validator\Text;
 
 class Upgrade extends Install
@@ -16,15 +17,16 @@ class Upgrade extends Install
     {
         $this
             ->desc('Upgrade Appwrite')
-            ->param('httpPort', '', new Text(4), 'Server HTTP port', true)
-            ->param('httpsPort', '', new Text(4), 'Server HTTPS port', true)
+            ->param('http-port', '', new Text(4), 'Server HTTP port', true)
+            ->param('https-port', '', new Text(4), 'Server HTTPS port', true)
             ->param('organization', 'appwrite', new Text(0), 'Docker Registry organization', true)
             ->param('image', 'appwrite', new Text(0), 'Main appwrite docker image', true)
             ->param('interactive', 'Y', new Text(1), 'Run an interactive session', true)
-            ->callback(fn ($httpPort, $httpsPort, $organization, $image, $interactive) => $this->action($httpPort, $httpsPort, $organization, $image, $interactive));
+            ->param('no-start', false, new Boolean(true), 'Run an interactive session', true)
+            ->callback(fn ($httpPort, $httpsPort, $organization, $image, $interactive, $noStart) => $this->action($httpPort, $httpsPort, $organization, $image, $interactive, $noStart));
     }
 
-    public function action(string $httpPort, string $httpsPort, string $organization, string $image, string $interactive): void
+    public function action(string $httpPort, string $httpsPort, string $organization, string $image, string $interactive, bool $noStart): void
     {
         // Check for previous installation
         $data = @file_get_contents($this->path . '/docker-compose.yml');
@@ -37,6 +39,6 @@ class Upgrade extends Install
             Console::log('      └── docker-compose.yml');
             Console::exit(1);
         }
-        parent::action($httpPort, $httpsPort, $organization, $image, $interactive);
+        parent::action($httpPort, $httpsPort, $organization, $image, $interactive, $noStart);
     }
 }

src/Appwrite/Platform/Tasks/Usage.php

@@ -1,60 +0,0 @@
-<?php
-
-namespace Appwrite\Platform\Tasks;
-
-use Appwrite\Usage\Calculators\TimeSeries;
-use InfluxDB\Database as InfluxDatabase;
-use Utopia\App;
-use Utopia\CLI\Console;
-use Utopia\Database\Database as UtopiaDatabase;
-use Throwable;
-use Utopia\Platform\Action;
-use Utopia\Registry\Registry;
-
-class Usage extends Action
-{
-    public static function getName(): string
-    {
-        return 'usage';
-    }
-
-    public function __construct()
-    {
-        $this
-            ->desc('Schedules syncing data from influxdb to Appwrite console db')
-            ->inject('dbForConsole')
-            ->inject('influxdb')
-            ->inject('register')
-            ->inject('getProjectDB')
-            ->inject('logError')
-            ->callback(fn ($dbForConsole, $influxDB, $register, $getProjectDB, $logError) => $this->action($dbForConsole, $influxDB, $register, $getProjectDB, $logError));
-    }
-
-    protected function aggregateTimeseries(UtopiaDatabase $database, InfluxDatabase $influxDB, callable $logError): void
-    {
-    }
-
-    public function action(UtopiaDatabase $dbForConsole, InfluxDatabase $influxDB, Registry $register, callable $getProjectDB, callable $logError)
-    {
-        Console::title('Usage Aggregation V1');
-        Console::success(APP_NAME . ' usage aggregation process v1 has started');
-
-        $errorLogger = fn(Throwable $error, string $action = 'syncUsageStats') => $logError($error, "usage", $action);
-
-        $interval = (int) App::getEnv('_APP_USAGE_AGGREGATION_INTERVAL', '30'); // 30 seconds (by default)
-        $region = App::getEnv('region', 'default');
-        $usage = new TimeSeries($region, $dbForConsole, $influxDB, $getProjectDB, $register, $errorLogger);
-
-        Console::loop(function () use ($interval, $usage) {
-            $now = date('d-m-Y H:i:s', time());
-            Console::info("[{$now}] Aggregating Timeseries Usage data every {$interval} seconds");
-            $loopStart = microtime(true);
-
-            $usage->collect();
-
-            $loopTook = microtime(true) - $loopStart;
-            $now = date('d-m-Y H:i:s', time());
-            Console::info("[{$now}] Aggregation took {$loopTook} seconds");
-        }, $interval);
-    }
-}

src/Appwrite/Platform/Workers/Builds.php

@@ -6,7 +6,6 @@ use Appwrite\Event\Event;
 use Appwrite\Event\Func;
 use Appwrite\Event\Usage;
 use Appwrite\Messaging\Adapter\Realtime;
-use Appwrite\Usage\Stats;
 use Appwrite\Utopia\Response\Model\Deployment;
 use Appwrite\Vcs\Comment;
 use Exception;
@@ -48,11 +47,11 @@ class Builds extends Action
             ->inject('dbForConsole')
             ->inject('queueForEvents')
             ->inject('queueForFunctions')
-            ->inject('usage')
+            ->inject('queueForUsage')
             ->inject('cache')
             ->inject('dbForProject')
             ->inject('getFunctionsDevice')
-            ->callback(fn($message, Database $dbForConsole, Event $queueForEvents, Func $queueForFunctions, Stats $usage, Cache $cache, Database $dbForProject, callable $getFunctionsDevice) => $this->action($message, $dbForConsole, $queueForEvents, $queueForFunctions, $usage, $cache, $dbForProject, $getFunctionsDevice));
+            ->callback(fn($message, Database $dbForConsole, Event $queueForEvents, Func $queueForFunctions, Usage $queueForUsage, Cache $cache, Database $dbForProject, callable $getFunctionsDevice) => $this->action($message, $dbForConsole, $queueForEvents, $queueForFunctions, $queueForUsage, $cache, $dbForProject, $getFunctionsDevice));
     }
 
     /**
@@ -60,14 +59,14 @@ class Builds extends Action
      * @param Database $dbForConsole
      * @param Event $queueForEvents
      * @param Func $queueForFunctions
-     * @param Stats $usage
+     * @param Usage $queueForUsage
      * @param Cache $cache
      * @param Database $dbForProject
      * @param callable $getFunctionsDevice
      * @return void
      * @throws \Utopia\Database\Exception
      */
-    public function action(Message $message, Database $dbForConsole, Event $queueForEvents, Func $queueForFunctions, Stats $usage, Cache $cache, Database $dbForProject, callable $getFunctionsDevice): void
+    public function action(Message $message, Database $dbForConsole, Event $queueForEvents, Func $queueForFunctions, Usage $queueForUsage, Cache $cache, Database $dbForProject, callable $getFunctionsDevice): void
     {
         $payload = $message->getPayload() ?? [];
 
@@ -86,7 +85,7 @@ class Builds extends Action
             case BUILD_TYPE_RETRY:
                 Console::info('Creating build for deployment: ' . $deployment->getId());
                 $github = new GitHub($cache);
-                $this->buildDeployment($getFunctionsDevice, $queueForFunctions, $queueForEvents, $usage, $dbForConsole, $dbForProject, $github, $project, $resource, $deployment, $template);
+                $this->buildDeployment($getFunctionsDevice, $queueForFunctions, $queueForEvents, $queueForUsage, $dbForConsole, $dbForProject, $github, $project, $resource, $deployment, $template);
                 break;
 
             default:
@@ -98,7 +97,7 @@ class Builds extends Action
      * @param callable $getFunctionsDevice
      * @param Func $queueForFunctions
      * @param Event $queueForEvents
-     * @param Stats $usage
+     * @param Usage $queueForUsage
      * @param Database $dbForConsole
      * @param Database $dbForProject
      * @param GitHub $github
@@ -110,7 +109,7 @@ class Builds extends Action
      * @throws \Utopia\Database\Exception
      * @throws Exception
      */
-    protected function buildDeployment(callable $getFunctionsDevice, Func $queueForFunctions, Event $queueForEvents, Stats $usage, Database $dbForConsole, Database $dbForProject, GitHub $github, Document $project, Document $function, Document $deployment, Document $template): void
+    protected function buildDeployment(callable $getFunctionsDevice, Func $queueForFunctions, Event $queueForEvents, Usage $queueForUsage, Database $dbForConsole, Database $dbForProject, GitHub $github, Document $project, Document $function, Document $deployment, Document $template): void
     {
         $executor = new Executor(App::getEnv('_APP_EXECUTOR_HOST'));
 
@@ -529,19 +528,16 @@ class Builds extends Action
                 roles: $target['roles']
             );
 
-            /** Update usage stats */
-            if (App::getEnv('_APP_USAGE_STATS', 'enabled') === 'enabled') {
-                $usage
-                    ->setParam('projectInternalId', $project->getInternalId())
-                    ->setParam('projectId', $project->getId())
-                    ->setParam('functionId', $function->getId())
-                    ->setParam('builds.{scope}.compute', 1)
-                    ->setParam('buildStatus', $build->getAttribute('status', ''))
-                    ->setParam('buildTime', $build->getAttribute('duration'))
-                    ->setParam('networkRequestSize', 0)
-                    ->setParam('networkResponseSize', 0)
-                    ->submit();
-            }
+            /** Trigger usage queue */
+            $queueForUsage
+                ->setProject($project)
+                ->addMetric(METRIC_BUILDS, 1) // per project
+                ->addMetric(METRIC_BUILDS_STORAGE, $build->getAttribute('size', 0))
+                ->addMetric(METRIC_BUILDS_COMPUTE, (int)$build->getAttribute('duration', 0) * 1000)
+                ->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_BUILDS), 1) // per function
+                ->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_BUILDS_STORAGE), $build->getAttribute('size', 0))
+                ->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_BUILDS_COMPUTE), (int)$build->getAttribute('duration', 0) * 1000)
+                ->trigger();
         }
     }
 

src/Appwrite/Platform/Workers/Certificates.php

@@ -23,6 +23,7 @@ use Utopia\Database\Helpers\ID;
 use Utopia\Database\Query;
 use Utopia\Domains\Domain;
 use Utopia\Locale\Locale;
+use Utopia\Logger\Log;
 use Utopia\Platform\Action;
 use Utopia\Queue\Message;
 
@@ -45,7 +46,8 @@ class Certificates extends Action
             ->inject('queueForMails')
             ->inject('queueForEvents')
             ->inject('queueForFunctions')
-            ->callback(fn(Message $message, Database $dbForConsole, Mail $queueForMails, Event $queueForEvents, Func $queueForFunctions) => $this->action($message, $dbForConsole, $queueForMails, $queueForEvents, $queueForFunctions));
+            ->inject('log')
+            ->callback(fn(Message $message, Database $dbForConsole, Mail $queueForMails, Event $queueForEvents, Func $queueForFunctions, Log $log) => $this->action($message, $dbForConsole, $queueForMails, $queueForEvents, $queueForFunctions, $log));
     }
 
     /**
@@ -58,7 +60,7 @@ class Certificates extends Action
      * @throws Throwable
      * @throws \Utopia\Database\Exception
      */
-    public function action(Message $message, Database $dbForConsole, Mail $queueForMails, Event $queueForEvents, Func $queueForFunctions): void
+    public function action(Message $message, Database $dbForConsole, Mail $queueForMails, Event $queueForEvents, Func $queueForFunctions, Log $log): void
     {
         $payload = $message->getPayload() ?? [];
 
@@ -70,7 +72,7 @@ class Certificates extends Action
         $domain   = new Domain($document->getAttribute('domain', ''));
         $skipRenewCheck = $payload['skipRenewCheck'] ?? false;
 
-        $this->execute($domain, $dbForConsole, $queueForMails, $queueForEvents, $queueForFunctions, $skipRenewCheck);
+        $this->execute($domain, $dbForConsole, $queueForMails, $queueForEvents, $queueForFunctions, $log, $skipRenewCheck);
     }
 
     /**
@@ -84,7 +86,7 @@ class Certificates extends Action
      * @throws Throwable
      * @throws \Utopia\Database\Exception
      */
-    private function execute(Domain $domain, Database $dbForConsole, Mail $queueForMails, Event $queueForEvents, Func $queueForFunctions, bool $skipRenewCheck = false): void
+    private function execute(Domain $domain, Database $dbForConsole, Mail $queueForMails, Event $queueForEvents, Func $queueForFunctions, Log $log, bool $skipRenewCheck = false): void
     {
         /**
          * 1. Read arguments and validate domain
@@ -138,11 +140,11 @@ class Certificates extends Action
             if (!$skipRenewCheck) {
                 $mainDomain = $this->getMainDomain();
                 $isMainDomain = !isset($mainDomain) || $domain->get() === $mainDomain;
-                $this->validateDomain($domain, $isMainDomain);
+                $this->validateDomain($domain, $isMainDomain, $log);
             }
 
             // If certificate exists already, double-check expiry date. Skip if job is forced
-            if (!$skipRenewCheck && !$this->isRenewRequired($domain->get())) {
+            if (!$skipRenewCheck && !$this->isRenewRequired($domain->get(), $log)) {
                 throw new Exception('Renew isn\'t required.');
             }
 
@@ -180,6 +182,8 @@ class Certificates extends Action
 
             // Send email to security email
             $this->notifyError($domain->get(), $e->getMessage(), $attempts, $queueForMails);
+
+            throw $e;
         } finally {
             // All actions result in new updatedAt date
             $certificate->setAttribute('updated', DateTime::now());
@@ -247,7 +251,7 @@ class Certificates extends Action
      * @return void
      * @throws Exception
      */
-    private function validateDomain(Domain $domain, bool $isMainDomain): void
+    private function validateDomain(Domain $domain, bool $isMainDomain, Log $log): void
     {
         if (empty($domain->get())) {
             throw new Exception('Missing certificate domain.');
@@ -267,8 +271,15 @@ class Certificates extends Action
             }
 
             // Verify domain with DNS records
+            $validationStart = \microtime(true);
             $validator = new CNAME($target->get());
             if (!$validator->isValid($domain->get())) {
+                $log->addExtra('dnsTiming', \strval(\microtime(true) - $validationStart));
+                $log->addTag('dnsDomain', $domain->get());
+
+                $error = $validator->getLogs();
+                $log->addExtra('dnsResponse', \is_array($error) ? \json_encode($error) : \strval($error));
+
                 throw new Exception('Failed to verify domain DNS records.');
             }
         } else {
@@ -284,7 +295,7 @@ class Certificates extends Action
      * @return bool True, if certificate needs to be renewed
      * @throws Exception
      */
-    private function isRenewRequired(string $domain): bool
+    private function isRenewRequired(string $domain, Log $log): bool
     {
         $certPath = APP_STORAGE_CERTIFICATES . '/' . $domain . '/cert.pem';
         if (\file_exists($certPath)) {
@@ -294,12 +305,15 @@ class Certificates extends Action
             $validTo = $certData['validTo_time_t'] ?? 0;
 
             if (empty($validTo)) {
+                $log->addTag('certificateDomain', $domain);
                 throw new Exception('Unable to read certificate file (cert.pem).');
             }
 
             // LetsEncrypt allows renewal 30 days before expiry
             $expiryInAdvance = (60 * 60 * 24 * 30);
             if ($validTo - $expiryInAdvance > \time()) {
+                $log->addTag('certificateDomain', $domain);
+                $log->addExtra('certificateData', \is_array($certData) ? \json_encode($certData) : \strval($certData));
                 return false;
             }
         }
@@ -430,7 +444,7 @@ class Certificates extends Action
 
         $message = Template::fromFile(__DIR__ . '/../../../../app/config/locale/templates/email-inner-base.tpl');
         $message
-            ->setParam('{{body}}', $locale->getText("emails.certificate.body"))
+            ->setParam('{{body}}', $locale->getText("emails.certificate.body"), escapeHtml: false)
             ->setParam('{{hello}}', $locale->getText("emails.certificate.hello"))
             ->setParam('{{footer}}', $locale->getText("emails.certificate.footer"))
             ->setParam('{{thanks}}', $locale->getText("emails.certificate.thanks"))

src/Appwrite/Platform/Workers/Deletes.php

@@ -2,6 +2,7 @@
 
 namespace Appwrite\Platform\Workers;
 
+use Appwrite\Auth\Auth;
 use Executor\Executor;
 use Throwable;
 use Utopia\Abuse\Abuse;
@@ -45,14 +46,17 @@ class Deletes extends Action
             ->inject('getFunctionsDevice')
             ->inject('getBuildsDevice')
             ->inject('getCacheDevice')
-            ->callback(fn ($message, $dbForConsole, callable $getProjectDB, callable $getFilesDevice, callable $getFunctionsDevice, callable $getBuildsDevice, callable $getCacheDevice) => $this->action($message, $dbForConsole, $getProjectDB, $getFilesDevice, $getFunctionsDevice, $getBuildsDevice, $getCacheDevice));
+            ->inject('abuseRetention')
+            ->inject('executionRetention')
+            ->inject('auditRetention')
+            ->callback(fn ($message, $dbForConsole, callable $getProjectDB, callable $getFilesDevice, callable $getFunctionsDevice, callable $getBuildsDevice, callable $getCacheDevice, string $abuseRetention, string $executionRetention, string $auditRetention) => $this->action($message, $dbForConsole, $getProjectDB, $getFilesDevice, $getFunctionsDevice, $getBuildsDevice, $getCacheDevice, $abuseRetention, $executionRetention, $auditRetention));
     }
 
     /**
      * @throws Exception
      * @throws Throwable
      */
-    public function action(Message $message, Database $dbForConsole, callable $getProjectDB, callable $getFilesDevice, callable $getFunctionsDevice, callable $getBuildsDevice, callable $getCacheDevice): void
+    public function action(Message $message, Database $dbForConsole, callable $getProjectDB, callable $getFilesDevice, callable $getFunctionsDevice, callable $getBuildsDevice, callable $getCacheDevice, string $abuseRetention, string $executionRetention, string $auditRetention): void
     {
         $payload = $message->getPayload() ?? [];
 
@@ -114,12 +118,12 @@ class Deletes extends Action
                 break;
 
             case DELETE_TYPE_EXECUTIONS:
-                $this->deleteExecutionLogs($dbForConsole, $getProjectDB, $datetime);
+                $this->deleteExecutionLogs($project, $getProjectDB, $executionRetention);
                 break;
 
             case DELETE_TYPE_AUDIT:
-                if (!empty($datetime)) {
-                    $this->deleteAuditLogs($dbForConsole, $getProjectDB, $datetime);
+                if (!$project->isEmpty()) {
+                    $this->deleteAuditLogs($project, $getProjectDB, $auditRetention);
                 }
 
                 if (!$document->isEmpty()) {
@@ -127,7 +131,7 @@ class Deletes extends Action
                 }
                 break;
             case DELETE_TYPE_ABUSE:
-                $this->deleteAbuseLogs($dbForConsole, $getProjectDB, $datetime);
+                $this->deleteAbuseLogs($project, $getProjectDB, $abuseRetention);
                 break;
 
             case DELETE_TYPE_REALTIME:
@@ -135,10 +139,10 @@ class Deletes extends Action
                 break;
 
             case DELETE_TYPE_SESSIONS:
-                $this->deleteExpiredSessions($dbForConsole, $getProjectDB);
+                $this->deleteExpiredSessions($project, $getProjectDB);
                 break;
             case DELETE_TYPE_USAGE:
-                $this->deleteUsageStats($dbForConsole, $getProjectDB, $hourlyUsageRetentionDatetime);
+                $this->deleteUsageStats($project, $getProjectDB, $hourlyUsageRetentionDatetime);
                 break;
             case DELETE_TYPE_CACHE_BY_RESOURCE:
                 $this->deleteCacheByResource($project, $getProjectDB, $resource);
@@ -337,16 +341,14 @@ class Deletes extends Action
      * @return void
      * @throws Exception
      */
-    private function deleteUsageStats(Database $dbForConsole, callable $getProjectDB, string $hourlyUsageRetentionDatetime): void
+    private function deleteUsageStats(Document $project, callable $getProjectDB, string $hourlyUsageRetentionDatetime): void
     {
-        $this->deleteForProjectIds($dbForConsole, function (Document $project) use ($getProjectDB, $hourlyUsageRetentionDatetime) {
-            $dbForProject = $getProjectDB($project);
-            // Delete Usage stats
-            $this->deleteByGroup('stats', [
-                Query::lessThan('time', $hourlyUsageRetentionDatetime),
-                Query::equal('period', ['1h']),
-            ], $dbForProject);
-        });
+        $dbForProject = $getProjectDB($project);
+        // Delete Usage stats
+        $this->deleteByGroup('stats_v2', [
+            Query::lessThan('time', $hourlyUsageRetentionDatetime),
+            Query::equal('period', ['1h']),
+        ], $dbForProject);
     }
 
     /**
@@ -387,6 +389,7 @@ class Deletes extends Action
      */
     private function deleteProjectsByTeam(Database $dbForConsole, callable $getProjectDB, callable $getFilesDevice, callable $getFunctionsDevice, callable $getBuildsDevice, callable $getCacheDevice, Document $document): void
     {
+
         $projects = $dbForConsole->find('projects', [
             Query::equal('teamInternalId', [$document->getInternalId()])
         ]);
@@ -514,12 +517,7 @@ class Deletes extends Action
                 $teamId = $document->getAttribute('teamId');
                 $team = $dbForProject->getDocument('teams', $teamId);
                 if (!$team->isEmpty()) {
-                    $team = $dbForProject->updateDocument(
-                        'teams',
-                        $teamId,
-                        // Ensure that total >= 0
-                        $team->setAttribute('total', \max($team->getAttribute('total', 0) - 1, 0))
-                    );
+                    $dbForProject->decreaseDocumentAttribute('teams', $teamId, 'total', 1, 0);
                 }
             }
         });
@@ -542,15 +540,13 @@ class Deletes extends Action
      * @return void
      * @throws Exception
      */
-    private function deleteExecutionLogs(database $dbForConsole, callable $getProjectDB, string $datetime): void
+    private function deleteExecutionLogs(Document $project, callable $getProjectDB, string $datetime): void
     {
-        $this->deleteForProjectIds($dbForConsole, function (Document $project) use ($getProjectDB, $datetime) {
-            $dbForProject = $getProjectDB($project);
-            // Delete Executions
-            $this->deleteByGroup('executions', [
-                Query::lessThan('$createdAt', $datetime)
-            ], $dbForProject);
-        });
+        $dbForProject = $getProjectDB($project);
+        // Delete Executions
+        $this->deleteByGroup('executions', [
+            Query::lessThan('$createdAt', $datetime)
+        ], $dbForProject);
     }
 
     /**
@@ -559,20 +555,16 @@ class Deletes extends Action
      * @return void
      * @throws Exception|Throwable
      */
-    private function deleteExpiredSessions(Database $dbForConsole, callable $getProjectDB): void
+    private function deleteExpiredSessions(Document $project, callable $getProjectDB): void
     {
+        $dbForProject = $getProjectDB($project);
+        $duration = $project->getAttribute('auths', [])['duration'] ?? Auth::TOKEN_EXPIRATION_LOGIN_LONG;
+        $expired = DateTime::addSeconds(new \DateTime(), -1 * $duration);
 
-        $this->deleteForProjectIds($dbForConsole, function (Document $project) use ($dbForConsole, $getProjectDB) {
-            $dbForProject = $getProjectDB($project);
-            $project = $dbForConsole->getDocument('projects', $project->getId());
-            $duration = $project->getAttribute('auths', [])['duration'] ?? Auth::TOKEN_EXPIRATION_LOGIN_LONG;
-            $expired = DateTime::addSeconds(new \DateTime(), -1 * $duration);
-
-            // Delete Sessions
-            $this->deleteByGroup('sessions', [
-                Query::lessThan('$createdAt', $expired)
-            ], $dbForProject);
-        });
+        // Delete Sessions
+        $this->deleteByGroup('sessions', [
+            Query::lessThan('$createdAt', $expired)
+        ], $dbForProject);
     }
 
     /**
@@ -596,22 +588,16 @@ class Deletes extends Action
      * @return void
      * @throws Exception
      */
-    private function deleteAbuseLogs(Database $dbForConsole, callable $getProjectDB, string $datetime): void
+    private function deleteAbuseLogs(Document $project, callable $getProjectDB, string $abuseRetention): void
     {
-        if (empty($datetime)) {
-            throw new Exception('Failed to delete audit logs. No datetime provided');
+        $projectId = $project->getId();
+        $dbForProject = $getProjectDB($project);
+        $timeLimit = new TimeLimit("", 0, 1, $dbForProject);
+        $abuse = new Abuse($timeLimit);
+        $status = $abuse->cleanup($abuseRetention);
+        if (!$status) {
+            throw new Exception('Failed to delete Abuse logs for project ' . $projectId);
         }
-
-        $this->deleteForProjectIds($dbForConsole, function (Document $project) use ($getProjectDB, $datetime) {
-            $projectId = $project->getId();
-            $dbForProject = $getProjectDB($project);
-            $timeLimit = new TimeLimit("", 0, 1, $dbForProject);
-            $abuse = new Abuse($timeLimit);
-            $status = $abuse->cleanup($datetime);
-            if (!$status) {
-                throw new Exception('Failed to delete Abuse logs for project ' . $projectId);
-            }
-        });
     }
 
     /**
@@ -621,21 +607,15 @@ class Deletes extends Action
      * @return void
      * @throws Exception
      */
-    private function deleteAuditLogs(Database $dbForConsole, callable $getProjectDB, string $datetime): void
+    private function deleteAuditLogs(Document $project, callable $getProjectDB, string $auditRetention): void
     {
-        if (empty($datetime)) {
-            throw new Exception('Failed to delete audit logs. No datetime provided');
+        $projectId = $project->getId();
+        $dbForProject = $getProjectDB($project);
+        $audit = new Audit($dbForProject);
+        $status = $audit->cleanup($auditRetention);
+        if (!$status) {
+            throw new Exception('Failed to delete Audit logs for project' . $projectId);
         }
-
-        $this->deleteForProjectIds($dbForConsole, function (Document $project) use ($getProjectDB, $datetime) {
-            $projectId = $project->getId();
-            $dbForProject = $getProjectDB($project);
-            $audit = new Audit($dbForProject);
-            $status = $audit->cleanup($datetime);
-            if (!$status) {
-                throw new Exception('Failed to delete Audit logs for project' . $projectId);
-            }
-        });
     }
 
     /**
@@ -870,39 +850,6 @@ class Deletes extends Action
         }
     }
 
-    /**
-     * @param Database $dbForConsole
-     * @param callable $callback
-     * @throws Exception
-     */
-    private function deleteForProjectIds(database $dbForConsole, callable $callback): void
-    {
-        // TODO: @Meldiron name of this method no longer matches. It does not delete, and it gives whole document
-        $count = 0;
-        $chunk = 0;
-        $limit = 50;
-        $sum = $limit;
-        $executionStart = \microtime(true);
-
-        while ($sum === $limit) {
-            $projects = $dbForConsole->find('projects', [Query::limit($limit), Query::offset($chunk * $limit)]);
-
-            $chunk++;
-
-            /** @var string[] $projectIds */
-            $sum = count($projects);
-
-            Console::info('Executing delete function for chunk #' . $chunk . '. Found ' . $sum . ' projects');
-            foreach ($projects as $project) {
-                $callback($project);
-                $count++;
-            }
-        }
-
-        $executionEnd = \microtime(true);
-        Console::info("Found {$count} projects " . ($executionEnd - $executionStart) . " seconds");
-    }
-
     /**
      * @param string $collection collectionID
      * @param array $queries

src/Appwrite/Platform/Workers/Functions.php

@@ -2,9 +2,9 @@
 
 namespace Appwrite\Platform\Workers;
 
-use Appwrite\Usage\Stats;
 use Appwrite\Event\Event;
 use Appwrite\Event\Func;
+use Appwrite\Event\Usage;
 use Appwrite\Messaging\Adapter\Realtime;
 use Appwrite\Utopia\Response\Model\Execution;
 use Exception;
@@ -39,13 +39,14 @@ class Functions extends Action
     {
         $this
             ->desc('Functions worker')
+            ->groups(['functions'])
             ->inject('message')
             ->inject('dbForProject')
             ->inject('queueForFunctions')
             ->inject('queueForEvents')
-            ->inject('usage')
+            ->inject('queueForUsage')
             ->inject('log')
-            ->callback(fn(Message $message, Database $dbForProject, Func $queueForFunctions, Event $queueForEvents, Stats $usage, Log $log) => $this->action($message, $dbForProject, $queueForFunctions, $queueForEvents, $usage, $log));
+            ->callback(fn(Message $message, Database $dbForProject, Func $queueForFunctions, Event $queueForEvents, Usage $queueForUsage, Log $log) => $this->action($message, $dbForProject, $queueForFunctions, $queueForEvents, $queueForUsage, $log));
     }
 
     /**
@@ -53,7 +54,7 @@ class Functions extends Action
      * @param Database $dbForProject
      * @param Func $queueForFunctions
      * @param Event $queueForEvents
-     * @param Stats $usage
+     * @param Usage $queueForUsage
      * @param Log $log
      * @return void
      * @throws Authorization
@@ -61,7 +62,7 @@ class Functions extends Action
      * @throws \Utopia\Database\Exception
      * @throws Conflict
      */
-    public function action(Message $message, Database $dbForProject, Func $queueForFunctions, Event $queueForEvents, Stats $usage, Log $log): void
+    public function action(Message $message, Database $dbForProject, Func $queueForFunctions, Event $queueForEvents, Usage $queueForUsage, Log $log): void
     {
         $payload = $message->getPayload() ?? [];
 
@@ -117,7 +118,7 @@ class Functions extends Action
                         log: $log,
                         dbForProject: $dbForProject,
                         queueForFunctions: $queueForFunctions,
-                        usage: $usage,
+                        queueForUsage: $queueForUsage,
                         queueForEvents: $queueForEvents,
                         project: $project,
                         function: $function,
@@ -153,7 +154,7 @@ class Functions extends Action
                     log: $log,
                     dbForProject: $dbForProject,
                     queueForFunctions: $queueForFunctions,
-                    usage: $usage,
+                    queueForUsage: $queueForUsage,
                     queueForEvents: $queueForEvents,
                     project: $project,
                     function: $function,
@@ -174,7 +175,7 @@ class Functions extends Action
                     log: $log,
                     dbForProject: $dbForProject,
                     queueForFunctions: $queueForFunctions,
-                    usage: $usage,
+                    queueForUsage: $queueForUsage,
                     queueForEvents: $queueForEvents,
                     project: $project,
                     function: $function,
@@ -197,7 +198,7 @@ class Functions extends Action
      * @param Log $log
      * @param Database $dbForProject
      * @param Func $queueForFunctions
-     * @param Stats $usage
+     * @param Usage $queueForUsage
      * @param Event $queueForEvents
      * @param Document $project
      * @param Document $function
@@ -221,7 +222,7 @@ class Functions extends Action
         Log $log,
         Database $dbForProject,
         Func $queueForFunctions,
-        stats $usage,
+        Usage $queueForUsage,
         Event $queueForEvents,
         Document $project,
         Document $function,
@@ -420,6 +421,16 @@ class Functions extends Action
 
             $error = $th->getMessage();
             $errorCode = $th->getCode();
+        } finally {
+            /** Trigger usage queue */
+            $queueForUsage
+                ->setProject($project)
+                ->addMetric(METRIC_EXECUTIONS, 1)
+                ->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS), 1)
+                ->addMetric(METRIC_EXECUTIONS_COMPUTE, (int)($execution->getAttribute('duration') * 1000))// per project
+                ->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS_COMPUTE), (int)($execution->getAttribute('duration') * 1000))
+                ->trigger()
+            ;
         }
 
         if ($function->getAttribute('logging')) {
@@ -471,19 +482,5 @@ class Functions extends Action
         if (!empty($error)) {
             throw new Exception($error, $errorCode);
         }
-
-        /** Update usage stats */
-        if (App::getEnv('_APP_USAGE_STATS', 'enabled') === 'enabled') {
-            $usage
-                ->setParam('projectId', $project->getId())
-                ->setParam('projectInternalId', $project->getInternalId())
-                ->setParam('functionId', $function->getId()) // TODO: We should use functionInternalId in usage stats
-                ->setParam('executions.{scope}.compute', 1)
-                ->setParam('executionStatus', $execution->getAttribute('status', ''))
-                ->setParam('executionTime', $execution->getAttribute('duration'))
-                ->setParam('networkRequestSize', 0)
-                ->setParam('networkResponseSize', 0)
-                ->submit();
-        }
     }
 }

src/Appwrite/Platform/Workers/Hamster.php

@@ -22,17 +22,18 @@ use Utopia\Pools\Group;
 class Hamster extends Action
 {
     private array $metrics = [
-        'usage_files' => 'files.$all.count.total',
-        'usage_buckets' => 'buckets.$all.count.total',
-        'usage_databases' => 'databases.$all.count.total',
-        'usage_documents' => 'documents.$all.count.total',
-        'usage_collections' => 'collections.$all.count.total',
-        'usage_storage' => 'project.$all.storage.size',
-        'usage_requests' => 'project.$all.network.requests',
-        'usage_bandwidth' => 'project.$all.network.bandwidth',
-        'usage_users' => 'users.$all.count.total',
-        'usage_sessions' => 'sessions.email.requests.create',
-        'usage_executions' => 'executions.$all.compute.total',
+        'usage_files' => 'files',
+        'usage_buckets' => 'buckets',
+        'usage_databases' => 'databases',
+        'usage_documents' => 'documents',
+        'usage_collections' => 'collections',
+        'usage_storage' => 'files.storage',
+        'usage_requests' => 'network.requests',
+        'usage_inbound' => 'network.inbound',
+        'usage_outbound' => 'network.outbound',
+        'usage_users' => 'users',
+        'usage_sessions' => 'sessions',
+        'usage_executions' => 'executions',
     ];
 
     protected Mixpanel $mixpanel;
@@ -217,6 +218,15 @@ class Hamster extends Action
                 }
             }
 
+            /** Add billing information to the project */
+            $organization = $dbForConsole->findOne('teams', [
+                Query::equal('$internalId', [$teamInternalId])
+            ]);
+
+            $billing = $this->getBillingDetails($organization);
+            $statsPerProject['billing_plan'] = $billing['billing_plan'] ?? null;
+            $statsPerProject['billing_start_date'] = $billing['billing_start_date'] ?? null;
+
             /** Get Domains */
             $statsPerProject['custom_domains'] = $dbForConsole->count('rules', [
                 Query::equal('projectInternalId', [$project->getInternalId()]),
@@ -276,7 +286,7 @@ class Hamster extends Action
                         $limit = $periodValue['limit'];
                         $period = $periodValue['period'];
 
-                        $requestDocs = $dbForProject->find('stats', [
+                        $requestDocs = $dbForProject->find('stats_v2', [
                             Query::equal('period', [$period]),
                             Query::equal('metric', [$metric]),
                             Query::limit($limit),
@@ -298,6 +308,17 @@ class Hamster extends Action
                 }
             });
 
+            /**
+             * Workaround to combine network.Inbound+network.outbound as bandwidth.
+             */
+            $statsPerProject["usage_bandwidth_infinity"]  = $statsPerProject["usage_inbound_infinity"] + $statsPerProject["usage_outbound_infinity"];
+            $statsPerProject["usage_bandwidth_24h"]  = $statsPerProject["usage_inbound_24h"] + $statsPerProject["usage_outbound_24h"];
+            unset($statsPerProject["usage_outbound_24h"]);
+            unset($statsPerProject["usage_inbound_24h"]);
+            unset($statsPerProject["usage_outbound_infinity"]);
+            unset($statsPerProject["usage_inbound_infinity"]);
+
+
             if (isset($statsPerProject['email'])) {
                 /** Send data to mixpanel */
                 $res = $this->mixpanel->createProfile($statsPerProject['email'], '', [
@@ -332,7 +353,6 @@ class Hamster extends Action
     /**
      * @param Document $organization
      * @param Database $dbForConsole
-     * @throws \Utopia\Database\Exception
      */
     private function getStatsForOrganization(Document $organization, Database $dbForConsole): void
     {
@@ -346,22 +366,26 @@ class Hamster extends Action
             /** Organization name */
             $statsPerOrganization['name'] = $organization->getAttribute('name');
 
-
             /** Get Email and of the organization owner */
             $membership = $dbForConsole->findOne('memberships', [
                 Query::equal('teamInternalId', [$organization->getInternalId()]),
             ]);
-
             if (!$membership || $membership->isEmpty()) {
                 throw new \Exception('Membership not found. Skipping organization : ' . $organization->getId());
             }
-
             $userId = $membership->getAttribute('userId', null);
             if ($userId) {
                 $user = $dbForConsole->getDocument('users', $userId);
                 $statsPerOrganization['email'] = $user->getAttribute('email', null);
             }
 
+            /** Add billing information */
+            $billing = $this->getBillingDetails($organization);
+            $statsPerOrganization['billing_plan'] = $billing['billing_plan'] ?? null;
+            $statsPerOrganization['billing_start_date'] = $billing['billing_start_date'] ?? null;
+            $statsPerOrganization['marked_for_deletion'] = $billing['markedForDeletion'] ?? 0;
+            $statsPerOrganization['billing_plan_downgrade'] = $billing['billing_plan_downgrade'] ?? null;
+
             /** Organization Creation Date */
             $statsPerOrganization['created'] = $organization->getAttribute('$createdAt');
 
@@ -400,6 +424,16 @@ class Hamster extends Action
 
             $statsPerUser['time'] = $user->getAttribute('$time');
 
+            /** Add billing information */
+            $organization = $dbForConsole->findOne('teams', [
+                Query::equal('userInternalId', [$user->getInternalId()])
+            ]);
+
+
+            $billing = $this->getBillingDetails($organization);
+            $statsPerUser['billing_plan'] = $billing['billing_plan'] ?? null;
+            $statsPerUser['billing_start_date'] = $billing['billing_start_date'] ?? null;
+
             /** Organization name */
             $statsPerUser['name'] = $user->getAttribute('name');
 
@@ -434,4 +468,28 @@ class Hamster extends Action
             Console::error($e->getMessage());
         }
     }
+
+    private function getBillingDetails(bool|Document $team): array
+    {
+        $billing = [];
+
+        if (!empty($team) && !$team->isEmpty()) {
+            $billingPlan = $team->getAttribute('billingPlan', null);
+            $billingPlanDowngrade = $team->getAttribute('billingPlanDowngrade', null);
+
+            if (!empty($billingPlan) && empty($billingPlanDowngrade)) {
+                $billing['billing_plan'] = $billingPlan;
+            }
+
+            if (in_array($billingPlan, ['tier-1', 'tier-2'])) {
+                $billingStartDate = $team->getAttribute('billingStartDate', null);
+                $billing['billing_start_date'] = $billingStartDate;
+            }
+
+            $billing['marked_for_deletion'] = $team->getAttribute('markedForDeletion', 0);
+            $billing['billing_plan_downgrade'] = $billingPlanDowngrade;
+        }
+
+        return $billing;
+    }
 }

src/Appwrite/Platform/Workers/Mails.php

@@ -59,11 +59,16 @@ class Mails extends Action
         $variables = $payload['variables'];
         $name = $payload['name'];
         $body = $payload['body'];
-
-        $bodyTemplate = Template::fromFile(__DIR__ . '/../../../../app/config/locale/templates/email-base.tpl');
-        $bodyTemplate->setParam('{{body}}', $body);
+        $attachment = $payload['attachment'] ?? [];
+        $bodyTemplate = $payload['bodyTemplate'];
+        if (empty($bodyTemplate)) {
+            $bodyTemplate = __DIR__ . '/../../../../app/config/locale/templates/email-base.tpl';
+        }
+        $bodyTemplate = Template::fromFile($bodyTemplate);
+        $bodyTemplate->setParam('{{body}}', $body, escapeHtml: false);
         foreach ($variables as $key => $value) {
-            $bodyTemplate->setParam('{{' . $key . '}}', $value);
+            // TODO: hotfix for redirect param
+            $bodyTemplate->setParam('{{' . $key . '}}', $value, escapeHtml: $key !== 'redirect');
         }
         $body = $bodyTemplate->render();
 
@@ -89,6 +94,14 @@ class Mails extends Action
         $mail->Subject = $subject;
         $mail->Body = $body;
         $mail->AltBody = \strip_tags($body);
+        if (!empty($attachment['content'] ?? '')) {
+            $mail->AddStringAttachment(
+                base64_decode($attachment['content']),
+                $attachment['filename'] ?? 'unknown.file',
+                $attachment['encoding'] ?? PHPMailer::ENCODING_BASE64,
+                $attachment['type'] ?? 'plain/text'
+            );
+        }
 
         try {
             $mail->send();

src/Appwrite/Platform/Workers/Messaging.php

@@ -5,8 +5,9 @@ namespace Appwrite\Platform\Workers;
 use Exception;
 use Utopia\App;
 use Utopia\CLI\Console;
+use Utopia\Database\Document;
 use Utopia\DSN\DSN;
-use Utopia\Messaging\Messages\Sms;
+use Utopia\Messaging\Messages\SMS;
 use Utopia\Messaging\Adapters\SMS\Mock;
 use Utopia\Messaging\Adapters\SMS\Msg91;
 use Utopia\Messaging\Adapters\SMS\Telesign;
@@ -15,6 +16,7 @@ use Utopia\Messaging\Adapters\SMS\Twilio;
 use Utopia\Messaging\Adapters\SMS\Vonage;
 use Utopia\Platform\Action;
 use Utopia\Queue\Message;
+use Appwrite\Event\Usage;
 
 class Messaging extends Action
 {
@@ -43,20 +45,36 @@ class Messaging extends Action
         $this
             ->desc('Messaging worker')
             ->inject('message')
-            ->callback(fn($message) => $this->action($message));
+            ->inject('queueForUsage')
+            ->callback(fn(Message $message, Usage $queueForUsage) => $this->action($message, $queueForUsage));
     }
 
     /**
      * @param Message $message
+     * @param Usage $queueForUsage
      * @return void
      * @throws Exception
      */
-    public function action(Message $message): void
+    public function action(Message $message, Usage $queueForUsage): void
     {
         $payload = $message->getPayload() ?? [];
 
         if (empty($payload)) {
-            Console::error('Payload arg not found');
+            throw new Exception('Missing payload');
+        }
+
+        if (empty($payload['project'])) {
+            throw new Exception('Project not set in payload');
+        }
+
+        $project = new Document($payload['project'] ?? []);
+
+        Console::log('Project: ' . $project->getId());
+
+        $denyList = App::getEnv('_APP_SMS_PROJECTS_DENY_LIST', '');
+        $denyList = explode(',', $denyList);
+        if (in_array($project->getId(), $denyList)) {
+            Console::error("Project is in the deny list. Skipping ...");
             return;
         }
 
@@ -70,14 +88,29 @@ class Messaging extends Action
             return;
         }
 
-        $sms =  match ($this->dsn->getHost()) {
-            'mock' => new Mock($this->user, $this->secret), // used for tests
-            'twilio' => new Twilio($this->user, $this->secret),
-            'text-magic' => new TextMagic($this->user, $this->secret),
-            'telesign' => new Telesign($this->user, $this->secret),
-            'msg91' => new Msg91($this->user, $this->secret),
-            'vonage' => new Vonage($this->user, $this->secret),
-            default => null
+
+        switch ($this->dsn->getHost()) {
+            case 'mock':
+                 $sms = new Mock($this->user, $this->secret); // used for tests
+                break;
+            case 'twilio':
+                 $sms = new Twilio($this->user, $this->secret);
+                break;
+            case 'text-magic':
+                $sms = new TextMagic($this->user, $this->secret);
+                break;
+            case 'telesign':
+                $sms = new Telesign($this->user, $this->secret);
+                break;
+            case 'msg91':
+                $sms = new Msg91($this->user, $this->secret);
+                $sms->setTemplate($this->dsn->getParam('template'));
+                break;
+            case 'vonage':
+                $sms = new Vonage($this->user, $this->secret);
+                break;
+            default:
+                $sms = null;
         };
 
         if (empty(App::getEnv('_APP_SMS_PROVIDER'))) {
@@ -100,6 +133,11 @@ class Messaging extends Action
 
         try {
             $sms->send($message);
+
+            $queueForUsage
+                ->setProject($project)
+                ->addMetric(METRIC_MESSAGES, 1)
+                ->trigger();
         } catch (\Exception $error) {
             throw new Exception('Error sending message: ' . $error->getMessage(), 500);
         }

src/Appwrite/Platform/Workers/Usage.php

@@ -0,0 +1,250 @@
+<?php
+
+namespace Appwrite\Platform\Workers;
+
+use Exception;
+use Utopia\App;
+use Utopia\CLI\Console;
+use Utopia\Database\DateTime;
+use Utopia\Database\Document;
+use Utopia\Platform\Action;
+use Appwrite\Event\UsageDump;
+use Utopia\Queue\Message;
+
+class Usage extends Action
+{
+    private array $stats = [];
+    private int $lastTriggeredTime = 0;
+    private int $keys = 0;
+    private const INFINITY_PERIOD = '_inf_';
+    private const KEYS_THRESHOLD = 10000;
+
+
+    public static function getName(): string
+    {
+        return 'usage';
+    }
+
+    /**
+     * @throws Exception
+     */
+    public function __construct()
+    {
+
+        $this
+        ->desc('Usage worker')
+        ->inject('message')
+        ->inject('getProjectDB')
+        ->inject('queueForUsageDump')
+        ->callback(function (Message $message, callable $getProjectDB, UsageDump $queueForUsageDump) {
+             $this->action($message, $getProjectDB, $queueForUsageDump);
+        });
+
+        $this->lastTriggeredTime = time();
+    }
+
+    /**
+     * @param Message $message
+     * @param callable $getProjectDB
+     * @param UsageDump $queueForUsageDump
+     * @return void
+     * @throws \Utopia\Database\Exception
+     * @throws Exception
+     */
+    public function action(Message $message, callable $getProjectDB, UsageDump $queueForUsageDump): void
+    {
+        $payload = $message->getPayload() ?? [];
+        if (empty($payload)) {
+            throw new Exception('Missing payload');
+        }
+        //Todo Figure out way to preserve keys when the container is being recreated @shimonewman
+
+        $aggregationInterval = (int) App::getEnv('_APP_USAGE_AGGREGATION_INTERVAL', '20');
+        $project = new Document($payload['project'] ?? []);
+        $projectId = $project->getInternalId();
+        foreach ($payload['reduce'] ?? [] as $document) {
+            if (empty($document)) {
+                continue;
+            }
+
+            $this->reduce(
+                project: $project,
+                document: new Document($document),
+                metrics:  $payload['metrics'],
+                getProjectDB: $getProjectDB
+            );
+        }
+
+        $this->stats[$projectId]['project'] = $project;
+        $this->stats[$projectId]['receivedAt'] = DateTime::now();
+        foreach ($payload['metrics'] ?? [] as $metric) {
+                 $this->keys++;
+            if (!isset($this->stats[$projectId]['keys'][$metric['key']])) {
+                $this->stats[$projectId]['keys'][$metric['key']] = $metric['value'];
+                continue;
+            }
+
+            $this->stats[$projectId]['keys'][$metric['key']] += $metric['value'];
+        }
+
+        // if keys crossed threshold or X time passed since the last send and there are some keys in the array ($this->stats)
+        if (
+            $this->keys >= self::KEYS_THRESHOLD ||
+            (time() - $this->lastTriggeredTime > $aggregationInterval  && $this->keys > 0)
+        ) {
+            console::warning('[' . DateTime::now() . '] Aggregated ' . $this->keys . ' keys');
+
+            $queueForUsageDump
+                ->setStats($this->stats)
+                ->trigger();
+
+            $this->stats = [];
+            $this->keys = 0;
+            $this->lastTriggeredTime = time();
+        }
+    }
+
+     /**
+     * On Documents that tied by relations like functions>deployments>build || documents>collection>database || buckets>files.
+     * When we remove a parent document we need to deduct his children aggregation from the project scope.
+     * @param Document $project
+     * @param Document $document
+     * @param array $metrics
+     * @param  callable $getProjectDB
+     * @return void
+     */
+    private function reduce(Document $project, Document $document, array &$metrics, callable $getProjectDB): void
+    {
+
+        $dbForProject = $getProjectDB($project);
+
+        try {
+            switch (true) {
+                case $document->getCollection() === 'users': // users
+                    $sessions = count($document->getAttribute(METRIC_SESSIONS, 0));
+                    if (!empty($sessions)) {
+                        $metrics[] = [
+                        'key' => METRIC_SESSIONS,
+                        'value' => ($sessions * -1),
+                        ];
+                    }
+                    break;
+                case $document->getCollection() === 'databases': // databases
+                    $collections = $dbForProject->getDocument('stats_v2', md5(self::INFINITY_PERIOD . str_replace('{databaseInternalId}', $document->getInternalId(), METRIC_DATABASE_ID_COLLECTIONS)));
+                    $documents = $dbForProject->getDocument('stats_v2', md5(self::INFINITY_PERIOD . str_replace('{databaseInternalId}', $document->getInternalId(), METRIC_DATABASE_ID_DOCUMENTS)));
+                    if (!empty($collections['value'])) {
+                        $metrics[] = [
+                        'key' => METRIC_COLLECTIONS,
+                        'value' => ($collections['value'] * -1),
+                        ];
+                    }
+
+                    if (!empty($documents['value'])) {
+                        $metrics[] = [
+                        'key' => METRIC_DOCUMENTS,
+                        'value' => ($documents['value'] * -1),
+                        ];
+                    }
+                    break;
+                case str_starts_with($document->getCollection(), 'database_') && !str_contains($document->getCollection(), 'collection'): //collections
+                    $parts = explode('_', $document->getCollection());
+                    $databaseInternalId = $parts[1] ?? 0;
+                    $documents = $dbForProject->getDocument('stats_v2', md5(self::INFINITY_PERIOD . str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$databaseInternalId, $document->getInternalId()], METRIC_DATABASE_ID_COLLECTION_ID_DOCUMENTS)));
+
+                    if (!empty($documents['value'])) {
+                        $metrics[] = [
+                        'key' => METRIC_DOCUMENTS,
+                        'value' => ($documents['value'] * -1),
+                        ];
+                        $metrics[] = [
+                        'key' => str_replace('{databaseInternalId}', $databaseInternalId, METRIC_DATABASE_ID_DOCUMENTS),
+                        'value' => ($documents['value'] * -1),
+                        ];
+                    }
+                    break;
+
+                case $document->getCollection() === 'buckets':
+                    $files = $dbForProject->getDocument('stats_v2', md5(self::INFINITY_PERIOD . str_replace('{bucketInternalId}', $document->getInternalId(), METRIC_BUCKET_ID_FILES)));
+                    $storage = $dbForProject->getDocument('stats_v2', md5(self::INFINITY_PERIOD . str_replace('{bucketInternalId}', $document->getInternalId(), METRIC_BUCKET_ID_FILES_STORAGE)));
+
+                    if (!empty($files['value'])) {
+                        $metrics[] = [
+                        'key' => METRIC_FILES,
+                        'value' => ($files['value'] * -1),
+                        ];
+                    }
+
+                    if (!empty($storage['value'])) {
+                        $metrics[] = [
+                        'key' => METRIC_FILES_STORAGE,
+                        'value' => ($storage['value'] * -1),
+                        ];
+                    }
+                    break;
+
+                case $document->getCollection() === 'functions':
+                    $deployments = $dbForProject->getDocument('stats_v2', md5(self::INFINITY_PERIOD . str_replace(['{resourceType}', '{resourceInternalId}'], ['functions', $document->getInternalId()], METRIC_FUNCTION_ID_DEPLOYMENTS)));
+                    $deploymentsStorage = $dbForProject->getDocument('stats_v2', md5(self::INFINITY_PERIOD . str_replace(['{resourceType}', '{resourceInternalId}'], ['functions', $document->getInternalId()], METRIC_FUNCTION_ID_DEPLOYMENTS_STORAGE)));
+                    $builds = $dbForProject->getDocument('stats_v2', md5(self::INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_BUILDS)));
+                    $buildsStorage = $dbForProject->getDocument('stats_v2', md5(self::INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_BUILDS_STORAGE)));
+                    $buildsCompute = $dbForProject->getDocument('stats_v2', md5(self::INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_BUILDS_COMPUTE)));
+                    $executions = $dbForProject->getDocument('stats_v2', md5(self::INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS)));
+                    $executionsCompute = $dbForProject->getDocument('stats_v2', md5(self::INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS_COMPUTE)));
+
+                    if (!empty($deployments['value'])) {
+                        $metrics[] = [
+                        'key' => METRIC_DEPLOYMENTS,
+                        'value' => ($deployments['value'] * -1),
+                        ];
+                    }
+
+                    if (!empty($deploymentsStorage['value'])) {
+                        $metrics[] = [
+                        'key' => METRIC_DEPLOYMENTS_STORAGE,
+                        'value' => ($deploymentsStorage['value'] * -1),
+                        ];
+                    }
+
+                    if (!empty($builds['value'])) {
+                        $metrics[] = [
+                        'key' => METRIC_BUILDS,
+                        'value' => ($builds['value'] * -1),
+                        ];
+                    }
+
+                    if (!empty($buildsStorage['value'])) {
+                        $metrics[] = [
+                        'key' => METRIC_BUILDS_STORAGE,
+                        'value' => ($buildsStorage['value'] * -1),
+                        ];
+                    }
+
+                    if (!empty($buildsCompute['value'])) {
+                        $metrics[] = [
+                        'key' => METRIC_BUILDS_COMPUTE,
+                        'value' => ($buildsCompute['value'] * -1),
+                        ];
+                    }
+
+                    if (!empty($executions['value'])) {
+                        $metrics[] = [
+                        'key' => METRIC_EXECUTIONS,
+                        'value' => ($executions['value'] * -1),
+                        ];
+                    }
+
+                    if (!empty($executionsCompute['value'])) {
+                        $metrics[] = [
+                        'key' => METRIC_EXECUTIONS_COMPUTE,
+                        'value' => ($executionsCompute['value'] * -1),
+                        ];
+                    }
+                    break;
+                default:
+                    break;
+            }
+        } catch (\Exception $e) {
+            console::error("[reducer] " . " {DateTime::now()} " . " {$project->getInternalId()} " . " {$e->getMessage()}");
+        }
+    }
+}

src/Appwrite/Platform/Workers/UsageDump.php

@@ -0,0 +1,113 @@
+<?php
+
+namespace Appwrite\Platform\Workers;
+
+use Appwrite\Extend\Exception;
+use Utopia\App;
+use Utopia\Database\Document;
+use Utopia\Database\Exception\Duplicate;
+use Utopia\Platform\Action;
+use Utopia\CLI\Console;
+use Utopia\Database\DateTime;
+use Utopia\Queue\Message;
+
+class UsageDump extends Action
+{
+    protected array $stats = [];
+    protected array $periods = [
+        '1h' => 'Y-m-d H:00',
+        '1d' => 'Y-m-d 00:00',
+        'inf' => '0000-00-00 00:00'
+    ];
+
+    public static function getName(): string
+    {
+        return 'usage-dump';
+    }
+
+    /**
+     * @throws \Exception
+     */
+    public function __construct()
+    {
+
+        $this
+            ->inject('message')
+            ->inject('getProjectDB')
+            ->callback(function (Message $message, callable $getProjectDB) {
+                $this->action($message, $getProjectDB);
+            })
+        ;
+    }
+
+    /**
+     * @param Message $message
+     * @param callable $getProjectDB
+     * @return void
+     * @throws Exception
+     * @throws \Utopia\Database\Exception
+     */
+    public function action(Message $message, callable $getProjectDB): void
+    {
+
+        $payload = $message->getPayload() ?? [];
+        if (empty($payload)) {
+            throw new Exception('Missing payload');
+        }
+
+        //Todo rename both usage workers @shimonewman
+        foreach ($payload['stats'] ?? [] as $stats) {
+            $project = new Document($stats['project'] ?? []);
+            $numberOfKeys = !empty($stats['keys']) ? count($stats['keys']) : 0;
+            $receivedAt = $stats['receivedAt'] ?? 'NONE';
+            if ($numberOfKeys === 0) {
+                continue;
+            }
+
+            console::log('[' . DateTime::now() . '] ProjectId [' . $project->getInternalId()  . '] ReceivedAt [' . $receivedAt . '] ' . $numberOfKeys . ' keys');
+
+            try {
+                $dbForProject = $getProjectDB($project);
+                foreach ($stats['keys'] ?? [] as $key => $value) {
+                    if ($value == 0) {
+                        continue;
+                    }
+
+                    foreach ($this->periods as $period => $format) {
+                        $time = 'inf' === $period ? null : date($format, time());
+                        $id = \md5("{$time}_{$period}_{$key}");
+
+                        try {
+                            $dbForProject->createDocument('stats_v2', new Document([
+                                '$id' => $id,
+                                'period' => $period,
+                                'time' => $time,
+                                'metric' => $key,
+                                'value' => $value,
+                                'region' => App::getEnv('_APP_REGION', 'default'),
+                            ]));
+                        } catch (Duplicate $th) {
+                            if ($value < 0) {
+                                $dbForProject->decreaseDocumentAttribute(
+                                    'stats_v2',
+                                    $id,
+                                    'value',
+                                    abs($value)
+                                );
+                            } else {
+                                $dbForProject->increaseDocumentAttribute(
+                                    'stats_v2',
+                                    $id,
+                                    'value',
+                                    $value
+                                );
+                            }
+                        }
+                    }
+                }
+            } catch (\Exception $e) {
+                console::error('[' . DateTime::now() . '] project [' . $project->getInternalId() . '] database [' . $project['database'] . '] ' . ' ' . $e->getMessage());
+            }
+        }
+    }
+}

src/Appwrite/Usage/Calculator.php

@@ -1,15 +0,0 @@
-<?php
-
-namespace Appwrite\Usage;
-
-abstract class Calculator
-{
-    protected string $region;
-
-    public function __construct(string $region)
-    {
-        $this->region = $region;
-    }
-
-    abstract public function collect(): void;
-}

src/Appwrite/Usage/Calculators/TimeSeries.php

@@ -1,560 +0,0 @@
-<?php
-
-namespace Appwrite\Usage\Calculators;
-
-use Utopia\App;
-use Appwrite\Usage\Calculator;
-use Utopia\Database\Database;
-use Utopia\Database\Document;
-use InfluxDB\Database as InfluxDatabase;
-use DateTime;
-use Utopia\Database\Validator\Authorization;
-use Utopia\Registry\Registry;
-
-class TimeSeries extends Calculator
-{
-    /**
-     * InfluxDB
-     *
-     * @var InfluxDatabase
-     */
-    protected InfluxDatabase $influxDB;
-
-    /**
-     * Utopia Database
-     *
-     * @var Database
-     */
-    protected Database $database;
-
-    /**
-     * Error Handler Callback
-     *
-     * @var callable
-     */
-    protected $errorHandler;
-
-    /**
-     * Callback to get project DB
-     *
-     * @var callable
-     */
-    protected mixed $getProjectDB;
-
-    /**
-     * Registry
-     *
-     * @var Registry
-     */
-    protected Registry $register;
-
-    /**
-     * Latest times for metric that was synced to the database
-     *
-     * @var array
-     */
-    private array $latestTime = [];
-
-    /**
-     * Periods the metrics are collected for
-     * @var array
-     */
-    protected array $periods = [
-        [
-            'key' => '1h',
-            'startTime' => '-24 hours'
-        ],
-        [
-            'key' => '1d',
-            'startTime' => '-30 days'
-        ]
-    ];
-
-    /**
-     * All the metrics that we are collecting
-     *
-     * @var array
-     */
-    protected array $metrics = [
-        'project.$all.network.requests' => [
-            'table' => 'appwrite_usage_project_{scope}_network_requests',
-        ],
-        'project.$all.network.bandwidth' => [
-            'table' => 'appwrite_usage_project_{scope}_network_bandwidth',
-        ],
-        'project.$all.network.inbound' => [
-            'table' => 'appwrite_usage_project_{scope}_network_inbound',
-        ],
-        'project.$all.network.outbound' => [
-            'table' => 'appwrite_usage_project_{scope}_network_outbound',
-        ],
-        /* Users service metrics */
-        'users.$all.requests.create' => [
-            'table' => 'appwrite_usage_users_{scope}_requests_create',
-        ],
-        'users.$all.requests.read' => [
-            'table' => 'appwrite_usage_users_{scope}_requests_read',
-        ],
-        'users.$all.requests.update' => [
-            'table' => 'appwrite_usage_users_{scope}_requests_update',
-        ],
-        'users.$all.requests.delete' => [
-            'table' => 'appwrite_usage_users_{scope}_requests_delete',
-        ],
-
-        'databases.$all.requests.create' => [
-            'table' => 'appwrite_usage_databases_{scope}_requests_create',
-        ],
-        'databases.$all.requests.read' => [
-            'table' => 'appwrite_usage_databases_{scope}_requests_read',
-        ],
-        'databases.$all.requests.update' => [
-            'table' => 'appwrite_usage_databases_{scope}_requests_update',
-        ],
-        'databases.$all.requests.delete' => [
-            'table' => 'appwrite_usage_databases_{scope}_requests_delete',
-        ],
-
-        'collections.$all.requests.create' => [
-            'table' => 'appwrite_usage_collections_{scope}_requests_create',
-        ],
-        'collections.$all.requests.read' => [
-            'table' => 'appwrite_usage_collections_{scope}_requests_read',
-        ],
-        'collections.$all.requests.update' => [
-            'table' => 'appwrite_usage_collections_{scope}_requests_update',
-        ],
-        'collections.$all.requests.delete' => [
-            'table' => 'appwrite_usage_collections_{scope}_requests_delete',
-        ],
-
-        'documents.$all.requests.create' => [
-            'table' => 'appwrite_usage_documents_{scope}_requests_create',
-        ],
-        'documents.$all.requests.read' => [
-            'table' => 'appwrite_usage_documents_{scope}_requests_read',
-        ],
-        'documents.$all.requests.update' => [
-            'table' => 'appwrite_usage_documents_{scope}_requests_update',
-        ],
-        'documents.$all.requests.delete' => [
-            'table' => 'appwrite_usage_documents_{scope}_requests_delete',
-        ],
-
-        'collections.databaseId.requests.create' => [
-            'table' => 'appwrite_usage_collections_{scope}_requests_create',
-            'groupBy' => ['databaseId'],
-        ],
-        'collections.databaseId.requests.read' => [
-            'table' => 'appwrite_usage_collections_{scope}_requests_read',
-            'groupBy' => ['databaseId'],
-        ],
-        'collections.databaseId.requests.update' => [
-            'table' => 'appwrite_usage_collections_{scope}_requests_update',
-            'groupBy' => ['databaseId'],
-        ],
-        'collections.databaseId.requests.delete' => [
-            'table' => 'appwrite_usage_collections_{scope}_requests_delete',
-            'groupBy' => ['databaseId'],
-        ],
-
-        'documents.databaseId.requests.create' => [
-            'table' => 'appwrite_usage_documents_{scope}_requests_create',
-            'groupBy' => ['databaseId'],
-        ],
-        'documents.databaseId.requests.read' => [
-            'table' => 'appwrite_usage_documents_{scope}_requests_read',
-            'groupBy' => ['databaseId'],
-        ],
-        'documents.databaseId.requests.update' => [
-            'table' => 'appwrite_usage_documents_{scope}_requests_update',
-            'groupBy' => ['databaseId'],
-        ],
-        'documents.databaseId.requests.delete' => [
-            'table' => 'appwrite_usage_documents_{scope}_requests_delete',
-            'groupBy' => ['databaseId'],
-        ],
-
-        'documents.databaseId/collectionId.requests.create' => [
-            'table' => 'appwrite_usage_documents_{scope}_requests_create',
-            'groupBy' => ['databaseId', 'collectionId'],
-        ],
-        'documents.databaseId/collectionId.requests.read' => [
-            'table' => 'appwrite_usage_documents_{scope}_requests_read',
-            'groupBy' => ['databaseId', 'collectionId'],
-        ],
-        'documents.databaseId/collectionId.requests.update' => [
-            'table' => 'appwrite_usage_documents_{scope}_requests_update',
-            'groupBy' => ['databaseId', 'collectionId'],
-        ],
-        'documents.databaseId/collectionId.requests.delete' => [
-            'table' => 'appwrite_usage_documents_{scope}_requests_delete',
-            'groupBy' => ['databaseId', 'collectionId'],
-        ],
-
-        'buckets.$all.requests.create' => [
-            'table' => 'appwrite_usage_buckets_{scope}_requests_create',
-        ],
-        'buckets.$all.requests.read' => [
-            'table' => 'appwrite_usage_buckets_{scope}_requests_read',
-        ],
-        'buckets.$all.requests.update' => [
-            'table' => 'appwrite_usage_buckets_{scope}_requests_update',
-        ],
-        'buckets.$all.requests.delete' => [
-            'table' => 'appwrite_usage_buckets_{scope}_requests_delete',
-        ],
-
-        'files.$all.requests.create' => [
-            'table' => 'appwrite_usage_files_{scope}_requests_create',
-        ],
-        'files.$all.requests.read' => [
-            'table' => 'appwrite_usage_files_{scope}_requests_read',
-        ],
-        'files.$all.requests.update' => [
-            'table' => 'appwrite_usage_files_{scope}_requests_update',
-        ],
-        'files.$all.requests.delete' => [
-            'table' => 'appwrite_usage_files_{scope}_requests_delete',
-        ],
-
-        'files.bucketId.requests.create' => [
-            'table' => 'appwrite_usage_files_{scope}_requests_create',
-            'groupBy' => ['bucketId'],
-        ],
-        'files.bucketId.requests.read' => [
-            'table' => 'appwrite_usage_files_{scope}_requests_read',
-            'groupBy' => ['bucketId'],
-        ],
-        'files.bucketId.requests.update' => [
-            'table' => 'appwrite_usage_files_{scope}_requests_update',
-            'groupBy' => ['bucketId'],
-        ],
-        'files.bucketId.requests.delete' => [
-            'table' => 'appwrite_usage_files_{scope}_requests_delete',
-            'groupBy' => ['bucketId'],
-        ],
-
-        'sessions.$all.requests.create' => [
-            'table' => 'appwrite_usage_sessions__{scope}_requests_create',
-        ],
-        'sessions.provider.requests.create' => [
-            'table' => 'appwrite_usage_sessions_{scope}_requests_create',
-            'groupBy' => ['provider'],
-        ],
-        'sessions.$all.requests.delete' => [
-            'table' => 'appwrite_usage_sessions_{scope}_requests_delete',
-        ],
-        'executions.$all.compute.total' => [
-            'table' => 'appwrite_usage_executions_{scope}_compute',
-        ],
-        'builds.$all.compute.total' => [
-            'table' => 'appwrite_usage_builds_{scope}_compute',
-        ],
-        'executions.$all.compute.failure' => [
-            'table' => 'appwrite_usage_executions_{scope}_compute',
-            'filters' => [
-                'functionStatus' => 'failed',
-            ],
-        ],
-        'builds.$all.compute.failure' => [
-            'table' => 'appwrite_usage_builds_{scope}_compute',
-            'filters' => [
-                'functionStatus' => 'failed',
-            ],
-        ],
-        'executions.$all.compute.success' => [
-            'table' => 'appwrite_usage_executions_{scope}_compute',
-            'filters' => [
-                'functionStatus' => 'success',
-            ],
-        ],
-        'builds.$all.compute.success' => [
-            'table' => 'appwrite_usage_builds_{scope}_compute',
-            'filters' => [
-                'functionStatus' => 'success',
-            ],
-        ],
-        'executions.functionId.compute.total' => [
-            'table' => 'appwrite_usage_executions_{scope}_compute',
-            'groupBy' => ['functionId'],
-        ],
-        'builds.functionId.compute.total' => [
-            'table' => 'appwrite_usage_builds_{scope}_compute',
-            'groupBy' => ['functionId'],
-        ],
-
-        'executions.functionId.compute.failure' => [
-            'table' => 'appwrite_usage_executions_{scope}_compute',
-            'groupBy' => ['functionId'],
-            'filters' => [
-                'functionStatus' => 'failed',
-            ],
-        ],
-        'builds.functionId.compute.failure' => [
-            'table' => 'appwrite_usage_builds_{scope}_compute',
-            'groupBy' => ['functionId'],
-            'filters' => [
-                'functionBuildStatus' => 'failed',
-            ],
-        ],
-        'executions.functionId.compute.success' => [
-            'table' => 'appwrite_usage_executions_{scope}_compute',
-            'groupBy' => ['functionId'],
-            'filters' => [
-                'functionStatus' => 'success',
-            ],
-        ],
-        'builds.functionId.compute.success' => [
-            'table' => 'appwrite_usage_builds_{scope}_compute',
-            'groupBy' => ['functionId'],
-            'filters' => [
-                'functionBuildStatus' => 'success',
-            ],
-        ],
-
-        // counters
-        'users.$all.count.total' => [
-            'table' => 'appwrite_usage_users_{scope}_count_total',
-        ],
-        'buckets.$all.count.total' => [
-            'table' => 'appwrite_usage_buckets_{scope}_count_total',
-        ],
-        'files.$all.count.total' => [
-            'table' => 'appwrite_usage_files_{scope}_count_total',
-        ],
-        'files.bucketId.count.total' => [
-            'table' => 'appwrite_usage_files_{scope}_count_total',
-            'groupBy' => ['bucketId']
-        ],
-        'databases.$all.count.total' => [
-            'table' => 'appwrite_usage_databases_{scope}_count_total',
-        ],
-        'collections.$all.count.total' => [
-            'table' => 'appwrite_usage_collections_{scope}_count_total',
-        ],
-        'documents.$all.count.total' => [
-            'table' => 'appwrite_usage_documents_{scope}_count_total',
-        ],
-        'collections.databaseId.count.total' => [
-            'table' => 'appwrite_usage_collections_{scope}_count_total',
-            'groupBy' => ['databaseId']
-        ],
-        'documents.databaseId.count.total' => [
-            'table' => 'appwrite_usage_documents_{scope}_count_total',
-            'groupBy' => ['databaseId']
-        ],
-        'documents.databaseId/collectionId.count.total' => [
-            'table' => 'appwrite_usage_documents_{scope}_count_total',
-            'groupBy' => ['databaseId', 'collectionId']
-        ],
-        'deployments.$all.storage.size' => [
-            'table' => 'appwrite_usage_deployments_{scope}_storage_size',
-        ],
-        'project.$all.storage.size' => [
-            'table' => 'appwrite_usage_project_{scope}_storage_size',
-        ],
-        'files.$all.storage.size' => [
-            'table' => 'appwrite_usage_files_{scope}_storage_size',
-        ],
-        'files.$bucketId.storage.size' => [
-            'table' => 'appwrite_usage_files_{scope}_storage_size',
-            'groupBy' => ['bucketId']
-        ],
-
-        'builds.$all.compute.time' => [
-            'table' => 'appwrite_usage_executions_{scope}_compute_time',
-        ],
-        'executions.$all.compute.time' => [
-            'table' => 'appwrite_usage_executions_{scope}_compute_time',
-        ],
-
-        'executions.functionId.compute.time' => [
-            'table' => 'appwrite_usage_executions_{scope}_compute_time',
-            'groupBy' => ['functionId'],
-        ],
-        'builds.functionId.compute.time' => [
-            'table' => 'appwrite_usage_builds_{scope}_compute_time',
-            'groupBy' => ['functionId'],
-        ],
-
-        'project.$all.compute.time' => [ // Built time + execution time
-            'table' => 'appwrite_usage_project_{scope}_compute_time',
-            'groupBy' => ['functionId'],
-        ],
-
-        'deployments.$all.storage.size' => [
-            'table' => 'appwrite_usage_deployments_{scope}_storage_size'
-        ],
-        'project.$all.storage.size' => [
-            'table' => 'appwrite_usage_project_{scope}_storage_size'
-        ],
-        'files.$all.storage.size' => [
-            'table' => 'appwrite_usage_files_{scope}_storage_size'
-        ],
-        'files.bucketId.storage.size' => [
-            'table' => 'appwrite_usage_files_{scope}_storage_size',
-            'groupBy' => ['bucketId']
-        ]
-    ];
-
-    public function __construct(string $region, Database $database, InfluxDatabase $influxDB, callable $getProjectDB, Registry $register, callable $errorHandler = null)
-    {
-        parent::__construct($region);
-        $this->database = $database;
-        $this->influxDB = $influxDB;
-        $this->getProjectDB = $getProjectDB;
-        $this->register = $register;
-        $this->errorHandler = $errorHandler;
-    }
-
-    /**
-     * Create or Update Mertic
-     * Create or update each metric in the stats collection for the given project
-     *
-     * @param string $projectId
-     * @param int $time
-     * @param string $period
-     * @param string $metric
-     * @param int $value
-     * @param int $type
-     *
-     * @return void
-     */
-    private function createOrUpdateMetric(string $projectId, string $time, string $period, string $metric, int $value, int $type): void
-    {
-        $id = \md5("{$time}_{$period}_{$metric}");
-        $project = $this->database->getDocument('projects', $projectId);
-        $database = call_user_func($this->getProjectDB, $project);
-
-        Authorization::skip(function () use ($database, $id, $period, $time, $metric, $value, $type, $projectId) {
-            try {
-                $document = $database->getDocument('stats', $id);
-                if ($document->isEmpty()) {
-                    $database->createDocument('stats', new Document([
-                        '$id' => $id,
-                        'period' => $period,
-                        'time' => $time,
-                        'metric' => $metric,
-                        'value' => $value,
-                        'type' => $type,
-                        'region' => $this->region,
-                    ]));
-                } else {
-                    $database->updateDocument(
-                        'stats',
-                        $document->getId(),
-                        $document->setAttribute('value', $value)
-                    );
-                }
-            } catch (\Exception $e) { // if projects are deleted this might fail
-                if (is_callable($this->errorHandler)) {
-                    call_user_func($this->errorHandler, $e, "sync_project_{$projectId}_metric_{$metric}");
-                } else {
-                    throw $e;
-                }
-            }
-        });
-
-        $this->register->get('pools')->reclaim();
-    }
-
-    /**
-     * Sync From InfluxDB
-     * Sync stats from influxDB to stats collection in the Appwrite database
-     *
-     * @param string $metric
-     * @param array $options
-     * @param array $period
-     *
-     * @return void
-     */
-    private function syncFromInfluxDB(string $metric, array $options, array $period): void
-    {
-        $start = DateTime::createFromFormat('U', \strtotime($period['startTime']))->format(DateTime::RFC3339);
-        if (!empty($this->latestTime[$metric][$period['key']])) {
-            $start = $this->latestTime[$metric][$period['key']];
-        }
-        $end = (new DateTime())->format(DateTime::RFC3339);
-
-        $table = $options['table']; //Which influxdb table to query for this metric
-        $groupBy = empty($options['groupBy']) ? '' : ', ' . implode(', ', array_map(fn($groupBy) => '"' . $groupBy . '" ', $options['groupBy'])); //Some sub level metrics may be grouped by other tags like collectionId, bucketId, etc
-
-        $filters = $options['filters'] ?? []; // Some metrics might have additional filters, like function's status
-        if (!empty($filters)) {
-            $filters = ' AND ' . implode(' AND ', array_map(fn ($filter, $value) => "\"{$filter}\"='{$value}'", array_keys($filters), array_values($filters)));
-        } else {
-            $filters = '';
-        }
-
-        $query = "SELECT sum(value) AS \"value\" ";
-        $query .= "FROM \"{$table}\" ";
-        $query .= "WHERE \"time\" > '{$start}' ";
-        $query .= "AND \"time\" < '{$end}' ";
-        $query .= "AND \"metric_type\"='counter' {$filters} ";
-        $query .= "GROUP BY time({$period['key']}), \"projectId\" {$groupBy} ";
-        $query .= "FILL(null)";
-
-        try {
-            $result = $this->influxDB->query($query);
-            $points = $result->getPoints();
-            foreach ($points as $point) {
-                $projectId = $point['projectId'];
-
-                if (!empty($projectId) && $projectId !== 'console') {
-                    $metricUpdated = $metric;
-                    if (!empty($groupBy)) {
-                        foreach ($options['groupBy'] as $groupBy) {
-                            $groupedBy = $point[$groupBy] ?? '';
-                            if (empty($groupedBy)) {
-                                continue;
-                            }
-                            $metricUpdated = str_replace($groupBy, $groupedBy, $metricUpdated);
-                        }
-                    }
-
-                    $value = (!empty($point['value'])) ? $point['value'] : 0;
-
-                    $this->createOrUpdateMetric(
-                        $point['projectId'],
-                        $point['time'],
-                        $period['key'],
-                        $metricUpdated,
-                        $value,
-                        0
-                    );
-                    $this->latestTime[$metric][$period['key']] = $point['time'];
-                }
-            }
-        } catch (\Exception $e) { // if projects are deleted this might fail
-            if (is_callable($this->errorHandler)) {
-                call_user_func($this->errorHandler, $e, "sync_metric_{$metric}_influxdb");
-            } else {
-                throw $e;
-            }
-        }
-    }
-
-    /**
-     * Collect Stats
-     * Collect all the stats from Influd DB to Database
-     *
-     * @return void
-     */
-    public function collect(): void
-    {
-        foreach ($this->periods as $period) {
-            foreach ($this->metrics as $metric => $options) { //for each metrics
-                try {
-                    $this->syncFromInfluxDB($metric, $options, $period);
-                } catch (\Exception $e) {
-                    if (is_callable($this->errorHandler)) {
-                        call_user_func($this->errorHandler, $e);
-                    } else {
-                        throw $e;
-                    }
-                }
-            }
-        }
-    }
-}

src/Appwrite/Usage/Stats.php

@@ -1,225 +0,0 @@
-<?php
-
-namespace Appwrite\Usage;
-
-use Utopia\App;
-
-class Stats
-{
-    /**
-     * @var array
-     */
-    protected $params = [];
-
-    /**
-     * @var mixed
-     */
-    protected $statsd;
-
-    /**
-     * @var string
-     */
-    protected $namespace = 'appwrite.usage';
-
-    /**
-     * Event constructor.
-     *
-     * @param mixed $statsd
-     */
-    public function __construct($statsd)
-    {
-        $this->statsd = $statsd;
-    }
-
-    /**
-     * @param string $key
-     * @param mixed  $value
-     *
-     * @return $this
-     */
-    public function setParam(string $key, $value): self
-    {
-        $this->params[$key] = $value;
-
-        return $this;
-    }
-
-    /**
-     * @param string $key
-     *
-     * @return mixed|null
-     */
-    public function getParam(string $key)
-    {
-        return (isset($this->params[$key])) ? $this->params[$key] : null;
-    }
-
-    /**
-     * @param string $namespace
-     *
-     * @return $this
-     */
-    public function setNamespace(string $namespace): self
-    {
-        $this->namespace = $namespace;
-
-        return $this;
-    }
-
-    /**
-     * @return string
-     */
-    public function getNamespace()
-    {
-        return $this->namespace;
-    }
-
-    /**
-     * Submit data to StatsD.
-     * Send various metrics to StatsD based on the parameters that are set
-     * @return void
-     */
-    public function submit(): void
-    {
-        $projectId = $this->params['projectId'] ?? '';
-        $projectInternalId = $this->params['projectInternalId'];
-        $tags = ",projectInternalId={$projectInternalId},projectId={$projectId},version=" . App::getEnv('_APP_VERSION', 'UNKNOWN');
-
-        // the global namespace is prepended to every key (optional)
-        $this->statsd->setNamespace($this->namespace);
-
-        $httpRequest = $this->params['project.{scope}.network.requests'] ?? 0;
-        $httpMethod = $this->params['httpMethod'] ?? '';
-        if ($httpRequest >= 1) {
-            $this->statsd->increment('project.{scope}.network.requests' . $tags . ',method=' . \strtolower($httpMethod));
-        }
-
-        $inbound = $this->params['project.{scope}.network.inbound'] ?? 0;
-        $outbound = $this->params['project.{scope}.network.outbound'] ?? 0;
-        $this->statsd->count('project.{scope}.network.inbound' . $tags, $inbound);
-        $this->statsd->count('project.{scope}.network.outbound' . $tags, $outbound);
-        $this->statsd->count('project.{scope}.network.bandwidth' . $tags, $inbound + $outbound);
-
-        $usersMetrics = [
-            'users.{scope}.requests.create',
-            'users.{scope}.requests.read',
-            'users.{scope}.requests.update',
-            'users.{scope}.requests.delete',
-            'users.{scope}.count.total',
-        ];
-
-        foreach ($usersMetrics as $metric) {
-            $value = $this->params[$metric] ?? 0;
-            if ($value === 1 || $value === -1) {
-                $this->statsd->count($metric . $tags, $value);
-            }
-        }
-
-        $dbMetrics = [
-            'databases.{scope}.requests.create',
-            'databases.{scope}.requests.read',
-            'databases.{scope}.requests.update',
-            'databases.{scope}.requests.delete',
-            'collections.{scope}.requests.create',
-            'collections.{scope}.requests.read',
-            'collections.{scope}.requests.update',
-            'collections.{scope}.requests.delete',
-            'documents.{scope}.requests.create',
-            'documents.{scope}.requests.read',
-            'documents.{scope}.requests.update',
-            'documents.{scope}.requests.delete',
-            'databases.{scope}.count.total',
-            'collections.{scope}.count.total',
-            'documents.{scope}.count.total'
-        ];
-
-        foreach ($dbMetrics as $metric) {
-            $value = $this->params[$metric] ?? 0;
-            if ($value === 1 || $value === -1) {
-                $dbTags = $tags . ",collectionId=" . ($this->params['collectionId'] ?? '') . ",databaseId=" . ($this->params['databaseId'] ?? '');
-                $this->statsd->count($metric . $dbTags, $value);
-            }
-        }
-
-        $storageMertics = [
-            'buckets.{scope}.requests.create',
-            'buckets.{scope}.requests.read',
-            'buckets.{scope}.requests.update',
-            'buckets.{scope}.requests.delete',
-            'files.{scope}.requests.create',
-            'files.{scope}.requests.read',
-            'files.{scope}.requests.update',
-            'files.{scope}.requests.delete',
-            'buckets.{scope}.count.total',
-            'files.{scope}.count.total',
-            'files.{scope}.storage.size'
-        ];
-
-        foreach ($storageMertics as $metric) {
-            $value = $this->params[$metric] ?? 0;
-            if ($value !== 0) {
-                $storageTags = $tags . ",bucketId=" . ($this->params['bucketId'] ?? '');
-                $this->statsd->count($metric . $storageTags, $value);
-            }
-        }
-
-        $sessionsMetrics = [
-            'sessions.{scope}.requests.create',
-            'sessions.{scope}.requests.update',
-            'sessions.{scope}.requests.delete',
-        ];
-
-        foreach ($sessionsMetrics as $metric) {
-            $value = $this->params[$metric] ?? 0;
-            if ($value >= 1) {
-                $sessionTags = $tags . ",provider=" . ($this->params['provider'] ?? '');
-                $this->statsd->count($metric . $sessionTags, $value);
-            }
-        }
-
-        $functionId = $this->params['functionId'] ?? '';
-        $functionExecution = $this->params['executions.{scope}.compute'] ?? 0;
-        $functionExecutionTime = ($this->params['executionTime'] ?? 0) * 1000; // ms
-        $functionExecutionStatus = $this->params['executionStatus'] ?? '';
-
-        $functionBuild = $this->params['builds.{scope}.compute'] ?? 0;
-        $functionBuildTime = ($this->params['buildTime'] ?? 0) * 1000; // ms
-        $functionBuildStatus = $this->params['buildStatus'] ?? '';
-        $functionCompute = $functionExecutionTime + $functionBuildTime;
-        $functionTags = $tags . ',functionId=' . $functionId;
-
-        $deploymentSize = $this->params['deployment.{scope}.storage.size'] ?? 0;
-        $storageSize = $this->params['files.{scope}.storage.size'] ?? 0;
-        if ($deploymentSize + $storageSize > 0 || $deploymentSize + $storageSize <= -1) {
-            $this->statsd->count('project.{scope}.storage.size' . $tags, $deploymentSize + $storageSize);
-        }
-
-        if ($deploymentSize !== 0) {
-            $this->statsd->count('deployments.{scope}.storage.size' . $functionTags, $deploymentSize);
-        }
-
-        if ($functionExecution >= 1) {
-            $this->statsd->increment('executions.{scope}.compute' . $functionTags . ',functionStatus=' . $functionExecutionStatus);
-            if ($functionExecutionTime > 0) {
-                $this->statsd->count('executions.{scope}.compute.time' . $functionTags, $functionExecutionTime);
-            }
-        }
-        if ($functionBuild >= 1) {
-            $this->statsd->increment('builds.{scope}.compute' . $functionTags . ',functionBuildStatus=' . $functionBuildStatus);
-            $this->statsd->count('builds.{scope}.compute.time' . $functionTags, $functionBuildTime);
-        }
-        if ($functionBuild + $functionExecution >= 1) {
-            $this->statsd->count('project.{scope}.compute.time' . $functionTags, $functionCompute);
-        }
-
-        $this->reset();
-    }
-
-    public function reset(): self
-    {
-        $this->params = [];
-        $this->namespace = 'appwrite.usage';
-
-        return $this;
-    }
-}

src/Appwrite/Utopia/Database/Validator/Queries/Base.php

@@ -25,7 +25,7 @@ class Base extends Queries
     public function __construct(string $collection, array $allowedAttributes)
     {
         $config = Config::getParam('collections', []);
-        $collections = array_merge($config['console'], $config['projects'], $config['buckets'], $config['databases']);
+        $collections = array_merge($config['projects'], $config['buckets'], $config['databases'], $config['console']);
         $collection = $collections[$collection];
         // array for constant lookup time
         $allowedAttributesLookup = [];

src/Appwrite/Utopia/Database/Validator/Queries/Teams.php

@@ -6,7 +6,8 @@ class Teams extends Base
 {
     public const ALLOWED_ATTRIBUTES = [
         'name',
-        'total'
+        'total',
+        'billingPlan'
     ];
 
     /**

src/Appwrite/Utopia/Response.php

@@ -3,7 +3,6 @@
 namespace Appwrite\Utopia;
 
 use Exception;
-use Swoole\Http\Request as SwooleRequest;
 use Utopia\Swoole\Response as SwooleResponse;
 use Swoole\Http\Response as SwooleHTTPResponse;
 use Utopia\Database\Document;
@@ -39,7 +38,6 @@ use Appwrite\Utopia\Response\Model\Continent;
 use Appwrite\Utopia\Response\Model\Country;
 use Appwrite\Utopia\Response\Model\Currency;
 use Appwrite\Utopia\Response\Model\Document as ModelDocument;
-use Appwrite\Utopia\Response\Model\Domain;
 use Appwrite\Utopia\Response\Model\Error;
 use Appwrite\Utopia\Response\Model\ErrorDev;
 use Appwrite\Utopia\Response\Model\Execution;
@@ -60,7 +58,6 @@ use Appwrite\Utopia\Response\Model\Locale;
 use Appwrite\Utopia\Response\Model\Log;
 use Appwrite\Utopia\Response\Model\Membership;
 use Appwrite\Utopia\Response\Model\Metric;
-use Appwrite\Utopia\Response\Model\Permissions;
 use Appwrite\Utopia\Response\Model\Phone;
 use Appwrite\Utopia\Response\Model\Platform;
 use Appwrite\Utopia\Response\Model\Project;
@@ -73,12 +70,14 @@ use Appwrite\Utopia\Response\Model\Token;
 use Appwrite\Utopia\Response\Model\Webhook;
 use Appwrite\Utopia\Response\Model\Preferences;
 use Appwrite\Utopia\Response\Model\HealthAntivirus;
+use Appwrite\Utopia\Response\Model\HealthCertificate;
 use Appwrite\Utopia\Response\Model\HealthQueue;
 use Appwrite\Utopia\Response\Model\HealthStatus;
 use Appwrite\Utopia\Response\Model\HealthTime;
 use Appwrite\Utopia\Response\Model\HealthVersion;
 use Appwrite\Utopia\Response\Model\Installation;
 use Appwrite\Utopia\Response\Model\LocaleCode;
+use Appwrite\Utopia\Response\Model\MetricBreakdown;
 use Appwrite\Utopia\Response\Model\Provider;
 use Appwrite\Utopia\Response\Model\ProviderRepository;
 use Appwrite\Utopia\Response\Model\Runtime;
@@ -113,6 +112,7 @@ class Response extends SwooleResponse
     public const MODEL_ERROR = 'error';
     public const MODEL_METRIC = 'metric';
     public const MODEL_METRIC_LIST = 'metricList';
+    public const MODEL_METRIC_BREAKDOWN = 'metricBreakdown';
     public const MODEL_ERROR_DEV = 'errorDev';
     public const MODEL_BASE_LIST = 'baseList';
     public const MODEL_USAGE_DATABASES = 'usageDatabases';
@@ -254,6 +254,7 @@ class Response extends SwooleResponse
     public const MODEL_HEALTH_QUEUE = 'healthQueue';
     public const MODEL_HEALTH_TIME = 'healthTime';
     public const MODEL_HEALTH_ANTIVIRUS = 'healthAntivirus';
+    public const MODEL_HEALTH_CERTIFICATE = 'healthCertificate';
     public const MODEL_HEALTH_STATUS_LIST = 'healthStatusList';
 
     // Console
@@ -391,9 +392,11 @@ class Response extends SwooleResponse
             ->setModel(new HealthAntivirus())
             ->setModel(new HealthQueue())
             ->setModel(new HealthStatus())
+            ->setModel(new HealthCertificate())
             ->setModel(new HealthTime())
             ->setModel(new HealthVersion())
             ->setModel(new Metric())
+            ->setModel(new MetricBreakdown())
             ->setModel(new UsageDatabases())
             ->setModel(new UsageDatabase())
             ->setModel(new UsageCollection())

src/Appwrite/Utopia/Response/Model/Bucket.php

@@ -4,6 +4,7 @@ namespace Appwrite\Utopia\Response\Model;
 
 use Appwrite\Utopia\Response;
 use Appwrite\Utopia\Response\Model;
+use Utopia\Storage\Compression\Compression;
 
 class Bucket extends Model
 {
@@ -68,7 +69,7 @@ class Bucket extends Model
             ])
             ->addRule('compression', [
                 'type' => self::TYPE_STRING,
-                'description' => 'Compression algorithm choosen for compression. Will be one of ' . COMPRESSION_TYPE_NONE . ', [' . COMPRESSION_TYPE_GZIP . '](https://en.wikipedia.org/wiki/Gzip), or [' . COMPRESSION_TYPE_ZSTD . '](https://en.wikipedia.org/wiki/Zstd).',
+                'description' => 'Compression algorithm choosen for compression. Will be one of ' . Compression::NONE . ', [' . Compression::GZIP . '](https://en.wikipedia.org/wiki/Gzip), or [' . Compression::ZSTD . '](https://en.wikipedia.org/wiki/Zstd).',
                 'default' => '',
                 'example' => 'gzip',
                 'array' => false

src/Appwrite/Utopia/Response/Model/HealthCertificate.php

@@ -0,0 +1,71 @@
+<?php
+
+namespace Appwrite\Utopia\Response\Model;
+
+use Appwrite\Utopia\Response;
+use Appwrite\Utopia\Response\Model;
+
+class HealthCertificate extends Model
+{
+    public function __construct()
+    {
+        $this
+            ->addRule('name', [
+                'type' => self::TYPE_STRING,
+                'description' => 'Certificate name',
+                'default' => '',
+                'example' => '/CN=www.google.com',
+            ])
+            ->addRule('subjectSN', [
+                'type' => self::TYPE_STRING,
+                'description' => 'Subject SN',
+                'default' => 'www.google.com',
+                'example' => '',
+            ])
+            ->addRule('issuerOrganisation', [
+                'type' => self::TYPE_STRING,
+                'description' => 'Issuer organisation',
+                'default' => 'Google Trust Services LLC',
+                'example' => '',
+            ])
+            ->addRule('validFrom', [
+                'type' => self::TYPE_STRING,
+                'description' => 'Valid from',
+                'default' => '',
+                'example' => '1704200998',
+            ])
+            ->addRule('validTo', [
+                'type' => self::TYPE_STRING,
+                'description' => 'Valid to',
+                'default' => '',
+                'example' => '1711458597',
+            ])
+            ->addRule('signatureTypeSN', [
+                'type' => self::TYPE_STRING,
+                'description' => 'Signature type SN',
+                'default' => '',
+                'example' => 'RSA-SHA256',
+            ])
+        ;
+    }
+
+    /**
+     * Get Name
+     *
+     * @return string
+     */
+    public function getName(): string
+    {
+        return 'Health Certificate';
+    }
+
+    /**
+     * Get Type
+     *
+     * @return string
+     */
+    public function getType(): string
+    {
+        return Response::MODEL_HEALTH_CERTIFICATE;
+    }
+}

src/Appwrite/Utopia/Response/Model/MetricBreakdown.php

@@ -0,0 +1,52 @@
+<?php
+
+namespace Appwrite\Utopia\Response\Model;
+
+use Appwrite\Utopia\Response;
+use Appwrite\Utopia\Response\Model;
+
+class MetricBreakdown extends Model
+{
+    public function __construct()
+    {
+        $this
+            ->addRule('resourceId', [
+                'type' => self::TYPE_STRING,
+                'description' => 'Resource ID.',
+                'default' => '',
+                'example' => '5e5ea5c16897e',
+            ])
+            ->addRule('name', [
+                'type' => self::TYPE_STRING,
+                'description' => 'Resource name.',
+                'default' => '',
+                'example' => 'Documents',
+            ])
+            ->addRule('value', [
+                'type' => self::TYPE_INTEGER,
+                'description' => 'The value of this metric at the timestamp.',
+                'default' => 0,
+                'example' => 1,
+            ]);
+    }
+
+    /**
+     * Get Name
+     *
+     * @return string
+     */
+    public function getName(): string
+    {
+        return 'Metric Breakdown';
+    }
+
+    /**
+     * Get Collection
+     *
+     * @return string
+     */
+    public function getType(): string
+    {
+        return Response::MODEL_METRIC_BREAKDOWN;
+    }
+}

src/Appwrite/Utopia/Response/Model/UsageBuckets.php

@@ -12,48 +12,32 @@ class UsageBuckets extends Model
         $this
             ->addRule('range', [
                 'type' => self::TYPE_STRING,
-                'description' => 'The time range of the usage stats.',
+                'description' => 'Time range of the usage stats.',
                 'default' => '',
                 'example' => '30d',
             ])
-            ->addRule('filesCount', [
+            ->addRule('filesTotal', [
+                'type' => self::TYPE_INTEGER,
+                'description' => 'Total aggregated number of bucket files.',
+                'default' => 0,
+                'example' => 0,
+            ])
+            ->addRule('filesStorageTotal', [
+                'type' => self::TYPE_INTEGER,
+                'description' => 'Total aggregated number of bucket files storage (in bytes).',
+                'default' => 0,
+                'example' => 0,
+            ])
+            ->addRule('files', [
                 'type' => Response::MODEL_METRIC,
-                'description' => 'Aggregated stats for total number of files in this bucket.',
+                'description' => 'Aggregated  number of bucket files per period.',
                 'default' => [],
                 'example' => [],
                 'array' => true
             ])
-            ->addRule('filesStorage', [
+            ->addRule('storage', [
                 'type' => Response::MODEL_METRIC,
-                'description' => 'Aggregated stats for total storage of files in this bucket.',
-                'default' => [],
-                'example' => [],
-                'array' => true
-            ])
-            ->addRule('filesCreate', [
-                'type' => Response::MODEL_METRIC,
-                'description' => 'Aggregated stats for files created.',
-                'default' => [],
-                'example' => [],
-                'array' => true
-            ])
-            ->addRule('filesRead', [
-                'type' => Response::MODEL_METRIC,
-                'description' => 'Aggregated stats for files read.',
-                'default' => [],
-                'example' => [],
-                'array' => true
-            ])
-            ->addRule('filesUpdate', [
-                'type' => Response::MODEL_METRIC,
-                'description' => 'Aggregated stats for files updated.',
-                'default' => [],
-                'example' => [],
-                'array' => true
-            ])
-            ->addRule('filesDelete', [
-                'type' => Response::MODEL_METRIC,
-                'description' => 'Aggregated stats for files deleted.',
+                'description' => 'Aggregated  number of bucket storage files (in bytes) per period.',
                 'default' => [],
                 'example' => [],
                 'array' => true

src/Appwrite/Utopia/Response/Model/UsageCollection.php

@@ -12,41 +12,19 @@ class UsageCollection extends Model
         $this
             ->addRule('range', [
                 'type' => self::TYPE_STRING,
-                'description' => 'The time range of the usage stats.',
+                'description' => 'Time range of the usage stats.',
                 'default' => '',
                 'example' => '30d',
             ])
-            ->addRule('documentsCount', [
-                'type' => Response::MODEL_METRIC,
-                'description' => 'Aggregated stats for total number of documents.',
-                'default' => [],
-                'example' => [],
-                'array' => true
+            ->addRule('documentsTotal', [
+                'type' => self::TYPE_INTEGER,
+                'description' => 'Total aggregated number of of documents.',
+                'default' => 0,
+                'example' => 0,
             ])
-            ->addRule('documentsCreate', [
+            ->addRule('documents', [
                 'type' => Response::MODEL_METRIC,
-                'description' => 'Aggregated stats for documents created.',
-                'default' => [],
-                'example' => [],
-                'array' => true
-            ])
-            ->addRule('documentsRead', [
-                'type' => Response::MODEL_METRIC,
-                'description' => 'Aggregated stats for documents read.',
-                'default' => [],
-                'example' => [],
-                'array' => true
-            ])
-            ->addRule('documentsUpdate', [
-                'type' => Response::MODEL_METRIC,
-                'description' => 'Aggregated stats for documents updated.',
-                'default' => [],
-                'example' => [],
-                'array' => true
-            ])
-            ->addRule('documentsDelete', [
-                'type' => Response::MODEL_METRIC,
-                'description' => 'Aggregated stats for documents deleted.',
+                'description' => 'Aggregated  number of documents per period.',
                 'default' => [],
                 'example' => [],
                 'array' => true

src/Appwrite/Utopia/Response/Model/UsageDatabase.php

@@ -12,76 +12,32 @@ class UsageDatabase extends Model
         $this
             ->addRule('range', [
                 'type' => self::TYPE_STRING,
-                'description' => 'The time range of the usage stats.',
+                'description' => 'Time range of the usage stats.',
                 'default' => '',
                 'example' => '30d',
             ])
-            ->addRule('documentsCount', [
+            ->addRule('collectionsTotal', [
+                'type' => self::TYPE_INTEGER,
+                'description' => 'Total aggregated number of collections.',
+                'default' => 0,
+                'example' => 0,
+            ])
+            ->addRule('documentsTotal', [
+                'type' => self::TYPE_INTEGER,
+                'description' => 'Total aggregated number of documents.',
+                'default' => 0,
+                'example' => 0,
+            ])
+            ->addRule('collections', [
                 'type' => Response::MODEL_METRIC,
-                'description' => 'Aggregated stats for total number of documents.',
+                'description' => 'Aggregated  number of collections per period.',
                 'default' => [],
                 'example' => [],
                 'array' => true
             ])
-            ->addRule('collectionsCount', [
+            ->addRule('documents', [
                 'type' => Response::MODEL_METRIC,
-                'description' => 'Aggregated stats for total number of collections.',
-                'default' => [],
-                'example' => [],
-                'array' => true
-            ])
-            ->addRule('documentsCreate', [
-                'type' => Response::MODEL_METRIC,
-                'description' => 'Aggregated stats for documents created.',
-                'default' => [],
-                'example' => [],
-                'array' => true
-            ])
-            ->addRule('documentsRead', [
-                'type' => Response::MODEL_METRIC,
-                'description' => 'Aggregated stats for documents read.',
-                'default' => [],
-                'example' => [],
-                'array' => true
-            ])
-            ->addRule('documentsUpdate', [
-                'type' => Response::MODEL_METRIC,
-                'description' => 'Aggregated stats for documents updated.',
-                'default' => [],
-                'example' => [],
-                'array' => true
-            ])
-            ->addRule('documentsDelete', [
-                'type' => Response::MODEL_METRIC,
-                'description' => 'Aggregated stats for documents deleted.',
-                'default' => [],
-                'example' => [],
-                'array' => true
-            ])
-            ->addRule('collectionsCreate', [
-                'type' => Response::MODEL_METRIC,
-                'description' => 'Aggregated stats for collections created.',
-                'default' => [],
-                'example' => [],
-                'array' => true
-            ])
-            ->addRule('collectionsRead', [
-                'type' => Response::MODEL_METRIC,
-                'description' => 'Aggregated stats for collections read.',
-                'default' => [],
-                'example' => [],
-                'array' => true
-            ])
-            ->addRule('collectionsUpdate', [
-                'type' => Response::MODEL_METRIC,
-                'description' => 'Aggregated stats for collections updated.',
-                'default' => [],
-                'example' => [],
-                'array' => true
-            ])
-            ->addRule('collectionsDelete', [
-                'type' => Response::MODEL_METRIC,
-                'description' => 'Aggregated stats for collections delete.',
+                'description' => 'Aggregated  number of documents per period.',
                 'default' => [],
                 'example' => [],
                 'array' => true

src/Appwrite/Utopia/Response/Model/UsageDatabases.php

@@ -12,111 +12,45 @@ class UsageDatabases extends Model
         $this
             ->addRule('range', [
                 'type' => self::TYPE_STRING,
-                'description' => 'The time range of the usage stats.',
+                'description' => 'Time range of the usage stats.',
                 'default' => '',
                 'example' => '30d',
             ])
-            ->addRule('databasesCount', [
+            ->addRule('databasesTotal', [
+                'type' => self::TYPE_INTEGER,
+                'description' => 'Total aggregated number of databases.',
+                'default' => 0,
+                'example' => 0,
+            ])
+            ->addRule('collectionsTotal', [
+                'type' => self::TYPE_INTEGER,
+                'description' => 'Total aggregated number  of collections.',
+                'default' => 0,
+                'example' => 0,
+            ])
+            ->addRule('documentsTotal', [
+                'type' => self::TYPE_INTEGER,
+                'description' => 'Total aggregated number of documents.',
+                'default' => 0,
+                'example' => 0,
+            ])
+            ->addRule('databases', [
                 'type' => Response::MODEL_METRIC,
-                'description' => 'Aggregated stats for total number of documents.',
+                'description' => 'Aggregated number of databases per period.',
                 'default' => [],
                 'example' => [],
                 'array' => true
             ])
-            ->addRule('documentsCount', [
+            ->addRule('collections', [
                 'type' => Response::MODEL_METRIC,
-                'description' => 'Aggregated stats for total number of documents.',
+                'description' => 'Aggregated number of collections per period.',
                 'default' => [],
                 'example' => [],
                 'array' => true
             ])
-            ->addRule('collectionsCount', [
+            ->addRule('documents', [
                 'type' => Response::MODEL_METRIC,
-                'description' => 'Aggregated stats for total number of collections.',
-                'default' => [],
-                'example' => [],
-                'array' => true
-            ])
-            ->addRule('databasesCreate', [
-                'type' => Response::MODEL_METRIC,
-                'description' => 'Aggregated stats for documents created.',
-                'default' => [],
-                'example' => [],
-                'array' => true
-            ])
-            ->addRule('databasesRead', [
-                'type' => Response::MODEL_METRIC,
-                'description' => 'Aggregated stats for documents read.',
-                'default' => [],
-                'example' => [],
-                'array' => true
-            ])
-            ->addRule('databasesUpdate', [
-                'type' => Response::MODEL_METRIC,
-                'description' => 'Aggregated stats for documents updated.',
-                'default' => [],
-                'example' => [],
-                'array' => true
-            ])
-            ->addRule('databasesDelete', [
-                'type' => Response::MODEL_METRIC,
-                'description' => 'Aggregated stats for total number of collections.',
-                'default' => [],
-                'example' => [],
-                'array' => true
-            ])
-            ->addRule('documentsCreate', [
-                'type' => Response::MODEL_METRIC,
-                'description' => 'Aggregated stats for documents created.',
-                'default' => [],
-                'example' => [],
-                'array' => true
-            ])
-            ->addRule('documentsRead', [
-                'type' => Response::MODEL_METRIC,
-                'description' => 'Aggregated stats for documents read.',
-                'default' => [],
-                'example' => [],
-                'array' => true
-            ])
-            ->addRule('documentsUpdate', [
-                'type' => Response::MODEL_METRIC,
-                'description' => 'Aggregated stats for documents updated.',
-                'default' => [],
-                'example' => [],
-                'array' => true
-            ])
-            ->addRule('documentsDelete', [
-                'type' => Response::MODEL_METRIC,
-                'description' => 'Aggregated stats for documents deleted.',
-                'default' => [],
-                'example' => [],
-                'array' => true
-            ])
-            ->addRule('collectionsCreate', [
-                'type' => Response::MODEL_METRIC,
-                'description' => 'Aggregated stats for collections created.',
-                'default' => [],
-                'example' => [],
-                'array' => true
-            ])
-            ->addRule('collectionsRead', [
-                'type' => Response::MODEL_METRIC,
-                'description' => 'Aggregated stats for collections read.',
-                'default' => [],
-                'example' => [],
-                'array' => true
-            ])
-            ->addRule('collectionsUpdate', [
-                'type' => Response::MODEL_METRIC,
-                'description' => 'Aggregated stats for collections updated.',
-                'default' => [],
-                'example' => [],
-                'array' => true
-            ])
-            ->addRule('collectionsDelete', [
-                'type' => Response::MODEL_METRIC,
-                'description' => 'Aggregated stats for collections delete.',
+                'description' => 'Aggregated number of documents per period.',
                 'default' => [],
                 'example' => [],
                 'array' => true

src/Appwrite/Utopia/Response/Model/UsageFunction.php

@@ -16,58 +16,94 @@ class UsageFunction extends Model
                 'default' => '',
                 'example' => '30d',
             ])
-            ->addRule('executionsTotal', [
-                'type' => Response::MODEL_METRIC,
-                'description' => 'Aggregated stats for number of function executions.',
-                'default' => [],
-                'example' => [],
-                'array' => true
+            ->addRule('deploymentsTotal', [
+                'type' => self::TYPE_INTEGER,
+                'description' => 'Total aggregated number of function deployments.',
+                'default' => 0,
+                'example' => 0,
             ])
-            ->addRule('executionsFailure', [
-                'type' => Response::MODEL_METRIC,
-                'description' => 'Aggregated stats for function execution failures.',
-                'default' => [],
-                'example' => [],
-                'array' => true
-            ])
-            ->addRule('executionsSuccess', [
-                'type' => Response::MODEL_METRIC,
-                'description' => 'Aggregated stats for function execution successes.',
-                'default' => [],
-                'example' => [],
-                'array' => true
-            ])
-            ->addRule('executionsTime', [
-                'type' => Response::MODEL_METRIC,
-                'description' => 'Aggregated stats for function execution duration.',
-                'default' => [],
-                'example' => [],
-                'array' => true
+            ->addRule('deploymentsStorageTotal', [
+                'type' => self::TYPE_INTEGER,
+                'description' => 'Total aggregated sum of function deployments storage.',
+                'default' => 0,
+                'example' => 0,
             ])
             ->addRule('buildsTotal', [
+                'type' => self::TYPE_INTEGER,
+                'description' => 'Total aggregated number of function builds.',
+                'default' => 0,
+                'example' => 0,
+            ])
+            ->addRule('buildsStorageTotal', [
+                'type' => self::TYPE_INTEGER,
+                'description' => 'total aggregated sum of function builds storage.',
+                'default' => 0,
+                'example' => 0,
+            ])
+            ->addRule('buildsTimeTotal', [
+                'type' => self::TYPE_INTEGER,
+                'description' => 'Total aggregated sum of function builds compute time.',
+                'default' => 0,
+                'example' => 0,
+            ])
+            ->addRule('executionsTotal', [
+                'type' => self::TYPE_INTEGER,
+                'description' => 'Total  aggregated number of function executions.',
+                'default' => 0,
+                'example' => 0,
+            ])
+            ->addRule('executionsTimeTotal', [
+                'type' => self::TYPE_INTEGER,
+                'description' => 'Total aggregated sum of function  executions compute time.',
+                'default' => 0,
+                'example' => 0,
+            ])
+            ->addRule('deployments', [
                 'type' => Response::MODEL_METRIC,
-                'description' => 'Aggregated stats for number of function builds.',
+                'description' => 'Aggregated number of function deployments per period.',
                 'default' => [],
                 'example' => [],
                 'array' => true
             ])
-            ->addRule('buildsFailure', [
+            ->addRule('deploymentsStorage', [
                 'type' => Response::MODEL_METRIC,
-                'description' => 'Aggregated stats for function build failures.',
+                'description' => 'Aggregated number of  function deployments storage per period.',
                 'default' => [],
                 'example' => [],
                 'array' => true
             ])
-            ->addRule('buildsSuccess', [
+            ->addRule('builds', [
                 'type' => Response::MODEL_METRIC,
-                'description' => 'Aggregated stats for function build successes.',
+                'description' => 'Aggregated number of function builds per period.',
+                'default' => [],
+                'example' => [],
+                'array' => true
+            ])
+            ->addRule('buildsStorage', [
+                'type' => Response::MODEL_METRIC,
+                'description' => 'Aggregated sum of function builds storage per period.',
                 'default' => [],
                 'example' => [],
                 'array' => true
             ])
             ->addRule('buildsTime', [
                 'type' => Response::MODEL_METRIC,
-                'description' => 'Aggregated stats for function build duration.',
+                'description' => 'Aggregated sum of function builds compute time per period.',
+                'default' => [],
+                'example' => [],
+                'array' => true
+            ])
+            ->addRule('executions', [
+                'type' => Response::MODEL_METRIC,
+                'description' => 'Aggregated number of function executions per period.',
+                'default' => [],
+                'example' => [],
+                'array' => true
+            ])
+
+            ->addRule('executionsTime', [
+                'type' => Response::MODEL_METRIC,
+                'description' => 'Aggregated number of function executions compute time per period.',
                 'default' => [],
                 'example' => [],
                 'array' => true

src/Appwrite/Utopia/Response/Model/UsageFunctions.php

@@ -12,62 +12,111 @@ class UsageFunctions extends Model
         $this
             ->addRule('range', [
                 'type' => self::TYPE_STRING,
-                'description' => 'The time range of the usage stats.',
+                'description' => 'Time range of the usage stats.',
                 'default' => '',
                 'example' => '30d',
             ])
-            ->addRule('executionsTotal', [
-                'type' => Response::MODEL_METRIC,
-                'description' => 'Aggregated stats for number of function executions.',
-                'default' => [],
-                'example' => [],
-                'array' => true
+            ->addRule('functionsTotal', [
+                'type' => self::TYPE_INTEGER,
+                'description' => 'Total aggregated number of functions.',
+                'default' => 0,
+                'example' => 0,
             ])
-            ->addRule('executionsFailure', [
-                'type' => Response::MODEL_METRIC,
-                'description' => 'Aggregated stats for function execution failures.',
-                'default' => [],
-                'example' => [],
-                'array' => true
+            ->addRule('deploymentsTotal', [
+                'type' => self::TYPE_INTEGER,
+                'description' => 'Total aggregated number of functions deployments.',
+                'default' => 0,
+                'example' => 0,
             ])
-            ->addRule('executionsSuccess', [
-                'type' => Response::MODEL_METRIC,
-                'description' => 'Aggregated stats for function execution successes.',
-                'default' => [],
-                'example' => [],
-                'array' => true
-            ])
-            ->addRule('executionsTime', [
-                'type' => Response::MODEL_METRIC,
-                'description' => 'Aggregated stats for function execution duration.',
-                'default' => [],
-                'example' => [],
-                'array' => true
+            ->addRule('deploymentsStorageTotal', [
+                'type' => self::TYPE_INTEGER,
+                'description' => 'Total aggregated sum of functions deployment storage.',
+                'default' => 0,
+                'example' => 0,
             ])
             ->addRule('buildsTotal', [
+                'type' => self::TYPE_INTEGER,
+                'description' => 'Total aggregated number of functions build.',
+                'default' => 0,
+                'example' => 0,
+            ])
+            ->addRule('buildsStorageTotal', [
+                'type' => self::TYPE_INTEGER,
+                'description' => 'total aggregated sum of functions build storage.',
+                'default' => 0,
+                'example' => 0,
+            ])
+            ->addRule('buildsTimeTotal', [
+                'type' => self::TYPE_INTEGER,
+                'description' => 'Total aggregated sum of functions build compute time.',
+                'default' => 0,
+                'example' => 0,
+            ])
+            ->addRule('executionsTotal', [
+                'type' => self::TYPE_INTEGER,
+                'description' => 'Total  aggregated number of functions execution.',
+                'default' => 0,
+                'example' => 0,
+            ])
+            ->addRule('executionsTimeTotal', [
+                'type' => self::TYPE_INTEGER,
+                'description' => 'Total aggregated sum of functions  execution compute time.',
+                'default' => 0,
+                'example' => 0,
+            ])
+            ->addRule('functions', [
                 'type' => Response::MODEL_METRIC,
-                'description' => 'Aggregated stats for number of function builds.',
+                'description' => 'Aggregated number of functions per period.',
+                'default' => 0,
+                'example' => 0,
+                'array' => true
+            ])
+            ->addRule('deployments', [
+                'type' => Response::MODEL_METRIC,
+                'description' => 'Aggregated number of functions deployment per period.',
                 'default' => [],
                 'example' => [],
                 'array' => true
             ])
-            ->addRule('buildsFailure', [
+            ->addRule('deploymentsStorage', [
                 'type' => Response::MODEL_METRIC,
-                'description' => 'Aggregated stats for function build failures.',
+                'description' => 'Aggregated number of  functions deployment storage per period.',
                 'default' => [],
                 'example' => [],
                 'array' => true
             ])
-            ->addRule('buildsSuccess', [
+            ->addRule('builds', [
                 'type' => Response::MODEL_METRIC,
-                'description' => 'Aggregated stats for function build successes.',
+                'description' => 'Aggregated number of functions build per period.',
+                'default' => [],
+                'example' => [],
+                'array' => true
+            ])
+            ->addRule('buildsStorage', [
+                'type' => Response::MODEL_METRIC,
+                'description' => 'Aggregated sum of functions build storage per period.',
                 'default' => [],
                 'example' => [],
                 'array' => true
             ])
             ->addRule('buildsTime', [
                 'type' => Response::MODEL_METRIC,
-                'description' => 'Aggregated stats for function build duration.',
+                'description' => 'Aggregated sum of  functions build compute time per period.',
+                'default' => [],
+                'example' => [],
+                'array' => true
+            ])
+            ->addRule('executions', [
+                'type' => Response::MODEL_METRIC,
+                'description' => 'Aggregated number of  functions execution per period.',
+                'default' => [],
+                'example' => [],
+                'array' => true
+            ])
+
+            ->addRule('executionsTime', [
+                'type' => Response::MODEL_METRIC,
+                'description' => 'Aggregated number of functions execution compute time per period.',
                 'default' => [],
                 'example' => [],
                 'array' => true

src/Appwrite/Utopia/Response/Model/UsageProject.php

@@ -10,64 +10,80 @@ class UsageProject extends Model
     public function __construct()
     {
         $this
-            ->addRule('range', [
-                'type' => self::TYPE_STRING,
-                'description' => 'The time range of the usage stats.',
-                'default' => '',
-                'example' => '30d',
+            ->addRule('executionsTotal', [
+                'type' => self::TYPE_INTEGER,
+                'description' => 'Total aggregated number of function executions.',
+                'default' => 0,
+                'example' => 0,
+            ])
+            ->addRule('documentsTotal', [
+                'type' => self::TYPE_INTEGER,
+                'description' => 'Total aggregated  number of documents.',
+                'default' => 0,
+                'example' => 0,
+            ])
+            ->addRule('databasesTotal', [
+                'type' => self::TYPE_INTEGER,
+                'description' => 'Total aggregated number of databases.',
+                'default' => 0,
+                'example' => 0,
+            ])
+            ->addRule('usersTotal', [
+                'type' => self::TYPE_INTEGER,
+                'description' => 'Total aggregated number of users.',
+                'default' => 0,
+                'example' => 0,
+            ])
+            ->addRule('filesStorageTotal', [
+                'type' => self::TYPE_INTEGER,
+                'description' => 'Total aggregated sum of files storage size (in bytes).',
+                'default' => 0,
+                'example' => 0,
+            ])
+            ->addRule('bucketsTotal', [
+                'type' => self::TYPE_INTEGER,
+                'description' => 'Total aggregated number of buckets.',
+                'default' => 0,
+                'example' => 0,
             ])
             ->addRule('requests', [
                 'type' => Response::MODEL_METRIC,
-                'description' => 'Aggregated stats for number of requests.',
+                'description' => 'Aggregated  number of requests per period.',
                 'default' => [],
                 'example' => [],
                 'array' => true
             ])
             ->addRule('network', [
                 'type' => Response::MODEL_METRIC,
-                'description' => 'Aggregated stats for consumed bandwidth.',
-                'default' => [],
-                'example' => [],
-                'array' => true
-            ])
-            ->addRule('executions', [
-                'type' => Response::MODEL_METRIC,
-                'description' => 'Aggregated stats for function executions.',
-                'default' => [],
-                'example' => [],
-                'array' => true
-            ])
-            ->addRule('documents', [
-                'type' => Response::MODEL_METRIC,
-                'description' => 'Aggregated stats for number of documents.',
-                'default' => [],
-                'example' => [],
-                'array' => true
-            ])
-            ->addRule('databases', [
-                'type' => Response::MODEL_METRIC,
-                'description' => 'Aggregated stats for number of databases.',
+                'description' => 'Aggregated number of consumed bandwidth per period.',
                 'default' => [],
                 'example' => [],
                 'array' => true
             ])
             ->addRule('users', [
                 'type' => Response::MODEL_METRIC,
-                'description' => 'Aggregated stats for number of users.',
+                'description' => 'Aggregated number of users per period.',
                 'default' => [],
                 'example' => [],
                 'array' => true
             ])
-            ->addRule('storage', [
+            ->addRule('executions', [
                 'type' => Response::MODEL_METRIC,
-                'description' => 'Aggregated stats for the occupied storage size (in bytes).',
+                'description' => 'Aggregated number of executions per period.',
                 'default' => [],
                 'example' => [],
                 'array' => true
             ])
-            ->addRule('buckets', [
-                'type' => Response::MODEL_METRIC,
-                'description' => 'Aggregated stats for number of buckets.',
+            ->addRule('executionsBreakdown', [
+                'type' => Response::MODEL_METRIC_BREAKDOWN,
+                'description' => 'Aggregated breakdown in totals of executions by functions.',
+                'default' => [],
+                'example' => [],
+                'array' => true
+            ])
+            ->addRule('bucketsBreakdown', [
+                'type' => Response::MODEL_METRIC_BREAKDOWN,
+                'description' => 'Aggregated breakdown in totals of usage by buckets.',
                 'default' => [],
                 'example' => [],
                 'array' => true

src/Appwrite/Utopia/Response/Model/UsageStorage.php

@@ -12,83 +12,45 @@ class UsageStorage extends Model
         $this
             ->addRule('range', [
                 'type' => self::TYPE_STRING,
-                'description' => 'The time range of the usage stats.',
+                'description' => 'Time range of the usage stats.',
                 'default' => '',
                 'example' => '30d',
             ])
+            ->addRule('bucketsTotal', [
+                'type' => self::TYPE_INTEGER,
+                'description' => 'Total aggregated number of buckets',
+                'default' => 0,
+                'example' => 0,
+            ])
+            ->addRule('filesTotal', [
+                'type' => self::TYPE_INTEGER,
+                'description' => 'Total aggregated number of files.',
+                'default' => 0,
+                'example' => 0,
+            ])
+            ->addRule('filesStorageTotal', [
+                'type' => self::TYPE_INTEGER,
+                'description' => 'Total aggregated number of files storage (in bytes).',
+                'default' => 0,
+                'example' => 0,
+            ])
+            ->addRule('buckets', [
+                'type' => Response::MODEL_METRIC,
+                'description' => 'Aggregated number of buckets per period.',
+                'default' => [],
+                'example' => [],
+                'array' => true
+            ])
+            ->addRule('files', [
+                'type' => Response::MODEL_METRIC,
+                'description' => 'Aggregated number of files per period.',
+                'default' => [],
+                'example' => [],
+                'array' => true
+            ])
             ->addRule('storage', [
                 'type' => Response::MODEL_METRIC,
-                'description' => 'Aggregated stats for the occupied storage size (in bytes).',
-                'default' => [],
-                'example' => [],
-                'array' => true
-            ])
-            ->addRule('filesCount', [
-                'type' => Response::MODEL_METRIC,
-                'description' => 'Aggregated stats for total number of files.',
-                'default' => [],
-                'example' => [],
-                'array' => true
-            ])
-            ->addRule('bucketsCount', [
-                'type' => Response::MODEL_METRIC,
-                'description' => 'Aggregated stats for total number of buckets.',
-                'default' => [],
-                'example' => [],
-                'array' => true
-            ])
-            ->addRule('bucketsCreate', [
-                'type' => Response::MODEL_METRIC,
-                'description' => 'Aggregated stats for buckets created.',
-                'default' => [],
-                'example' => [],
-                'array' => true
-            ])
-            ->addRule('bucketsRead', [
-                'type' => Response::MODEL_METRIC,
-                'description' => 'Aggregated stats for buckets read.',
-                'default' => [],
-                'example' => [],
-                'array' => true
-            ])
-            ->addRule('bucketsUpdate', [
-                'type' => Response::MODEL_METRIC,
-                'description' => 'Aggregated stats for buckets updated.',
-                'default' => [],
-                'example' => [],
-                'array' => true
-            ])
-            ->addRule('bucketsDelete', [
-                'type' => Response::MODEL_METRIC,
-                'description' => 'Aggregated stats for buckets deleted.',
-                'default' => [],
-                'example' => [],
-                'array' => true
-            ])
-            ->addRule('filesCreate', [
-                'type' => Response::MODEL_METRIC,
-                'description' => 'Aggregated stats for files created.',
-                'default' => [],
-                'example' => [],
-                'array' => true
-            ])
-            ->addRule('filesRead', [
-                'type' => Response::MODEL_METRIC,
-                'description' => 'Aggregated stats for files read.',
-                'default' => [],
-                'example' => [],
-                'array' => true
-            ])
-            ->addRule('filesUpdate', [
-                'type' => Response::MODEL_METRIC,
-                'description' => 'Aggregated stats for files updated.',
-                'default' => [],
-                'example' => [],
-                'array' => true
-            ])
-            ->addRule('filesDelete', [
-                'type' => Response::MODEL_METRIC,
-                'description' => 'Aggregated stats for files deleted.',
+                'description' => 'Aggregated number of files storage (in bytes) per period .',
                 'default' => [],
                 'example' => [],
                 'array' => true

src/Appwrite/Utopia/Response/Model/UsageUsers.php

@@ -12,62 +12,34 @@ class UsageUsers extends Model
         $this
             ->addRule('range', [
                 'type' => self::TYPE_STRING,
-                'description' => 'The time range of the usage stats.',
+                'description' => 'Time range of the usage stats.',
                 'default' => '',
                 'example' => '30d',
             ])
-            ->addRule('usersCount', [
+            ->addRule('usersTotal', [
+                'type' => self::TYPE_INTEGER,
+                'description' => 'Total aggregated number of statistics of users.',
+                'default' => 0,
+                'example' => 0,
+            ])
+
+            ->addRule('sessionsTotal', [
+                'type' => self::TYPE_INTEGER,
+                'description' => 'Total aggregated number of active sessions.',
+                'default' => 0,
+                'example' => 0,
+            ])
+            ->addRule('users', [
                 'type' => Response::MODEL_METRIC,
-                'description' => 'Aggregated stats for total number of users.',
+                'description' => 'Aggregated number of users per period.',
                 'default' => [],
                 'example' => [],
                 'array' => true
             ])
-            ->addRule('usersCreate', [
+
+            ->addRule('sessions', [
                 'type' => Response::MODEL_METRIC,
-                'description' => 'Aggregated stats for users created.',
-                'default' => [],
-                'example' => [],
-                'array' => true
-            ])
-            ->addRule('usersRead', [
-                'type' => Response::MODEL_METRIC,
-                'description' => 'Aggregated stats for users read.',
-                'default' => [],
-                'example' => [],
-                'array' => true
-            ])
-            ->addRule('usersUpdate', [
-                'type' => Response::MODEL_METRIC,
-                'description' => 'Aggregated stats for users updated.',
-                'default' => [],
-                'example' => [],
-                'array' => true
-            ])
-            ->addRule('usersDelete', [
-                'type' => Response::MODEL_METRIC,
-                'description' => 'Aggregated stats for users deleted.',
-                'default' => [],
-                'example' => [],
-                'array' => true
-            ])
-            ->addRule('sessionsCreate', [
-                'type' => Response::MODEL_METRIC,
-                'description' => 'Aggregated stats for sessions created.',
-                'default' => [],
-                'example' => [],
-                'array' => true
-            ])
-            ->addRule('sessionsProviderCreate', [
-                'type' => Response::MODEL_METRIC,
-                'description' => 'Aggregated stats for sessions created for a provider ( email, anonymous or oauth2 ).',
-                'default' => [],
-                'example' => [],
-                'array' => true
-            ])
-            ->addRule('sessionsDelete', [
-                'type' => Response::MODEL_METRIC,
-                'description' => 'Aggregated stats for sessions deleted.',
+                'description' => 'Aggregated number of active sessions  per period.',
                 'default' => [],
                 'example' => [],
                 'array' => true

src/Executor/Executor.php

@@ -72,6 +72,7 @@ class Executor
     ) {
         $runtimeId = "$projectId-$deploymentId-build";
         $route = "/runtimes";
+        $timeout = (int) App::getEnv('_APP_FUNCTIONS_BUILD_TIMEOUT', 900);
         $params = [
             'runtimeId' => $runtimeId,
             'source' => $source,
@@ -84,10 +85,9 @@ class Executor
             'cpus' => $this->cpus,
             'memory' => $this->memory,
             'version' => $version,
+            'timeout' => $timeout,
         ];
 
-        $timeout  = (int) App::getEnv('_APP_FUNCTIONS_BUILD_TIMEOUT', 900);
-
         $response = $this->call(self::METHOD_POST, $route, [ 'x-opr-runtime-id' => $runtimeId ], $params, true, $timeout);
 
         $status = $response['headers']['status-code'];
@@ -111,7 +111,7 @@ class Executor
         string $projectId,
         callable $callback
     ) {
-        $timeout  = (int) App::getEnv('_APP_FUNCTIONS_BUILD_TIMEOUT', 900);
+        $timeout = (int) App::getEnv('_APP_FUNCTIONS_BUILD_TIMEOUT', 900);
 
         $runtimeId = "$projectId-$deploymentId-build";
         $route = "/runtimes/{$runtimeId}/logs";