Merge branch '1.6.x' into pla-1883

src/Appwrite/Platform/Tasks/StatsResources.php

@@ -62,7 +62,7 @@ class StatsResources extends Action
             Authorization::disable();
             Authorization::setDefaultStatus(false);
 
-            $last24Hours = (new \DateTime())->sub(\DateInterval::createFromDateString('24 hours'));
+            $last24Hours = (new \DateTime())->sub(\DateInterval::createFromDateString('3 hours'));
             /**
              * For each project that were accessed in last 24 hours
              */

src/Appwrite/Platform/Workers/Audits.php

@@ -7,7 +7,6 @@ use Exception;
 use Throwable;
 use Utopia\Audit\Audit;
 use Utopia\CLI\Console;
-use Utopia\Database\Database;
 use Utopia\Database\DateTime;
 use Utopia\Database\Document;
 use Utopia\Database\Exception\Authorization;
@@ -46,8 +45,9 @@ class Audits extends Action
         $this
             ->desc('Audits worker')
             ->inject('message')
-            ->inject('dbForProject')
-            ->callback(fn ($message, $dbForProject) => $this->action($message, $dbForProject));
+            ->inject('getProjectDB')
+            ->inject('project')
+            ->callback([$this, 'action']);
 
         $this->lastTriggeredTime = time();
     }
@@ -55,14 +55,15 @@ class Audits extends Action
 
     /**
      * @param Message $message
-     * @param Database $dbForProject
+     * @param callable $getProjectDB
+     * @param Document $project
      * @return void
      * @throws Throwable
      * @throws \Utopia\Database\Exception
      * @throws Authorization
      * @throws Structure
      */
-    public function action(Message $message, Database $dbForProject): void
+    public function action(Message $message, callable $getProjectDB, Document $project): void
     {
         $payload = $message->getPayload() ?? [];
 
@@ -103,27 +104,42 @@ class Audits extends Action
             'timestamp' => DateTime::formatTz(DateTime::now())
         ];
 
-        $this->logs[] = $eventData;
+        if (isset($this->logs[$project->getInternalId()])) {
+            $this->logs[$project->getInternalId()]['logs'][] = $eventData;
+        } else {
+            $this->logs[$project->getInternalId()] = [
+                'project' => new Document([
+                    '$id' => $project->getId(),
+                    '$internalId' => $project->getInternalId(),
+                    'database' => $project->getAttribute('database'),
+                ]),
+                'logs' => [$eventData]
+            ];
+        }
 
         // Check if we should process the batch by checking both for the batch size and the elapsed time
         $batchSize = $this->getBatchSize();
-        $shouldProcessBatch = count($this->logs) >= $batchSize;
-        if (!$shouldProcessBatch && count($this->logs) > 0) {
-            $shouldProcessBatch = (time() - $this->lastTriggeredTime) >= self::BATCH_AGGREGATION_INTERVAL;
+        $shouldProcessBatch = \count($this->logs) >= $batchSize;
+        if (!$shouldProcessBatch && \count($this->logs) > 0) {
+            $shouldProcessBatch = (\time() - $this->lastTriggeredTime) >= self::BATCH_AGGREGATION_INTERVAL;
         }
 
         if ($shouldProcessBatch) {
-            Console::log('Processing batch with ' . count($this->logs) . ' events');
-            $audit = new Audit($dbForProject);
-
             try {
-                $audit->logBatch($this->logs);
-                Console::success('Audit logs processed successfully');
+                foreach ($this->logs as $internalId => $projectLogs) {
+                    $dbForProject = $getProjectDB($projectLogs['project']);
+
+                    Console::log('Processing batch with ' . count($projectLogs['logs']) . ' events');
+                    $audit = new Audit($dbForProject);
+
+                    $audit->logBatch($projectLogs['logs']);
+                    Console::success('Audit logs processed successfully');
+
+                    unset($this->logs[$internalId]);
+                }
             } catch (Throwable $e) {
                 Console::error('Error processing audit logs: ' . $e->getMessage());
             } finally {
-                // Clear the pending events after successful batch processing
-                $this->logs = [];
                 $this->lastTriggeredTime = time();
             }
         }

src/Appwrite/Platform/Workers/StatsResources.php

@@ -129,9 +129,20 @@ class StatsResources extends Action
             ]);
             $teams = $dbForProject->count('teams');
             $functions = $dbForProject->count('functions');
+
             $messages = $dbForProject->count('messages');
             $providers = $dbForProject->count('providers');
             $topics = $dbForProject->count('topics');
+            $targets = $dbForProject->count('targets');
+            $emailTargets = $dbForProject->count('targets', [
+                Query::equal('providerType', [MESSAGE_TYPE_EMAIL])
+            ]);
+            $pushTargets = $dbForProject->count('targets', [
+                Query::equal('providerType', [MESSAGE_TYPE_PUSH])
+            ]);
+            $smsTargets = $dbForProject->count('targets', [
+                Query::equal('providerType', [MESSAGE_TYPE_SMS])
+            ]);
 
             $metrics = [
                 METRIC_DATABASES => $databases,
@@ -148,6 +159,10 @@ class StatsResources extends Action
                 METRIC_PROVIDERS => $providers,
                 METRIC_TOPICS => $topics,
                 METRIC_KEYS => $keys,
+                METRIC_TARGETS => $targets,
+                str_replace('{providerType}', MESSAGE_TYPE_EMAIL, METRIC_PROVIDER_TYPE_TARGETS) => $emailTargets,
+                str_replace('{providerType}', MESSAGE_TYPE_PUSH, METRIC_PROVIDER_TYPE_TARGETS) => $pushTargets,
+                str_replace('{providerType}', MESSAGE_TYPE_SMS, METRIC_PROVIDER_TYPE_TARGETS) => $smsTargets,
             ];
 
             foreach ($metrics as $metric => $value) {

src/Appwrite/Platform/Workers/StatsUsage.php

@@ -17,13 +17,20 @@ class StatsUsage extends Action
     private int $lastTriggeredTime = 0;
     private int $keys = 0;
     private const INFINITY_PERIOD = '_inf_';
-    private const KEYS_THRESHOLD = 10000;
+    private const BATCH_SIZE_DEVELOPMENT = 1;
+    private const BATCH_SIZE_PRODUCTION = 10_000;
 
     public static function getName(): string
     {
         return 'stats-usage';
     }
 
+    private function getBatchSize(): int
+    {
+        return System::getEnv('_APP_ENV', 'development') === 'development'
+            ? self::BATCH_SIZE_DEVELOPMENT
+            : self::BATCH_SIZE_PRODUCTION;
+    }
     /**
      * @throws Exception
      */
@@ -86,7 +93,7 @@ class StatsUsage extends Action
 
         // If keys crossed threshold or X time passed since the last send and there are some keys in the array ($this->stats)
         if (
-            $this->keys >= self::KEYS_THRESHOLD ||
+            $this->keys >= $this->getBatchSize() ||
             (time() - $this->lastTriggeredTime > $aggregationInterval  && $this->keys > 0)
         ) {
             Console::warning('[' . DateTime::now() . '] Aggregated ' . $this->keys . ' keys');

src/Appwrite/Utopia/Database/Validator/Queries/Base.php

@@ -24,7 +24,15 @@ class Base extends Queries
     public function __construct(string $collection, array $allowedAttributes)
     {
         $config = Config::getParam('collections', []);
-        $collections = array_merge($config['projects'], $config['buckets'], $config['databases'], $config['console']);
+
+        $collections = array_merge(
+            $config['projects'],
+            $config['buckets'],
+            $config['databases'],
+            $config['console'],
+            $config['logs']
+        );
+
         $collection = $collections[$collection];
         // array for constant lookup time
         $allowedAttributesLookup = [];
@@ -35,6 +43,7 @@ class Base extends Queries
         $attributes = [];
         foreach ($collection['attributes'] as $attribute) {
             $key = $attribute['$id'];
+
             if (!isset($allowedAttributesLookup[$key])) {
                 continue;
             }