Merge branch '1.6.x' into feat-remove-deprecated-usage-stuff

src/Appwrite/Platform/Workers/Audits.php

@@ -6,15 +6,33 @@ use Appwrite\Auth\Auth;
 use Exception;
 use Throwable;
 use Utopia\Audit\Audit;
+use Utopia\CLI\Console;
 use Utopia\Database\Database;
+use Utopia\Database\DateTime;
 use Utopia\Database\Document;
 use Utopia\Database\Exception\Authorization;
 use Utopia\Database\Exception\Structure;
 use Utopia\Platform\Action;
 use Utopia\Queue\Message;
+use Utopia\System\System;
 
 class Audits extends Action
 {
+    private const BATCH_SIZE_DEVELOPMENT = 1; // smaller batch size for development
+    private const BATCH_SIZE_PRODUCTION = 5_000;
+    private const BATCH_AGGREGATION_INTERVAL = 60; // in seconds
+
+    private int $lastTriggeredTime = 0;
+
+    private array $logs = [];
+
+    private function getBatchSize(): int
+    {
+        return System::getEnv('_APP_ENV', 'development') === 'development'
+            ? self::BATCH_SIZE_DEVELOPMENT
+            : self::BATCH_SIZE_PRODUCTION;
+    }
+
     public static function getName(): string
     {
         return 'audits';
@@ -30,6 +48,8 @@ class Audits extends Action
             ->inject('message')
             ->inject('dbForProject')
             ->callback(fn ($message, $dbForProject) => $this->action($message, $dbForProject));
+
+        $this->lastTriggeredTime = time();
     }
 
 
@@ -44,13 +64,14 @@ class Audits extends Action
      */
     public function action(Message $message, Database $dbForProject): void
     {
-
         $payload = $message->getPayload() ?? [];
 
         if (empty($payload)) {
             throw new Exception('Missing payload');
         }
 
+        Console::info('Aggregating audit logs');
+
         $event = $payload['event'] ?? '';
         $auditPayload = $payload['payload'] ?? '';
         $mode = $payload['mode'] ?? '';
@@ -63,23 +84,48 @@ class Audits extends Action
         $userEmail = $user->getAttribute('email', '');
         $userType = $user->getAttribute('type', Auth::ACTIVITY_TYPE_USER);
 
-        $audit = new Audit($dbForProject);
-        $audit->log(
-            userId: $user->getInternalId(),
-            // Pass first, most verbose event pattern
-            event: $event,
-            resource: $resource,
-            userAgent: $userAgent,
-            ip: $ip,
-            location: '',
-            data: [
+        // Create event data
+        $eventData = [
+            'userId' => $user->getInternalId(),
+            'event' => $event,
+            'resource' => $resource,
+            'userAgent' => $userAgent,
+            'ip' => $ip,
+            'location' => '',
+            'data' => [
                 'userId' => $user->getId(),
                 'userName' => $userName,
                 'userEmail' => $userEmail,
                 'userType' => $userType,
                 'mode' => $mode,
                 'data' => $auditPayload,
-            ]
-        );
+            ],
+            'timestamp' => DateTime::formatTz(DateTime::now())
+        ];
+
+        $this->logs[] = $eventData;
+
+        // Check if we should process the batch by checking both for the batch size and the elapsed time
+        $batchSize = $this->getBatchSize();
+        $shouldProcessBatch = count($this->logs) >= $batchSize;
+        if (!$shouldProcessBatch && count($this->logs) > 0) {
+            $shouldProcessBatch = (time() - $this->lastTriggeredTime) >= self::BATCH_AGGREGATION_INTERVAL;
+        }
+
+        if ($shouldProcessBatch) {
+            Console::log('Processing batch with ' . count($this->logs) . ' events');
+            $audit = new Audit($dbForProject);
+
+            try {
+                $audit->logBatch($this->logs);
+                Console::success('Audit logs processed successfully');
+            } catch (Throwable $e) {
+                Console::error('Error processing audit logs: ' . $e->getMessage());
+            } finally {
+                // Clear the pending events after successful batch processing
+                $this->logs = [];
+                $this->lastTriggeredTime = time();
+            }
+        }
     }
 }

src/Appwrite/Platform/Workers/Builds.php

@@ -54,8 +54,10 @@ class Builds extends Action
             ->inject('cache')
             ->inject('dbForProject')
             ->inject('deviceForFunctions')
+            ->inject('isResourceBlocked')
             ->inject('log')
-            ->callback(fn ($message, Document $project, Database $dbForPlatform, Event $queueForEvents, Func $queueForFunctions, StatsUsage $usage, Cache $cache, Database $dbForProject, Device $deviceForFunctions, Log $log) => $this->action($message, $project, $dbForPlatform, $queueForEvents, $queueForFunctions, $usage, $cache, $dbForProject, $deviceForFunctions, $log));
+            ->callback(fn ($message, Document $project, Database $dbForPlatform, Event $queueForEvents, Func $queueForFunctions, StatsUsage $usage, Cache $cache, Database $dbForProject, Device $deviceForFunctions, callable $isResourceBlocked, Log $log) =>
+                $this->action($message, $project, $dbForPlatform, $queueForEvents, $queueForFunctions, $usage, $cache, $dbForProject, $deviceForFunctions, $isResourceBlocked, $log));
     }
 
     /**
@@ -72,7 +74,7 @@ class Builds extends Action
      * @return void
      * @throws \Utopia\Database\Exception
      */
-    public function action(Message $message, Document $project, Database $dbForPlatform, Event $queueForEvents, Func $queueForFunctions, StatsUsage $queueForStatsUsage, Cache $cache, Database $dbForProject, Device $deviceForFunctions, Log $log): void
+    public function action(Message $message, Document $project, Database $dbForPlatform, Event $queueForEvents, Func $queueForFunctions, StatsUsage $queueForStatsUsage, Cache $cache, Database $dbForProject, Device $deviceForFunctions, callable $isResourceBlocked, Log $log): void
     {
         $payload = $message->getPayload() ?? [];
 
@@ -93,7 +95,7 @@ class Builds extends Action
             case BUILD_TYPE_RETRY:
                 Console::info('Creating build for deployment: ' . $deployment->getId());
                 $github = new GitHub($cache);
-                $this->buildDeployment($deviceForFunctions, $queueForFunctions, $queueForEvents, $queueForStatsUsage, $dbForPlatform, $dbForProject, $github, $project, $resource, $deployment, $template, $log);
+                $this->buildDeployment($deviceForFunctions, $queueForFunctions, $queueForEvents, $queueForStatsUsage, $dbForPlatform, $dbForProject, $github, $project, $resource, $deployment, $template, $isResourceBlocked, $log);
                 break;
 
             default:
@@ -118,7 +120,7 @@ class Builds extends Action
      * @throws \Utopia\Database\Exception
      * @throws Exception
      */
-    protected function buildDeployment(Device $deviceForFunctions, Func $queueForFunctions, Event $queueForEvents, StatsUsage $queueForStatsUsage, Database $dbForPlatform, Database $dbForProject, GitHub $github, Document $project, Document $function, Document $deployment, Document $template, Log $log): void
+    protected function buildDeployment(Device $deviceForFunctions, Func $queueForFunctions, Event $queueForEvents, StatsUsage $queueForStatsUsage, Database $dbForPlatform, Database $dbForProject, GitHub $github, Document $project, Document $function, Document $deployment, Document $template, callable $isResourceBlocked, Log $log): void
     {
         $executor = new Executor(System::getEnv('_APP_EXECUTOR_HOST'));
 
@@ -127,7 +129,11 @@ class Builds extends Action
 
         $function = $dbForProject->getDocument('functions', $functionId);
         if ($function->isEmpty()) {
-            throw new \Exception('Function not found', 404);
+            throw new \Exception('Function not found');
+        }
+
+        if ($isResourceBlocked($project, RESOURCE_TYPE_FUNCTIONS, $functionId)) {
+            throw new \Exception('Function blocked');
         }
 
         $deploymentId = $deployment->getId();
@@ -135,11 +141,11 @@ class Builds extends Action
 
         $deployment = $dbForProject->getDocument('deployments', $deploymentId);
         if ($deployment->isEmpty()) {
-            throw new \Exception('Deployment not found', 404);
+            throw new \Exception('Deployment not found');
         }
 
         if (empty($deployment->getAttribute('entrypoint', ''))) {
-            throw new \Exception('Entrypoint for your Appwrite Function is missing. Please specify it when making deployment or update the entrypoint under your function\'s "Settings" > "Configuration" > "Entrypoint".', 500);
+            throw new \Exception('Entrypoint for your Appwrite Function is missing. Please specify it when making deployment or update the entrypoint under your function\'s "Settings" > "Configuration" > "Entrypoint".');
         }
 
         $version = $function->getAttribute('version', 'v2');
@@ -571,7 +577,7 @@ class Builds extends Action
                                     $build = $dbForProject->getDocument('builds', $build->getId());
 
                                     if ($build->isEmpty()) {
-                                        throw new \Exception('Build not found', 404);
+                                        throw new \Exception('Build not found');
                                     }
 
                                     if ($build->getAttribute('status') === 'canceled') {

src/Appwrite/Platform/Workers/Deletes.php

@@ -7,6 +7,7 @@ use Appwrite\Certificates\Adapter as CertificatesAdapter;
 use Appwrite\Extend\Exception;
 use Executor\Executor;
 use Throwable;
+use Utopia\Abuse\Adapters\TimeLimit\Database as AbuseDatabase;
 use Utopia\Audit\Audit;
 use Utopia\Cache\Adapter\Filesystem;
 use Utopia\Cache\Cache;
@@ -505,7 +506,8 @@ class Deletes extends Action
 
         $projectCollectionIds = [
             ...\array_keys(Config::getParam('collections', [])['projects']),
-            Audit::COLLECTION
+            Audit::COLLECTION,
+            AbuseDatabase::COLLECTION,
         ];
 
         $limit = \count($projectCollectionIds) + 25;

src/Appwrite/Platform/Workers/StatsResources.php

@@ -160,6 +160,12 @@ class StatsResources extends Action
                 call_user_func_array($this->logError, [$th, "StatsResources", "count_for_buckets_{$project->getId()}"]);
             }
 
+            try {
+                $this->countImageTransformations($dbForProject, $dbForLogs, $region);
+            } catch (Throwable $th) {
+                call_user_func_array($this->logError, [$th, "StatsResources", "count_for_buckets_{$project->getId()}"]);
+            }
+
             try {
                 $this->countForDatabase($dbForProject, $dbForLogs, $region);
             } catch (Throwable $th) {
@@ -246,6 +252,8 @@ class StatsResources extends Action
         });
 
         $this->createStatsDocuments($region, METRIC_FILES_IMAGES_TRANSFORMED, $totalImageTransformations, 'inf');
+        $this->createStatsDocuments($region, METRIC_FILES_IMAGES_TRANSFORMED, $totalDailyImageTransformations, '1d');
+        $this->createStatsDocuments($region, METRIC_FILES_IMAGES_TRANSFORMED, $totalHourlyImageTransformations, '1h');
 
     }