Merge branch '1.6.x' into 1.7.x

2026-05-26 13:51:13 +00:00 · 2025-04-08 10:24:09 +02:00
parent e0a54a2ea6 a20ff31ba3
commit 69a7c23d2b
26 changed files with 447 additions and 508 deletions
@@ -10,7 +10,6 @@ use Utopia\Database\DateTime;
 use Utopia\Database\Document;
 use Utopia\Database\Exception;
 use Utopia\Database\Query;
-use Utopia\System\System;

 class V19 extends Migration
 {
@@ -731,7 +730,7 @@ class V19 extends Migration

                if (empty($document->getAttribute('scheduleId', null))) {
                    $schedule = $this->consoleDB->createDocument('schedules', new Document([
-                        'region' => System::getEnv('_APP_REGION', 'default'), // Todo replace with projects region
+                        'region' => $project->getAttribute('region'),
                        'resourceType' => 'function',
                        'resourceId' => $document->getId(),
                        'resourceInternalId' => $document->getInternalId(),
@@ -103,8 +103,15 @@ abstract class ScheduleBase extends Action
                $paginationQueries[] = Query::cursorAfter($latestDocument);
            }

+            // Temporarly accepting both 'fra' and 'default'
+            // When all migrated, only use _APP_REGION with 'default' as default value
+            $regions = [System::getEnv('_APP_REGION', 'default')];
+            if (!in_array('default', $regions)) {
+                $regions[] = 'default';
+            }
+
            $results = $dbForPlatform->find('schedules', \array_merge($paginationQueries, [
-                Query::equal('region', [System::getEnv('_APP_REGION', 'default')]),
+                Query::equal('region', $regions),
                Query::equal('resourceType', [static::getSupportedResource()]),
                Query::equal('active', [true]),
            ]));
@@ -153,8 +160,15 @@ abstract class ScheduleBase extends Action
                        $paginationQueries[] = Query::cursorAfter($latestDocument);
                    }

+                    // Temporarly accepting both 'fra' and 'default'
+                    // When all migrated, only use _APP_REGION with 'default' as default value
+                    $regions = [System::getEnv('_APP_REGION', 'default')];
+                    if (!in_array('default', $regions)) {
+                        $regions[] = 'default';
+                    }
+
                    $results = $dbForPlatform->find('schedules', \array_merge($paginationQueries, [
-                        Query::equal('region', [System::getEnv('_APP_REGION', 'default')]),
+                        Query::equal('region', $regions),
                        Query::equal('resourceType', [static::getSupportedResource()]),
                        Query::greaterThanEqual('resourceUpdatedAt', $lastSyncUpdate),
                    ]));
@@ -74,7 +74,11 @@ class Audits extends Action
        Console::info('Aggregating audit logs');

        $event = $payload['event'] ?? '';
-        $auditPayload = $payload['payload'] ?? '';
+
+        $auditPayload = '';
+        if ($project->getId() === 'console') {
+            $auditPayload = $payload['payload'] ?? '';
+        }
        $mode = $payload['mode'] ?? '';
        $resource = $payload['resource'] ?? '';
        $userAgent = $payload['userAgent'] ?? '';
@@ -59,8 +59,9 @@ class Builds extends Action
            ->inject('deviceForFunctions')
            ->inject('isResourceBlocked')
            ->inject('log')
-            ->callback(fn ($message, Document $project, Database $dbForPlatform, Event $queueForEvents, Webhook $queueForWebhooks, Func $queueForFunctions, Realtime $queueForRealtime, StatsUsage $usage, Cache $cache, Database $dbForProject, Device $deviceForFunctions, callable $isResourceBlocked, Log $log) =>
-                $this->action($message, $project, $dbForPlatform, $queueForEvents, $queueForWebhooks, $queueForFunctions, $queueForRealtime, $usage, $cache, $dbForProject, $deviceForFunctions, $isResourceBlocked, $log));
+            ->inject('executor')
+            ->callback(fn ($message, Document $project, Database $dbForPlatform, Event $queueForEvents, Webhook $queueForWebhooks, Func $queueForFunctions, Realtime $queueForRealtime, StatsUsage $usage, Cache $cache, Database $dbForProject, Device $deviceForFunctions, callable $isResourceBlocked, Log $log, Executor $executor) =>
+                $this->action($message, $project, $dbForPlatform, $queueForEvents, $queueForWebhooks, $queueForFunctions, $queueForRealtime, $usage, $cache, $dbForProject, $deviceForFunctions, $isResourceBlocked, $log, $executor));
    }

    /**
@@ -76,10 +77,11 @@ class Builds extends Action
     * @param Database $dbForProject
     * @param Device $deviceForFunctions
     * @param Log $log
+     * @param Executor $executor
     * @return void
     * @throws \Utopia\Database\Exception
     */
-    public function action(Message $message, Document $project, Database $dbForPlatform, Event $queueForEvents, Webhook $queueForWebhooks, Func $queueForFunctions, Realtime $queueForRealtime, StatsUsage $queueForStatsUsage, Cache $cache, Database $dbForProject, Device $deviceForFunctions, callable $isResourceBlocked, Log $log): void
+    public function action(Message $message, Document $project, Database $dbForPlatform, Event $queueForEvents, Webhook $queueForWebhooks, Func $queueForFunctions, Realtime $queueForRealtime, StatsUsage $queueForStatsUsage, Cache $cache, Database $dbForProject, Device $deviceForFunctions, callable $isResourceBlocked, Log $log, Executor $executor): void
    {
        $payload = $message->getPayload() ?? [];

@@ -100,7 +102,7 @@ class Builds extends Action
            case BUILD_TYPE_RETRY:
                Console::info('Creating build for deployment: ' . $deployment->getId());
                $github = new GitHub($cache);
-                $this->buildDeployment($deviceForFunctions, $queueForWebhooks, $queueForFunctions, $queueForRealtime, $queueForEvents, $queueForStatsUsage, $dbForPlatform, $dbForProject, $github, $project, $resource, $deployment, $template, $isResourceBlocked, $log);
+                $this->buildDeployment($deviceForFunctions, $queueForWebhooks, $queueForFunctions, $queueForRealtime, $queueForEvents, $queueForStatsUsage, $dbForPlatform, $dbForProject, $github, $project, $resource, $deployment, $template, $isResourceBlocked, $log, $executor);
                break;

            default:
@@ -123,14 +125,13 @@ class Builds extends Action
     * @param Document $deployment
     * @param Document $template
     * @param Log $log
+     * @param Executor $executor
     * @return void
     * @throws \Utopia\Database\Exception
     * @throws Exception
     */
-    protected function buildDeployment(Device $deviceForFunctions, Webhook $queueForWebhooks, Func $queueForFunctions, Realtime $queueForRealtime, Event $queueForEvents, StatsUsage $queueForStatsUsage, Database $dbForPlatform, Database $dbForProject, GitHub $github, Document $project, Document $function, Document $deployment, Document $template, callable $isResourceBlocked, Log $log): void
+    protected function buildDeployment(Device $deviceForFunctions, Webhook $queueForWebhooks, Func $queueForFunctions, Realtime $queueForRealtime, Event $queueForEvents, StatsUsage $queueForStatsUsage, Database $dbForPlatform, Database $dbForProject, GitHub $github, Document $project, Document $function, Document $deployment, Document $template, callable $isResourceBlocked, Log $log, Executor $executor): void
    {
-        $executor = new Executor(System::getEnv('_APP_EXECUTOR_HOST'));
-
        $functionId = $function->getId();
        $log->addTag('functionId', $function->getId());

@@ -565,7 +565,8 @@ class Databases extends Action
        try {
            $documents = $database->deleteDocuments($collectionId, $queries);
        } catch (\Throwable $th) {
-            Console::error('Failed to delete documents for collection ' . $collectionId . ': ' . $th->getMessage());
+            $tenant = $database->getSharedTables() ? 'Tenant:'.$database->getTenant() : '';
+            Console::error("Failed to delete documents for collection:{$database->getNamespace()}_{$collectionId} {$tenant} :{$th->getMessage()}");
            return;
        }

@@ -31,6 +31,8 @@ use Utopia\System\System;

 class Deletes extends Action
 {
+    protected array $selects = ['$internalId', '$id', '$collection', '$permissions', '$updatedAt'];
+
    public static function getName(): string
    {
        return 'deletes';
@@ -53,12 +55,13 @@ class Deletes extends Action
            ->inject('deviceForBuilds')
            ->inject('deviceForCache')
            ->inject('certificates')
+            ->inject('executor')
            ->inject('executionRetention')
            ->inject('auditRetention')
            ->inject('log')
            ->callback(
-                fn ($message, Document $project, Database $dbForPlatform, callable $getProjectDB, callable $getLogsDB, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, CertificatesAdapter $certificates, string $executionRetention, string $auditRetention, Log $log) =>
-                    $this->action($message, $project, $dbForPlatform, $getProjectDB, $getLogsDB, $deviceForFiles, $deviceForFunctions, $deviceForBuilds, $deviceForCache, $certificates, $executionRetention, $auditRetention, $log)
+                fn ($message, Document $project, Database $dbForPlatform, callable $getProjectDB, callable $getLogsDB, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, CertificatesAdapter $certificates, Executor $executor, string $executionRetention, string $auditRetention, Log $log) =>
+                    $this->action($message, $project, $dbForPlatform, $getProjectDB, $getLogsDB, $deviceForFiles, $deviceForFunctions, $deviceForBuilds, $deviceForCache, $certificates, $executor, $executionRetention, $auditRetention, $log)
            );
    }

@@ -66,7 +69,7 @@ class Deletes extends Action
     * @throws Exception
     * @throws Throwable
     */
-    public function action(Message $message, Document $project, Database $dbForPlatform, callable $getProjectDB, callable $getLogsDB, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, CertificatesAdapter $certificates, string $executionRetention, string $auditRetention, Log $log): void
+    public function action(Message $message, Document $project, Database $dbForPlatform, callable $getProjectDB, callable $getLogsDB, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, CertificatesAdapter $certificates, Executor $executor, string $executionRetention, string $auditRetention, Log $log): void
    {
        $payload = $message->getPayload() ?? [];

@@ -91,10 +94,10 @@ class Deletes extends Action
                        $this->deleteProject($dbForPlatform, $getProjectDB, $deviceForFiles, $deviceForFunctions, $deviceForBuilds, $deviceForCache, $certificates, $document);
                        break;
                    case DELETE_TYPE_FUNCTIONS:
-                        $this->deleteFunction($dbForPlatform, $getProjectDB, $deviceForFunctions, $deviceForBuilds, $certificates, $document, $project);
+                        $this->deleteFunction($dbForPlatform, $getProjectDB, $deviceForFunctions, $deviceForBuilds, $certificates, $document, $project, $executor);
                        break;
                    case DELETE_TYPE_DEPLOYMENTS:
-                        $this->deleteDeployment($getProjectDB, $deviceForFunctions, $deviceForBuilds, $document, $project);
+                        $this->deleteDeployment($getProjectDB, $deviceForFunctions, $deviceForBuilds, $document, $project, $executor);
                        break;
                    case DELETE_TYPE_USERS:
                        $this->deleteUser($getProjectDB, $document, $project);
@@ -180,10 +183,17 @@ class Deletes extends Action
     */
    private function deleteSchedules(Database $dbForPlatform, callable $getProjectDB, string $datetime): void
    {
+        // Temporarly accepting both 'fra' and 'default'
+        // When all migrated, only use _APP_REGION with 'default' as default value
+        $regions = [System::getEnv('_APP_REGION', 'default')];
+        if (!in_array('default', $regions)) {
+            $regions[] = 'default';
+        }
+
        $this->listByGroup(
            'schedules',
            [
-                Query::equal('region', [System::getEnv('_APP_REGION', 'default')]),
+                Query::equal('region', $regions),
                Query::lessThanEqual('resourceUpdatedAt', $datetime),
                Query::equal('active', [false]),
            ],
@@ -248,7 +258,8 @@ class Deletes extends Action
        $this->deleteByGroup(
            'subscribers',
            [
-                Query::equal('topicInternalId', [$topic->getInternalId()])
+                Query::equal('topicInternalId', [$topic->getInternalId()]),
+                Query::orderAsc(),
            ],
            $getProjectDB($project)
        );
@@ -269,7 +280,8 @@ class Deletes extends Action
        $this->deleteByGroup(
            'subscribers',
            [
-                Query::equal('targetInternalId', [$target->getInternalId()])
+                Query::equal('targetInternalId', [$target->getInternalId()]),
+                Query::orderAsc(),
            ],
            $dbForProject,
            function (Document $subscriber) use ($dbForProject, $target) {
@@ -306,7 +318,8 @@ class Deletes extends Action
        $this->deleteByGroup(
            'targets',
            [
-                Query::equal('expired', [true])
+                Query::equal('expired', [true]),
+                Query::orderAsc(),
            ],
            $getProjectDB($project),
            function (Document $target) use ($getProjectDB, $project) {
@@ -320,7 +333,8 @@ class Deletes extends Action
        $this->deleteByGroup(
            'targets',
            [
-                Query::equal('sessionInternalId', [$session->getInternalId()])
+                Query::equal('sessionInternalId', [$session->getInternalId()]),
+                Query::orderAsc(),
            ],
            $getProjectDB($project),
            function (Document $target) use ($getProjectDB, $project) {
@@ -347,14 +361,20 @@ class Deletes extends Action
            new Filesystem(APP_STORAGE_CACHE . DIRECTORY_SEPARATOR . 'app-' . $projectId)
        );

-        $query[] = Query::equal('resource', [$resource]);
+        $queries = [
+            Query::equal('resource', [$resource])
+        ];
+
        if (!empty($resourceType)) {
-            $query[] = Query::equal('resourceType', [$resourceType]);
+            $queries[] = Query::equal('resourceType', [$resourceType]);
        }

+        $queries[] = Query::select($this->selects);
+        $queries[] = Query::orderAsc();
+
        $this->deleteByGroup(
            'cache',
-            $query,
+            $queries,
            $dbForProject,
            function (Document $document) use ($cache, $projectId) {
                $path = APP_STORAGE_CACHE . DIRECTORY_SEPARATOR . 'app-' . $projectId . DIRECTORY_SEPARATOR . $document->getId();
@@ -385,15 +405,16 @@ class Deletes extends Action
            new Filesystem(APP_STORAGE_CACHE . DIRECTORY_SEPARATOR . 'app-' . $projectId)
        );

-        $query = [
+        $queries = [
+            Query::select([...$this->selects, 'accessedAt']),
            Query::lessThan('accessedAt', $datetime),
            Query::orderDesc('accessedAt'),
-            Query::orderDesc('$internalId'),
+            Query::orderDesc(),
        ];

        $this->deleteByGroup(
            'cache',
-            $query,
+            $queries,
            $dbForProject,
            function (Document $document) use ($cache, $projectId) {
                $path = APP_STORAGE_CACHE . DIRECTORY_SEPARATOR . 'app-' . $projectId . DIRECTORY_SEPARATOR . $document->getId();
@@ -419,12 +440,15 @@ class Deletes extends Action
        /** @var Database $dbForProject*/
        $dbForProject = $getProjectDB($project);

+        $selects = [...$this->selects, 'time'];
+
        // Delete Usage stats from projectDB
        $this->deleteByGroup('stats', [
+            Query::select($selects),
+            Query::equal('period', ['1h']),
            Query::lessThan('time', $hourlyUsageRetentionDatetime),
            Query::orderDesc('time'),
-            Query::orderDesc('$internalId'),
-            Query::equal('period', ['1h']),
+            Query::orderDesc(),
        ], $dbForProject);

        if ($project->getId() !== 'console') {
@@ -433,10 +457,11 @@ class Deletes extends Action

            // Delete Usage stats from logsDB
            $this->deleteByGroup('stats', [
+                Query::select($selects),
+                Query::equal('period', ['1h']),
                Query::lessThan('time', $hourlyUsageRetentionDatetime),
                Query::orderDesc('time'),
-                Query::orderDesc('$internalId'),
-                Query::equal('period', ['1h']),
+                Query::orderDesc(),
            ], $dbForLogs);
        }
    }
@@ -457,7 +482,8 @@ class Deletes extends Action
        $this->deleteByGroup(
            'memberships',
            [
-                Query::equal('teamInternalId', [$teamInternalId])
+                Query::equal('teamInternalId', [$teamInternalId]),
+                Query::orderAsc()
            ],
            $dbForProject,
            function (Document $membership) use ($dbForProject) {
@@ -547,7 +573,13 @@ class Deletes extends Action
                    if ($projectTables || !\in_array($collection->getId(), $projectCollectionIds)) {
                        $dbForProject->deleteCollection($collection->getId());
                    } else {
-                        $this->deleteByGroup($collection->getId(), [], database: $dbForProject);
+                        $this->deleteByGroup(
+                            $collection->getId(),
+                            [
+                                Query::orderAsc()
+                            ],
+                            database: $dbForProject
+                        );
                    }
                } catch (Throwable $e) {
                    Console::error('Error deleting '.$collection->getId().' '.$e->getMessage());
@@ -567,58 +599,78 @@ class Deletes extends Action

        // Delete Platforms
        $this->deleteByGroup('platforms', [
-            Query::equal('projectInternalId', [$projectInternalId])
+            Query::equal('projectInternalId', [$projectInternalId]),
+            Query::orderAsc()
        ], $dbForPlatform);

        // Delete project and function rules
        $this->deleteByGroup('rules', [
-            Query::equal('projectInternalId', [$projectInternalId])
+            Query::equal('projectInternalId', [$projectInternalId]),
+            Query::orderAsc()
        ], $dbForPlatform, function (Document $document) use ($dbForPlatform, $certificates) {
            $this->deleteRule($dbForPlatform, $document, $certificates);
        });

        // Delete Keys
        $this->deleteByGroup('keys', [
-            Query::equal('projectInternalId', [$projectInternalId])
+            Query::equal('projectInternalId', [$projectInternalId]),
+            Query::orderAsc()
        ], $dbForPlatform);

        // Delete Webhooks
        $this->deleteByGroup('webhooks', [
-            Query::equal('projectInternalId', [$projectInternalId])
+            Query::equal('projectInternalId', [$projectInternalId]),
+            Query::orderAsc()
        ], $dbForPlatform);

        // Delete VCS Installations
        $this->deleteByGroup('installations', [
-            Query::equal('projectInternalId', [$projectInternalId])
+            Query::equal('projectInternalId', [$projectInternalId]),
+            Query::orderAsc()
        ], $dbForPlatform);

        // Delete VCS Repositories
        $this->deleteByGroup('repositories', [
            Query::equal('projectInternalId', [$projectInternalId]),
+            Query::orderAsc()
        ], $dbForPlatform);

        // Delete VCS comments
        $this->deleteByGroup('vcsComments', [
            Query::equal('projectInternalId', [$projectInternalId]),
+            Query::orderAsc()
        ], $dbForPlatform);

-        // Delete Schedules (No projectInternalId in this collection)
+        // Delete Schedules
        $this->deleteByGroup('schedules', [
            Query::equal('projectId', [$projectId]),
+            Query::orderAsc()
        ], $dbForPlatform);

        // Delete metadata table
        if ($projectTables) {
            $dbForProject->deleteCollection(Database::METADATA);
        } elseif ($sharedTablesV1) {
-            $this->deleteByGroup(Database::METADATA, [], $dbForProject);
+            $this->deleteByGroup(
+                Database::METADATA,
+                [
+                    Query::orderAsc()
+                ],
+                $dbForProject
+            );
        } elseif ($sharedTablesV2) {
            $queries = \array_map(
                fn ($id) => Query::notEqual('$id', $id),
                $projectCollectionIds
            );

-            $this->deleteByGroup(Database::METADATA, $queries, $dbForProject);
+            $queries[] = Query::orderAsc();
+
+            $this->deleteByGroup(
+                Database::METADATA,
+                $queries,
+                $dbForProject
+            );
        }

        // Delete all storage directories
@@ -643,14 +695,16 @@ class Deletes extends Action

        // Delete all sessions of this user from the sessions table and update the sessions field of the user record
        $this->deleteByGroup('sessions', [
-            Query::equal('userInternalId', [$userInternalId])
+            Query::equal('userInternalId', [$userInternalId]),
+            Query::orderAsc()
        ], $dbForProject);

        $dbForProject->purgeCachedDocument('users', $userId);

        // Delete Memberships and decrement team membership counts
        $this->deleteByGroup('memberships', [
-            Query::equal('userInternalId', [$userInternalId])
+            Query::equal('userInternalId', [$userInternalId]),
+            Query::orderAsc()
        ], $dbForProject, function (Document $document) use ($dbForProject) {
            if ($document->getAttribute('confirm')) { // Count only confirmed members
                $teamId = $document->getAttribute('teamId');
@@ -663,19 +717,22 @@ class Deletes extends Action

        // Delete tokens
        $this->deleteByGroup('tokens', [
-            Query::equal('userInternalId', [$userInternalId])
+            Query::equal('userInternalId', [$userInternalId]),
+            Query::orderAsc()
        ], $dbForProject);

        // Delete identities
        $this->deleteByGroup('identities', [
-            Query::equal('userInternalId', [$userInternalId])
+            Query::equal('userInternalId', [$userInternalId]),
+            Query::orderAsc()
        ], $dbForProject);

        // Delete targets
        $this->deleteByGroup(
            'targets',
            [
-                Query::equal('userInternalId', [$userInternalId])
+                Query::equal('userInternalId', [$userInternalId]),
+                Query::orderAsc()
            ],
            $dbForProject,
            function (Document $target) use ($getProjectDB, $project) {
@@ -697,9 +754,10 @@ class Deletes extends Action

        // Delete Executions
        $this->deleteByGroup('executions', [
+            Query::select([...$this->selects, '$createdAt']),
            Query::lessThan('$createdAt', $datetime),
            Query::orderDesc('$createdAt'),
-            Query::orderDesc('$internalId'),
+            Query::orderDesc(),
        ], $dbForProject);
    }

@@ -717,9 +775,10 @@ class Deletes extends Action

        // Delete Sessions
        $this->deleteByGroup('sessions', [
+            Query::select([...$this->selects, '$createdAt']),
            Query::lessThan('$createdAt', $expired),
            Query::orderDesc('$createdAt'),
-            Query::orderDesc('$internalId'),
+            Query::orderDesc(),
        ], $dbForProject);
    }

@@ -735,14 +794,14 @@ class Deletes extends Action
        $this->deleteByGroup('realtime', [
            Query::lessThan('timestamp', $datetime),
            Query::orderDesc('timestamp'),
-            Query::orderDesc('$internalId'),
+            Query::orderAsc(),
        ], $dbForPlatform);
    }

    /**
     * @param Database $dbForPlatform
     * @param callable $getProjectDB
-     * @param string $datetime
+     * @param string $auditRetention
     * @return void
     * @throws Exception
     */
@@ -753,9 +812,10 @@ class Deletes extends Action

        try {
            $this->deleteByGroup(Audit::COLLECTION, [
+                Query::select([...$this->selects, 'time']),
                Query::lessThan('time', $auditRetention),
                Query::orderDesc('time'),
-                Query::orderDesc('$internalId'),
+                Query::orderAsc(),
            ], $dbForProject);
        } catch (DatabaseException $e) {
            Console::error('Failed to delete audit logs for project ' . $projectId . ': ' . $e->getMessage());
@@ -768,10 +828,11 @@ class Deletes extends Action
     * @param Device $deviceForBuilds
     * @param Document $document function document
     * @param Document $project
+     * @param Executor $executor
     * @return void
     * @throws Exception
     */
-    private function deleteFunction(Database $dbForPlatform, callable $getProjectDB, Device $deviceForFunctions, Device $deviceForBuilds, CertificatesAdapter $certificates, Document $document, Document $project): void
+    private function deleteFunction(Database $dbForPlatform, callable $getProjectDB, Device $deviceForFunctions, Device $deviceForBuilds, CertificatesAdapter $certificates, Document $document, Document $project, Executor $executor): void
    {
        $projectId = $project->getId();
        $dbForProject = $getProjectDB($project);
@@ -783,9 +844,10 @@ class Deletes extends Action
         */
        Console::info("Deleting rules for function " . $functionId);
        $this->deleteByGroup('rules', [
-            Query::equal('resourceType', ['function']),
+            Query::equal('projectInternalId', [$project->getInternalId()]),
            Query::equal('resourceInternalId', [$functionInternalId]),
-            Query::equal('projectInternalId', [$project->getInternalId()])
+            Query::equal('resourceType', ['function']),
+            Query::orderAsc()
        ], $dbForPlatform, function (Document $document) use ($project, $dbForPlatform, $certificates) {
            $this->deleteRule($dbForPlatform, $document, $certificates);
        });
@@ -795,8 +857,9 @@ class Deletes extends Action
         */
        Console::info("Deleting variables for function " . $functionId);
        $this->deleteByGroup('variables', [
+            Query::equal('resourceInternalId', [$functionInternalId]),
            Query::equal('resourceType', ['function']),
-            Query::equal('resourceInternalId', [$functionInternalId])
+            Query::orderAsc()
        ], $dbForProject);

        /**
@@ -806,7 +869,8 @@ class Deletes extends Action

        $deploymentInternalIds = [];
        $this->deleteByGroup('deployments', [
-            Query::equal('resourceInternalId', [$functionInternalId])
+            Query::equal('resourceInternalId', [$functionInternalId]),
+            Query::orderAsc()
        ], $dbForProject, function (Document $document) use ($deviceForFunctions, &$deploymentInternalIds) {
            $deploymentInternalIds[] = $document->getInternalId();
            $this->deleteDeploymentFiles($deviceForFunctions, $document);
@@ -819,7 +883,8 @@ class Deletes extends Action

        foreach ($deploymentInternalIds as $deploymentInternalId) {
            $this->deleteByGroup('builds', [
-                Query::equal('deploymentInternalId', [$deploymentInternalId])
+                Query::equal('deploymentInternalId', [$deploymentInternalId]),
+                Query::orderAsc()
            ], $dbForProject, function (Document $document) use ($deviceForBuilds) {
                $this->deleteBuildFiles($deviceForBuilds, $document);
            });
@@ -830,7 +895,9 @@ class Deletes extends Action
         */
        Console::info("Deleting executions for function " . $functionId);
        $this->deleteByGroup('executions', [
-            Query::equal('functionInternalId', [$functionInternalId])
+            Query::select($this->selects),
+            Query::equal('functionInternalId', [$functionInternalId]),
+            Query::orderAsc()
        ], $dbForProject);

        /**
@@ -841,12 +908,15 @@ class Deletes extends Action
            Query::equal('projectInternalId', [$project->getInternalId()]),
            Query::equal('resourceInternalId', [$functionInternalId]),
            Query::equal('resourceType', ['function']),
+            Query::orderAsc()
        ], $dbForPlatform, function (Document $document) use ($dbForPlatform) {
            $providerRepositoryId = $document->getAttribute('providerRepositoryId', '');
            $projectInternalId = $document->getAttribute('projectInternalId', '');
+
            $this->deleteByGroup('vcsComments', [
                Query::equal('providerRepositoryId', [$providerRepositoryId]),
                Query::equal('projectInternalId', [$projectInternalId]),
+                Query::orderAsc()
            ], $dbForPlatform);
        });

@@ -854,7 +924,7 @@ class Deletes extends Action
         * Request executor to delete all deployment containers
         */
        Console::info("Requesting executor to delete all deployment containers for function " . $functionId);
-        $this->deleteRuntimes($getProjectDB, $document, $project);
+        $this->deleteRuntimes($getProjectDB, $document, $project, $executor);
    }

    /**
@@ -925,10 +995,11 @@ class Deletes extends Action
     * @param Device $deviceForBuilds
     * @param Document $document
     * @param Document $project
+     * @param Executor $executor
     * @return void
     * @throws Exception
     */
-    private function deleteDeployment(callable $getProjectDB, Device $deviceForFunctions, Device $deviceForBuilds, Document $document, Document $project): void
+    private function deleteDeployment(callable $getProjectDB, Device $deviceForFunctions, Device $deviceForBuilds, Document $document, Document $project, Executor $executor): void
    {
        $projectId = $project->getId();
        $dbForProject = $getProjectDB($project);
@@ -946,7 +1017,8 @@ class Deletes extends Action
        Console::info("Deleting builds for deployment " . $deploymentId);

        $this->deleteByGroup('builds', [
-            Query::equal('deploymentInternalId', [$deploymentInternalId])
+            Query::equal('deploymentInternalId', [$deploymentInternalId]),
+            Query::orderAsc()
        ], $dbForProject, function (Document $document) use ($deviceForBuilds) {
            $this->deleteBuildFiles($deviceForBuilds, $document);
        });
@@ -955,7 +1027,7 @@ class Deletes extends Action
         * Request executor to delete all deployment containers
         */
        Console::info("Requesting executor to delete deployment container for deployment " . $deploymentId);
-        $this->deleteRuntimes($getProjectDB, $document, $project);
+        $this->deleteRuntimes($getProjectDB, $document, $project, $executor);
    }

    /**
@@ -974,10 +1046,15 @@ class Deletes extends Action
    ): void {
        $start = \microtime(true);

+        /**
+         * deleteDocuments uses a cursor, we need to add a unique order by field or use default
+         */
+
        try {
            $documents = $database->deleteDocuments($collection, $queries);
        } catch (Throwable $th) {
-            Console::error('Failed to delete documents for collection ' . $collection . ': ' . $th->getMessage());
+            $tenant = $database->getSharedTables() ? 'Tenant:'.$database->getTenant() : '';
+            Console::error("Failed to delete documents for collection:{$database->getNamespace()}_{$collection} {$tenant} :{$th->getMessage()}");
            return;
        }

@@ -1106,13 +1183,12 @@ class Deletes extends Action
     * @param callable $getProjectDB
     * @param ?Document $function
     * @param Document $project
+     * @param Executor $executor
     * @return void
     * @throws Exception
     */
-    private function deleteRuntimes(callable $getProjectDB, ?Document $function, Document $project): void
+    private function deleteRuntimes(callable $getProjectDB, ?Document $function, Document $project, Executor $executor): void
    {
-        $executor = new Executor(System::getEnv('_APP_EXECUTOR_HOST'));
-
        $deleteByFunction = function (Document $function) use ($getProjectDB, $project, $executor) {
            $this->listByGroup(
                'deployments',
@@ -51,11 +51,12 @@ class Functions extends Action
            ->inject('queueForEvents')
            ->inject('queueForStatsUsage')
            ->inject('log')
+            ->inject('executor')
            ->inject('isResourceBlocked')
-            ->callback(fn (Document $project, Message $message, Database $dbForProject, Webhook $queueForWebhooks, Func $queueForFunctions, Realtime $queueForRealtime, Event $queueForEvents, StatsUsage $queueForStatsUsage, Log $log, callable $isResourceBlocked) => $this->action($project, $message, $dbForProject, $queueForWebhooks, $queueForFunctions, $queueForRealtime, $queueForEvents, $queueForStatsUsage, $log, $isResourceBlocked));
+            ->callback(fn (Document $project, Message $message, Database $dbForProject, Webhook $queueForWebhooks, Func $queueForFunctions, Realtime $queueForRealtime, Event $queueForEvents, StatsUsage $queueForStatsUsage, Log $log, Executor $executor, callable $isResourceBlocked) => $this->action($project, $message, $dbForProject, $queueForWebhooks, $queueForFunctions, $queueForRealtime, $queueForEvents, $queueForStatsUsage, $log, $executor, $isResourceBlocked));
    }

-    public function action(Document $project, Message $message, Database $dbForProject, Webhook $queueForWebhooks, Func $queueForFunctions, Realtime $queueForRealtime, Event $queueForEvents, StatsUsage $queueForStatsUsage, Log $log, callable $isResourceBlocked): void
+    public function action(Document $project, Message $message, Database $dbForProject, Webhook $queueForWebhooks, Func $queueForFunctions, Realtime $queueForRealtime, Event $queueForEvents, StatsUsage $queueForStatsUsage, Log $log, Executor $executor, callable $isResourceBlocked): void
    {
        $payload = $message->getPayload() ?? [];

@@ -146,6 +147,7 @@ class Functions extends Action
                        queueForEvents: $queueForEvents,
                        project: $project,
                        function: $function,
+                        executor:  $executor,
                        trigger: 'event',
                        path: '/',
                        method: 'POST',
@@ -188,6 +190,7 @@ class Functions extends Action
                    queueForEvents: $queueForEvents,
                    project: $project,
                    function: $function,
+                    executor:  $executor,
                    trigger: 'http',
                    path: $path,
                    method: $method,
@@ -212,6 +215,7 @@ class Functions extends Action
                    queueForEvents: $queueForEvents,
                    project: $project,
                    function: $function,
+                    executor:  $executor,
                    trigger: 'schedule',
                    path: $path,
                    method: $method,
@@ -298,6 +302,7 @@ class Functions extends Action
     * @param Event $queueForEvents
     * @param Document $project
     * @param Document $function
+     * @param Executor $executor
     * @param string $trigger
     * @param string $path
     * @param string $method
@@ -324,6 +329,7 @@ class Functions extends Action
        Event $queueForEvents,
        Document $project,
        Document $function,
+        Executor $executor,
        string $trigger,
        string $path,
        string $method,
@@ -514,7 +520,6 @@ class Functions extends Action
        try {
            $version = $function->getAttribute('version', 'v2');
            $command = $runtime['startCommand'];
-            $executor = new Executor(System::getEnv('_APP_EXECUTOR_HOST'));
            $command = $version === 'v2' ? '' : 'cp /tmp/code.tar.gz /mnt/code/code.tar.gz && nohup helpers/start.sh "' . $command . '"';
            $executionResponse = $executor->createExecution(
                projectId: $project->getId(),
@@ -71,12 +71,18 @@ class Base extends Queries
            'array' => false,
        ]);

+        $internalId = new Document([
+            'key' => '$internalId',
+            'type' => Database::VAR_STRING,
+            'array' => false,
+        ]);
+
        $validators = [
            new Limit(),
            new Offset(),
            new Cursor(),
            new Filter($attributes, APP_DATABASE_QUERY_MAX_VALUES),
-            new Order($attributes),
+            new Order([...$attributes, $internalId]),
        ];

        parent::__construct($validators);
@@ -21,17 +21,19 @@ class Executor

    private bool $selfSigned = false;

-    private string $endpoint;
+    /**
+     * @var callable(string, string): string  $endpoint
+     */
+    private $endpointSelector;

    protected array $headers;

-    public function __construct(string $endpoint)
+    /**
+     * @param callable(string, string): string $endpointSelector
+     */
+    public function __construct(callable $endpointSelector)
    {
-        if (!filter_var($endpoint, FILTER_VALIDATE_URL)) {
-            throw new Exception('Unsupported endpoint');
-        }
-
-        $this->endpoint = $endpoint;
+        $this->endpointSelector = $endpointSelector;
        $this->headers = [
            'content-type' => 'application/json',
            'authorization' => 'Bearer ' . System::getEnv('_APP_EXECUTOR_SECRET', ''),
@@ -92,7 +94,8 @@ class Executor
            'timeout' => $timeout,
        ];

-        $response = $this->call(self::METHOD_POST, $route, [ 'x-opr-runtime-id' => $runtimeId ], $params, true, $timeout);
+        $endpoint = $this->selectEndpoint($projectId, $deploymentId);
+        $response = $this->call($endpoint, self::METHOD_POST, $route, [ 'x-opr-runtime-id' => $runtimeId ], $params, true, $timeout);

        $status = $response['headers']['status-code'];
        if ($status >= 400) {
@@ -123,7 +126,8 @@ class Executor
            'timeout' => $timeout
        ];

-        $this->call(self::METHOD_GET, $route, [ 'x-opr-runtime-id' => $runtimeId ], $params, true, $timeout, $callback);
+        $endpoint = $this->selectEndpoint($projectId, $deploymentId);
+        $this->call($endpoint, self::METHOD_GET, $route, [ 'x-opr-runtime-id' => $runtimeId ], $params, true, $timeout, $callback);
    }

    /**
@@ -139,7 +143,8 @@ class Executor
        $runtimeId = "$projectId-$deploymentId";
        $route = "/runtimes/$runtimeId";

-        $response = $this->call(self::METHOD_DELETE, $route, [
+        $endpoint = $this->selectEndpoint($projectId, $deploymentId);
+        $response = $this->call($endpoint, self::METHOD_DELETE, $route, [
            'x-opr-addressing-method' => 'broadcast'
        ], [], true, 30);

@@ -227,7 +232,8 @@ class Executor
            $requestTimeout = $timeout + 15;
        }

-        $response = $this->call(self::METHOD_POST, $route, [ 'x-opr-runtime-id' => $runtimeId, 'content-type' => 'multipart/form-data', 'accept' => 'multipart/form-data' ], $params, true, $requestTimeout);
+        $endpoint = $this->selectEndpoint($projectId, $deploymentId);
+        $response = $this->call($endpoint, self::METHOD_POST, $route, [ 'x-opr-runtime-id' => $runtimeId, 'content-type' => 'multipart/form-data', 'accept' => 'multipart/form-data' ], $params, true, $requestTimeout);

        $status = $response['headers']['status-code'];
        if ($status >= 400) {
@@ -235,7 +241,11 @@ class Executor
            throw new \Exception($message, $status);
        }

-        $response['body']['headers'] = \json_decode($response['body']['headers'] ?? '{}', true);
+        $headers = $response['body']['headers'] ?? [];
+        if (is_string($headers)) {
+            $headers = \json_decode($headers, true);
+        }
+        $response['body']['headers'] = $headers;
        $response['body']['statusCode'] = \intval($response['body']['statusCode'] ?? 500);
        $response['body']['duration'] = \floatval($response['body']['duration'] ?? 0);
        $response['body']['startTime'] = \floatval($response['body']['startTime'] ?? \microtime(true));
@@ -256,10 +266,10 @@ class Executor
     * @return array|string
     * @throws Exception
     */
-    public function call(string $method, string $path = '', array $headers = [], array $params = [], bool $decode = true, int $timeout = 15, callable $callback = null)
+    private function call(string $endpoint, string $method, string $path = '', array $headers = [], array $params = [], bool $decode = true, int $timeout = 15, callable $callback = null)
    {
        $headers            = array_merge($this->headers, $headers);
-        $ch                 = curl_init($this->endpoint . $path . (($method == self::METHOD_GET && !empty($params)) ? '?' . http_build_query($params) : ''));
+        $ch                 = curl_init($endpoint . $path . (($method == self::METHOD_GET && !empty($params)) ? '?' . http_build_query($params) : ''));
        $responseHeaders    = [];
        $responseStatus     = -1;
        $responseType       = '';
@@ -422,4 +432,9 @@ class Executor

        return $output;
    }
+
+    private function selectEndpoint(string $projectId, string $deploymentId): string
+    {
+        return call_user_func($this->endpointSelector, $projectId, $deploymentId);
+    }
 }