Add expired log deletion

2026-05-26 13:51:13 +00:00 · 2025-09-03 01:31:53 +12:00
parent f836fbeba5
commit c9cf38d630
4 changed files with 183 additions and 68 deletions
@@ -109,6 +109,7 @@ const DELETE_TYPE_DATABASES = 'databases';
 const DELETE_TYPE_DOCUMENT = 'document';
 const DELETE_TYPE_COLLECTIONS = 'collections';
 const DELETE_TYPE_TRANSACTION = 'transaction';
+const DELETE_TYPE_EXPIRED_TRANSACTIONS = 'expired_transactions';
 const DELETE_TYPE_PROJECTS = 'projects';
 const DELETE_TYPE_SITES = 'sites';
 const DELETE_TYPE_FUNCTIONS = 'functions';
@@ -2599,16 +2599,16 @@
        },
        {
            "name": "symfony/http-client",
-            "version": "v7.3.2",
+            "version": "v7.3.3",
            "source": {
                "type": "git",
                "url": "https://github.com/symfony/http-client.git",
-                "reference": "1c064a0c67749923483216b081066642751cc2c7"
+                "reference": "333b9bd7639cbdaecd25a3a48a9d2dcfaa86e019"
            },
            "dist": {
                "type": "zip",
-                "url": "https://api.github.com/repos/symfony/http-client/zipball/1c064a0c67749923483216b081066642751cc2c7",
-                "reference": "1c064a0c67749923483216b081066642751cc2c7",
+                "url": "https://api.github.com/repos/symfony/http-client/zipball/333b9bd7639cbdaecd25a3a48a9d2dcfaa86e019",
+                "reference": "333b9bd7639cbdaecd25a3a48a9d2dcfaa86e019",
                "shasum": ""
            },
            "require": {
@@ -2616,6 +2616,7 @@
                "psr/log": "^1|^2|^3",
                "symfony/deprecation-contracts": "^2.5|^3",
                "symfony/http-client-contracts": "~3.4.4|^3.5.2",
+                "symfony/polyfill-php83": "^1.29",
                "symfony/service-contracts": "^2.5|^3"
            },
            "conflict": {
@@ -2674,7 +2675,7 @@
                "http"
            ],
            "support": {
-                "source": "https://github.com/symfony/http-client/tree/v7.3.2"
+                "source": "https://github.com/symfony/http-client/tree/v7.3.3"
            },
            "funding": [
                {
@@ -2694,7 +2695,7 @@
                    "type": "tidelift"
                }
            ],
-            "time": "2025-07-15T11:36:08+00:00"
+            "time": "2025-08-27T07:45:05+00:00"
        },
        {
            "name": "symfony/http-client-contracts",
@@ -2939,6 +2940,86 @@
            ],
            "time": "2024-09-09T11:45:10+00:00"
        },
+        {
+            "name": "symfony/polyfill-php83",
+            "version": "v1.33.0",
+            "source": {
+                "type": "git",
+                "url": "https://github.com/symfony/polyfill-php83.git",
+                "reference": "17f6f9a6b1735c0f163024d959f700cfbc5155e5"
+            },
+            "dist": {
+                "type": "zip",
+                "url": "https://api.github.com/repos/symfony/polyfill-php83/zipball/17f6f9a6b1735c0f163024d959f700cfbc5155e5",
+                "reference": "17f6f9a6b1735c0f163024d959f700cfbc5155e5",
+                "shasum": ""
+            },
+            "require": {
+                "php": ">=7.2"
+            },
+            "type": "library",
+            "extra": {
+                "thanks": {
+                    "url": "https://github.com/symfony/polyfill",
+                    "name": "symfony/polyfill"
+                }
+            },
+            "autoload": {
+                "files": [
+                    "bootstrap.php"
+                ],
+                "psr-4": {
+                    "Symfony\\Polyfill\\Php83\\": ""
+                },
+                "classmap": [
+                    "Resources/stubs"
+                ]
+            },
+            "notification-url": "https://packagist.org/downloads/",
+            "license": [
+                "MIT"
+            ],
+            "authors": [
+                {
+                    "name": "Nicolas Grekas",
+                    "email": "p@tchwork.com"
+                },
+                {
+                    "name": "Symfony Community",
+                    "homepage": "https://symfony.com/contributors"
+                }
+            ],
+            "description": "Symfony polyfill backporting some PHP 8.3+ features to lower PHP versions",
+            "homepage": "https://symfony.com",
+            "keywords": [
+                "compatibility",
+                "polyfill",
+                "portable",
+                "shim"
+            ],
+            "support": {
+                "source": "https://github.com/symfony/polyfill-php83/tree/v1.33.0"
+            },
+            "funding": [
+                {
+                    "url": "https://symfony.com/sponsor",
+                    "type": "custom"
+                },
+                {
+                    "url": "https://github.com/fabpot",
+                    "type": "github"
+                },
+                {
+                    "url": "https://github.com/nicolas-grekas",
+                    "type": "github"
+                },
+                {
+                    "url": "https://tidelift.com/funding/github/packagist/symfony/symfony",
+                    "type": "tidelift"
+                }
+            ],
+            "time": "2025-07-08T02:45:35+00:00"
+        },
        {
            "name": "symfony/service-contracts",
            "version": "v3.6.0",
@@ -3557,16 +3638,16 @@
        },
        {
            "name": "utopia-php/database",
-            "version": "1.2.3",
+            "version": "1.2.4",
            "source": {
                "type": "git",
                "url": "https://github.com/utopia-php/database.git",
-                "reference": "8a536fead840d9da6ee819fe6b80e0f047997f69"
+                "reference": "87fb55e86892eecd726635eb1829acb743c2c156"
            },
            "dist": {
                "type": "zip",
-                "url": "https://api.github.com/repos/utopia-php/database/zipball/8a536fead840d9da6ee819fe6b80e0f047997f69",
-                "reference": "8a536fead840d9da6ee819fe6b80e0f047997f69",
+                "url": "https://api.github.com/repos/utopia-php/database/zipball/87fb55e86892eecd726635eb1829acb743c2c156",
+                "reference": "87fb55e86892eecd726635eb1829acb743c2c156",
                "shasum": ""
            },
            "require": {
@@ -3607,9 +3688,9 @@
            ],
            "support": {
                "issues": "https://github.com/utopia-php/database/issues",
-                "source": "https://github.com/utopia-php/database/tree/1.2.3"
+                "source": "https://github.com/utopia-php/database/tree/1.2.4"
            },
-            "time": "2025-08-27T11:47:04+00:00"
+            "time": "2025-09-01T06:01:09+00:00"
        },
        {
            "name": "utopia-php/detector",
@@ -4109,16 +4190,16 @@
        },
        {
            "name": "utopia-php/migration",
-            "version": "1.0.0",
+            "version": "1.0.1",
            "source": {
                "type": "git",
                "url": "https://github.com/utopia-php/migration.git",
-                "reference": "0e4499d9dd2c90c2be188cc5fb7a32d9a892b569"
+                "reference": "38171023efd3abe650d2abc5ac65f5df52311da6"
            },
            "dist": {
                "type": "zip",
-                "url": "https://api.github.com/repos/utopia-php/migration/zipball/0e4499d9dd2c90c2be188cc5fb7a32d9a892b569",
-                "reference": "0e4499d9dd2c90c2be188cc5fb7a32d9a892b569",
+                "url": "https://api.github.com/repos/utopia-php/migration/zipball/38171023efd3abe650d2abc5ac65f5df52311da6",
+                "reference": "38171023efd3abe650d2abc5ac65f5df52311da6",
                "shasum": ""
            },
            "require": {
@@ -4159,9 +4240,9 @@
            ],
            "support": {
                "issues": "https://github.com/utopia-php/migration/issues",
-                "source": "https://github.com/utopia-php/migration/tree/1.0.0"
+                "source": "https://github.com/utopia-php/migration/tree/1.0.1"
            },
-            "time": "2025-08-13T09:15:53+00:00"
+            "time": "2025-08-28T13:41:25+00:00"
        },
        {
            "name": "utopia-php/orchestration",
@@ -7410,16 +7491,16 @@
        },
        {
            "name": "symfony/console",
-            "version": "v7.3.2",
+            "version": "v7.3.3",
            "source": {
                "type": "git",
                "url": "https://github.com/symfony/console.git",
-                "reference": "5f360ebc65c55265a74d23d7fe27f957870158a1"
+                "reference": "cb0102a1c5ac3807cf3fdf8bea96007df7fdbea7"
            },
            "dist": {
                "type": "zip",
-                "url": "https://api.github.com/repos/symfony/console/zipball/5f360ebc65c55265a74d23d7fe27f957870158a1",
-                "reference": "5f360ebc65c55265a74d23d7fe27f957870158a1",
+                "url": "https://api.github.com/repos/symfony/console/zipball/cb0102a1c5ac3807cf3fdf8bea96007df7fdbea7",
+                "reference": "cb0102a1c5ac3807cf3fdf8bea96007df7fdbea7",
                "shasum": ""
            },
            "require": {
@@ -7484,7 +7565,7 @@
                "terminal"
            ],
            "support": {
-                "source": "https://github.com/symfony/console/tree/v7.3.2"
+                "source": "https://github.com/symfony/console/tree/v7.3.3"
            },
            "funding": [
                {
@@ -7504,7 +7585,7 @@
                    "type": "tidelift"
                }
            ],
-            "time": "2025-07-30T17:13:41+00:00"
+            "time": "2025-08-25T06:35:40+00:00"
        },
        {
            "name": "symfony/filesystem",
@@ -7646,16 +7727,16 @@
        },
        {
            "name": "symfony/options-resolver",
-            "version": "v7.3.2",
+            "version": "v7.3.3",
            "source": {
                "type": "git",
                "url": "https://github.com/symfony/options-resolver.git",
-                "reference": "119bcf13e67dbd188e5dbc74228b1686f66acd37"
+                "reference": "0ff2f5c3df08a395232bbc3c2eb7e84912df911d"
            },
            "dist": {
                "type": "zip",
-                "url": "https://api.github.com/repos/symfony/options-resolver/zipball/119bcf13e67dbd188e5dbc74228b1686f66acd37",
-                "reference": "119bcf13e67dbd188e5dbc74228b1686f66acd37",
+                "url": "https://api.github.com/repos/symfony/options-resolver/zipball/0ff2f5c3df08a395232bbc3c2eb7e84912df911d",
+                "reference": "0ff2f5c3df08a395232bbc3c2eb7e84912df911d",
                "shasum": ""
            },
            "require": {
@@ -7693,7 +7774,7 @@
                "options"
            ],
            "support": {
-                "source": "https://github.com/symfony/options-resolver/tree/v7.3.2"
+                "source": "https://github.com/symfony/options-resolver/tree/v7.3.3"
            },
            "funding": [
                {
@@ -7713,7 +7794,7 @@
                    "type": "tidelift"
                }
            ],
-            "time": "2025-07-15T11:36:08+00:00"
+            "time": "2025-08-05T10:16:07+00:00"
        },
        {
            "name": "symfony/polyfill-ctype",
@@ -8047,16 +8128,16 @@
        },
        {
            "name": "symfony/process",
-            "version": "v7.3.0",
+            "version": "v7.3.3",
            "source": {
                "type": "git",
                "url": "https://github.com/symfony/process.git",
-                "reference": "40c295f2deb408d5e9d2d32b8ba1dd61e36f05af"
+                "reference": "32241012d521e2e8a9d713adb0812bb773b907f1"
            },
            "dist": {
                "type": "zip",
-                "url": "https://api.github.com/repos/symfony/process/zipball/40c295f2deb408d5e9d2d32b8ba1dd61e36f05af",
-                "reference": "40c295f2deb408d5e9d2d32b8ba1dd61e36f05af",
+                "url": "https://api.github.com/repos/symfony/process/zipball/32241012d521e2e8a9d713adb0812bb773b907f1",
+                "reference": "32241012d521e2e8a9d713adb0812bb773b907f1",
                "shasum": ""
            },
            "require": {
@@ -8088,7 +8169,7 @@
            "description": "Executes commands in sub-processes",
            "homepage": "https://symfony.com",
            "support": {
-                "source": "https://github.com/symfony/process/tree/v7.3.0"
+                "source": "https://github.com/symfony/process/tree/v7.3.3"
            },
            "funding": [
                {
@@ -8099,25 +8180,29 @@
                    "url": "https://github.com/fabpot",
                    "type": "github"
                },
+                {
+                    "url": "https://github.com/nicolas-grekas",
+                    "type": "github"
+                },
                {
                    "url": "https://tidelift.com/funding/github/packagist/symfony/symfony",
                    "type": "tidelift"
                }
            ],
-            "time": "2025-04-17T09:11:12+00:00"
+            "time": "2025-08-18T09:42:54+00:00"
        },
        {
            "name": "symfony/string",
-            "version": "v7.3.2",
+            "version": "v7.3.3",
            "source": {
                "type": "git",
                "url": "https://github.com/symfony/string.git",
-                "reference": "42f505aff654e62ac7ac2ce21033818297ca89ca"
+                "reference": "17a426cce5fd1f0901fefa9b2a490d0038fd3c9c"
            },
            "dist": {
                "type": "zip",
-                "url": "https://api.github.com/repos/symfony/string/zipball/42f505aff654e62ac7ac2ce21033818297ca89ca",
-                "reference": "42f505aff654e62ac7ac2ce21033818297ca89ca",
+                "url": "https://api.github.com/repos/symfony/string/zipball/17a426cce5fd1f0901fefa9b2a490d0038fd3c9c",
+                "reference": "17a426cce5fd1f0901fefa9b2a490d0038fd3c9c",
                "shasum": ""
            },
            "require": {
@@ -8175,7 +8260,7 @@
                "utf8"
            ],
            "support": {
-                "source": "https://github.com/symfony/string/tree/v7.3.2"
+                "source": "https://github.com/symfony/string/tree/v7.3.3"
            },
            "funding": [
                {
@@ -8195,7 +8280,7 @@
                    "type": "tidelift"
                }
            ],
-            "time": "2025-07-10T08:47:49+00:00"
+            "time": "2025-08-25T06:35:40+00:00"
        },
        {
            "name": "textalk/websocket",
@@ -193,6 +193,7 @@ class Update extends Action
                    $queueForDeletes
                        ->setType(DELETE_TYPE_DOCUMENT)
                        ->setDocument($transaction);
+
                } catch (DuplicateException|ConflictException) {
                    $dbForProject->updateDocument('transactions', $transactionId, new Document([
                        'status' => 'failed',
@@ -86,14 +86,15 @@ class Deletes extends Action
        string $executionRetention,
        string $auditRetention,
        Log $log
-    ): void {
+    ): void
+    {
        $payload = $message->getPayload() ?? [];

        if (empty($payload)) {
            throw new Exception('Missing payload');
        }

-        $type     = $payload['type'] ?? '';
+        $type = $payload['type'] ?? '';
        $datetime = $payload['datetime'] ?? null;
        $hourlyUsageRetentionDatetime = $payload['hourlyUsageRetentionDatetime'] ?? null;
        $resource = $payload['resource'] ?? null;
@@ -185,6 +186,7 @@ class Deletes extends Action
                $this->deleteAuditLogs($project, $getProjectDB, $auditRetention);
                $this->deleteUsageStats($project, $getProjectDB, $getLogsDB, $hourlyUsageRetentionDatetime);
                $this->deleteExpiredSessions($project, $getProjectDB);
+                $this->deleteExpiredTransactions($project, $getProjectDB);
                break;
            default:
                throw new \Exception('No delete operation for type: ' . \strval($type));
@@ -308,9 +310,9 @@ class Deletes extends Action
     * @param Document $project
     * @param callable $getProjectDB
     * @param string $resource
+     * @param string|null $resourceType
     * @return void
     * @throws Authorization
-     * @param string|null $resourceType
     * @throws Exception
     */
    private function deleteCacheByResource(Document $project, callable $getProjectDB, string $resource, string $resourceType = null): void
@@ -398,7 +400,7 @@ class Deletes extends Action
     */
    private function deleteUsageStats(Document $project, callable $getProjectDB, callable $getLogsDB, string $hourlyUsageRetentionDatetime): void
    {
-        /** @var Database $dbForProject*/
+        /** @var Database $dbForProject */
        $dbForProject = $getProjectDB($project);

        $selects = [...$this->selects, 'time'];
@@ -413,7 +415,7 @@ class Deletes extends Action
        ], $dbForProject);

        if ($project->getId() !== 'console') {
-            /** @var Database $dbForLogs*/
+            /** @var Database $dbForLogs */
            $dbForLogs = call_user_func($getLogsDB, $project);

            // Delete Usage stats from logsDB
@@ -455,16 +457,16 @@ class Deletes extends Action
    }

    /**
-    * @param Database $dbForPlatform
-    * @param Document $document
-    * @return void
-    * @throws Authorization
-    * @throws DatabaseException
-    * @throws Conflict
-    * @throws Restricted
-    * @throws Structure
-    * @throws Exception
-    */
+     * @param Database $dbForPlatform
+     * @param Document $document
+     * @return void
+     * @throws Authorization
+     * @throws DatabaseException
+     * @throws Conflict
+     * @throws Restricted
+     * @throws Structure
+     * @throws Exception
+     */
    protected function deleteProjectsByTeam(Database $dbForPlatform, callable $getProjectDB, CertificatesAdapter $certificates, Document $document): void
    {

@@ -542,7 +544,7 @@ class Deletes extends Action
                    );
                }
            } catch (Throwable $e) {
-                Console::error('Error deleting '.$collection->getId().' '.$e->getMessage());
+                Console::error('Error deleting ' . $collection->getId() . ' ' . $e->getMessage());
            }
        });

@@ -609,7 +611,7 @@ class Deletes extends Action
            );
        } elseif ($sharedTablesV2) {
            $queries = \array_map(
-                fn ($id) => Query::notEqual('$id', $id),
+                fn($id) => Query::notEqual('$id', $id),
                $projectCollectionIds
            );

@@ -953,14 +955,14 @@ class Deletes extends Action
        }
        Console::info("Deleting screenshots for deployment " . $deployment->getId());

-        $bucket = ValidatorAuthorization::skip(fn () => $dbForPlatform->getDocument('buckets', 'screenshots'));
+        $bucket = ValidatorAuthorization::skip(fn() => $dbForPlatform->getDocument('buckets', 'screenshots'));
        if ($bucket->isEmpty()) {
            Console::error('Failed to get bucket for deployment screenshots');
            return;
        }

        foreach ($screenshotIds as $id) {
-            $file = ValidatorAuthorization::skip(fn () => $dbForPlatform->getDocument('bucket_' . $bucket->getSequence(), $id));
+            $file = ValidatorAuthorization::skip(fn() => $dbForPlatform->getDocument('bucket_' . $bucket->getSequence(), $id));

            if ($file->isEmpty()) {
                Console::error('Failed to get deployment screenshot: ' . $id);
@@ -1117,12 +1119,10 @@ class Deletes extends Action
        array $queries,
        Database $database,
        ?callable $callback = null
-    ): void {
+    ): void
+    {
        $start = \microtime(true);

-        $deleteBatchSize = Database::DELETE_BATCH_SIZE;
-        $deleteBatchSize = 500; // TODO: Set right value in DB library after investigation
-
        /**
         * deleteDocuments uses a cursor, we need to add a unique order by field or use default
         */
@@ -1130,11 +1130,10 @@ class Deletes extends Action
            $count = $database->deleteDocuments(
                $collection,
                $queries,
-                $deleteBatchSize,
-                $callback
+                onNext: $callback
            );
        } catch (Throwable $th) {
-            $tenant = $database->getSharedTables() ? 'Tenant:'.$database->getTenant() : '';
+            $tenant = $database->getSharedTables() ? 'Tenant:' . $database->getTenant() : '';
            Console::error("Failed to delete documents for collection:{$database->getNamespace()}_{$collection} {$tenant} :{$th->getMessage()}");
            return;
        }
@@ -1318,4 +1317,33 @@ class Deletes extends Action
            Console::error("Failed to delete transaction logs for {$transactionId}: " . $th->getMessage());
        }
    }
+
+    private function deleteExpiredTransactions(Document $project, callable $getProjectDB): void
+    {
+        $dbForProject = $getProjectDB($project);
+        $transactionInternalIds = [];
+
+        try {
+            $dbForProject->deleteDocuments('transactions', [
+                Query::equal('status', ['pending']),
+                Query::lessThan('expiresAt', DateTime::format(new \DateTime())),
+            ], onNext: function (Document $transaction) use ($dbForProject, $project, &$transactionInternalIds) {
+                $transactionInternalIds[] = $transaction->getSequence();
+            }, onError: function (Throwable $th) use ($project) {
+                // Swallow errors to avoid breaking the cleanup process
+            });
+        } catch (Throwable $th) {
+            Console::error("Failed to find expired transactions for project {$project->getId()}: " . $th->getMessage());
+        }
+
+        if (empty($transactionInternalIds)) {
+            return;
+        }
+
+        $dbForProject->deleteDocuments('transactionLogs', [
+            Query::equal('transactionInternalId', $transactionInternalIds),
+        ], onError: function (Throwable $th) use ($project) {
+            // Swallow errors to avoid breaking the cleanup process
+        });
+    }
 }