From c9cf38d6304bac6edf5ec62bdf0e7617f915abc5 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Wed, 3 Sep 2025 01:31:53 +1200 Subject: [PATCH] Add expired log deletion --- app/init/constants.php | 1 + composer.lock | 169 +++++++++++++----- .../Databases/Http/Transactions/Update.php | 1 + src/Appwrite/Platform/Workers/Deletes.php | 80 ++++++--- 4 files changed, 183 insertions(+), 68 deletions(-) diff --git a/app/init/constants.php b/app/init/constants.php index 66933149a8..2e22a4ca3c 100644 --- a/app/init/constants.php +++ b/app/init/constants.php @@ -109,6 +109,7 @@ const DELETE_TYPE_DATABASES = 'databases'; const DELETE_TYPE_DOCUMENT = 'document'; const DELETE_TYPE_COLLECTIONS = 'collections'; const DELETE_TYPE_TRANSACTION = 'transaction'; +const DELETE_TYPE_EXPIRED_TRANSACTIONS = 'expired_transactions'; const DELETE_TYPE_PROJECTS = 'projects'; const DELETE_TYPE_SITES = 'sites'; const DELETE_TYPE_FUNCTIONS = 'functions'; diff --git a/composer.lock b/composer.lock index 3bf17bc228..90bb2f12f6 100644 --- a/composer.lock +++ b/composer.lock @@ -2599,16 +2599,16 @@ }, { "name": "symfony/http-client", - "version": "v7.3.2", + "version": "v7.3.3", "source": { "type": "git", "url": "https://github.com/symfony/http-client.git", - "reference": "1c064a0c67749923483216b081066642751cc2c7" + "reference": "333b9bd7639cbdaecd25a3a48a9d2dcfaa86e019" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/http-client/zipball/1c064a0c67749923483216b081066642751cc2c7", - "reference": "1c064a0c67749923483216b081066642751cc2c7", + "url": "https://api.github.com/repos/symfony/http-client/zipball/333b9bd7639cbdaecd25a3a48a9d2dcfaa86e019", + "reference": "333b9bd7639cbdaecd25a3a48a9d2dcfaa86e019", "shasum": "" }, "require": { @@ -2616,6 +2616,7 @@ "psr/log": "^1|^2|^3", "symfony/deprecation-contracts": "^2.5|^3", "symfony/http-client-contracts": "~3.4.4|^3.5.2", + "symfony/polyfill-php83": "^1.29", "symfony/service-contracts": "^2.5|^3" }, "conflict": { @@ -2674,7 +2675,7 @@ "http" ], "support": { - "source": "https://github.com/symfony/http-client/tree/v7.3.2" + "source": "https://github.com/symfony/http-client/tree/v7.3.3" }, "funding": [ { @@ -2694,7 +2695,7 @@ "type": "tidelift" } ], - "time": "2025-07-15T11:36:08+00:00" + "time": "2025-08-27T07:45:05+00:00" }, { "name": "symfony/http-client-contracts", @@ -2939,6 +2940,86 @@ ], "time": "2024-09-09T11:45:10+00:00" }, + { + "name": "symfony/polyfill-php83", + "version": "v1.33.0", + "source": { + "type": "git", + "url": "https://github.com/symfony/polyfill-php83.git", + "reference": "17f6f9a6b1735c0f163024d959f700cfbc5155e5" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/symfony/polyfill-php83/zipball/17f6f9a6b1735c0f163024d959f700cfbc5155e5", + "reference": "17f6f9a6b1735c0f163024d959f700cfbc5155e5", + "shasum": "" + }, + "require": { + "php": ">=7.2" + }, + "type": "library", + "extra": { + "thanks": { + "url": "https://github.com/symfony/polyfill", + "name": "symfony/polyfill" + } + }, + "autoload": { + "files": [ + "bootstrap.php" + ], + "psr-4": { + "Symfony\\Polyfill\\Php83\\": "" + }, + "classmap": [ + "Resources/stubs" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Nicolas Grekas", + "email": "p@tchwork.com" + }, + { + "name": "Symfony Community", + "homepage": "https://symfony.com/contributors" + } + ], + "description": "Symfony polyfill backporting some PHP 8.3+ features to lower PHP versions", + "homepage": "https://symfony.com", + "keywords": [ + "compatibility", + "polyfill", + "portable", + "shim" + ], + "support": { + "source": "https://github.com/symfony/polyfill-php83/tree/v1.33.0" + }, + "funding": [ + { + "url": "https://symfony.com/sponsor", + "type": "custom" + }, + { + "url": "https://github.com/fabpot", + "type": "github" + }, + { + "url": "https://github.com/nicolas-grekas", + "type": "github" + }, + { + "url": "https://tidelift.com/funding/github/packagist/symfony/symfony", + "type": "tidelift" + } + ], + "time": "2025-07-08T02:45:35+00:00" + }, { "name": "symfony/service-contracts", "version": "v3.6.0", @@ -3557,16 +3638,16 @@ }, { "name": "utopia-php/database", - "version": "1.2.3", + "version": "1.2.4", "source": { "type": "git", "url": "https://github.com/utopia-php/database.git", - "reference": "8a536fead840d9da6ee819fe6b80e0f047997f69" + "reference": "87fb55e86892eecd726635eb1829acb743c2c156" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/database/zipball/8a536fead840d9da6ee819fe6b80e0f047997f69", - "reference": "8a536fead840d9da6ee819fe6b80e0f047997f69", + "url": "https://api.github.com/repos/utopia-php/database/zipball/87fb55e86892eecd726635eb1829acb743c2c156", + "reference": "87fb55e86892eecd726635eb1829acb743c2c156", "shasum": "" }, "require": { @@ -3607,9 +3688,9 @@ ], "support": { "issues": "https://github.com/utopia-php/database/issues", - "source": "https://github.com/utopia-php/database/tree/1.2.3" + "source": "https://github.com/utopia-php/database/tree/1.2.4" }, - "time": "2025-08-27T11:47:04+00:00" + "time": "2025-09-01T06:01:09+00:00" }, { "name": "utopia-php/detector", @@ -4109,16 +4190,16 @@ }, { "name": "utopia-php/migration", - "version": "1.0.0", + "version": "1.0.1", "source": { "type": "git", "url": "https://github.com/utopia-php/migration.git", - "reference": "0e4499d9dd2c90c2be188cc5fb7a32d9a892b569" + "reference": "38171023efd3abe650d2abc5ac65f5df52311da6" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/migration/zipball/0e4499d9dd2c90c2be188cc5fb7a32d9a892b569", - "reference": "0e4499d9dd2c90c2be188cc5fb7a32d9a892b569", + "url": "https://api.github.com/repos/utopia-php/migration/zipball/38171023efd3abe650d2abc5ac65f5df52311da6", + "reference": "38171023efd3abe650d2abc5ac65f5df52311da6", "shasum": "" }, "require": { @@ -4159,9 +4240,9 @@ ], "support": { "issues": "https://github.com/utopia-php/migration/issues", - "source": "https://github.com/utopia-php/migration/tree/1.0.0" + "source": "https://github.com/utopia-php/migration/tree/1.0.1" }, - "time": "2025-08-13T09:15:53+00:00" + "time": "2025-08-28T13:41:25+00:00" }, { "name": "utopia-php/orchestration", @@ -7410,16 +7491,16 @@ }, { "name": "symfony/console", - "version": "v7.3.2", + "version": "v7.3.3", "source": { "type": "git", "url": "https://github.com/symfony/console.git", - "reference": "5f360ebc65c55265a74d23d7fe27f957870158a1" + "reference": "cb0102a1c5ac3807cf3fdf8bea96007df7fdbea7" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/console/zipball/5f360ebc65c55265a74d23d7fe27f957870158a1", - "reference": "5f360ebc65c55265a74d23d7fe27f957870158a1", + "url": "https://api.github.com/repos/symfony/console/zipball/cb0102a1c5ac3807cf3fdf8bea96007df7fdbea7", + "reference": "cb0102a1c5ac3807cf3fdf8bea96007df7fdbea7", "shasum": "" }, "require": { @@ -7484,7 +7565,7 @@ "terminal" ], "support": { - "source": "https://github.com/symfony/console/tree/v7.3.2" + "source": "https://github.com/symfony/console/tree/v7.3.3" }, "funding": [ { @@ -7504,7 +7585,7 @@ "type": "tidelift" } ], - "time": "2025-07-30T17:13:41+00:00" + "time": "2025-08-25T06:35:40+00:00" }, { "name": "symfony/filesystem", @@ -7646,16 +7727,16 @@ }, { "name": "symfony/options-resolver", - "version": "v7.3.2", + "version": "v7.3.3", "source": { "type": "git", "url": "https://github.com/symfony/options-resolver.git", - "reference": "119bcf13e67dbd188e5dbc74228b1686f66acd37" + "reference": "0ff2f5c3df08a395232bbc3c2eb7e84912df911d" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/options-resolver/zipball/119bcf13e67dbd188e5dbc74228b1686f66acd37", - "reference": "119bcf13e67dbd188e5dbc74228b1686f66acd37", + "url": "https://api.github.com/repos/symfony/options-resolver/zipball/0ff2f5c3df08a395232bbc3c2eb7e84912df911d", + "reference": "0ff2f5c3df08a395232bbc3c2eb7e84912df911d", "shasum": "" }, "require": { @@ -7693,7 +7774,7 @@ "options" ], "support": { - "source": "https://github.com/symfony/options-resolver/tree/v7.3.2" + "source": "https://github.com/symfony/options-resolver/tree/v7.3.3" }, "funding": [ { @@ -7713,7 +7794,7 @@ "type": "tidelift" } ], - "time": "2025-07-15T11:36:08+00:00" + "time": "2025-08-05T10:16:07+00:00" }, { "name": "symfony/polyfill-ctype", @@ -8047,16 +8128,16 @@ }, { "name": "symfony/process", - "version": "v7.3.0", + "version": "v7.3.3", "source": { "type": "git", "url": "https://github.com/symfony/process.git", - "reference": "40c295f2deb408d5e9d2d32b8ba1dd61e36f05af" + "reference": "32241012d521e2e8a9d713adb0812bb773b907f1" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/process/zipball/40c295f2deb408d5e9d2d32b8ba1dd61e36f05af", - "reference": "40c295f2deb408d5e9d2d32b8ba1dd61e36f05af", + "url": "https://api.github.com/repos/symfony/process/zipball/32241012d521e2e8a9d713adb0812bb773b907f1", + "reference": "32241012d521e2e8a9d713adb0812bb773b907f1", "shasum": "" }, "require": { @@ -8088,7 +8169,7 @@ "description": "Executes commands in sub-processes", "homepage": "https://symfony.com", "support": { - "source": "https://github.com/symfony/process/tree/v7.3.0" + "source": "https://github.com/symfony/process/tree/v7.3.3" }, "funding": [ { @@ -8099,25 +8180,29 @@ "url": "https://github.com/fabpot", "type": "github" }, + { + "url": "https://github.com/nicolas-grekas", + "type": "github" + }, { "url": "https://tidelift.com/funding/github/packagist/symfony/symfony", "type": "tidelift" } ], - "time": "2025-04-17T09:11:12+00:00" + "time": "2025-08-18T09:42:54+00:00" }, { "name": "symfony/string", - "version": "v7.3.2", + "version": "v7.3.3", "source": { "type": "git", "url": "https://github.com/symfony/string.git", - "reference": "42f505aff654e62ac7ac2ce21033818297ca89ca" + "reference": "17a426cce5fd1f0901fefa9b2a490d0038fd3c9c" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/string/zipball/42f505aff654e62ac7ac2ce21033818297ca89ca", - "reference": "42f505aff654e62ac7ac2ce21033818297ca89ca", + "url": "https://api.github.com/repos/symfony/string/zipball/17a426cce5fd1f0901fefa9b2a490d0038fd3c9c", + "reference": "17a426cce5fd1f0901fefa9b2a490d0038fd3c9c", "shasum": "" }, "require": { @@ -8175,7 +8260,7 @@ "utf8" ], "support": { - "source": "https://github.com/symfony/string/tree/v7.3.2" + "source": "https://github.com/symfony/string/tree/v7.3.3" }, "funding": [ { @@ -8195,7 +8280,7 @@ "type": "tidelift" } ], - "time": "2025-07-10T08:47:49+00:00" + "time": "2025-08-25T06:35:40+00:00" }, { "name": "textalk/websocket", diff --git a/src/Appwrite/Platform/Modules/Databases/Http/Transactions/Update.php b/src/Appwrite/Platform/Modules/Databases/Http/Transactions/Update.php index 4775fd06c9..dfae4f3a50 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/Transactions/Update.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/Transactions/Update.php @@ -193,6 +193,7 @@ class Update extends Action $queueForDeletes ->setType(DELETE_TYPE_DOCUMENT) ->setDocument($transaction); + } catch (DuplicateException|ConflictException) { $dbForProject->updateDocument('transactions', $transactionId, new Document([ 'status' => 'failed', diff --git a/src/Appwrite/Platform/Workers/Deletes.php b/src/Appwrite/Platform/Workers/Deletes.php index c7dda8ccf0..b68c758169 100644 --- a/src/Appwrite/Platform/Workers/Deletes.php +++ b/src/Appwrite/Platform/Workers/Deletes.php @@ -86,14 +86,15 @@ class Deletes extends Action string $executionRetention, string $auditRetention, Log $log - ): void { + ): void + { $payload = $message->getPayload() ?? []; if (empty($payload)) { throw new Exception('Missing payload'); } - $type = $payload['type'] ?? ''; + $type = $payload['type'] ?? ''; $datetime = $payload['datetime'] ?? null; $hourlyUsageRetentionDatetime = $payload['hourlyUsageRetentionDatetime'] ?? null; $resource = $payload['resource'] ?? null; @@ -185,6 +186,7 @@ class Deletes extends Action $this->deleteAuditLogs($project, $getProjectDB, $auditRetention); $this->deleteUsageStats($project, $getProjectDB, $getLogsDB, $hourlyUsageRetentionDatetime); $this->deleteExpiredSessions($project, $getProjectDB); + $this->deleteExpiredTransactions($project, $getProjectDB); break; default: throw new \Exception('No delete operation for type: ' . \strval($type)); @@ -308,9 +310,9 @@ class Deletes extends Action * @param Document $project * @param callable $getProjectDB * @param string $resource + * @param string|null $resourceType * @return void * @throws Authorization - * @param string|null $resourceType * @throws Exception */ private function deleteCacheByResource(Document $project, callable $getProjectDB, string $resource, string $resourceType = null): void @@ -398,7 +400,7 @@ class Deletes extends Action */ private function deleteUsageStats(Document $project, callable $getProjectDB, callable $getLogsDB, string $hourlyUsageRetentionDatetime): void { - /** @var Database $dbForProject*/ + /** @var Database $dbForProject */ $dbForProject = $getProjectDB($project); $selects = [...$this->selects, 'time']; @@ -413,7 +415,7 @@ class Deletes extends Action ], $dbForProject); if ($project->getId() !== 'console') { - /** @var Database $dbForLogs*/ + /** @var Database $dbForLogs */ $dbForLogs = call_user_func($getLogsDB, $project); // Delete Usage stats from logsDB @@ -455,16 +457,16 @@ class Deletes extends Action } /** - * @param Database $dbForPlatform - * @param Document $document - * @return void - * @throws Authorization - * @throws DatabaseException - * @throws Conflict - * @throws Restricted - * @throws Structure - * @throws Exception - */ + * @param Database $dbForPlatform + * @param Document $document + * @return void + * @throws Authorization + * @throws DatabaseException + * @throws Conflict + * @throws Restricted + * @throws Structure + * @throws Exception + */ protected function deleteProjectsByTeam(Database $dbForPlatform, callable $getProjectDB, CertificatesAdapter $certificates, Document $document): void { @@ -542,7 +544,7 @@ class Deletes extends Action ); } } catch (Throwable $e) { - Console::error('Error deleting '.$collection->getId().' '.$e->getMessage()); + Console::error('Error deleting ' . $collection->getId() . ' ' . $e->getMessage()); } }); @@ -609,7 +611,7 @@ class Deletes extends Action ); } elseif ($sharedTablesV2) { $queries = \array_map( - fn ($id) => Query::notEqual('$id', $id), + fn($id) => Query::notEqual('$id', $id), $projectCollectionIds ); @@ -953,14 +955,14 @@ class Deletes extends Action } Console::info("Deleting screenshots for deployment " . $deployment->getId()); - $bucket = ValidatorAuthorization::skip(fn () => $dbForPlatform->getDocument('buckets', 'screenshots')); + $bucket = ValidatorAuthorization::skip(fn() => $dbForPlatform->getDocument('buckets', 'screenshots')); if ($bucket->isEmpty()) { Console::error('Failed to get bucket for deployment screenshots'); return; } foreach ($screenshotIds as $id) { - $file = ValidatorAuthorization::skip(fn () => $dbForPlatform->getDocument('bucket_' . $bucket->getSequence(), $id)); + $file = ValidatorAuthorization::skip(fn() => $dbForPlatform->getDocument('bucket_' . $bucket->getSequence(), $id)); if ($file->isEmpty()) { Console::error('Failed to get deployment screenshot: ' . $id); @@ -1117,12 +1119,10 @@ class Deletes extends Action array $queries, Database $database, ?callable $callback = null - ): void { + ): void + { $start = \microtime(true); - $deleteBatchSize = Database::DELETE_BATCH_SIZE; - $deleteBatchSize = 500; // TODO: Set right value in DB library after investigation - /** * deleteDocuments uses a cursor, we need to add a unique order by field or use default */ @@ -1130,11 +1130,10 @@ class Deletes extends Action $count = $database->deleteDocuments( $collection, $queries, - $deleteBatchSize, - $callback + onNext: $callback ); } catch (Throwable $th) { - $tenant = $database->getSharedTables() ? 'Tenant:'.$database->getTenant() : ''; + $tenant = $database->getSharedTables() ? 'Tenant:' . $database->getTenant() : ''; Console::error("Failed to delete documents for collection:{$database->getNamespace()}_{$collection} {$tenant} :{$th->getMessage()}"); return; } @@ -1318,4 +1317,33 @@ class Deletes extends Action Console::error("Failed to delete transaction logs for {$transactionId}: " . $th->getMessage()); } } + + private function deleteExpiredTransactions(Document $project, callable $getProjectDB): void + { + $dbForProject = $getProjectDB($project); + $transactionInternalIds = []; + + try { + $dbForProject->deleteDocuments('transactions', [ + Query::equal('status', ['pending']), + Query::lessThan('expiresAt', DateTime::format(new \DateTime())), + ], onNext: function (Document $transaction) use ($dbForProject, $project, &$transactionInternalIds) { + $transactionInternalIds[] = $transaction->getSequence(); + }, onError: function (Throwable $th) use ($project) { + // Swallow errors to avoid breaking the cleanup process + }); + } catch (Throwable $th) { + Console::error("Failed to find expired transactions for project {$project->getId()}: " . $th->getMessage()); + } + + if (empty($transactionInternalIds)) { + return; + } + + $dbForProject->deleteDocuments('transactionLogs', [ + Query::equal('transactionInternalId', $transactionInternalIds), + ], onError: function (Throwable $th) use ($project) { + // Swallow errors to avoid breaking the cleanup process + }); + } }