From 3018b478ba281f998cac8bf3f701de1cdc5723da Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Thu, 2 Apr 2026 17:25:13 +0530 Subject: [PATCH] Fix database transaction and vectors migration flakiness --- .../Http/Databases/Transactions/Update.php | 59 +++++++++++-------- .../Http/VectorsDB/Collections/Create.php | 35 +++++++++++ 2 files changed, 69 insertions(+), 25 deletions(-) diff --git a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Transactions/Update.php b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Transactions/Update.php index 0c8c6a8520..9f0839a14b 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Transactions/Update.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Transactions/Update.php @@ -180,21 +180,35 @@ class Update extends Action } $dbForDatabases = $getDatabasesDB($databaseDoc); + $collections = []; try { - $dbForDatabases->withTransaction(function () use ($dbForDatabases, $dbForProject, $transactionState, $queueForDeletes, $transactionId, &$transaction, &$operations, &$totalOperations, &$databaseOperations, &$currentDocumentId, $authorization) { - $authorization->skip(fn () => $dbForProject->updateDocument('transactions', $transactionId, new Document([ - 'status' => 'committing', - ]))); + $transaction = $authorization->skip(fn () => $dbForProject->updateDocument( + 'transactions', + $transactionId, + new Document(['status' => 'committing']) + )); - $operations = $authorization->skip(fn () => $dbForProject->find('transactionLogs', [ - Query::equal('transactionInternalId', [$transaction->getSequence()]), - Query::orderAsc(), - Query::limit(PHP_INT_MAX), - ])); + $operations = $authorization->skip(fn () => $dbForProject->find('transactionLogs', [ + Query::equal('transactionInternalId', [$transaction->getSequence()]), + Query::orderAsc(), + Query::limit(PHP_INT_MAX), + ])); + foreach ($operations as $operation) { + $databaseInternalId = $operation['databaseInternalId']; + $collectionInternalId = $operation['collectionInternalId']; + $collectionId = "database_{$databaseInternalId}_collection_{$collectionInternalId}"; + + if (!isset($collections[$collectionId])) { + $collections[$collectionId] = $authorization->skip( + fn () => $dbForProject->getCollection($collectionId) + ); + } + } + + $dbForDatabases->withTransaction(function () use ($dbForDatabases, $transactionState, $operations, $collections, &$totalOperations, &$databaseOperations, &$currentDocumentId) { $state = []; - $collections = []; foreach ($operations as $operation) { $databaseInternalId = $operation['databaseInternalId']; @@ -210,11 +224,6 @@ class Update extends Action $data = $data->getArrayCopy(); } - if (!isset($collections[$collectionId])) { - $collections[$collectionId] = $authorization->skip( - fn () => $dbForProject->getCollection($collectionId) - ); - } $collection = $collections[$collectionId]; if (\is_array($data) && !empty($data)) { @@ -275,17 +284,17 @@ class Update extends Action break; } } - - $transaction = $authorization->skip(fn () => $dbForProject->updateDocument( - 'transactions', - $transactionId, - new Document(['status' => 'committed']) - )); - - $queueForDeletes - ->setType(DELETE_TYPE_DOCUMENT) - ->setDocument($transaction); }); + + $transaction = $authorization->skip(fn () => $dbForProject->updateDocument( + 'transactions', + $transactionId, + new Document(['status' => 'committed']) + )); + + $queueForDeletes + ->setType(DELETE_TYPE_DOCUMENT) + ->setDocument($transaction); } catch (NotFoundException $e) { $authorization->skip(fn () => $dbForProject->updateDocument('transactions', $transactionId, new Document([ 'status' => 'failed', diff --git a/src/Appwrite/Platform/Modules/Databases/Http/VectorsDB/Collections/Create.php b/src/Appwrite/Platform/Modules/Databases/Http/VectorsDB/Collections/Create.php index a7e2d68eac..baa31c4ef7 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/VectorsDB/Collections/Create.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/VectorsDB/Collections/Create.php @@ -20,6 +20,7 @@ use Utopia\Database\Exception\Limit as LimitException; use Utopia\Database\Exception\NotFound as NotFoundException; use Utopia\Database\Helpers\ID; use Utopia\Database\Helpers\Permission; +use Utopia\Database\Query; use Utopia\Database\Validator\Authorization; use Utopia\Database\Validator\Permissions; use Utopia\Database\Validator\UID; @@ -116,6 +117,30 @@ class Create extends CollectionAction } /** @var Database $dbForDatabases */ $dbForDatabases = $getDatabasesDB($database); + $cleanupCollection = function () use ($authorization, $dbForProject, $database, $collection): void { + try { + $authorization->skip(fn () => $dbForProject->deleteDocument( + 'database_' . $database->getSequence(), + $collection->getId() + )); + } catch (\Throwable) { + } + + $queries = [ + Query::equal('databaseInternalId', [$database->getSequence()]), + Query::equal('collectionInternalId', [$collection->getSequence()]), + ]; + + try { + $authorization->skip(fn () => $dbForProject->deleteDocuments('attributes', $queries)); + } catch (\Throwable) { + } + + try { + $authorization->skip(fn () => $dbForProject->deleteDocuments('indexes', $queries)); + } catch (\Throwable) { + } + }; $attributes = []; $indexes = []; @@ -134,6 +159,10 @@ class Create extends CollectionAction try { $dbForDatabases->create(); } catch (DuplicateException) { + } catch (\Throwable $e) { + if (!$dbForDatabases->exists(null, Database::METADATA)) { + throw $e; + } } } $dbForDatabases->createCollection( @@ -191,11 +220,17 @@ class Create extends CollectionAction $dbForProject->createDocuments('indexes', $indexDocs); } } catch (DuplicateException) { + $cleanupCollection(); throw new Exception($this->getDuplicateException()); } catch (IndexException) { + $cleanupCollection(); throw new Exception($this->getInvalidIndexException()); } catch (LimitException) { + $cleanupCollection(); throw new Exception($this->getLimitException()); + } catch (\Throwable $e) { + $cleanupCollection(); + throw $e; } $queueForEvents