From bf0d49cc7669012a8e155ce2df3fe79fc64921e2 Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Tue, 10 Mar 2026 15:41:32 +0530 Subject: [PATCH] Refactor embedding timeout and clean database logic - Updated embedding agent timeout to be configurable via environment variable. - Removed commented code in XList for clarity. - Refactored database cleaning logic into separate methods for better readability and maintainability. --- app/init/resources.php | 2 +- .../Databases/Http/Databases/XList.php | 1 - .../Http/VectorDB/Embeddings/Text/Create.php | 4 +- src/Appwrite/Platform/Workers/Deletes.php | 76 ++++++++++++------- 4 files changed, 53 insertions(+), 30 deletions(-) diff --git a/app/init/resources.php b/app/init/resources.php index 8d8af49107..0fa116d5e7 100644 --- a/app/init/resources.php +++ b/app/init/resources.php @@ -1615,6 +1615,6 @@ Http::setResource('executionsRetentionCount', function (Document $project, array Http::setResource('embeddingAgent', function ($register) { $adapter = new Ollama(); $adapter->setEndpoint(System::getEnv('_APP_EMBEDDING_ENDPOINT', 'http://ollama:11434/api/embed')); - $adapter->setTimeout(90000); + $adapter->setTimeout((int) System::getEnv('_APP_EMBEDDING_TIMEOUT', '30000')); return new Agent($adapter); }, ['register']); diff --git a/src/Appwrite/Platform/Modules/Databases/Http/Databases/XList.php b/src/Appwrite/Platform/Modules/Databases/Http/Databases/XList.php index 989e488ccf..e0ffeb8149 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/Databases/XList.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/Databases/XList.php @@ -91,7 +91,6 @@ class XList extends Action $cursor->setValue($cursorDocument); } - // use base action to get all types, contexts, etc! $queries[] = Query::equal('type', [$this->getDatabaseType()]); try { diff --git a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Embeddings/Text/Create.php b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Embeddings/Text/Create.php index c14c708539..2311dda6ce 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Embeddings/Text/Create.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Embeddings/Text/Create.php @@ -74,7 +74,7 @@ class Create extends CreateDocumentAction ] ) ]) - ->param('texts', [], fn (array $plan) => new ArrayList(new Text(0), $plan['databasesBatchSize'] ?? APP_LIMIT_DATABASE_BATCH), 'Array of text to generate embeddings.', false, ['plan']) + ->param('texts', [], fn (array $plan) => new ArrayList(new Text(0), $plan['databasesMaxEmbeddingTexts'] ?? APP_LIMIT_DATABASE_BATCH), 'Array of text to generate embeddings.', false, ['plan']) ->param('model', Ollama::MODEL_EMBEDDING_GEMMA, new WhiteList(Ollama::MODELS), 'The embedding model to use for generating vector embeddings.', true) ->inject('response') ->inject('project') @@ -88,7 +88,7 @@ class Create extends CreateDocumentAction public function action(array $texts, string $model, UtopiaResponse $response, Document $project, Agent $embeddingAgent, StatsUsage $queueForStatsUsage, Log $log, ?Logger $logger): void { $results = []; - $embeddingAgent->getAdapter()->setModel($model)->setTimeout(60000); + $embeddingAgent->getAdapter()->setModel($model); $dimension = $embeddingAgent->getAdapter()->getEmbeddingDimension(); $totalDuration = 0; diff --git a/src/Appwrite/Platform/Workers/Deletes.php b/src/Appwrite/Platform/Workers/Deletes.php index 45f50eea07..7770e5bc90 100644 --- a/src/Appwrite/Platform/Workers/Deletes.php +++ b/src/Appwrite/Platform/Workers/Deletes.php @@ -218,6 +218,50 @@ class Deletes extends Action } } + private function cleanDatabase( + Document $databaseDoc, + callable $executionActionPerDatabase, + bool $projectTables, + array $projectCollectionIds + ): void { + $executionActionPerDatabase( + $databaseDoc, + fn (Database $dbForDatabases) => $this->cleanDatabaseCollections( + $dbForDatabases, + $projectTables, + $projectCollectionIds + ) + ); + } + + private function cleanDatabaseCollections( + Database $dbForDatabases, + bool $projectTables, + array $projectCollectionIds + ): void { + $dbForDatabases->foreach( + Database::METADATA, + function (Document $collection) use ($dbForDatabases, $projectTables, $projectCollectionIds) { + $collectionId = $collection->getId(); + + try { + if ($projectTables || !\in_array($collectionId, $projectCollectionIds, true)) { + $dbForDatabases->deleteCollection($collectionId); + return; + } + + $this->deleteByGroup( + $collectionId, + [Query::orderAsc()], + database: $dbForDatabases + ); + } catch (Throwable $e) { + Console::error('Error deleting ' . $collectionId . ' ' . $e->getMessage()); + } + } + ); + } + /** * @param Database $dbForPlatform * @param callable $getProjectDB @@ -590,32 +634,12 @@ class Deletes extends Action }; batch(array_map( - fn ($databaseDoc) => - fn () => $executionActionPerDatabase( - $databaseDoc, - function (Database $dbForDatabases) use ($projectTables, $projectCollectionIds) { - $dbForDatabases->foreach( - Database::METADATA, - function (Document $collection) use ($dbForDatabases, $projectTables, $projectCollectionIds) { - try { - if ($projectTables || !\in_array($collection->getId(), $projectCollectionIds, true)) { - $dbForDatabases->deleteCollection($collection->getId()); - } else { - $this->deleteByGroup( - $collection->getId(), - [Query::orderAsc()], - database: $dbForDatabases - ); - } - } catch (Throwable $e) { - Console::error( - 'Error deleting ' . $collection->getId() . ' ' . $e->getMessage() - ); - } - } - ); - } - ), + fn ($databaseDoc) => fn () => $this->cleanDatabase( + $databaseDoc, + $executionActionPerDatabase, + $projectTables, + $projectCollectionIds + ), $databasesToClean ));