From 7af361fa164510ecaae6996f9616539ea160d5eb Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Wed, 3 Dec 2025 13:38:24 +0530 Subject: [PATCH] * added logger in the create text embedding * aadded error text metric --- app/controllers/api/project.php | 3 ++ app/init/constants.php | 1 + .../VectorDB/Collections/Documents/Create.php | 2 +- .../Http/VectorDB/Embeddings/Text/Create.php | 51 +++++++++++++++---- .../Utopia/Response/Model/UsageProject.php | 7 +++ tests/e2e/General/UsageTest.php | 9 ++-- 6 files changed, 60 insertions(+), 13 deletions(-) diff --git a/app/controllers/api/project.php b/app/controllers/api/project.php index aefe9d883a..55d6fb8003 100644 --- a/app/controllers/api/project.php +++ b/app/controllers/api/project.php @@ -74,6 +74,7 @@ App::get('/v1/project/usage') METRIC_EMBEDDINGS_TEXT, METRIC_EMBEDDINGS_TEXT_TOTAL_TOKENS, METRIC_EMBEDDINGS_TEXT_TOTAL_DURATION, + METRIC_EMBEDDINGS_TEXT_TOTAL_ERROR ], 'period' => [ METRIC_NETWORK_REQUESTS, @@ -90,6 +91,7 @@ App::get('/v1/project/usage') METRIC_EMBEDDINGS_TEXT, METRIC_EMBEDDINGS_TEXT_TOTAL_TOKENS, METRIC_EMBEDDINGS_TEXT_TOTAL_DURATION, + METRIC_EMBEDDINGS_TEXT_TOTAL_ERROR ] ]; @@ -388,6 +390,7 @@ App::get('/v1/project/usage') 'embeddingsText' => $usage[METRIC_EMBEDDINGS_TEXT] ?? [], 'embeddingsTextTokens' => $usage[METRIC_EMBEDDINGS_TEXT_TOTAL_TOKENS] ?? [], 'embeddingsTextDuration' => $usage[METRIC_EMBEDDINGS_TEXT_TOTAL_DURATION] ?? [], + 'embeddingsTextErrors' => $usage[METRIC_EMBEDDINGS_TEXT_TOTAL_ERROR] ?? [], ]), Response::MODEL_USAGE_PROJECT); }); diff --git a/app/init/constants.php b/app/init/constants.php index 244d4441a9..e0b72bf1ad 100644 --- a/app/init/constants.php +++ b/app/init/constants.php @@ -338,6 +338,7 @@ const METRIC_AVATARS_SCREENSHOTS_GENERATED = 'avatars.screenshotsGenerated'; const METRIC_FUNCTIONS_RUNTIME = 'functions.runtimes.{runtime}'; const METRIC_SITES_FRAMEWORK = 'sites.frameworks.{framework}'; const METRIC_EMBEDDINGS_TEXT = 'embeddings.text.{embeddingModel}'; +const METRIC_EMBEDDINGS_TEXT_TOTAL_ERROR = 'embeddings.text.{embeddingModel}.totalErrors'; const METRIC_EMBEDDINGS_TEXT_TOTAL_DURATION = 'embeddings.text.{embeddingModel}.totalDuration'; const METRIC_EMBEDDINGS_TEXT_TOTAL_TOKENS = 'embeddings.text.{embeddingModel}.totalTokens'; diff --git a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Create.php b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Create.php index 6c331e40dd..5550e64fab 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Create.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Create.php @@ -95,7 +95,7 @@ class Create extends DocumentCreate ->param('databaseId', '', new UID(), 'Database ID.') ->param('documentId', '', new CustomId(), 'Document ID. Choose a custom ID or generate a random ID with `ID.unique()`. Valid chars are a-z, A-Z, 0-9, period, hyphen, and underscore. Can\'t start with a special char. Max length is 36 chars.', true) ->param('collectionId', '', new UID(), 'Collection ID. You can create a new collection using the Database service [server integration](https://appwrite.io/docs/server/databases#databasesCreateCollection). Make sure to define attributes before creating documents.') - ->param('data', [], new JSON(), 'Document data as JSON object.', true, example: '{"username":"walter.obrien","email":"walter.obrien@example.com","fullName":"Walter O\'Brien","age":30,"isAdmin":false}') + ->param('data', [], new JSON(), 'Document data as JSON object.', true, example: '{"embeddings": [0.12, -0.55, 0.88, 1.02], "metadata": {"key":"value"} }') ->param('permissions', null, new Permissions(APP_LIMIT_ARRAY_PARAMS_SIZE, [Database::PERMISSION_READ, Database::PERMISSION_UPDATE, Database::PERMISSION_DELETE, Database::PERMISSION_WRITE]), 'An array of permissions strings. By default, only the current user is granted all permissions. [Learn more about permissions](https://appwrite.io/docs/permissions).', true) ->param('documents', [], fn (array $plan) => new ArrayList(new JSON(), $plan['databasesBatchSize'] ?? APP_LIMIT_DATABASE_BATCH), 'Array of documents data as JSON objects.', true, ['plan']) ->param('transactionId', null, new UID(), 'Transaction ID for staging the operation.', true) diff --git a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Embeddings/Text/Create.php b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Embeddings/Text/Create.php index 704d009fe7..ba6a4c9a22 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Embeddings/Text/Create.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Embeddings/Text/Create.php @@ -13,7 +13,10 @@ use Appwrite\Utopia\Response as UtopiaResponse; use Utopia\Agents\Adapters\Ollama; use Utopia\Agents\Agent; use Utopia\Database\Document; +use Utopia\Logger\Log; +use Utopia\Logger\Logger; use Utopia\Swoole\Response as SwooleResponse; +use Utopia\System\System; use Utopia\Validator\ArrayList; use Utopia\Validator\Text; use Utopia\Validator\WhiteList; @@ -71,22 +74,26 @@ class Create extends CreateDocumentAction ] ) ]) - ->param('embeddingModel', Ollama::MODEL_EMBEDDING_GEMMA, new WhiteList(Ollama::MODELS), 'The embedding model to use for generating vector embeddings.', false) - ->param('texts', [], fn (array $plan) => new ArrayList(new Text(0), $plan['databasesBatchSize'] ?? APP_LIMIT_DATABASE_BATCH), 'Array of text to generate embeddings.', true, ['plan']) + ->param('texts', [], fn (array $plan) => new ArrayList(new Text(0), $plan['databasesBatchSize'] ?? APP_LIMIT_DATABASE_BATCH), 'Array of text to generate embeddings.', false, ['plan']) + ->param('model', Ollama::MODEL_EMBEDDING_GEMMA, new WhiteList(Ollama::MODELS), 'The embedding model to use for generating vector embeddings.', true) ->inject('response') + ->inject('project') ->inject('embeddingAgent') ->inject('queueForStatsUsage') + ->inject('log') + ->inject('logger') ->callback($this->action(...)); } - public function action(string $embeddingModel, array $texts, UtopiaResponse $response, Agent $embeddingAgent, StatsUsage $queueForStatsUsage): void + public function action(array $texts, string $model, UtopiaResponse $response, Document $project, Agent $embeddingAgent, StatsUsage $queueForStatsUsage, Log $log, ?Logger $logger): void { $results = []; - $embeddingAgent->getAdapter()->setModel($embeddingModel); + $embeddingAgent->getAdapter()->setModel($model); $dimension = $embeddingAgent->getAdapter()->getEmbeddingDimension(); $totalDuration = 0; $totalTokens = 0; + $totalErrors = 0; foreach ($texts as $text) { $embedding = []; $error = ''; @@ -95,12 +102,30 @@ class Create extends CreateDocumentAction $embedding = $embedResult['embedding'] ?? []; $totalDuration += $embedResult['totalDuration'] ?? 0; $totalTokens += $embedResult['tokensProcessed'] ?? 0; - } catch (\Exception) { + } catch (\Exception $e) { $error = 'Error while generating embedding'; + $totalErrors += 1; + if ($logger) { + $log->setNamespace("http"); + $log->setServer(System::getEnv('_APP_LOGGING_SERVICE_IDENTIFIER', \gethostname())); + $log->setVersion(System::getEnv('_APP_VERSION', 'UNKNOWN')); + $log->setType(Log::TYPE_ERROR); + $log->setMessage($e->getMessage()); + + $log->addTag('embeddingModel', $model); + $log->addTag('code', $e->getCode()); + $log->addTag('projectId', $project->getId()); + + $log->addExtra('file', $e->getFile()); + $log->addExtra('line', $e->getLine()); + $log->addExtra('trace', $e->getTraceAsString()); + + $logger->addLog($log); + } } $results[] = new Document([ - 'model' => $embeddingModel, + 'model' => $model, 'dimension' => $dimension, 'embedding' => $embedding, 'error' => $error @@ -119,7 +144,7 @@ class Create extends CreateDocumentAction ->addMetric( \str_replace( '{embeddingModel}', - $embeddingModel, + $model, METRIC_EMBEDDINGS_TEXT ), \count($texts) @@ -127,7 +152,7 @@ class Create extends CreateDocumentAction ->addMetric( \str_replace( '{embeddingModel}', - $embeddingModel, + $model, METRIC_EMBEDDINGS_TEXT_TOTAL_TOKENS ), $totalTokens @@ -135,11 +160,19 @@ class Create extends CreateDocumentAction ->addMetric( \str_replace( '{embeddingModel}', - $embeddingModel, + $model, METRIC_EMBEDDINGS_TEXT_TOTAL_DURATION ), $totalDuration ) + ->addMetric( + \str_replace( + '{embeddingModel}', + $model, + METRIC_EMBEDDINGS_TEXT_TOTAL_ERROR + ), + $totalErrors + ) ->trigger(); $queueForStatsUsage->reset(); } diff --git a/src/Appwrite/Utopia/Response/Model/UsageProject.php b/src/Appwrite/Utopia/Response/Model/UsageProject.php index d9fec1e4a1..fcb073cf61 100644 --- a/src/Appwrite/Utopia/Response/Model/UsageProject.php +++ b/src/Appwrite/Utopia/Response/Model/UsageProject.php @@ -237,6 +237,13 @@ class UsageProject extends Model 'example' => [], 'array' => true ]) + ->addRule('embeddingsTextErrors', [ + 'type' => Response::MODEL_METRIC, + 'description' => 'Aggregated number of errors while generating text embeddings per period.', + 'default' => [], + 'example' => [], + 'array' => true + ]) ; } diff --git a/tests/e2e/General/UsageTest.php b/tests/e2e/General/UsageTest.php index ff9e0f0ec8..6c9024c80f 100644 --- a/tests/e2e/General/UsageTest.php +++ b/tests/e2e/General/UsageTest.php @@ -1634,13 +1634,16 @@ class UsageTest extends Scope // New embeddings metrics should be present after calls above $this->assertArrayHasKey('embeddingsText', $response['body']); + $this->assertArrayHasKey('embeddingsTextErrors', $response['body']); $this->assertArrayHasKey('embeddingsTextTokens', $response['body']); $this->assertArrayHasKey('embeddingsTextDuration', $response['body']); - $this->assertGreaterThanOrEqual(0, $response['body']['embeddingsText']); - $this->assertGreaterThanOrEqual(0, $response['body']['embeddingsTextTokens']); - $this->assertGreaterThanOrEqual(0, $response['body']['embeddingsTextDuration']); + $this->assertGreaterThanOrEqual(0, count($response['body']['embeddingsText'])); + $this->assertGreaterThanOrEqual(0, count($response['body']['embeddingsTextErrors'])); + $this->assertGreaterThanOrEqual(0, count($response['body']['embeddingsTextTokens'])); + $this->assertGreaterThanOrEqual(0, count($response['body']['embeddingsTextDuration'])); $this->validateDates($response['body']['embeddingsText']); + $this->validateDates($response['body']['embeddingsTextErrors']); $this->validateDates($response['body']['embeddingsTextTokens']); $this->validateDates($response['body']['embeddingsTextDuration']); });