From ca4c7b536197bb81a871799b32dbd55277a78ee5 Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Tue, 2 Dec 2025 18:13:49 +0530 Subject: [PATCH] added stats usage for text embeddings --- app/controllers/api/project.php | 9 +++ app/init/constants.php | 4 ++ composer.lock | 12 ++-- .../Http/VectorDB/Embeddings/Text/Create.php | 43 ++++++++++++-- .../Utopia/Response/Model/UsageProject.php | 21 +++++++ tests/e2e/General/UsageTest.php | 57 +++++++++++++++++++ 6 files changed, 135 insertions(+), 11 deletions(-) diff --git a/app/controllers/api/project.php b/app/controllers/api/project.php index 41200aed35..aefe9d883a 100644 --- a/app/controllers/api/project.php +++ b/app/controllers/api/project.php @@ -71,6 +71,9 @@ App::get('/v1/project/usage') METRIC_DATABASES_OPERATIONS_READS, METRIC_DATABASES_OPERATIONS_WRITES, METRIC_FILES_IMAGES_TRANSFORMED, + METRIC_EMBEDDINGS_TEXT, + METRIC_EMBEDDINGS_TEXT_TOTAL_TOKENS, + METRIC_EMBEDDINGS_TEXT_TOTAL_DURATION, ], 'period' => [ METRIC_NETWORK_REQUESTS, @@ -84,6 +87,9 @@ App::get('/v1/project/usage') METRIC_DATABASES_OPERATIONS_READS, METRIC_DATABASES_OPERATIONS_WRITES, METRIC_FILES_IMAGES_TRANSFORMED, + METRIC_EMBEDDINGS_TEXT, + METRIC_EMBEDDINGS_TEXT_TOTAL_TOKENS, + METRIC_EMBEDDINGS_TEXT_TOTAL_DURATION, ] ]; @@ -379,6 +385,9 @@ App::get('/v1/project/usage') 'authPhoneCountryBreakdown' => $authPhoneCountryBreakdown, 'imageTransformations' => $usage[METRIC_FILES_IMAGES_TRANSFORMED], 'imageTransformationsTotal' => $total[METRIC_FILES_IMAGES_TRANSFORMED], + 'embeddingsText' => $usage[METRIC_EMBEDDINGS_TEXT] ?? [], + 'embeddingsTextTokens' => $usage[METRIC_EMBEDDINGS_TEXT_TOTAL_TOKENS] ?? [], + 'embeddingsTextDuration' => $usage[METRIC_EMBEDDINGS_TEXT_TOTAL_DURATION] ?? [], ]), Response::MODEL_USAGE_PROJECT); }); diff --git a/app/init/constants.php b/app/init/constants.php index 7146e6e118..6c0058f9b0 100644 --- a/app/init/constants.php +++ b/app/init/constants.php @@ -274,6 +274,9 @@ const METRIC_SITES_ID_OUTBOUND = 'sites.{siteInternalId}.outbound'; const METRIC_AVATARS_SCREENSHOTS_GENERATED = 'avatars.screenshotsGenerated'; const METRIC_FUNCTIONS_RUNTIME = 'functions.runtimes.{runtime}'; const METRIC_SITES_FRAMEWORK = 'sites.frameworks.{framework}'; +const METRIC_EMBEDDINGS_TEXT = 'embeddings.text.{embeddingModel}'; +const METRIC_EMBEDDINGS_TEXT_TOTAL_DURATION = 'embeddings.text.{embeddingModel}.totalDuration'; +const METRIC_EMBEDDINGS_TEXT_TOTAL_TOKENS = 'embeddings.text.{embeddingModel}.totalTokens'; // Resource types const RESOURCE_TYPE_PROJECTS = 'projects'; @@ -286,6 +289,7 @@ const RESOURCE_TYPE_TOPICS = 'topics'; const RESOURCE_TYPE_SUBSCRIBERS = 'subscribers'; const RESOURCE_TYPE_MESSAGES = 'messages'; const RESOURCE_TYPE_EXECUTIONS = 'executions'; +const RESOURCE_TYPE_EMBEDDINGS_TEXT = 'embeddingsText'; // Resource types for Tokens const TOKENS_RESOURCE_TYPE_FILES = 'files'; diff --git a/composer.lock b/composer.lock index 32dfedd3be..09d29b9418 100644 --- a/composer.lock +++ b/composer.lock @@ -3500,16 +3500,16 @@ }, { "name": "utopia-php/agents", - "version": "0.5.0", + "version": "0.6.0", "source": { "type": "git", "url": "https://github.com/utopia-php/agents.git", - "reference": "7bbc77d4936ee7e87394a239690c9059b5cdaf1b" + "reference": "ab026e44d332598852b4606808d9cd3bf1de9c96" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/agents/zipball/7bbc77d4936ee7e87394a239690c9059b5cdaf1b", - "reference": "7bbc77d4936ee7e87394a239690c9059b5cdaf1b", + "url": "https://api.github.com/repos/utopia-php/agents/zipball/ab026e44d332598852b4606808d9cd3bf1de9c96", + "reference": "ab026e44d332598852b4606808d9cd3bf1de9c96", "shasum": "" }, "require": { @@ -3548,9 +3548,9 @@ ], "support": { "issues": "https://github.com/utopia-php/agents/issues", - "source": "https://github.com/utopia-php/agents/tree/0.5.0" + "source": "https://github.com/utopia-php/agents/tree/0.6.0" }, - "time": "2025-11-12T06:06:41+00:00" + "time": "2025-12-02T10:51:32+00:00" }, { "name": "utopia-php/analytics", diff --git a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Embeddings/Text/Create.php b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Embeddings/Text/Create.php index b5d1be8f98..704d009fe7 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Embeddings/Text/Create.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Embeddings/Text/Create.php @@ -2,6 +2,7 @@ namespace Appwrite\Platform\Modules\Databases\Http\VectorDB\Embeddings\Text; +use Appwrite\Event\StatsUsage; use Appwrite\Platform\Modules\Databases\Http\Databases\Collections\Documents\Action as CreateDocumentAction; use Appwrite\SDK\AuthType; use Appwrite\SDK\ContentType; @@ -41,8 +42,8 @@ class Create extends CreateDocumentAction ->setHttpPath('/v1/vectordb/embeddings/text') ->desc('Create Text Embeddings') ->groups(['api', 'database']) - ->label('scope', 'documents.read') - ->label('resourceType', RESOURCE_TYPE_DATABASES) + ->label('scope', 'documents.write') + ->label('resourceType', RESOURCE_TYPE_EMBEDDINGS_TEXT) ->label('audits.event', 'embedding.create') ->label('audits.resource', 'vectordb/embeddings/text') ->label('abuse-key', 'ip:{ip},method:{method},url:{url},userId:{userId}') @@ -74,23 +75,27 @@ class Create extends CreateDocumentAction ->param('texts', [], fn (array $plan) => new ArrayList(new Text(0), $plan['databasesBatchSize'] ?? APP_LIMIT_DATABASE_BATCH), 'Array of text to generate embeddings.', true, ['plan']) ->inject('response') ->inject('embeddingAgent') + ->inject('queueForStatsUsage') ->callback($this->action(...)); } - public function action(string $embeddingModel, array $texts, UtopiaResponse $response, Agent $embeddingAgent): void + public function action(string $embeddingModel, array $texts, UtopiaResponse $response, Agent $embeddingAgent, StatsUsage $queueForStatsUsage): void { $results = []; $embeddingAgent->getAdapter()->setModel($embeddingModel); $dimension = $embeddingAgent->getAdapter()->getEmbeddingDimension(); + $totalDuration = 0; + $totalTokens = 0; foreach ($texts as $text) { - $embedding = []; $error = ''; try { $embedResult = $embeddingAgent->embed($text); $embedding = $embedResult['embedding'] ?? []; - } catch (\Exception $e) { + $totalDuration += $embedResult['totalDuration'] ?? 0; + $totalTokens += $embedResult['tokensProcessed'] ?? 0; + } catch (\Exception) { $error = 'Error while generating embedding'; } @@ -109,5 +114,33 @@ class Create extends CreateDocumentAction $response ->setStatusCode(SwooleResponse::STATUS_CODE_OK) ->dynamic($embeddings, $this->getBulkResponseModel()); + + $queueForStatsUsage + ->addMetric( + \str_replace( + '{embeddingModel}', + $embeddingModel, + METRIC_EMBEDDINGS_TEXT + ), + \count($texts) + ) + ->addMetric( + \str_replace( + '{embeddingModel}', + $embeddingModel, + METRIC_EMBEDDINGS_TEXT_TOTAL_TOKENS + ), + $totalTokens + ) + ->addMetric( + \str_replace( + '{embeddingModel}', + $embeddingModel, + METRIC_EMBEDDINGS_TEXT_TOTAL_DURATION + ), + $totalDuration + ) + ->trigger(); + $queueForStatsUsage->reset(); } } diff --git a/src/Appwrite/Utopia/Response/Model/UsageProject.php b/src/Appwrite/Utopia/Response/Model/UsageProject.php index ee644aa845..d9fec1e4a1 100644 --- a/src/Appwrite/Utopia/Response/Model/UsageProject.php +++ b/src/Appwrite/Utopia/Response/Model/UsageProject.php @@ -216,6 +216,27 @@ class UsageProject extends Model 'default' => 0, 'example' => 0, ]) + ->addRule('embeddingsText', [ + 'type' => Response::MODEL_METRIC, + 'description' => 'Aggregated number of text embedding calls per period.', + 'default' => [], + 'example' => [], + 'array' => true + ]) + ->addRule('embeddingsTextTokens', [ + 'type' => Response::MODEL_METRIC, + 'description' => 'Aggregated number of tokens processed by text embeddings per period.', + 'default' => [], + 'example' => [], + 'array' => true + ]) + ->addRule('embeddingsTextDuration', [ + 'type' => Response::MODEL_METRIC, + 'description' => 'Aggregated duration spent generating text embeddings per period.', + 'default' => [], + 'example' => [], + 'array' => true + ]) ; } diff --git a/tests/e2e/General/UsageTest.php b/tests/e2e/General/UsageTest.php index b384d4812b..ff9e0f0ec8 100644 --- a/tests/e2e/General/UsageTest.php +++ b/tests/e2e/General/UsageTest.php @@ -1589,6 +1589,63 @@ class UsageTest extends Scope }); } + public function testEmbeddingsTextUsageDoesNotBreakProjectUsage(): void + { + // Trigger embeddings endpoint a few times so stats usage worker has data to aggregate + for ($i = 0; $i < 3; $i++) { + $response = $this->client->call( + Client::METHOD_POST, + '/vectordb/embeddings/text', + array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ], $this->getHeaders()), + [ + 'embeddingModel' => 'embeddinggemma', + 'texts' => [ + 'usage test text ' . $i, + ], + ] + ); + + $this->assertEquals(200, $response['headers']['status-code']); + $this->assertIsArray($response['body']['embeddings']); + $this->assertGreaterThan(0, $response['body']['total']); + } + + // Ensure project usage endpoint still responds correctly after embeddings calls + $this->assertEventually(function () { + $response = $this->client->call( + Client::METHOD_GET, + '/project/usage', + $this->getConsoleHeaders(), + [ + 'period' => '1h', + 'startDate' => self::getToday(), + 'endDate' => self::getTomorrow(), + ] + ); + + $this->assertEquals(200, $response['headers']['status-code']); + $this->assertArrayHasKey('requests', $response['body']); + $this->assertArrayHasKey('network', $response['body']); + $this->assertArrayHasKey('executionsTotal', $response['body']); + + // New embeddings metrics should be present after calls above + $this->assertArrayHasKey('embeddingsText', $response['body']); + $this->assertArrayHasKey('embeddingsTextTokens', $response['body']); + $this->assertArrayHasKey('embeddingsTextDuration', $response['body']); + $this->assertGreaterThanOrEqual(0, $response['body']['embeddingsText']); + $this->assertGreaterThanOrEqual(0, $response['body']['embeddingsTextTokens']); + $this->assertGreaterThanOrEqual(0, $response['body']['embeddingsTextDuration']); + + $this->validateDates($response['body']['embeddingsText']); + $this->validateDates($response['body']['embeddingsTextTokens']); + $this->validateDates($response['body']['embeddingsTextDuration']); + }); + } + public function tearDown(): void { $this->projectId = '';