From 3901f34e8e89de883aa3cd65e68a9411403a02c7 Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Fri, 29 May 2026 15:58:16 +0530 Subject: [PATCH 01/11] refactor: update embedding model and endpoint to use 'nomic' and 'appwrite-embedding' --- .env | 4 +- app/init/constants.php | 3 + app/init/resources/request.php | 6 +- composer.lock | 12 +-- docker-compose.yml | 50 +++++----- .../Http/VectorsDB/Embeddings/Text/Create.php | 92 ++++++++++++------- .../Utopia/Response/Model/Embedding.php | 2 +- tests/e2e/General/UsageTest.php | 4 +- .../VectorsDB/DatabasesCustomServerTest.php | 4 +- .../Databases/VectorsDBCustomServerTest.php | 4 +- 10 files changed, 104 insertions(+), 77 deletions(-) diff --git a/.env b/.env index 4a6a3ac344..5478608c60 100644 --- a/.env +++ b/.env @@ -55,8 +55,8 @@ _APP_DB_PORT_DOCUMENTSDB=27017 _APP_DB_ADAPTER_VECTORSDB=postgresql _APP_DB_HOST_VECTORSDB=postgresql _APP_DB_PORT_VECTORSDB=5432 -_APP_EMBEDDING_MODELS=embeddinggemma -_APP_EMBEDDING_ENDPOINT='http://ollama:11434/api/embed' +_APP_EMBEDDING_MODELS=nomic +_APP_EMBEDDING_ENDPOINT='http://appwrite-embedding:11434/embed' _APP_EMBEDDING_TIMEOUT=30000 _APP_STORAGE_DEVICE=Local _APP_STORAGE_S3_ACCESS_KEY= diff --git a/app/init/constants.php b/app/init/constants.php index 4bb607e2fb..d426e51fb6 100644 --- a/app/init/constants.php +++ b/app/init/constants.php @@ -108,6 +108,9 @@ const APP_VCS_GITHUB_EMAIL = 'team@appwrite.io'; const APP_VCS_GITHUB_URL = 'https://github.com/TeamAppwrite'; const APP_BRANDED_EMAIL_BASE_TEMPLATE = 'email-base-styled'; +// Embeddings +const APP_EMBEDDING_BATCH_LIMIT = 100; + /** * JWT for Resource Tokens. */ diff --git a/app/init/resources/request.php b/app/init/resources/request.php index 97fb417d8b..f10794a323 100644 --- a/app/init/resources/request.php +++ b/app/init/resources/request.php @@ -22,7 +22,7 @@ use Appwrite\Usage\Context as UsageContext; use Appwrite\Utopia\Database\Documents\User; use Appwrite\Utopia\Request; use Appwrite\Utopia\Response; -use Utopia\Agents\Adapters\Ollama; +use Utopia\Agents\Adapters\Appwrite as AppwriteAdapter; use Utopia\Agents\Agent; use Utopia\Audit\Adapter\Database as AdapterDatabase; use Utopia\Audit\Audit; @@ -1277,8 +1277,8 @@ return function (Container $context): void { $context->set('deviceForBuilds', fn ($project, Telemetry $telemetry) => new Device\Telemetry($telemetry, getDevice(APP_STORAGE_BUILDS . '/app-' . $project->getId())), ['project', 'telemetry']); $context->set('embeddingAgent', function ($register) { - $adapter = new Ollama(); - $adapter->setEndpoint(System::getEnv('_APP_EMBEDDING_ENDPOINT', 'http://ollama:11434/api/embed')); + $adapter = new AppwriteAdapter(); + $adapter->setEndpoint(System::getEnv('_APP_EMBEDDING_ENDPOINT', 'http://appwrite-embedding:11434/embed')); $adapter->setTimeout((int) System::getEnv('_APP_EMBEDDING_TIMEOUT', '30000')); return new Agent($adapter); }, ['register']); diff --git a/composer.lock b/composer.lock index 6787411985..2823faf618 100644 --- a/composer.lock +++ b/composer.lock @@ -3411,16 +3411,16 @@ }, { "name": "utopia-php/agents", - "version": "1.2.2", + "version": "1.2.3", "source": { "type": "git", "url": "https://github.com/utopia-php/agents.git", - "reference": "0703f4cae02261e09a1bf0d39a4b1ce649cae634" + "reference": "47fb6282909fb8dfbb196f1a9834e4d858e41dcb" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/agents/zipball/0703f4cae02261e09a1bf0d39a4b1ce649cae634", - "reference": "0703f4cae02261e09a1bf0d39a4b1ce649cae634", + "url": "https://api.github.com/repos/utopia-php/agents/zipball/47fb6282909fb8dfbb196f1a9834e4d858e41dcb", + "reference": "47fb6282909fb8dfbb196f1a9834e4d858e41dcb", "shasum": "" }, "require": { @@ -3458,9 +3458,9 @@ ], "support": { "issues": "https://github.com/utopia-php/agents/issues", - "source": "https://github.com/utopia-php/agents/tree/1.2.2" + "source": "https://github.com/utopia-php/agents/tree/1.2.3" }, - "time": "2026-05-08T10:38:23+00:00" + "time": "2026-05-28T11:22:16+00:00" }, { "name": "utopia-php/analytics", diff --git a/docker-compose.yml b/docker-compose.yml index 7d3f6bdc5f..da518fb6e9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -112,7 +112,7 @@ services: condition: service_healthy coredns: condition: service_started - ollama: + appwrite-embedding: condition: service_started entrypoint: - php @@ -304,7 +304,7 @@ services: depends_on: - ${_APP_DB_HOST:-mongodb} - redis - - ollama + - appwrite-embedding environment: - _APP_ENV - _APP_WORKER_PER_CORE @@ -346,7 +346,7 @@ services: depends_on: - redis - ${_APP_DB_HOST:-mongodb} - - ollama + - appwrite-embedding environment: - _APP_ENV - _APP_WORKER_PER_CORE @@ -380,7 +380,7 @@ services: - ${_APP_DB_HOST:-mongodb} - request-catcher-sms - request-catcher-webhook - - ollama + - appwrite-embedding environment: - _APP_ENV - _APP_WORKER_PER_CORE @@ -411,7 +411,7 @@ services: depends_on: - redis - ${_APP_DB_HOST:-mongodb} - - ollama + - appwrite-embedding volumes: - appwrite-uploads:/storage/uploads:rw - appwrite-cache:/storage/cache:rw @@ -481,7 +481,7 @@ services: depends_on: - redis - ${_APP_DB_HOST:-mongodb} - - ollama + - appwrite-embedding environment: - _APP_ENV - _APP_WORKER_PER_CORE @@ -524,7 +524,7 @@ services: depends_on: - redis - ${_APP_DB_HOST:-mongodb} - - ollama + - appwrite-embedding environment: - _APP_ENV - _APP_WORKER_PER_CORE @@ -657,7 +657,7 @@ services: depends_on: - redis - ${_APP_DB_HOST:-mongodb} - - ollama + - appwrite-embedding volumes: - appwrite-config:/storage/config:rw - appwrite-certificates:/storage/certificates:rw @@ -1075,7 +1075,7 @@ services: depends_on: - redis - ${_APP_DB_HOST:-mongodb} - - ollama + - appwrite-embedding environment: - _APP_ENV - _APP_WORKER_PER_CORE @@ -1109,7 +1109,7 @@ services: depends_on: - ${_APP_DB_HOST:-mongodb} - redis - - ollama + - appwrite-embedding environment: - _APP_ENV - _APP_WORKER_PER_CORE @@ -1147,7 +1147,7 @@ services: depends_on: - ${_APP_DB_HOST:-mongodb} - redis - - ollama + - appwrite-embedding environment: - _APP_ENV - _APP_WORKER_PER_CORE @@ -1185,7 +1185,7 @@ services: depends_on: - ${_APP_DB_HOST:-mongodb} - redis - - ollama + - appwrite-embedding environment: - _APP_ENV - _APP_WORKER_PER_CORE @@ -1358,19 +1358,19 @@ services: start_period: 10s command: "postgres" - ollama: - image: appwrite/ollama:0.1.1 - container_name: ollama - ports: - - "11434:11434" - restart: unless-stopped - environment: - MODELS: ${_APP_EMBEDDING_MODELS:-embeddinggemma} - OLLAMA_KEEP_ALIVE: 24h - volumes: - - appwrite-models:/root/.ollama - networks: - - appwrite + appwrite-embedding: + image: appwrite/embedding + environment: + EMBEDDING_MODELS: "nomic" + container_name: appwrite-embedding + ports: + - "11430:11434" + restart: unless-stopped + # persistent for caching models across restarts and preloading + volumes: + - embedding_models:/home/embedder/models + networks: + - appwrite redis: image: redis:7.4.7-alpine diff --git a/src/Appwrite/Platform/Modules/Databases/Http/VectorsDB/Embeddings/Text/Create.php b/src/Appwrite/Platform/Modules/Databases/Http/VectorsDB/Embeddings/Text/Create.php index b5480f39e3..90e477d401 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/VectorsDB/Embeddings/Text/Create.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/VectorsDB/Embeddings/Text/Create.php @@ -10,7 +10,7 @@ use Appwrite\SDK\Parameter; use Appwrite\SDK\Response as SDKResponse; use Appwrite\Usage\Context; use Appwrite\Utopia\Response as UtopiaResponse; -use Utopia\Agents\Adapters\Ollama; +use Utopia\Agents\Adapters\Appwrite as AppwriteAdapter; use Utopia\Agents\Agent; use Utopia\Database\Document; use Utopia\Http\Adapter\Swoole\Response as SwooleResponse; @@ -75,7 +75,7 @@ class Create extends CreateDocumentAction ) ]) ->param('texts', [], fn (array $plan) => new ArrayList(new Text(0), $plan['databasesMaxEmbeddingTexts'] ?? APP_LIMIT_DATABASE_BATCH), 'Array of text to generate embeddings.', false, ['plan']) - ->param('model', Ollama::MODEL_EMBEDDING_GEMMA, new WhiteList(Ollama::MODELS), 'The embedding model to use for generating vector embeddings.', true, enum: new Enum(name: 'EmbeddingModel')) + ->param('model', AppwriteAdapter::MODEL_NOMIC_EMBED_TEXT, new WhiteList(AppwriteAdapter::MODELS), 'The embedding model to use for generating vector embeddings.', true, enum: new Enum(name: 'EmbeddingModel')) ->inject('response') ->inject('project') ->inject('embeddingAgent') @@ -87,50 +87,36 @@ class Create extends CreateDocumentAction public function action(array $texts, string $model, UtopiaResponse $response, Document $project, Agent $embeddingAgent, Context $usage, Log $log, ?Logger $logger): void { - $results = []; - $embeddingAgent->getAdapter()->setModel($model); - $dimension = $embeddingAgent->getAdapter()->getEmbeddingDimension(); + $adapter = $embeddingAgent->getAdapter(); + $adapter->setModel($model); + $dimension = $adapter->getEmbeddingDimension(); + $results = []; $totalDuration = 0; $totalTokens = 0; $totalErrors = 0; - foreach ($texts as $text) { - $embedding = []; - $error = ''; + + foreach (array_chunk($texts, APP_EMBEDDING_BATCH_LIMIT) as $batch) { try { - $embedResult = $embeddingAgent->embed($text); - $embedding = $embedResult['embedding']; + // TODO: Have the interface in the agents lib itself + $embedResult = $adapter->bulkEmbed($batch); $totalDuration += $embedResult['totalDuration'] ?? 0; $totalTokens += $embedResult['tokensProcessed'] ?? 0; + + foreach ($embedResult['embeddings'] as $embedding) { + $results[] = $this->embeddingResult($model, $dimension, $embedding); + } } catch (\Exception $e) { - $error = 'Error while generating embedding'; - $totalErrors += 1; - if ($logger) { - $log->setNamespace("http"); - $log->setServer(System::getEnv('_APP_LOGGING_SERVICE_IDENTIFIER', \gethostname())); - $log->setVersion(System::getEnv('_APP_VERSION', 'UNKNOWN')); - $log->setType(Log::TYPE_ERROR); - $log->setMessage($e->getMessage()); + $totalErrors++; + $this->logError($e, $model, $project, $log, $logger); - $log->addTag('embeddingModel', $model); - $log->addTag('code', $e->getCode()); - $log->addTag('projectId', $project->getId()); - - $log->addExtra('file', $e->getFile()); - $log->addExtra('line', $e->getLine()); - $log->addExtra('trace', $e->getTraceAsString()); - - $logger->addLog($log); + // One error result per text in the failed batch. + foreach ($batch as $ignored) { + $results[] = $this->embeddingResult($model, $dimension, [], 'Error while generating embedding'); } } - - $results[] = new Document([ - 'model' => $model, - 'dimension' => $dimension, - 'embedding' => $embedding, - 'error' => $error - ]); } + $embeddings = new Document([ 'embeddings' => $results, 'total' => \count($results), @@ -150,4 +136,42 @@ class Create extends CreateDocumentAction ->addMetric(METRIC_EMBEDDINGS_TEXT_TOTAL_ERROR, $totalErrors) ->addMetric(\str_replace('{embeddingModel}', $model, METRIC_EMBEDDINGS_MODEL_TEXT_TOTAL_ERROR), $totalErrors); } + + /** + * Build a single embedding response document. + * + * @param float[] $embedding + */ + private function embeddingResult(string $model, int $dimension, array $embedding, string $error = ''): Document + { + return new Document([ + 'model' => $model, + 'dimension' => $dimension, + 'embedding' => $embedding, + 'error' => $error, + ]); + } + + private function logError(\Throwable $e, string $model, Document $project, Log $log, ?Logger $logger): void + { + if ($logger === null) { + return; + } + + $log->setNamespace('http'); + $log->setServer(System::getEnv('_APP_LOGGING_SERVICE_IDENTIFIER', \gethostname())); + $log->setVersion(System::getEnv('_APP_VERSION', 'UNKNOWN')); + $log->setType(Log::TYPE_ERROR); + $log->setMessage($e->getMessage()); + + $log->addTag('embeddingModel', $model); + $log->addTag('code', $e->getCode()); + $log->addTag('projectId', $project->getId()); + + $log->addExtra('file', $e->getFile()); + $log->addExtra('line', $e->getLine()); + $log->addExtra('trace', $e->getTraceAsString()); + + $logger->addLog($log); + } } diff --git a/src/Appwrite/Utopia/Response/Model/Embedding.php b/src/Appwrite/Utopia/Response/Model/Embedding.php index 9fce913723..00367316c4 100644 --- a/src/Appwrite/Utopia/Response/Model/Embedding.php +++ b/src/Appwrite/Utopia/Response/Model/Embedding.php @@ -22,7 +22,7 @@ class Embedding extends Model ->addRule('model', [ 'type' => self::TYPE_STRING, 'description' => 'Embedding model used to generate embeddings.', - 'example' => 'embeddinggemma' + 'example' => 'nomic' ]) ->addRule('dimension', [ 'type' => self::TYPE_INTEGER, diff --git a/tests/e2e/General/UsageTest.php b/tests/e2e/General/UsageTest.php index dddcb14491..b54f390db5 100644 --- a/tests/e2e/General/UsageTest.php +++ b/tests/e2e/General/UsageTest.php @@ -2035,7 +2035,7 @@ class UsageTest extends Scope 'x-appwrite-key' => $this->getProject()['apiKey'], ], $this->getHeaders()), [ - 'model' => 'embeddinggemma', + 'model' => 'nomic', 'texts' => ['usage test warm-up ' . $callCount], ] ); @@ -2059,7 +2059,7 @@ class UsageTest extends Scope 'x-appwrite-key' => $this->getProject()['apiKey'], ], $this->getHeaders()), [ - 'model' => 'embeddinggemma', + 'model' => 'nomic', 'texts' => ['usage test text ' . $i], ] ); diff --git a/tests/e2e/Services/Databases/VectorsDB/DatabasesCustomServerTest.php b/tests/e2e/Services/Databases/VectorsDB/DatabasesCustomServerTest.php index 9564b76079..57730cd9b3 100644 --- a/tests/e2e/Services/Databases/VectorsDB/DatabasesCustomServerTest.php +++ b/tests/e2e/Services/Databases/VectorsDB/DatabasesCustomServerTest.php @@ -582,7 +582,7 @@ class DatabasesCustomServerTest extends Scope 'x-appwrite-project' => $this->getProject()['$id'], 'x-appwrite-key' => $this->getProject()['apiKey'] ], [ - 'model' => 'embeddinggemma', + 'model' => 'nomic', 'texts' => [ 'hello world', 'second sentence', @@ -616,7 +616,7 @@ class DatabasesCustomServerTest extends Scope 'x-appwrite-project' => $this->getProject()['$id'], 'x-appwrite-key' => $this->getProject()['apiKey'] ], [ - 'model' => 'embeddinggemma', + 'model' => 'nomic', 'texts' => [ 'valid text', 123, // invalid, not a string diff --git a/tests/e2e/Services/Databases/VectorsDBCustomServerTest.php b/tests/e2e/Services/Databases/VectorsDBCustomServerTest.php index ceb672443e..114de12e82 100644 --- a/tests/e2e/Services/Databases/VectorsDBCustomServerTest.php +++ b/tests/e2e/Services/Databases/VectorsDBCustomServerTest.php @@ -581,7 +581,7 @@ class VectorsDBCustomServerTest extends Scope 'x-appwrite-project' => $this->getProject()['$id'], 'x-appwrite-key' => $this->getProject()['apiKey'] ], [ - 'model' => 'embeddinggemma', + 'model' => 'nomic', 'texts' => [ 'hello world', 'second sentence', @@ -615,7 +615,7 @@ class VectorsDBCustomServerTest extends Scope 'x-appwrite-project' => $this->getProject()['$id'], 'x-appwrite-key' => $this->getProject()['apiKey'] ], [ - 'model' => 'embeddinggemma', + 'model' => 'nomic', 'texts' => [ 'valid text', 123, // invalid, not a string From 8e897f18108bc1c097b916919286f53b8e5aed2d Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Fri, 29 May 2026 16:05:09 +0530 Subject: [PATCH 02/11] updated --- docker-compose.yml | 26 +++++++++---------- .../Http/VectorsDB/Embeddings/Text/Create.php | 2 +- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index da518fb6e9..d3950fb891 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1358,19 +1358,19 @@ services: start_period: 10s command: "postgres" - appwrite-embedding: - image: appwrite/embedding - environment: - EMBEDDING_MODELS: "nomic" - container_name: appwrite-embedding - ports: - - "11430:11434" - restart: unless-stopped - # persistent for caching models across restarts and preloading - volumes: - - embedding_models:/home/embedder/models - networks: - - appwrite + appwrite-embedding: + image: appwrite/embedding + environment: + EMBEDDING_MODELS: "nomic" + container_name: appwrite-embedding + ports: + - "11430:11434" + restart: unless-stopped + # persistent for caching models across restarts and preloading + volumes: + - embedding_models:/home/embedder/models + networks: + - appwrite redis: image: redis:7.4.7-alpine diff --git a/src/Appwrite/Platform/Modules/Databases/Http/VectorsDB/Embeddings/Text/Create.php b/src/Appwrite/Platform/Modules/Databases/Http/VectorsDB/Embeddings/Text/Create.php index 90e477d401..d71715b921 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/VectorsDB/Embeddings/Text/Create.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/VectorsDB/Embeddings/Text/Create.php @@ -107,7 +107,7 @@ class Create extends CreateDocumentAction $results[] = $this->embeddingResult($model, $dimension, $embedding); } } catch (\Exception $e) { - $totalErrors++; + $totalErrors += \count($batch); $this->logError($e, $model, $project, $log, $logger); // One error result per text in the failed batch. From 91b6babd0ff62528c3f7b5049a907e6ac90893f2 Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Fri, 29 May 2026 16:07:24 +0530 Subject: [PATCH 03/11] updated --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index d3950fb891..7f9ec55eb3 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1368,7 +1368,7 @@ services: restart: unless-stopped # persistent for caching models across restarts and preloading volumes: - - embedding_models:/home/embedder/models + - appwrite-models:/home/embedder/models networks: - appwrite From 34b064ca7a3e1cdb793fbd7698ea2eaae78bb586 Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Fri, 29 May 2026 16:17:13 +0530 Subject: [PATCH 04/11] refactor: update embedding model configuration and improve agent usage in Create.php --- composer.lock | 12 ++++++------ docker-compose.yml | 2 +- .../Http/VectorsDB/Embeddings/Text/Create.php | 3 +-- 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/composer.lock b/composer.lock index 2823faf618..1bdcb81140 100644 --- a/composer.lock +++ b/composer.lock @@ -3411,16 +3411,16 @@ }, { "name": "utopia-php/agents", - "version": "1.2.3", + "version": "1.2.4", "source": { "type": "git", "url": "https://github.com/utopia-php/agents.git", - "reference": "47fb6282909fb8dfbb196f1a9834e4d858e41dcb" + "reference": "c9785833948639c09c8ccc3e954178c66ed9a896" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/agents/zipball/47fb6282909fb8dfbb196f1a9834e4d858e41dcb", - "reference": "47fb6282909fb8dfbb196f1a9834e4d858e41dcb", + "url": "https://api.github.com/repos/utopia-php/agents/zipball/c9785833948639c09c8ccc3e954178c66ed9a896", + "reference": "c9785833948639c09c8ccc3e954178c66ed9a896", "shasum": "" }, "require": { @@ -3458,9 +3458,9 @@ ], "support": { "issues": "https://github.com/utopia-php/agents/issues", - "source": "https://github.com/utopia-php/agents/tree/1.2.3" + "source": "https://github.com/utopia-php/agents/tree/1.2.4" }, - "time": "2026-05-28T11:22:16+00:00" + "time": "2026-05-29T10:44:54+00:00" }, { "name": "utopia-php/analytics", diff --git a/docker-compose.yml b/docker-compose.yml index 7f9ec55eb3..ccced2670d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1361,7 +1361,7 @@ services: appwrite-embedding: image: appwrite/embedding environment: - EMBEDDING_MODELS: "nomic" + EMBEDDING_MODELS: ${_APP_EMBEDDING_MODELS:-nomic} container_name: appwrite-embedding ports: - "11430:11434" diff --git a/src/Appwrite/Platform/Modules/Databases/Http/VectorsDB/Embeddings/Text/Create.php b/src/Appwrite/Platform/Modules/Databases/Http/VectorsDB/Embeddings/Text/Create.php index d71715b921..ea49709b3b 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/VectorsDB/Embeddings/Text/Create.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/VectorsDB/Embeddings/Text/Create.php @@ -87,7 +87,6 @@ class Create extends CreateDocumentAction public function action(array $texts, string $model, UtopiaResponse $response, Document $project, Agent $embeddingAgent, Context $usage, Log $log, ?Logger $logger): void { - $adapter = $embeddingAgent->getAdapter(); $adapter->setModel($model); $dimension = $adapter->getEmbeddingDimension(); @@ -99,7 +98,7 @@ class Create extends CreateDocumentAction foreach (array_chunk($texts, APP_EMBEDDING_BATCH_LIMIT) as $batch) { try { // TODO: Have the interface in the agents lib itself - $embedResult = $adapter->bulkEmbed($batch); + $embedResult = $embeddingAgent->bulkEmbed($batch); $totalDuration += $embedResult['totalDuration'] ?? 0; $totalTokens += $embedResult['tokensProcessed'] ?? 0; From 1f38fe578bc954f76453587642f0b53db0dd8605 Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Fri, 29 May 2026 16:22:42 +0530 Subject: [PATCH 05/11] refactor: initialize adapter in action method and remove TODO comment --- .../Modules/Databases/Http/VectorsDB/Embeddings/Text/Create.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Appwrite/Platform/Modules/Databases/Http/VectorsDB/Embeddings/Text/Create.php b/src/Appwrite/Platform/Modules/Databases/Http/VectorsDB/Embeddings/Text/Create.php index ea49709b3b..41fafd8460 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/VectorsDB/Embeddings/Text/Create.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/VectorsDB/Embeddings/Text/Create.php @@ -87,6 +87,7 @@ class Create extends CreateDocumentAction public function action(array $texts, string $model, UtopiaResponse $response, Document $project, Agent $embeddingAgent, Context $usage, Log $log, ?Logger $logger): void { + $adapter = $embeddingAgent->getAdapter(); $adapter->setModel($model); $dimension = $adapter->getEmbeddingDimension(); @@ -97,7 +98,6 @@ class Create extends CreateDocumentAction foreach (array_chunk($texts, APP_EMBEDDING_BATCH_LIMIT) as $batch) { try { - // TODO: Have the interface in the agents lib itself $embedResult = $embeddingAgent->bulkEmbed($batch); $totalDuration += $embedResult['totalDuration'] ?? 0; $totalTokens += $embedResult['tokensProcessed'] ?? 0; From 8e4770e19ed2f7ea950b8175035ef3ffd395a05d Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Fri, 29 May 2026 16:31:27 +0530 Subject: [PATCH 06/11] added tag --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index ccced2670d..0dfe8db78f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1359,7 +1359,7 @@ services: command: "postgres" appwrite-embedding: - image: appwrite/embedding + image: appwrite/embedding:0.1.0 environment: EMBEDDING_MODELS: ${_APP_EMBEDDING_MODELS:-nomic} container_name: appwrite-embedding From 805bfd04e3915bfce9bb44d45385e085c6a3676c Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Fri, 29 May 2026 16:39:39 +0530 Subject: [PATCH 07/11] fix: handle partial success in embedding generation by filling missing results with errors --- .../Databases/Http/VectorsDB/Embeddings/Text/Create.php | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/Appwrite/Platform/Modules/Databases/Http/VectorsDB/Embeddings/Text/Create.php b/src/Appwrite/Platform/Modules/Databases/Http/VectorsDB/Embeddings/Text/Create.php index 41fafd8460..05512e3151 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/VectorsDB/Embeddings/Text/Create.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/VectorsDB/Embeddings/Text/Create.php @@ -105,6 +105,14 @@ class Create extends CreateDocumentAction foreach ($embedResult['embeddings'] as $embedding) { $results[] = $this->embeddingResult($model, $dimension, $embedding); } + + // Partial success: backend returned fewer embeddings than inputs. + // Fill the gap with error results so the response stays 1:1 with texts. + $missing = \count($batch) - \count($embedResult['embeddings']); + for ($i = 0; $i < $missing; $i++) { + $totalErrors++; + $results[] = $this->embeddingResult($model, $dimension, [], 'Error while generating embedding'); + } } catch (\Exception $e) { $totalErrors += \count($batch); $this->logError($e, $model, $project, $log, $logger); From 961d00b6a653268cb6ae957634bcdaf8d899139f Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Fri, 29 May 2026 18:48:01 +0530 Subject: [PATCH 08/11] fix: update embedding model name to 'nomic-embed-text' in configuration and tests --- .env | 2 +- docker-compose.yml | 2 +- src/Appwrite/Utopia/Response/Model/Embedding.php | 2 +- tests/e2e/General/UsageTest.php | 4 ++-- .../Databases/VectorsDB/DatabasesCustomServerTest.php | 4 ++-- tests/e2e/Services/Databases/VectorsDBCustomServerTest.php | 4 ++-- 6 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.env b/.env index 5478608c60..db6c73e762 100644 --- a/.env +++ b/.env @@ -55,7 +55,7 @@ _APP_DB_PORT_DOCUMENTSDB=27017 _APP_DB_ADAPTER_VECTORSDB=postgresql _APP_DB_HOST_VECTORSDB=postgresql _APP_DB_PORT_VECTORSDB=5432 -_APP_EMBEDDING_MODELS=nomic +_APP_EMBEDDING_MODELS=nomic-embed-text _APP_EMBEDDING_ENDPOINT='http://appwrite-embedding:11434/embed' _APP_EMBEDDING_TIMEOUT=30000 _APP_STORAGE_DEVICE=Local diff --git a/docker-compose.yml b/docker-compose.yml index 0dfe8db78f..028208b468 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1361,7 +1361,7 @@ services: appwrite-embedding: image: appwrite/embedding:0.1.0 environment: - EMBEDDING_MODELS: ${_APP_EMBEDDING_MODELS:-nomic} + EMBEDDING_MODELS: ${_APP_EMBEDDING_MODELS:-nomic-embed-text} container_name: appwrite-embedding ports: - "11430:11434" diff --git a/src/Appwrite/Utopia/Response/Model/Embedding.php b/src/Appwrite/Utopia/Response/Model/Embedding.php index 00367316c4..1ac2abb002 100644 --- a/src/Appwrite/Utopia/Response/Model/Embedding.php +++ b/src/Appwrite/Utopia/Response/Model/Embedding.php @@ -22,7 +22,7 @@ class Embedding extends Model ->addRule('model', [ 'type' => self::TYPE_STRING, 'description' => 'Embedding model used to generate embeddings.', - 'example' => 'nomic' + 'example' => 'nomic-embed-text' ]) ->addRule('dimension', [ 'type' => self::TYPE_INTEGER, diff --git a/tests/e2e/General/UsageTest.php b/tests/e2e/General/UsageTest.php index b54f390db5..017131bfbe 100644 --- a/tests/e2e/General/UsageTest.php +++ b/tests/e2e/General/UsageTest.php @@ -2035,7 +2035,7 @@ class UsageTest extends Scope 'x-appwrite-key' => $this->getProject()['apiKey'], ], $this->getHeaders()), [ - 'model' => 'nomic', + 'model' => 'nomic-embed-text', 'texts' => ['usage test warm-up ' . $callCount], ] ); @@ -2059,7 +2059,7 @@ class UsageTest extends Scope 'x-appwrite-key' => $this->getProject()['apiKey'], ], $this->getHeaders()), [ - 'model' => 'nomic', + 'model' => 'nomic-embed-text', 'texts' => ['usage test text ' . $i], ] ); diff --git a/tests/e2e/Services/Databases/VectorsDB/DatabasesCustomServerTest.php b/tests/e2e/Services/Databases/VectorsDB/DatabasesCustomServerTest.php index 57730cd9b3..b34aaef99e 100644 --- a/tests/e2e/Services/Databases/VectorsDB/DatabasesCustomServerTest.php +++ b/tests/e2e/Services/Databases/VectorsDB/DatabasesCustomServerTest.php @@ -582,7 +582,7 @@ class DatabasesCustomServerTest extends Scope 'x-appwrite-project' => $this->getProject()['$id'], 'x-appwrite-key' => $this->getProject()['apiKey'] ], [ - 'model' => 'nomic', + 'model' => 'nomic-embed-text', 'texts' => [ 'hello world', 'second sentence', @@ -616,7 +616,7 @@ class DatabasesCustomServerTest extends Scope 'x-appwrite-project' => $this->getProject()['$id'], 'x-appwrite-key' => $this->getProject()['apiKey'] ], [ - 'model' => 'nomic', + 'model' => 'nomic-embed-text', 'texts' => [ 'valid text', 123, // invalid, not a string diff --git a/tests/e2e/Services/Databases/VectorsDBCustomServerTest.php b/tests/e2e/Services/Databases/VectorsDBCustomServerTest.php index 114de12e82..12fef2ddd2 100644 --- a/tests/e2e/Services/Databases/VectorsDBCustomServerTest.php +++ b/tests/e2e/Services/Databases/VectorsDBCustomServerTest.php @@ -581,7 +581,7 @@ class VectorsDBCustomServerTest extends Scope 'x-appwrite-project' => $this->getProject()['$id'], 'x-appwrite-key' => $this->getProject()['apiKey'] ], [ - 'model' => 'nomic', + 'model' => 'nomic-embed-text', 'texts' => [ 'hello world', 'second sentence', @@ -615,7 +615,7 @@ class VectorsDBCustomServerTest extends Scope 'x-appwrite-project' => $this->getProject()['$id'], 'x-appwrite-key' => $this->getProject()['apiKey'] ], [ - 'model' => 'nomic', + 'model' => 'nomic-embed-text', 'texts' => [ 'valid text', 123, // invalid, not a string From 6d540f9b8e7cac565c76c0b4ed51798aa005e11d Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Mon, 1 Jun 2026 11:38:58 +0530 Subject: [PATCH 09/11] fix: update embedding endpoint and port in configuration --- .env | 2 +- app/init/resources/request.php | 2 +- docker-compose.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.env b/.env index db6c73e762..aa7438f6bb 100644 --- a/.env +++ b/.env @@ -56,7 +56,7 @@ _APP_DB_ADAPTER_VECTORSDB=postgresql _APP_DB_HOST_VECTORSDB=postgresql _APP_DB_PORT_VECTORSDB=5432 _APP_EMBEDDING_MODELS=nomic-embed-text -_APP_EMBEDDING_ENDPOINT='http://appwrite-embedding:11434/embed' +_APP_EMBEDDING_ENDPOINT='http://appwrite-embedding:3000/embed' _APP_EMBEDDING_TIMEOUT=30000 _APP_STORAGE_DEVICE=Local _APP_STORAGE_S3_ACCESS_KEY= diff --git a/app/init/resources/request.php b/app/init/resources/request.php index f10794a323..416c294140 100644 --- a/app/init/resources/request.php +++ b/app/init/resources/request.php @@ -1278,7 +1278,7 @@ return function (Container $context): void { $context->set('embeddingAgent', function ($register) { $adapter = new AppwriteAdapter(); - $adapter->setEndpoint(System::getEnv('_APP_EMBEDDING_ENDPOINT', 'http://appwrite-embedding:11434/embed')); + $adapter->setEndpoint(System::getEnv('_APP_EMBEDDING_ENDPOINT', 'http://appwrite-embedding:3000/embed')); $adapter->setTimeout((int) System::getEnv('_APP_EMBEDDING_TIMEOUT', '30000')); return new Agent($adapter); }, ['register']); diff --git a/docker-compose.yml b/docker-compose.yml index 028208b468..ab51ad32c0 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1364,7 +1364,7 @@ services: EMBEDDING_MODELS: ${_APP_EMBEDDING_MODELS:-nomic-embed-text} container_name: appwrite-embedding ports: - - "11430:11434" + - "3000:3000" restart: unless-stopped # persistent for caching models across restarts and preloading volumes: From c79a23b037b659d02404037293311ce79a00480e Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Mon, 1 Jun 2026 11:58:39 +0530 Subject: [PATCH 10/11] fix: update utopia-php/agents version to 2.* in composer files --- composer.json | 2 +- composer.lock | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/composer.json b/composer.json index 53c9287f23..3588743d06 100644 --- a/composer.json +++ b/composer.json @@ -52,7 +52,7 @@ "appwrite/php-runtimes": "0.20.*", "appwrite/php-clamav": "2.0.*", "utopia-php/abuse": "1.3.*", - "utopia-php/agents": "1.2.*", + "utopia-php/agents": "2.*", "utopia-php/analytics": "0.15.*", "utopia-php/audit": "^2.4", "utopia-php/auth": "0.5.*", diff --git a/composer.lock b/composer.lock index 4ae70bb542..6174709d56 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "53a96b051328aa5572b1008496733f85", + "content-hash": "95a83cce1d03ceb2bfc777309ebe79e9", "packages": [ { "name": "adhocore/jwt", @@ -3411,7 +3411,7 @@ }, { "name": "utopia-php/agents", - "version": "1.2.4", + "version": "2.1.0", "source": { "type": "git", "url": "https://github.com/utopia-php/agents.git", @@ -3458,7 +3458,7 @@ ], "support": { "issues": "https://github.com/utopia-php/agents/issues", - "source": "https://github.com/utopia-php/agents/tree/1.2.4" + "source": "https://github.com/utopia-php/agents/tree/2.1.0" }, "time": "2026-05-29T10:44:54+00:00" }, From 32759377b13772d703c3d169647d66ea7aefbeda Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Tue, 2 Jun 2026 13:30:17 +0530 Subject: [PATCH 11/11] updated composer lock --- composer.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/composer.lock b/composer.lock index 6bcfda13f1..70c843efe4 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "5e251e32e77d0eeac8d0aad25188f08a", + "content-hash": "ec80b1465e57bc223903fa102249f216", "packages": [ { "name": "adhocore/jwt",