mirror of
https://github.com/appwrite/appwrite.git
synced 2026-11-18 20:36:03 +00:00
Merge pull request #12427 from appwrite/appwrite-embeddings
refactor: update embedding model and endpoint to use 'nomic' and 'app…
This commit is contained in:
11 files changed
+109
-75
No files matched your search
@@ -55,8 +55,8 @@ _APP_DB_PORT_DOCUMENTSDB=27017
|
||||
_APP_DB_ADAPTER_VECTORSDB=postgresql
|
||||
_APP_DB_HOST_VECTORSDB=postgresql
|
||||
_APP_DB_PORT_VECTORSDB=5432
|
||||
_APP_EMBEDDING_MODELS=embeddinggemma
|
||||
_APP_EMBEDDING_ENDPOINT='http://ollama:11434/api/embed'
|
||||
_APP_EMBEDDING_MODELS=nomic-embed-text
|
||||
_APP_EMBEDDING_ENDPOINT='http://appwrite-embedding:3000/embed'
|
||||
_APP_EMBEDDING_TIMEOUT=30000
|
||||
_APP_STORAGE_DEVICE=Local
|
||||
_APP_STORAGE_S3_ACCESS_KEY=
|
||||
|
||||
@@ -108,6 +108,9 @@ const APP_VCS_GITHUB_EMAIL = 'team@appwrite.io';
|
||||
const APP_VCS_GITHUB_URL = 'https://github.com/TeamAppwrite';
|
||||
const APP_BRANDED_EMAIL_BASE_TEMPLATE = 'email-base-styled';
|
||||
|
||||
// Embeddings
|
||||
const APP_EMBEDDING_BATCH_LIMIT = 100;
|
||||
|
||||
/**
|
||||
* JWT for Resource Tokens.
|
||||
*/
|
||||
|
||||
@@ -22,7 +22,7 @@ use Appwrite\Usage\Context as UsageContext;
|
||||
use Appwrite\Utopia\Database\Documents\User;
|
||||
use Appwrite\Utopia\Request;
|
||||
use Appwrite\Utopia\Response;
|
||||
use Utopia\Agents\Adapters\Ollama;
|
||||
use Utopia\Agents\Adapters\Appwrite as AppwriteAdapter;
|
||||
use Utopia\Agents\Agent;
|
||||
use Utopia\Audit\Adapter\Database as AdapterDatabase;
|
||||
use Utopia\Audit\Audit;
|
||||
@@ -1277,8 +1277,8 @@ return function (Container $context): void {
|
||||
$context->set('deviceForBuilds', fn ($project, Telemetry $telemetry) => new Device\Telemetry($telemetry, getDevice(APP_STORAGE_BUILDS . '/app-' . $project->getId())), ['project', 'telemetry']);
|
||||
|
||||
$context->set('embeddingAgent', function ($register) {
|
||||
$adapter = new Ollama();
|
||||
$adapter->setEndpoint(System::getEnv('_APP_EMBEDDING_ENDPOINT', 'http://ollama:11434/api/embed'));
|
||||
$adapter = new AppwriteAdapter();
|
||||
$adapter->setEndpoint(System::getEnv('_APP_EMBEDDING_ENDPOINT', 'http://appwrite-embedding:3000/embed'));
|
||||
$adapter->setTimeout((int) System::getEnv('_APP_EMBEDDING_TIMEOUT', '30000'));
|
||||
return new Agent($adapter);
|
||||
}, ['register']);
|
||||
|
||||
+1
-1
@@ -52,7 +52,7 @@
|
||||
"appwrite/php-runtimes": "0.20.*",
|
||||
"appwrite/php-clamav": "2.0.*",
|
||||
"utopia-php/abuse": "1.3.*",
|
||||
"utopia-php/agents": "1.2.*",
|
||||
"utopia-php/agents": "2.*",
|
||||
"utopia-php/analytics": "0.15.*",
|
||||
"utopia-php/audit": "^2.4",
|
||||
"utopia-php/auth": "0.5.*",
|
||||
|
||||
Generated
+7
-7
@@ -4,7 +4,7 @@
|
||||
"Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies",
|
||||
"This file is @generated automatically"
|
||||
],
|
||||
"content-hash": "5e251e32e77d0eeac8d0aad25188f08a",
|
||||
"content-hash": "ec80b1465e57bc223903fa102249f216",
|
||||
"packages": [
|
||||
{
|
||||
"name": "adhocore/jwt",
|
||||
@@ -3411,16 +3411,16 @@
|
||||
},
|
||||
{
|
||||
"name": "utopia-php/agents",
|
||||
"version": "1.2.2",
|
||||
"version": "2.1.0",
|
||||
"source": {
|
||||
"type": "git",
|
||||
"url": "https://github.com/utopia-php/agents.git",
|
||||
"reference": "0703f4cae02261e09a1bf0d39a4b1ce649cae634"
|
||||
"reference": "c9785833948639c09c8ccc3e954178c66ed9a896"
|
||||
},
|
||||
"dist": {
|
||||
"type": "zip",
|
||||
"url": "https://api.github.com/repos/utopia-php/agents/zipball/0703f4cae02261e09a1bf0d39a4b1ce649cae634",
|
||||
"reference": "0703f4cae02261e09a1bf0d39a4b1ce649cae634",
|
||||
"url": "https://api.github.com/repos/utopia-php/agents/zipball/c9785833948639c09c8ccc3e954178c66ed9a896",
|
||||
"reference": "c9785833948639c09c8ccc3e954178c66ed9a896",
|
||||
"shasum": ""
|
||||
},
|
||||
"require": {
|
||||
@@ -3458,9 +3458,9 @@
|
||||
],
|
||||
"support": {
|
||||
"issues": "https://github.com/utopia-php/agents/issues",
|
||||
"source": "https://github.com/utopia-php/agents/tree/1.2.2"
|
||||
"source": "https://github.com/utopia-php/agents/tree/2.1.0"
|
||||
},
|
||||
"time": "2026-05-08T10:38:23+00:00"
|
||||
"time": "2026-05-29T10:44:54+00:00"
|
||||
},
|
||||
{
|
||||
"name": "utopia-php/analytics",
|
||||
|
||||
+21
-21
@@ -112,7 +112,7 @@ services:
|
||||
condition: service_healthy
|
||||
coredns:
|
||||
condition: service_started
|
||||
ollama:
|
||||
appwrite-embedding:
|
||||
condition: service_started
|
||||
entrypoint:
|
||||
- php
|
||||
@@ -304,7 +304,7 @@ services:
|
||||
depends_on:
|
||||
- ${_APP_DB_HOST:-mongodb}
|
||||
- redis
|
||||
- ollama
|
||||
- appwrite-embedding
|
||||
environment:
|
||||
- _APP_ENV
|
||||
- _APP_WORKER_PER_CORE
|
||||
@@ -346,7 +346,7 @@ services:
|
||||
depends_on:
|
||||
- redis
|
||||
- ${_APP_DB_HOST:-mongodb}
|
||||
- ollama
|
||||
- appwrite-embedding
|
||||
environment:
|
||||
- _APP_ENV
|
||||
- _APP_WORKER_PER_CORE
|
||||
@@ -380,7 +380,7 @@ services:
|
||||
- ${_APP_DB_HOST:-mongodb}
|
||||
- request-catcher-sms
|
||||
- request-catcher-webhook
|
||||
- ollama
|
||||
- appwrite-embedding
|
||||
environment:
|
||||
- _APP_ENV
|
||||
- _APP_WORKER_PER_CORE
|
||||
@@ -411,7 +411,7 @@ services:
|
||||
depends_on:
|
||||
- redis
|
||||
- ${_APP_DB_HOST:-mongodb}
|
||||
- ollama
|
||||
- appwrite-embedding
|
||||
volumes:
|
||||
- appwrite-uploads:/storage/uploads:rw
|
||||
- appwrite-cache:/storage/cache:rw
|
||||
@@ -481,7 +481,7 @@ services:
|
||||
depends_on:
|
||||
- redis
|
||||
- ${_APP_DB_HOST:-mongodb}
|
||||
- ollama
|
||||
- appwrite-embedding
|
||||
environment:
|
||||
- _APP_ENV
|
||||
- _APP_WORKER_PER_CORE
|
||||
@@ -524,7 +524,7 @@ services:
|
||||
depends_on:
|
||||
- redis
|
||||
- ${_APP_DB_HOST:-mongodb}
|
||||
- ollama
|
||||
- appwrite-embedding
|
||||
environment:
|
||||
- _APP_ENV
|
||||
- _APP_WORKER_PER_CORE
|
||||
@@ -657,7 +657,7 @@ services:
|
||||
depends_on:
|
||||
- redis
|
||||
- ${_APP_DB_HOST:-mongodb}
|
||||
- ollama
|
||||
- appwrite-embedding
|
||||
volumes:
|
||||
- appwrite-config:/storage/config:rw
|
||||
- appwrite-certificates:/storage/certificates:rw
|
||||
@@ -1075,7 +1075,7 @@ services:
|
||||
depends_on:
|
||||
- redis
|
||||
- ${_APP_DB_HOST:-mongodb}
|
||||
- ollama
|
||||
- appwrite-embedding
|
||||
environment:
|
||||
- _APP_ENV
|
||||
- _APP_WORKER_PER_CORE
|
||||
@@ -1109,7 +1109,7 @@ services:
|
||||
depends_on:
|
||||
- ${_APP_DB_HOST:-mongodb}
|
||||
- redis
|
||||
- ollama
|
||||
- appwrite-embedding
|
||||
environment:
|
||||
- _APP_ENV
|
||||
- _APP_WORKER_PER_CORE
|
||||
@@ -1147,7 +1147,7 @@ services:
|
||||
depends_on:
|
||||
- ${_APP_DB_HOST:-mongodb}
|
||||
- redis
|
||||
- ollama
|
||||
- appwrite-embedding
|
||||
environment:
|
||||
- _APP_ENV
|
||||
- _APP_WORKER_PER_CORE
|
||||
@@ -1185,7 +1185,7 @@ services:
|
||||
depends_on:
|
||||
- ${_APP_DB_HOST:-mongodb}
|
||||
- redis
|
||||
- ollama
|
||||
- appwrite-embedding
|
||||
environment:
|
||||
- _APP_ENV
|
||||
- _APP_WORKER_PER_CORE
|
||||
@@ -1358,17 +1358,17 @@ services:
|
||||
start_period: 10s
|
||||
command: "postgres"
|
||||
|
||||
ollama:
|
||||
image: appwrite/ollama:0.1.1
|
||||
container_name: ollama
|
||||
ports:
|
||||
- "11434:11434"
|
||||
restart: unless-stopped
|
||||
appwrite-embedding:
|
||||
image: appwrite/embedding:0.1.0
|
||||
environment:
|
||||
MODELS: ${_APP_EMBEDDING_MODELS:-embeddinggemma}
|
||||
OLLAMA_KEEP_ALIVE: 24h
|
||||
EMBEDDING_MODELS: ${_APP_EMBEDDING_MODELS:-nomic-embed-text}
|
||||
container_name: appwrite-embedding
|
||||
ports:
|
||||
- "3000:3000"
|
||||
restart: unless-stopped
|
||||
# persistent for caching models across restarts and preloading
|
||||
volumes:
|
||||
- appwrite-models:/root/.ollama
|
||||
- appwrite-models:/home/embedder/models
|
||||
networks:
|
||||
- appwrite
|
||||
|
||||
|
||||
@@ -10,7 +10,7 @@ use Appwrite\SDK\Parameter;
|
||||
use Appwrite\SDK\Response as SDKResponse;
|
||||
use Appwrite\Usage\Context;
|
||||
use Appwrite\Utopia\Response as UtopiaResponse;
|
||||
use Utopia\Agents\Adapters\Ollama;
|
||||
use Utopia\Agents\Adapters\Appwrite as AppwriteAdapter;
|
||||
use Utopia\Agents\Agent;
|
||||
use Utopia\Database\Document;
|
||||
use Utopia\Http\Adapter\Swoole\Response as SwooleResponse;
|
||||
@@ -75,7 +75,7 @@ class Create extends CreateDocumentAction
|
||||
)
|
||||
])
|
||||
->param('texts', [], fn (array $plan) => new ArrayList(new Text(0), $plan['databasesMaxEmbeddingTexts'] ?? APP_LIMIT_DATABASE_BATCH), 'Array of text to generate embeddings.', false, ['plan'])
|
||||
->param('model', Ollama::MODEL_EMBEDDING_GEMMA, new WhiteList(Ollama::MODELS), 'The embedding model to use for generating vector embeddings.', true, enum: new Enum(name: 'EmbeddingModel'))
|
||||
->param('model', AppwriteAdapter::MODEL_NOMIC_EMBED_TEXT, new WhiteList(AppwriteAdapter::MODELS), 'The embedding model to use for generating vector embeddings.', true, enum: new Enum(name: 'EmbeddingModel'))
|
||||
->inject('response')
|
||||
->inject('project')
|
||||
->inject('embeddingAgent')
|
||||
@@ -87,50 +87,43 @@ class Create extends CreateDocumentAction
|
||||
|
||||
public function action(array $texts, string $model, UtopiaResponse $response, Document $project, Agent $embeddingAgent, Context $usage, Log $log, ?Logger $logger): void
|
||||
{
|
||||
$results = [];
|
||||
$embeddingAgent->getAdapter()->setModel($model);
|
||||
$dimension = $embeddingAgent->getAdapter()->getEmbeddingDimension();
|
||||
$adapter = $embeddingAgent->getAdapter();
|
||||
$adapter->setModel($model);
|
||||
$dimension = $adapter->getEmbeddingDimension();
|
||||
|
||||
$results = [];
|
||||
$totalDuration = 0;
|
||||
$totalTokens = 0;
|
||||
$totalErrors = 0;
|
||||
foreach ($texts as $text) {
|
||||
$embedding = [];
|
||||
$error = '';
|
||||
|
||||
foreach (array_chunk($texts, APP_EMBEDDING_BATCH_LIMIT) as $batch) {
|
||||
try {
|
||||
$embedResult = $embeddingAgent->embed($text);
|
||||
$embedding = $embedResult['embedding'];
|
||||
$embedResult = $embeddingAgent->bulkEmbed($batch);
|
||||
$totalDuration += $embedResult['totalDuration'] ?? 0;
|
||||
$totalTokens += $embedResult['tokensProcessed'] ?? 0;
|
||||
|
||||
foreach ($embedResult['embeddings'] as $embedding) {
|
||||
$results[] = $this->embeddingResult($model, $dimension, $embedding);
|
||||
}
|
||||
|
||||
// Partial success: backend returned fewer embeddings than inputs.
|
||||
// Fill the gap with error results so the response stays 1:1 with texts.
|
||||
$missing = \count($batch) - \count($embedResult['embeddings']);
|
||||
for ($i = 0; $i < $missing; $i++) {
|
||||
$totalErrors++;
|
||||
$results[] = $this->embeddingResult($model, $dimension, [], 'Error while generating embedding');
|
||||
}
|
||||
} catch (\Exception $e) {
|
||||
$error = 'Error while generating embedding';
|
||||
$totalErrors += 1;
|
||||
if ($logger) {
|
||||
$log->setNamespace("http");
|
||||
$log->setServer(System::getEnv('_APP_LOGGING_SERVICE_IDENTIFIER', \gethostname()));
|
||||
$log->setVersion(System::getEnv('_APP_VERSION', 'UNKNOWN'));
|
||||
$log->setType(Log::TYPE_ERROR);
|
||||
$log->setMessage($e->getMessage());
|
||||
$totalErrors += \count($batch);
|
||||
$this->logError($e, $model, $project, $log, $logger);
|
||||
|
||||
$log->addTag('embeddingModel', $model);
|
||||
$log->addTag('code', $e->getCode());
|
||||
$log->addTag('projectId', $project->getId());
|
||||
|
||||
$log->addExtra('file', $e->getFile());
|
||||
$log->addExtra('line', $e->getLine());
|
||||
$log->addExtra('trace', $e->getTraceAsString());
|
||||
|
||||
$logger->addLog($log);
|
||||
// One error result per text in the failed batch.
|
||||
foreach ($batch as $ignored) {
|
||||
$results[] = $this->embeddingResult($model, $dimension, [], 'Error while generating embedding');
|
||||
}
|
||||
}
|
||||
|
||||
$results[] = new Document([
|
||||
'model' => $model,
|
||||
'dimension' => $dimension,
|
||||
'embedding' => $embedding,
|
||||
'error' => $error
|
||||
]);
|
||||
}
|
||||
|
||||
$embeddings = new Document([
|
||||
'embeddings' => $results,
|
||||
'total' => \count($results),
|
||||
@@ -150,4 +143,42 @@ class Create extends CreateDocumentAction
|
||||
->addMetric(METRIC_EMBEDDINGS_TEXT_TOTAL_ERROR, $totalErrors)
|
||||
->addMetric(\str_replace('{embeddingModel}', $model, METRIC_EMBEDDINGS_MODEL_TEXT_TOTAL_ERROR), $totalErrors);
|
||||
}
|
||||
|
||||
/**
|
||||
* Build a single embedding response document.
|
||||
*
|
||||
* @param float[] $embedding
|
||||
*/
|
||||
private function embeddingResult(string $model, int $dimension, array $embedding, string $error = ''): Document
|
||||
{
|
||||
return new Document([
|
||||
'model' => $model,
|
||||
'dimension' => $dimension,
|
||||
'embedding' => $embedding,
|
||||
'error' => $error,
|
||||
]);
|
||||
}
|
||||
|
||||
private function logError(\Throwable $e, string $model, Document $project, Log $log, ?Logger $logger): void
|
||||
{
|
||||
if ($logger === null) {
|
||||
return;
|
||||
}
|
||||
|
||||
$log->setNamespace('http');
|
||||
$log->setServer(System::getEnv('_APP_LOGGING_SERVICE_IDENTIFIER', \gethostname()));
|
||||
$log->setVersion(System::getEnv('_APP_VERSION', 'UNKNOWN'));
|
||||
$log->setType(Log::TYPE_ERROR);
|
||||
$log->setMessage($e->getMessage());
|
||||
|
||||
$log->addTag('embeddingModel', $model);
|
||||
$log->addTag('code', $e->getCode());
|
||||
$log->addTag('projectId', $project->getId());
|
||||
|
||||
$log->addExtra('file', $e->getFile());
|
||||
$log->addExtra('line', $e->getLine());
|
||||
$log->addExtra('trace', $e->getTraceAsString());
|
||||
|
||||
$logger->addLog($log);
|
||||
}
|
||||
}
|
||||
@@ -22,7 +22,7 @@ class Embedding extends Model
|
||||
->addRule('model', [
|
||||
'type' => self::TYPE_STRING,
|
||||
'description' => 'Embedding model used to generate embeddings.',
|
||||
'example' => 'embeddinggemma'
|
||||
'example' => 'nomic-embed-text'
|
||||
])
|
||||
->addRule('dimension', [
|
||||
'type' => self::TYPE_INTEGER,
|
||||
|
||||
@@ -2035,7 +2035,7 @@ class UsageTest extends Scope
|
||||
'x-appwrite-key' => $this->getProject()['apiKey'],
|
||||
], $this->getHeaders()),
|
||||
[
|
||||
'model' => 'embeddinggemma',
|
||||
'model' => 'nomic-embed-text',
|
||||
'texts' => ['usage test warm-up ' . $callCount],
|
||||
]
|
||||
);
|
||||
@@ -2059,7 +2059,7 @@ class UsageTest extends Scope
|
||||
'x-appwrite-key' => $this->getProject()['apiKey'],
|
||||
], $this->getHeaders()),
|
||||
[
|
||||
'model' => 'embeddinggemma',
|
||||
'model' => 'nomic-embed-text',
|
||||
'texts' => ['usage test text ' . $i],
|
||||
]
|
||||
);
|
||||
|
||||
@@ -582,7 +582,7 @@ class DatabasesCustomServerTest extends Scope
|
||||
'x-appwrite-project' => $this->getProject()['$id'],
|
||||
'x-appwrite-key' => $this->getProject()['apiKey']
|
||||
], [
|
||||
'model' => 'embeddinggemma',
|
||||
'model' => 'nomic-embed-text',
|
||||
'texts' => [
|
||||
'hello world',
|
||||
'second sentence',
|
||||
@@ -616,7 +616,7 @@ class DatabasesCustomServerTest extends Scope
|
||||
'x-appwrite-project' => $this->getProject()['$id'],
|
||||
'x-appwrite-key' => $this->getProject()['apiKey']
|
||||
], [
|
||||
'model' => 'embeddinggemma',
|
||||
'model' => 'nomic-embed-text',
|
||||
'texts' => [
|
||||
'valid text',
|
||||
123, // invalid, not a string
|
||||
|
||||
@@ -581,7 +581,7 @@ class VectorsDBCustomServerTest extends Scope
|
||||
'x-appwrite-project' => $this->getProject()['$id'],
|
||||
'x-appwrite-key' => $this->getProject()['apiKey']
|
||||
], [
|
||||
'model' => 'embeddinggemma',
|
||||
'model' => 'nomic-embed-text',
|
||||
'texts' => [
|
||||
'hello world',
|
||||
'second sentence',
|
||||
@@ -615,7 +615,7 @@ class VectorsDBCustomServerTest extends Scope
|
||||
'x-appwrite-project' => $this->getProject()['$id'],
|
||||
'x-appwrite-key' => $this->getProject()['apiKey']
|
||||
], [
|
||||
'model' => 'embeddinggemma',
|
||||
'model' => 'nomic-embed-text',
|
||||
'texts' => [
|
||||
'valid text',
|
||||
123, // invalid, not a string
|
||||
|
||||
Reference in new issue
Block a user