diff --git a/app/init/resources.php b/app/init/resources.php index 2a563ddeae..7ff41a996c 100644 --- a/app/init/resources.php +++ b/app/init/resources.php @@ -60,6 +60,8 @@ use Utopia\Telemetry\Adapter\None as NoTelemetry; use Utopia\Validator\Hostname; use Utopia\Validator\WhiteList; use Utopia\VCS\Adapter\Git\GitHub as VcsGitHub; +use Utopia\Agents\Agent; +use Utopia\Agents\Adapters\Ollama; // Runtime Execution App::setResource('log', fn () => new Log()); @@ -1167,3 +1169,9 @@ App::setResource('httpReferrerSafe', function (Request $request, string $httpRef App::setResource('transactionState', function (Database $dbForProject, callable $getDatabasesDB) { return new TransactionState($dbForProject, $getDatabasesDB); }, ['dbForProject', 'getDatabasesDB']); + +App::setResource('embeddingAgent', function () { + // TODO: ollama endpoint should be taken from env in cloud(for autoscaling) + $adapter = new Ollama(); + return new Agent($adapter); +}); \ No newline at end of file diff --git a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Create.php b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Create.php index 86a5c18490..1515ff9421 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Create.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Create.php @@ -27,6 +27,8 @@ use Utopia\Swoole\Response as SwooleResponse; use Utopia\Validator\Boolean; use Utopia\Validator\Integer; use Utopia\Validator\Text; +use Utopia\Validator\WhiteList; +use Utopia\Agents\Adapters\Ollama; class Create extends CollectionAction { @@ -70,7 +72,7 @@ class Create extends CollectionAction ->param('collectionId', '', fn (Database $dbForProject) => new CustomId(false, $dbForProject->getAdapter()->getMaxUIDLength()), 'Unique Id. Choose a custom ID or generate a random ID with `ID.unique()`. Valid chars are a-z, A-Z, 0-9, period, hyphen, and underscore. Can\'t start with a special char. Max length is 36 chars.', false, ['dbForProject']) ->param('name', '', new Text(128), 'Collection name. Max length: 128 chars.') ->param('dimensions', 0, new Integer(), 'Embedding dimensions.') - ->param('embeddingModel', '', new Text(256), 'Embedding model identifier.') + ->param('embeddingModel', '', new WhiteList((new Ollama())->getModels()), 'Embedding model identifier.') ->param('permissions', null, new Permissions(APP_LIMIT_ARRAY_PARAMS_SIZE), 'An array of permissions strings. By default, no user is granted with any permissions. [Learn more about permissions](https://appwrite.io/docs/permissions).', true) ->param('documentSecurity', false, new Boolean(true), 'Enables configuring permissions for individual documents. A user needs one of document or collection level permissions to access a document. [Learn more about permissions](https://appwrite.io/docs/permissions).', true) ->param('enabled', true, new Boolean(), 'Is collection enabled? When set to \'disabled\', users cannot access the collection but Server SDKs with and API key can still read and write to the collection. No data is lost when this is toggled.', true) diff --git a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Attribute/Increment.php b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Attribute/Increment.php deleted file mode 100644 index 297ec03338..0000000000 --- a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Attribute/Increment.php +++ /dev/null @@ -1,72 +0,0 @@ -setHttpMethod(self::HTTP_REQUEST_METHOD_PATCH) - ->setHttpPath('/v1/vectordb/:databaseId/collections/:collectionId/documents/:documentId/:attribute/increment') - ->desc('Increment document attribute') - ->groups(['api', 'database']) - ->label('event', 'vectordb.[databaseId].collections.[collectionId].documents.[documentId].update') - ->label('scope', 'documents.write') - ->label('resourceType', RESOURCE_TYPE_DATABASES) - ->label('audits.event', 'documents.update') - ->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}') - ->label('abuse-key', 'ip:{ip},method:{method},url:{url},userId:{userId}') - ->label('abuse-limit', APP_LIMIT_WRITE_RATE_DEFAULT * 2) - ->label('abuse-time', APP_LIMIT_WRITE_RATE_PERIOD_DEFAULT) - ->label('sdk', new Method( - namespace: 'documentsDB', - group: $this->getSdkGroup(), - name: 'incrementDocumentAttribute', - description: '/docs/references/vectordb/increment-document-attribute.md', - auth: [AuthType::SESSION, AuthType::JWT, AuthType::ADMIN, AuthType::KEY], - responses: [ - new SDKResponse( - code: SwooleResponse::STATUS_CODE_OK, - model: $this->getResponseModel(), - ) - ], - contentType: ContentType::JSON - )) - ->param('databaseId', '', new UID(), 'Database ID.') - ->param('collectionId', '', new UID(), 'Collection ID.') - ->param('documentId', '', new UID(), 'Document ID.') - ->param('attribute', '', new Key(), 'Attribute key.') - ->param('value', 1, new Numeric(), 'Value to increment the attribute by. The value must be a number.', true) - ->param('max', null, new Numeric(), 'Maximum value for the attribute. If the current value is greater than this value, an error will be thrown.', true) - ->param('transactionId', null, new UID(), 'Transaction ID for staging the operation.', true) - ->inject('response') - ->inject('dbForProject') - ->inject('getDatabasesDB') - ->inject('queueForEvents') - ->inject('queueForStatsUsage') - ->inject('plan') - ->callback($this->action(...)); - } -} diff --git a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Embedding/Create.php b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Embedding/Create.php new file mode 100644 index 0000000000..10ae664299 --- /dev/null +++ b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Embedding/Create.php @@ -0,0 +1,114 @@ +setHttpMethod(self::HTTP_REQUEST_METHOD_POST) + ->setHttpPath('/v1/vectordb/:databaseId/collections/:collectionId/documents') + ->desc('Create document') + ->groups(['api', 'database']) + ->label('scope', 'documents.write') + ->label('resourceType', RESOURCE_TYPE_DATABASES) + ->label('audits.event', 'document.create') + ->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}') + ->label('abuse-key', 'ip:{ip},method:{method},url:{url},userId:{userId}') + ->label('abuse-limit', APP_LIMIT_WRITE_RATE_DEFAULT * 2) + ->label('abuse-time', APP_LIMIT_WRITE_RATE_PERIOD_DEFAULT) + ->label('sdk', [ + new Method( + namespace: 'vectorDB', + group: $this->getSdkGroup(), + name: 'createEmbeddingDocument', + desc: 'Create document', + description: '/docs/references/vectordb/create-document.md', + auth: [AuthType::SESSION, AuthType::KEY, AuthType::JWT], + responses: [ + new SDKResponse( + code: SwooleResponse::STATUS_CODE_CREATED, + model: $this->getResponseModel(), + ) + ], + contentType: ContentType::JSON, + parameters: [ + new Parameter('databaseId', optional: false), + new Parameter('collectionId', optional: false), + new Parameter('documentId', optional: false), + new Parameter('data', optional: false), + new Parameter('permissions', optional: true), + ] + ), + new Method( + namespace: 'documentsDB', + group: $this->getSdkGroup(), + name: 'createDocuments', + desc: 'Create documents', + description: '/docs/references/vectordb/create-documents.md', + auth: [AuthType::ADMIN, AuthType::KEY], + responses: [ + new SDKResponse( + code: SwooleResponse::STATUS_CODE_CREATED, + model: $this->getBulkResponseModel(), + ) + ], + contentType: ContentType::JSON, + parameters: [ + new Parameter('databaseId', optional: false), + new Parameter('collectionId', optional: false), + new Parameter('documents', optional: false), + ] + ) + ]) + ->param('databaseId', '', new UID(), 'Database ID.') + ->param('documentId', '', new CustomId(), 'Document ID. Choose a custom ID or generate a random ID with `ID.unique()`. Valid chars are a-z, A-Z, 0-9, period, hyphen, and underscore. Can\'t start with a special char. Max length is 36 chars.', true) + ->param('collectionId', '', new UID(), 'Collection ID. You can create a new collection using the Database service [server integration](https://appwrite.io/docs/server/databases#databasesCreateCollection). Make sure to define attributes before creating documents.') + ->param('data', [], new JSON(), 'Document data as JSON object.', true, example: '{"username":"walter.obrien","email":"walter.obrien@example.com","fullName":"Walter O\'Brien","age":30,"isAdmin":false}') + ->param('permissions', null, new Permissions(APP_LIMIT_ARRAY_PARAMS_SIZE, [Database::PERMISSION_READ, Database::PERMISSION_UPDATE, Database::PERMISSION_DELETE, Database::PERMISSION_WRITE]), 'An array of permissions strings. By default, only the current user is granted all permissions. [Learn more about permissions](https://appwrite.io/docs/permissions).', true) + ->param('documents', [], fn (array $plan) => new ArrayList(new JSON(), $plan['databasesBatchSize'] ?? APP_LIMIT_DATABASE_BATCH), 'Array of documents data as JSON objects.', true, ['plan']) + ->param('transactionId', null, new UID(), 'Transaction ID for staging the operation.', true) + ->inject('response') + ->inject('dbForProject') + ->inject('getDatabasesDB') + ->inject('user') + ->inject('queueForEvents') + ->inject('queueForStatsUsage') + ->inject('queueForRealtime') + ->inject('queueForFunctions') + ->inject('queueForWebhooks') + ->inject('plan') + ->callback($this->action(...)); + } +} diff --git a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Attribute/Decrement.php b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Embedding/Update.php similarity index 60% rename from src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Attribute/Decrement.php rename to src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Embedding/Update.php index 7c429483e8..23f874d387 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Attribute/Decrement.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Embedding/Update.php @@ -1,23 +1,24 @@ setHttpMethod(self::HTTP_REQUEST_METHOD_PATCH) - ->setHttpPath('/v1/vectordb/:databaseId/collections/:collectionId/documents/:documentId/:attribute/decrement') - ->desc('Decrement document attribute') + ->setHttpPath('/v1/vectordb/:databaseId/collections/:collectionId/documents/:documentId') + ->desc('Update document') ->groups(['api', 'database']) - ->label('event', 'vectordb.[databaseId].collections.[collectionId].documents.[documentId].update') + ->label('event', 'databases.[databaseId].collections.[collectionId].documents.[documentId].update') ->label('scope', 'documents.write') ->label('resourceType', RESOURCE_TYPE_DATABASES) - ->label('audits.event', 'documents.update') - ->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}') + ->label('audits.event', 'document.update') + ->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}/document/{response.$id}') ->label('abuse-key', 'ip:{ip},method:{method},url:{url},userId:{userId}') ->label('abuse-limit', APP_LIMIT_WRITE_RATE_DEFAULT * 2) ->label('abuse-time', APP_LIMIT_WRITE_RATE_PERIOD_DEFAULT) ->label('sdk', new Method( - namespace: 'documentsDB', + namespace: 'vectorDB', group: $this->getSdkGroup(), - name: 'decrementDocumentAttribute', - description: '/docs/references/vectordb/decrement-document-attribute.md', - auth: [AuthType::SESSION, AuthType::JWT, AuthType::ADMIN, AuthType::KEY], + name: 'updateEmbeddingDocument', + description: '/docs/references/vectordb/update-document.md', + auth: [AuthType::SESSION, AuthType::KEY, AuthType::JWT], responses: [ new SDKResponse( code: SwooleResponse::STATUS_CODE_OK, @@ -57,15 +58,16 @@ class Decrement extends DecrementDocumentAttribute ->param('databaseId', '', new UID(), 'Database ID.') ->param('collectionId', '', new UID(), 'Collection ID.') ->param('documentId', '', new UID(), 'Document ID.') - ->param('attribute', '', new Key(), 'Attribute key.') - ->param('value', 1, new Numeric(), 'Value to decrement the attribute by. The value must be a number.', true) - ->param('min', null, new Numeric(), 'Minimum value for the attribute. If the current value is lesser than this value, an exception will be thrown.', true) + ->param('data', [], new JSON(), 'Document data as JSON object. Include only fields and value pairs to be updated.', true) + ->param('permissions', null, new Permissions(APP_LIMIT_ARRAY_PARAMS_SIZE, [Database::PERMISSION_READ, Database::PERMISSION_UPDATE, Database::PERMISSION_DELETE, Database::PERMISSION_WRITE]), 'An array of permissions strings. By default, the current permissions are inherited. [Learn more about permissions](https://appwrite.io/docs/permissions).', true) ->param('transactionId', null, new UID(), 'Transaction ID for staging the operation.', true) + ->inject('requestTimestamp') ->inject('response') ->inject('dbForProject') ->inject('getDatabasesDB') ->inject('queueForEvents') ->inject('queueForStatsUsage') + ->inject('transactionState') ->inject('plan') ->callback($this->action(...)); } diff --git a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Create.php b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Text/Create.php similarity index 98% rename from src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Create.php rename to src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Text/Create.php index d65266690f..4ea9d30849 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Create.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Text/Create.php @@ -1,6 +1,6 @@ setHttpMethod(self::HTTP_REQUEST_METHOD_POST) - ->setHttpPath('/v1/vectordb/:databaseId/collections/:collectionId/documents') + ->setHttpPath('/v1/vectordb/:databaseId/collections/:collectionId/text') ->desc('Create document') ->groups(['api', 'database']) ->label('scope', 'documents.write') @@ -52,7 +52,7 @@ class Create extends DocumentCreate new Method( namespace: 'documentsDB', group: $this->getSdkGroup(), - name: 'createDocument', + name: 'createTextDocument', desc: 'Create document', description: '/docs/references/vectordb/create-document.md', auth: [AuthType::SESSION, AuthType::KEY, AuthType::JWT], diff --git a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Update.php b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Text/Update.php similarity index 100% rename from src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Update.php rename to src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Text/Update.php diff --git a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Update.php b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Update.php index e833da21ea..c236b976ab 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Update.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Update.php @@ -16,7 +16,9 @@ use Utopia\Database\Validator\UID; use Utopia\Swoole\Response as SwooleResponse; use Utopia\Validator\Boolean; use Utopia\Validator\Integer; +use Utopia\Validator\WhiteList; use Utopia\Validator\Text; +use Utopia\Agents\Adapters\Ollama; class Update extends CollectionAction { @@ -61,7 +63,7 @@ class Update extends CollectionAction ->param('name', null, new Text(128), 'Collection name. Max length: 128 chars.') ->param('description', null, new Text(1024), 'Collection description. Max length: 1024 chars.', true) ->param('dimensions', null, new Integer(), 'Embedding dimensions.', true) - ->param('embeddingModel', null, new Text(256), 'Embedding model identifier.', true) + ->param('embeddingModel', '', new WhiteList((new Ollama())->getModels()), 'Embedding model identifier.') ->param('permissions', null, new Permissions(APP_LIMIT_ARRAY_PARAMS_SIZE), 'An array of permission strings. By default, the current permissions are inherited. [Learn more about permissions](https://appwrite.io/docs/permissions).', true) ->param('documentSecurity', false, new Boolean(true), 'Enables configuring permissions for individual documents. A user needs one of document or collection level permissions to access a document. [Learn more about permissions](https://appwrite.io/docs/permissions).', true) ->param('enabled', true, new Boolean(), 'Is collection enabled? When set to \'disabled\', users cannot access the collection but Server SDKs with and API key can still read and write to the collection. No data is lost when this is toggled.', true) diff --git a/src/Appwrite/Platform/Modules/Databases/Services/Registry/VectorDB.php b/src/Appwrite/Platform/Modules/Databases/Services/Registry/VectorDB.php index 9b526f2ca2..f570dbdbc5 100644 --- a/src/Appwrite/Platform/Modules/Databases/Services/Registry/VectorDB.php +++ b/src/Appwrite/Platform/Modules/Databases/Services/Registry/VectorDB.php @@ -16,6 +16,8 @@ use Appwrite\Platform\Modules\Databases\Http\VectorDB\Update as UpdateVectorData use Appwrite\Platform\Modules\Databases\Http\VectorDB\Usage\Get as GetVectorDatabaseUsage; use Appwrite\Platform\Modules\Databases\Http\VectorDB\Usage\XList as ListVectorDatabaseUsage; use Appwrite\Platform\Modules\Databases\Http\VectorDB\XList as ListVectorDatabases; +use Appwrite\Platform\Modules\Databases\Http\VectorDB\Collections\Documents\Embedding\Create as CreateEmbeddingDocument; +use Appwrite\Platform\Modules\Databases\Http\VectorDB\Collections\Documents\Embedding\Update as UpdateEmbeddingDocument; use Utopia\Platform\Service; class VectorDB extends Base @@ -25,7 +27,7 @@ class VectorDB extends Base $this->registerDatabaseActions($service); $this->registerCollectionActions($service); // $this->registerIndexActions($service); - // $this->registerRowActions($service); + $this->registerDocumentActions($service); // $this->registerTransactionActions($service); } @@ -50,4 +52,9 @@ class VectorDB extends Base $service->addAction(ListCollectionLogs::getName(), new ListCollectionLogs()); $service->addAction(GetCollectionUsage::getName(), new GetCollectionUsage()); } + + private function registerDocumentActions(Service $service):void{ + $service->addAction(CreateEmbeddingDocument::getName(), new CreateEmbeddingDocument()); + $service->addAction(UpdateEmbeddingDocument::getName(), new UpdateEmbeddingDocument()); + } } diff --git a/src/Appwrite/Utopia/Response.php b/src/Appwrite/Utopia/Response.php index 533eb273db..6c5c8e2f4c 100644 --- a/src/Appwrite/Utopia/Response.php +++ b/src/Appwrite/Utopia/Response.php @@ -27,6 +27,8 @@ use Appwrite\Utopia\Response\Model\AttributeLine; use Appwrite\Utopia\Response\Model\AttributeList; use Appwrite\Utopia\Response\Model\AttributePoint; use Appwrite\Utopia\Response\Model\AttributePolygon; +use Appwrite\Utopia\Response\Model\AttributeObject; +use Appwrite\Utopia\Response\Model\AttributeVector; use Appwrite\Utopia\Response\Model\AttributeRelationship; use Appwrite\Utopia\Response\Model\AttributeString; use Appwrite\Utopia\Response\Model\AttributeURL; @@ -215,6 +217,8 @@ class Response extends SwooleResponse public const MODEL_ATTRIBUTE_POINT = 'attributePoint'; public const MODEL_ATTRIBUTE_LINE = 'attributeLine'; public const MODEL_ATTRIBUTE_POLYGON = 'attributePolygon'; + public const MODEL_ATTRIBUTE_OBJECT = 'attributeObject'; + public const MODEL_ATTRIBUTE_VECTOR = 'attributeVector'; // Database Columns public const MODEL_COLUMN = 'column'; @@ -511,6 +515,8 @@ class Response extends SwooleResponse ->setModel(new AttributePoint()) ->setModel(new AttributeLine()) ->setModel(new AttributePolygon()) + ->setModel(new AttributeObject()) + ->setModel(new AttributeVector()) // Table API Models ->setModel(new Table()) ->setModel(new Column()) diff --git a/src/Appwrite/Utopia/Response/Model/AttributeObject.php b/src/Appwrite/Utopia/Response/Model/AttributeObject.php new file mode 100644 index 0000000000..14ff818b3c --- /dev/null +++ b/src/Appwrite/Utopia/Response/Model/AttributeObject.php @@ -0,0 +1,29 @@ + 'object', + ]; + + public function getName(): string + { + return 'AttributeObject'; + } + + public function getType(): string + { + return Response::MODEL_ATTRIBUTE_OBJECT; + } +} + + diff --git a/src/Appwrite/Utopia/Response/Model/AttributeVector.php b/src/Appwrite/Utopia/Response/Model/AttributeVector.php new file mode 100644 index 0000000000..56c3317cd0 --- /dev/null +++ b/src/Appwrite/Utopia/Response/Model/AttributeVector.php @@ -0,0 +1,37 @@ +addRule('size', [ + 'type' => self::TYPE_INTEGER, + 'description' => 'Vector dimensions.', + 'default' => 0, + 'example' => 1536, + ]); + } + + public array $conditions = [ + 'type' => 'vector', + ]; + + public function getName(): string + { + return 'AttributeVector'; + } + + public function getType(): string + { + return Response::MODEL_ATTRIBUTE_VECTOR; + } +} + + diff --git a/src/Appwrite/Utopia/Response/Model/VectorDBCollection.php b/src/Appwrite/Utopia/Response/Model/VectorDBCollection.php index b0cab1f4c7..97b2e75224 100644 --- a/src/Appwrite/Utopia/Response/Model/VectorDBCollection.php +++ b/src/Appwrite/Utopia/Response/Model/VectorDBCollection.php @@ -79,19 +79,8 @@ class VectorDBCollection extends Model ]) ->addRule('attributes', [ 'type' => [ - Response::MODEL_ATTRIBUTE_BOOLEAN, - Response::MODEL_ATTRIBUTE_INTEGER, - Response::MODEL_ATTRIBUTE_FLOAT, - Response::MODEL_ATTRIBUTE_EMAIL, - Response::MODEL_ATTRIBUTE_ENUM, - Response::MODEL_ATTRIBUTE_URL, - Response::MODEL_ATTRIBUTE_IP, - Response::MODEL_ATTRIBUTE_DATETIME, - Response::MODEL_ATTRIBUTE_RELATIONSHIP, - Response::MODEL_ATTRIBUTE_POINT, - Response::MODEL_ATTRIBUTE_LINE, - Response::MODEL_ATTRIBUTE_POLYGON, - Response::MODEL_ATTRIBUTE_STRING, + Response::MODEL_ATTRIBUTE_OBJECT, + Response::MODEL_ATTRIBUTE_VECTOR, ], 'description' => 'Collection attributes.', 'default' => [], diff --git a/tests/e2e/Services/Databases/VectorDB/DatabasesBase.php b/tests/e2e/Services/Databases/VectorDB/DatabasesBase.php index 44efbabcbe..fede56e078 100644 --- a/tests/e2e/Services/Databases/VectorDB/DatabasesBase.php +++ b/tests/e2e/Services/Databases/VectorDB/DatabasesBase.php @@ -7,6 +7,7 @@ use Utopia\Database\Database; use Utopia\Database\Helpers\ID; use Utopia\Database\Helpers\Permission; use Utopia\Database\Helpers\Role; +use Utopia\Agents\Adapters\Ollama; trait DatabasesBase { @@ -50,7 +51,7 @@ trait DatabasesBase 'name' => 'Movies', 'documentSecurity' => true, 'dimensions' => 1536, - 'embeddingModel' => 'gemma', + 'embeddingModel' => Ollama::MODEL_EMBEDDING_GEMMA, 'permissions' => [ Permission::create(Role::user($this->getUser()['$id'])), ], @@ -68,7 +69,7 @@ trait DatabasesBase 'name' => 'Actors', 'documentSecurity' => true, 'dimensions' => 1536, - 'embeddingModel' => 'gemma', + 'embeddingModel' => Ollama::MODEL_EMBEDDING_GEMMA, 'permissions' => [ Permission::create(Role::user($this->getUser()['$id'])), ], @@ -166,7 +167,7 @@ trait DatabasesBase /** * Helper to create a collection */ - $createCollection = function (string $databaseId, string $name) use ($projectId, $apiKey, $userId) { + $createCollection = function (string $databaseId, string $name, int $dimensions = 1536, string $embeddingModel = Ollama::MODEL_EMBEDDING_GEMMA) use ($projectId, $apiKey, $userId) { $res = $this->client->call(Client::METHOD_POST, '/vectordb/' . $databaseId . '/collections', [ 'content-type' => 'application/json', 'x-appwrite-project' => $projectId, @@ -175,8 +176,8 @@ trait DatabasesBase 'collectionId' => ID::unique(), 'name' => $name, 'documentSecurity' => true, - 'dimensions' => 1536, - 'embeddingModel' => 'gemma', + 'dimensions' => $dimensions, + 'embeddingModel' => $embeddingModel, 'permissions' => [ Permission::create(Role::user($userId)), ], @@ -212,11 +213,45 @@ trait DatabasesBase $contentCollectionIds[$col] = $createCollection($contentDbId, $col); } + // Create a tiny-dimension collection and insert a document to validate vector and object attributes + $tinyCollectionName = 'VectorsTiny'; + $tinyDimensions = 8; + $tinyCollectionId = $createCollection($mediaDbId, $tinyCollectionName, $tinyDimensions, Ollama::MODEL_EMBEDDING_GEMMA); + + // Build embeddings vector of correct length and metadata object + $embeddings = []; + for ($i = 0; $i < $tinyDimensions; $i++) { + $embeddings[] = (float)($i + 1) / 10.0; + } + $metadata = [ + 'genre' => 'drama', + 'score' => 9, + 'tags' => ['award', 'festival'] + ]; + + $docRes = $this->client->call(Client::METHOD_POST, '/vectordb/' . $mediaDbId . '/collections/' . $tinyCollectionId . '/documents', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $projectId, + 'x-appwrite-key' => $apiKey + ], [ + 'documentId' => ID::unique(), + 'data' => [ + 'embeddings' => $embeddings, + 'metadata' => $metadata, + ], + 'permissions' => [ + Permission::read(Role::user($userId)), + Permission::write(Role::user($userId)), + ], + ]); + + $this->assertEquals(201, $docRes['headers']['status-code']); + return [ 'databases' => [ 'MediaDB' => [ 'id' => $mediaDbId, - 'collections' => $mediaCollectionIds, + 'collections' => $mediaCollectionIds + ['VectorsTiny' => $tinyCollectionId], ], 'ContentDB' => [ 'id' => $contentDbId,