From 5a6561f1585b7129c648576bebb15942e0f4251e Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Thu, 16 Oct 2025 12:53:39 +0530 Subject: [PATCH 01/27] vectordb (in progress) * pools , env setup * collections, db endpoints --- .env | 3 + app/config/collections/vectordb.php | 162 ++++++++++++++++++ app/init/registers.php | 27 +++ docker-compose.yml | 35 ++++ postgres.dockerfile | 8 + .../Databases/Http/Databases/Action.php | 4 + .../Databases/Http/Databases/Create.php | 15 +- .../Http/VectorDB/Collections/Create.php | 68 ++++++++ .../Http/VectorDB/Collections/Delete.php | 61 +++++++ .../Documents/Attribute/Decrement.php | 72 ++++++++ .../Documents/Attribute/Increment.php | 72 ++++++++ .../Collections/Documents/Bulk/Delete.php | 71 ++++++++ .../Collections/Documents/Bulk/Update.php | 73 ++++++++ .../Collections/Documents/Bulk/Upsert.php | 73 ++++++++ .../VectorDB/Collections/Documents/Create.php | 114 ++++++++++++ .../VectorDB/Collections/Documents/Delete.php | 75 ++++++++ .../VectorDB/Collections/Documents/Get.php | 63 +++++++ .../Collections/Documents/Logs/XList.php | 56 ++++++ .../VectorDB/Collections/Documents/Update.php | 74 ++++++++ .../VectorDB/Collections/Documents/Upsert.php | 77 +++++++++ .../VectorDB/Collections/Documents/XList.php | 62 +++++++ .../Http/VectorDB/Collections/Get.php | 55 ++++++ .../VectorDB/Collections/Indexes/Create.php | 72 ++++++++ .../VectorDB/Collections/Indexes/Delete.php | 66 +++++++ .../Http/VectorDB/Collections/Indexes/Get.php | 57 ++++++ .../VectorDB/Collections/Indexes/XList.php | 57 ++++++ .../Http/VectorDB/Collections/Logs/XList.php | 55 ++++++ .../Http/VectorDB/Collections/Update.php | 67 ++++++++ .../Http/VectorDB/Collections/Usage/Get.php | 58 +++++++ .../Http/VectorDB/Collections/XList.php | 58 +++++++ .../Databases/Http/VectorDB/Create.php | 58 +++++++ .../Databases/Http/VectorDB/Delete.php | 55 ++++++ .../Modules/Databases/Http/VectorDB/Get.php | 49 ++++++ .../Databases/Http/VectorDB/Logs/XList.php | 57 ++++++ .../Databases/Http/VectorDB/Update.php | 57 ++++++ .../Databases/Http/VectorDB/Usage/Get.php | 53 ++++++ .../Databases/Http/VectorDB/Usage/XList.php | 51 ++++++ .../Modules/Databases/Http/VectorDB/XList.php | 51 ++++++ 38 files changed, 2240 insertions(+), 1 deletion(-) create mode 100644 app/config/collections/vectordb.php create mode 100644 postgres.dockerfile create mode 100644 src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Create.php create mode 100644 src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Delete.php create mode 100644 src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Attribute/Decrement.php create mode 100644 src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Attribute/Increment.php create mode 100644 src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Bulk/Delete.php create mode 100644 src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Bulk/Update.php create mode 100644 src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Bulk/Upsert.php create mode 100644 src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Create.php create mode 100644 src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Delete.php create mode 100644 src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Get.php create mode 100644 src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Logs/XList.php create mode 100644 src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Update.php create mode 100644 src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Upsert.php create mode 100644 src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/XList.php create mode 100644 src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Get.php create mode 100644 src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Indexes/Create.php create mode 100644 src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Indexes/Delete.php create mode 100644 src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Indexes/Get.php create mode 100644 src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Indexes/XList.php create mode 100644 src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Logs/XList.php create mode 100644 src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Update.php create mode 100644 src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Usage/Get.php create mode 100644 src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/XList.php create mode 100644 src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Create.php create mode 100644 src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Delete.php create mode 100644 src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Get.php create mode 100644 src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Logs/XList.php create mode 100644 src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Update.php create mode 100644 src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Usage/Get.php create mode 100644 src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Usage/XList.php create mode 100644 src/Appwrite/Platform/Modules/Databases/Http/VectorDB/XList.php diff --git a/.env b/.env index 6d9ec13227..c757e4731a 100644 --- a/.env +++ b/.env @@ -45,6 +45,9 @@ _APP_DB_ROOT_PASS=rootsecretpassword _APP_DB_ADAPTER_DOCUMENTSDB=mongodb _APP_DB_HOST_DOCUMENTSDB=mongodb _APP_DB_PORT_DOCUMENTSDB=27017 +_APP_DB_ADAPTER_VECTORDB=postgresql +_APP_DB_HOST_VECTORDB=postgresql +_APP_DB_PORT_VECTORDB=5432 _APP_STORAGE_DEVICE=Local _APP_STORAGE_S3_ACCESS_KEY= _APP_STORAGE_S3_SECRET= diff --git a/app/config/collections/vectordb.php b/app/config/collections/vectordb.php new file mode 100644 index 0000000000..decde40639 --- /dev/null +++ b/app/config/collections/vectordb.php @@ -0,0 +1,162 @@ + [ + '$collection' => ID::custom('databases'), + '$id' => ID::custom('collections'), + 'name' => 'Collections', + 'attributes' => [ + [ + '$id' => ID::custom('databaseInternalId'), + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => Database::LENGTH_KEY, + 'signed' => true, + 'required' => true, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => ID::custom('databaseId'), + 'type' => Database::VAR_STRING, + 'signed' => true, + 'size' => Database::LENGTH_KEY, + 'format' => '', + 'filters' => [], + 'required' => true, + 'default' => null, + 'array' => false, + ], + [ + '$id' => ID::custom('name'), + 'type' => Database::VAR_STRING, + 'size' => 256, + 'required' => true, + 'signed' => true, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => ID::custom('description'), + 'type' => Database::VAR_STRING, + 'size' => 1024, + 'required' => true, + 'signed' => true, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => ID::custom('dimensions'), + 'type' => Database::VAR_INTEGER, + 'size' => 0, + 'required' => true, + 'signed' => false, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => ID::custom('metric'), + 'type' => Database::VAR_STRING, + 'size' => 256, + 'default' => 'cosine', + 'signed' => true, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => ID::custom('embeddingModel'), + 'type' => Database::VAR_STRING, + 'size' => 256, + 'required' => true, + 'signed' => true, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => ID::custom('enabled'), + 'type' => Database::VAR_BOOLEAN, + 'signed' => true, + 'size' => 0, + 'format' => '', + 'filters' => [], + 'required' => true, + 'default' => null, + 'array' => false, + ], + [ + '$id' => ID::custom('documentSecurity'), + 'type' => Database::VAR_BOOLEAN, + 'signed' => true, + 'size' => 0, + 'format' => '', + 'filters' => [], + 'required' => true, + 'default' => null, + 'array' => false, + ], + [ + '$id' => ID::custom('attributes'), + 'type' => Database::VAR_STRING, + 'size' => 1000000, + 'required' => false, + 'signed' => true, + 'array' => false, + 'filters' => ['subQueryAttributes'], + ], + [ + '$id' => ID::custom('indexes'), + 'type' => Database::VAR_STRING, + 'size' => 1000000, + 'required' => false, + 'signed' => true, + 'array' => false, + 'filters' => ['subQueryIndexes'], + ], + [ + '$id' => ID::custom('search'), + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => 16384, + 'signed' => true, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + ], + 'indexes' => [ + [ + '$id' => ID::custom('_fulltext_search'), + 'type' => Database::INDEX_FULLTEXT, + 'attributes' => ['search'], + 'lengths' => [], + 'orders' => [], + ], + [ + '$id' => ID::custom('_key_name'), + 'type' => Database::INDEX_KEY, + 'attributes' => ['name'], + 'lengths' => [256], + 'orders' => [Database::ORDER_ASC], + ], + [ + '$id' => ID::custom('_key_enabled'), + 'type' => Database::INDEX_KEY, + 'attributes' => ['enabled'], + 'lengths' => [], + 'orders' => [Database::ORDER_ASC], + ], + [ + '$id' => ID::custom('_key_documentSecurity'), + 'type' => Database::INDEX_KEY, + 'attributes' => ['documentSecurity'], + 'lengths' => [], + 'orders' => [Database::ORDER_ASC], + ], + ], + ] +]; diff --git a/app/init/registers.php b/app/init/registers.php index 654bb67d2e..9a127a20ca 100644 --- a/app/init/registers.php +++ b/app/init/registers.php @@ -15,6 +15,7 @@ use Utopia\Config\Config; use Utopia\Database\Adapter\MariaDB; use Utopia\Database\Adapter\Mongo; use Utopia\Database\Adapter\MySQL; +use Utopia\Database\Adapter\Postgres; use Utopia\Database\Adapter\SQL; use Utopia\Database\PDO; use Utopia\Domains\Validator\PublicDomain; @@ -118,6 +119,14 @@ $register->set('pools', function () { 'pass' => System::getEnv('_APP_DB_PASS', ''), 'path' => System::getEnv('_APP_DB_SCHEMA', ''), ]); + $fallbackForVECTORDB = 'db_main=' . AppwriteURL::unparse([ + 'scheme' => System::getEnv('_APP_DB_ADAPTER_VECTORDB', 'postgresql'), + 'host' => System::getEnv('_APP_DB_HOST_VECTORDB', 'postgresql'), + 'port' => System::getEnv('_APP_DB_PORT_VECTORDB', '5432'), + 'user' => System::getEnv('_APP_DB_USER', ''), + 'pass' => System::getEnv('_APP_DB_PASS', ''), + 'path' => System::getEnv('_APP_DB_SCHEMA', ''), + ]); $fallbackForRedis = 'redis_main=' . AppwriteURL::unparse([ 'scheme' => 'redis', 'host' => System::getEnv('_APP_REDIS_HOST', 'redis'), @@ -145,6 +154,12 @@ $register->set('pools', function () { 'multiple' => true, 'schemes' => ['mongodb'], ], + 'vectordb' => [ + 'type' => 'database', + 'dsns' => System::getEnv('_APP_CONNECTIONS_DATABASE_VECTORDB', $fallbackForVECTORDB), + 'multiple' => true, + 'schemes' => ['postgresql'], + ], 'logs' => [ 'type' => 'database', 'dsns' => System::getEnv('_APP_CONNECTIONS_DB_LOGS', $fallbackForDB), @@ -253,6 +268,17 @@ $register->set('pools', function () { throw new Exception(Exception::GENERAL_SERVER_ERROR, "MongoDB connection failed: " . $e->getMessage()); } }, + 'postgresql' => function () use ($dsnHost, $dsnPort, $dsnUser, $dsnPass, $dsnDatabase) { + return new PDOProxy(function () use ($dsnHost, $dsnPort, $dsnUser, $dsnPass, $dsnDatabase) { + return new PDO("pgsql:host={$dsnHost};port={$dsnPort};dbname={$dsnDatabase}", $dsnUser, $dsnPass, array( + \PDO::ATTR_TIMEOUT => 3, // Seconds + \PDO::ATTR_PERSISTENT => false, + \PDO::ATTR_DEFAULT_FETCH_MODE => \PDO::FETCH_ASSOC, + \PDO::ATTR_EMULATE_PREPARES => true, + \PDO::ATTR_STRINGIFY_FETCHES => true + )); + }); + }, 'redis' => function () use ($dsnHost, $dsnPort, $dsnPass) { $redis = new \Redis(); @$redis->pconnect($dsnHost, (int)$dsnPort); @@ -274,6 +300,7 @@ $register->set('pools', function () { 'mariadb' => new MariaDB($resource()), 'mysql' => new MySQL($resource()), 'mongodb' => new Mongo($resource()), + 'postgresql' => new Postgres($resource()), default => null }; diff --git a/docker-compose.yml b/docker-compose.yml index de8ed35244..1aae9a3e70 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -92,6 +92,7 @@ services: depends_on: - ${_APP_DB_HOST:-mongodb} + - postgresql - redis # - clamav entrypoint: @@ -269,6 +270,7 @@ services: - ./src:/usr/src/code/src depends_on: - ${_APP_DB_HOST:-mongodb} + - postgresql - redis environment: - _APP_ENV @@ -303,6 +305,7 @@ services: depends_on: - redis - ${_APP_DB_HOST:-mongodb} + - postgresql environment: - _APP_ENV - _APP_WORKER_PER_CORE @@ -333,6 +336,7 @@ services: depends_on: - redis - ${_APP_DB_HOST:-mongodb} + - postgresql - request-catcher-sms - request-catcher-webhook environment: @@ -364,6 +368,7 @@ services: depends_on: - redis - ${_APP_DB_HOST:-mongodb} + - postgresql volumes: - appwrite-uploads:/storage/uploads:rw - appwrite-cache:/storage/cache:rw @@ -431,6 +436,7 @@ services: depends_on: - redis - ${_APP_DB_HOST:-mongodb} + - postgresql environment: - _APP_ENV - _APP_WORKER_PER_CORE @@ -467,6 +473,7 @@ services: depends_on: - redis - ${_APP_DB_HOST:-mongodb} + - postgresql environment: - _APP_BROWSER_HOST - _APP_ENV @@ -535,6 +542,7 @@ services: depends_on: - redis - ${_APP_DB_HOST:-mongodb} + - postgresql volumes: - appwrite-config:/storage/config:rw - appwrite-certificates:/storage/certificates:rw @@ -578,6 +586,7 @@ services: depends_on: - redis - ${_APP_DB_HOST:-mongodb} + - postgresql - openruntimes-executor environment: - _APP_ENV @@ -714,6 +723,7 @@ services: - ./tests:/usr/src/code/tests depends_on: - ${_APP_DB_HOST:-mongodb} + - postgresql environment: - _APP_ENV - _APP_WORKER_PER_CORE @@ -797,6 +807,7 @@ services: depends_on: - redis - ${_APP_DB_HOST:-mongodb} + - postgresql environment: - _APP_ENV - _APP_WORKER_PER_CORE @@ -829,6 +840,7 @@ services: depends_on: - redis - ${_APP_DB_HOST:-mongodb} + - postgresql environment: - _APP_ENV - _APP_WORKER_PER_CORE @@ -861,6 +873,7 @@ services: depends_on: - redis - ${_APP_DB_HOST:-mongodb} + - postgresql environment: - _APP_ENV - _APP_WORKER_PER_CORE @@ -892,6 +905,7 @@ services: - ./src:/usr/src/code/src depends_on: - ${_APP_DB_HOST:-mongodb} + - postgresql - redis environment: - _APP_ENV @@ -921,6 +935,7 @@ services: - ./src:/usr/src/code/src depends_on: - ${_APP_DB_HOST:-mongodb} + - postgresql - redis environment: - _APP_ENV @@ -949,6 +964,7 @@ services: - ./src:/usr/src/code/src depends_on: - ${_APP_DB_HOST:-mongodb} + - postgresql - redis environment: - _APP_ENV @@ -1112,6 +1128,24 @@ services: ME_CONFIG_BASICAUTH_PASSWORD: ${_APP_DB_PASS} + postgresql: + build: + context: . + dockerfile: postgres.dockerfile + container_name: appwrite-postgresql + <<: *x-logging + networks: + - appwrite + volumes: + - appwrite-postgresql:/var/lib/postgresql/data:rw + ports: + - "5432:5432" + environment: + - POSTGRES_DB=${_APP_DB_SCHEMA} + - POSTGRES_USER=${_APP_DB_USER} + - POSTGRES_PASSWORD=${_APP_DB_PASS} + command: "postgres" + redis: image: redis:7.2.4-alpine <<: *x-logging @@ -1211,6 +1245,7 @@ networks: volumes: appwrite-mariadb: appwrite-mongodb: + appwrite-postgresql: appwrite-redis: appwrite-cache: appwrite-uploads: diff --git a/postgres.dockerfile b/postgres.dockerfile new file mode 100644 index 0000000000..3214121ef7 --- /dev/null +++ b/postgres.dockerfile @@ -0,0 +1,8 @@ +FROM postgres:16 + +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + postgresql-16-postgis-3 \ + postgresql-16-postgis-3-scripts \ + postgresql-16-pgvector \ + && rm -rf /var/lib/apt/lists/* \ No newline at end of file diff --git a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Action.php b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Action.php index 5270bc8229..688540277e 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Action.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Action.php @@ -21,6 +21,10 @@ class Action extends UtopiaAction if (\str_contains($path, '/documentsdb')) { $this->context = 'documentsdb'; } + + if (\str_contains($path, '/vectordb')) { + $this->context = 'vectordb'; + } return parent::setHttpPath($path); } } diff --git a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Create.php b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Create.php index cd45d7763f..e1c2e4307f 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Create.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Create.php @@ -49,6 +49,12 @@ class Create extends Action $databaseOverride = System::getEnv('_APP_DATABASE_DOCUMENTSDB_OVERRIDE'); $dbScheme = System::getEnv('_APP_DB_HOST_DOCUMENTSDB', 'mongodb'); break; + case 'vectordb': + $databases = Config::getParam('pools-vectordb', []); + $databaseKeys = System::getEnv('_APP_DATABASE_VECTORDB_KEYS', ''); + $databaseOverride = System::getEnv('_APP_DATABASE_VECTORB_OVERRIDE'); + $dbScheme = System::getEnv('_APP_DB_HOST_VECTORDB', 'pstgres'); + break; default: // legacy/tablesdb case where projects having the location of the database return $project->getAttribute('database'); @@ -88,6 +94,13 @@ class Create extends Action return $dsn; } + protected function getDatabaseCollection() + { + return match ($this->getDatabaseType()) { + 'vectordb' => (Config::getParam('collections', [])['vectordb'] ?? [])['collections'] ?? [], + default => (Config::getParam('collections', [])['databases'] ?? [])['collections'] ?? [], + }; + } public function __construct() { $this @@ -151,7 +164,7 @@ class Create extends Action $database = $dbForProject->getDocument('databases', $databaseId); - $collections = (Config::getParam('collections', [])['databases'] ?? [])['collections'] ?? []; + $collections = $this->getDatabaseCollection(); if (empty($collections)) { throw new Exception(Exception::GENERAL_SERVER_ERROR, 'The "collections" collection is not configured.'); } diff --git a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Create.php b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Create.php new file mode 100644 index 0000000000..18e8c63996 --- /dev/null +++ b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Create.php @@ -0,0 +1,68 @@ +setHttpMethod(self::HTTP_REQUEST_METHOD_POST) + ->setHttpPath('/v1/vectordb/:databaseId/collections') + ->desc('Create collection') + ->groups(['api', 'database']) + ->label('event', 'databases.[databaseId].collections.[collectionId].create') + ->label('scope', 'collections.write') + ->label('resourceType', RESOURCE_TYPE_DATABASES) + ->label('audits.event', 'collections.create') + ->label('audits.resource', 'database/{request.databaseId}/collection/{response.$id}') + ->label('sdk', new Method( + namespace: 'documentsDB', + group: 'collections', + name: 'createCollection', + description: '/docs/references/vectordb/create-collection.md', + auth: [AuthType::KEY], + responses: [ + new SDKResponse( + code: SwooleResponse::STATUS_CODE_CREATED, + model: $this->getResponseModel(), + ) + ], + contentType: ContentType::JSON + )) + ->param('databaseId', '', new UID(), 'Database ID.') + ->param('collectionId', '', new CustomId(), 'Unique Id. Choose a custom ID or generate a random ID with `ID.unique()`. Valid chars are a-z, A-Z, 0-9, period, hyphen, and underscore. Can\'t start with a special char. Max length is 36 chars.') + ->param('name', '', new Text(128), 'Collection name. Max length: 128 chars.') + ->param('permissions', null, new Permissions(APP_LIMIT_ARRAY_PARAMS_SIZE), 'An array of permissions strings. By default, no user is granted with any permissions. [Learn more about permissions](https://appwrite.io/docs/permissions).', true) + ->param('documentSecurity', false, new Boolean(true), 'Enables configuring permissions for individual documents. A user needs one of document or collection level permissions to access a document. [Learn more about permissions](https://appwrite.io/docs/permissions).', true) + ->param('enabled', true, new Boolean(), 'Is collection enabled? When set to \'disabled\', users cannot access the collection but Server SDKs with and API key can still read and write to the collection. No data is lost when this is toggled.', true) + ->inject('response') + ->inject('dbForProject') + ->inject('getDatabasesDB') + ->inject('queueForEvents') + ->callback($this->action(...)); + } +} diff --git a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Delete.php b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Delete.php new file mode 100644 index 0000000000..4ba64769b1 --- /dev/null +++ b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Delete.php @@ -0,0 +1,61 @@ +setHttpMethod(self::HTTP_REQUEST_METHOD_DELETE) + ->setHttpPath('/v1/vectordb/:databaseId/collections/:collectionId') + ->desc('Delete collection') + ->groups(['api', 'database', 'schema']) + ->label('scope', 'collections.write') + ->label('resourceType', RESOURCE_TYPE_DATABASES) + ->label('event', 'databases.[databaseId].collections.[collectionId].delete') + ->label('audits.event', 'collection.delete') + ->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}') + ->label('sdk', new Method( + namespace: 'documentsDB', + group: 'collections', + name: 'deleteCollection', + description: '/docs/references/vectordb/delete-collection.md', + auth: [AuthType::KEY], + responses: [ + new SDKResponse( + code: SwooleResponse::STATUS_CODE_NOCONTENT, + model: UtopiaResponse::MODEL_NONE, + ) + ], + contentType: ContentType::NONE + )) + ->param('databaseId', '', new UID(), 'Database ID.') + ->param('collectionId', '', new UID(), 'Collection ID.') + ->inject('response') + ->inject('dbForProject') + ->inject('getDatabasesDB') + ->inject('queueForDatabase') + ->inject('queueForEvents') + ->callback($this->action(...)); + } +} diff --git a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Attribute/Decrement.php b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Attribute/Decrement.php new file mode 100644 index 0000000000..7c429483e8 --- /dev/null +++ b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Attribute/Decrement.php @@ -0,0 +1,72 @@ +setHttpMethod(self::HTTP_REQUEST_METHOD_PATCH) + ->setHttpPath('/v1/vectordb/:databaseId/collections/:collectionId/documents/:documentId/:attribute/decrement') + ->desc('Decrement document attribute') + ->groups(['api', 'database']) + ->label('event', 'vectordb.[databaseId].collections.[collectionId].documents.[documentId].update') + ->label('scope', 'documents.write') + ->label('resourceType', RESOURCE_TYPE_DATABASES) + ->label('audits.event', 'documents.update') + ->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}') + ->label('abuse-key', 'ip:{ip},method:{method},url:{url},userId:{userId}') + ->label('abuse-limit', APP_LIMIT_WRITE_RATE_DEFAULT * 2) + ->label('abuse-time', APP_LIMIT_WRITE_RATE_PERIOD_DEFAULT) + ->label('sdk', new Method( + namespace: 'documentsDB', + group: $this->getSdkGroup(), + name: 'decrementDocumentAttribute', + description: '/docs/references/vectordb/decrement-document-attribute.md', + auth: [AuthType::SESSION, AuthType::JWT, AuthType::ADMIN, AuthType::KEY], + responses: [ + new SDKResponse( + code: SwooleResponse::STATUS_CODE_OK, + model: $this->getResponseModel(), + ) + ], + contentType: ContentType::JSON + )) + ->param('databaseId', '', new UID(), 'Database ID.') + ->param('collectionId', '', new UID(), 'Collection ID.') + ->param('documentId', '', new UID(), 'Document ID.') + ->param('attribute', '', new Key(), 'Attribute key.') + ->param('value', 1, new Numeric(), 'Value to decrement the attribute by. The value must be a number.', true) + ->param('min', null, new Numeric(), 'Minimum value for the attribute. If the current value is lesser than this value, an exception will be thrown.', true) + ->param('transactionId', null, new UID(), 'Transaction ID for staging the operation.', true) + ->inject('response') + ->inject('dbForProject') + ->inject('getDatabasesDB') + ->inject('queueForEvents') + ->inject('queueForStatsUsage') + ->inject('plan') + ->callback($this->action(...)); + } +} diff --git a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Attribute/Increment.php b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Attribute/Increment.php new file mode 100644 index 0000000000..297ec03338 --- /dev/null +++ b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Attribute/Increment.php @@ -0,0 +1,72 @@ +setHttpMethod(self::HTTP_REQUEST_METHOD_PATCH) + ->setHttpPath('/v1/vectordb/:databaseId/collections/:collectionId/documents/:documentId/:attribute/increment') + ->desc('Increment document attribute') + ->groups(['api', 'database']) + ->label('event', 'vectordb.[databaseId].collections.[collectionId].documents.[documentId].update') + ->label('scope', 'documents.write') + ->label('resourceType', RESOURCE_TYPE_DATABASES) + ->label('audits.event', 'documents.update') + ->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}') + ->label('abuse-key', 'ip:{ip},method:{method},url:{url},userId:{userId}') + ->label('abuse-limit', APP_LIMIT_WRITE_RATE_DEFAULT * 2) + ->label('abuse-time', APP_LIMIT_WRITE_RATE_PERIOD_DEFAULT) + ->label('sdk', new Method( + namespace: 'documentsDB', + group: $this->getSdkGroup(), + name: 'incrementDocumentAttribute', + description: '/docs/references/vectordb/increment-document-attribute.md', + auth: [AuthType::SESSION, AuthType::JWT, AuthType::ADMIN, AuthType::KEY], + responses: [ + new SDKResponse( + code: SwooleResponse::STATUS_CODE_OK, + model: $this->getResponseModel(), + ) + ], + contentType: ContentType::JSON + )) + ->param('databaseId', '', new UID(), 'Database ID.') + ->param('collectionId', '', new UID(), 'Collection ID.') + ->param('documentId', '', new UID(), 'Document ID.') + ->param('attribute', '', new Key(), 'Attribute key.') + ->param('value', 1, new Numeric(), 'Value to increment the attribute by. The value must be a number.', true) + ->param('max', null, new Numeric(), 'Maximum value for the attribute. If the current value is greater than this value, an error will be thrown.', true) + ->param('transactionId', null, new UID(), 'Transaction ID for staging the operation.', true) + ->inject('response') + ->inject('dbForProject') + ->inject('getDatabasesDB') + ->inject('queueForEvents') + ->inject('queueForStatsUsage') + ->inject('plan') + ->callback($this->action(...)); + } +} diff --git a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Bulk/Delete.php b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Bulk/Delete.php new file mode 100644 index 0000000000..acbbe74f5c --- /dev/null +++ b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Bulk/Delete.php @@ -0,0 +1,71 @@ +setHttpMethod(self::HTTP_REQUEST_METHOD_DELETE) + ->setHttpPath('/v1/vectordb/:databaseId/collections/:collectionId/documents') + ->desc('Delete documents') + ->groups(['api', 'database']) + ->label('scope', 'documents.write') + ->label('resourceType', RESOURCE_TYPE_DATABASES) + ->label('audits.event', 'documents.delete') + ->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}') + ->label('abuse-key', 'ip:{ip},method:{method},url:{url},userId:{userId}') + ->label('abuse-limit', APP_LIMIT_WRITE_RATE_DEFAULT) + ->label('abuse-time', APP_LIMIT_WRITE_RATE_PERIOD_DEFAULT) + ->label('sdk', new Method( + namespace: 'documentsDB', + group: $this->getSdkGroup(), + name: 'deleteDocuments', + description: '/docs/references/vectordb/delete-documents.md', + auth: [AuthType::ADMIN, AuthType::KEY], + responses: [ + new SDKResponse( + code: SwooleResponse::STATUS_CODE_OK, + model: $this->getResponseModel(), + ) + ], + contentType: ContentType::JSON + )) + ->param('databaseId', '', new UID(), 'Database ID.') + ->param('collectionId', '', new UID(), 'Collection ID. You can create a new collection using the Database service [server integration](https://appwrite.io/docs/server/databases#databasesCreateCollection).') + ->param('queries', [], new ArrayList(new Text(APP_LIMIT_ARRAY_ELEMENT_SIZE), APP_LIMIT_ARRAY_PARAMS_SIZE), 'Array of query strings generated using the Query class provided by the SDK. [Learn more about queries](https://appwrite.io/docs/queries). Maximum of ' . APP_LIMIT_ARRAY_PARAMS_SIZE . ' queries are allowed, each ' . APP_LIMIT_ARRAY_ELEMENT_SIZE . ' characters long.', true) + ->param('transactionId', null, new UID(), 'Transaction ID for staging the operation.', true) + ->inject('response') + ->inject('dbForProject') + ->inject('getDatabasesDB') + ->inject('queueForStatsUsage') + ->inject('queueForEvents') + ->inject('queueForRealtime') + ->inject('queueForFunctions') + ->inject('queueForWebhooks') + ->inject('plan') + ->callback($this->action(...)); + } +} diff --git a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Bulk/Update.php b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Bulk/Update.php new file mode 100644 index 0000000000..55dc18e84a --- /dev/null +++ b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Bulk/Update.php @@ -0,0 +1,73 @@ +setHttpMethod(self::HTTP_REQUEST_METHOD_PATCH) + ->setHttpPath('/v1/vectordb/:databaseId/collections/:collectionId/documents') + ->desc('Update documents') + ->groups(['api', 'database']) + ->label('scope', 'documents.write') + ->label('resourceType', RESOURCE_TYPE_DATABASES) + ->label('audits.event', 'documents.update') + ->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}') + ->label('abuse-key', 'ip:{ip},method:{method},url:{url},userId:{userId}') + ->label('abuse-limit', APP_LIMIT_WRITE_RATE_DEFAULT * 2) + ->label('abuse-time', APP_LIMIT_WRITE_RATE_PERIOD_DEFAULT) + ->label('sdk', new Method( + namespace: 'documentsDB', + group: $this->getSdkGroup(), + name: 'updateDocuments', + description: '/docs/references/vectordb/update-documents.md', + auth: [AuthType::ADMIN, AuthType::KEY], + responses: [ + new SDKResponse( + code: SwooleResponse::STATUS_CODE_OK, + model: $this->getResponseModel(), + ) + ], + contentType: ContentType::JSON + )) + ->param('databaseId', '', new UID(), 'Database ID.') + ->param('collectionId', '', new UID(), 'Collection ID.') + ->param('data', [], new JSON(), 'Document data as JSON object. Include only attribute and value pairs to be updated.', true) + ->param('queries', [], new ArrayList(new Text(APP_LIMIT_ARRAY_ELEMENT_SIZE), APP_LIMIT_ARRAY_PARAMS_SIZE), 'Array of query strings generated using the Query class provided by the SDK. [Learn more about queries](https://appwrite.io/docs/queries). Maximum of ' . APP_LIMIT_ARRAY_PARAMS_SIZE . ' queries are allowed, each ' . APP_LIMIT_ARRAY_ELEMENT_SIZE . ' characters long.', true) + ->param('transactionId', null, new UID(), 'Transaction ID for staging the operation.', true) + ->inject('response') + ->inject('dbForProject') + ->inject('getDatabasesDB') + ->inject('queueForStatsUsage') + ->inject('queueForEvents') + ->inject('queueForRealtime') + ->inject('queueForFunctions') + ->inject('queueForWebhooks') + ->inject('plan') + ->callback($this->action(...)); + } +} diff --git a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Bulk/Upsert.php b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Bulk/Upsert.php new file mode 100644 index 0000000000..d2206de4da --- /dev/null +++ b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Bulk/Upsert.php @@ -0,0 +1,73 @@ +setHttpMethod(self::HTTP_REQUEST_METHOD_PUT) + ->setHttpPath('/v1/vectordb/:databaseId/collections/:collectionId/documents') + ->desc('Upsert documents') + ->groups(['api', 'database']) + ->label('scope', 'documents.write') + ->label('resourceType', RESOURCE_TYPE_DATABASES) + ->label('audits.event', 'document.create') + ->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}') + ->label('abuse-key', 'ip:{ip},method:{method},url:{url},userId:{userId}') + ->label('abuse-limit', APP_LIMIT_WRITE_RATE_DEFAULT * 2) + ->label('abuse-time', APP_LIMIT_WRITE_RATE_PERIOD_DEFAULT) + ->label('sdk', [ + new Method( + namespace: 'documentsDB', + group: $this->getSdkGroup(), + name: 'upsertDocuments', + description: '/docs/references/vectordb/upsert-documents.md', + auth: [AuthType::ADMIN, AuthType::KEY], + responses: [ + new SDKResponse( + code: SwooleResponse::STATUS_CODE_CREATED, + model: $this->getResponseModel(), + ) + ], + contentType: ContentType::JSON, + ) + ]) + ->param('databaseId', '', new UID(), 'Database ID.') + ->param('collectionId', '', new UID(), 'Collection ID.') + ->param('documents', [], fn (array $plan) => new ArrayList(new JSON(), $plan['databasesBatchSize'] ?? APP_LIMIT_DATABASE_BATCH), 'Array of document data as JSON objects. May contain partial documents.', false, ['plan']) + ->param('transactionId', null, new UID(), 'Transaction ID for staging the operation.', true) + ->inject('response') + ->inject('dbForProject') + ->inject('getDatabasesDB') + ->inject('queueForStatsUsage') + ->inject('queueForEvents') + ->inject('queueForRealtime') + ->inject('queueForFunctions') + ->inject('queueForWebhooks') + ->inject('plan') + ->callback($this->action(...)); + } +} diff --git a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Create.php b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Create.php new file mode 100644 index 0000000000..d65266690f --- /dev/null +++ b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Create.php @@ -0,0 +1,114 @@ +setHttpMethod(self::HTTP_REQUEST_METHOD_POST) + ->setHttpPath('/v1/vectordb/:databaseId/collections/:collectionId/documents') + ->desc('Create document') + ->groups(['api', 'database']) + ->label('scope', 'documents.write') + ->label('resourceType', RESOURCE_TYPE_DATABASES) + ->label('audits.event', 'document.create') + ->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}') + ->label('abuse-key', 'ip:{ip},method:{method},url:{url},userId:{userId}') + ->label('abuse-limit', APP_LIMIT_WRITE_RATE_DEFAULT * 2) + ->label('abuse-time', APP_LIMIT_WRITE_RATE_PERIOD_DEFAULT) + ->label('sdk', [ + new Method( + namespace: 'documentsDB', + group: $this->getSdkGroup(), + name: 'createDocument', + desc: 'Create document', + description: '/docs/references/vectordb/create-document.md', + auth: [AuthType::SESSION, AuthType::KEY, AuthType::JWT], + responses: [ + new SDKResponse( + code: SwooleResponse::STATUS_CODE_CREATED, + model: $this->getResponseModel(), + ) + ], + contentType: ContentType::JSON, + parameters: [ + new Parameter('databaseId', optional: false), + new Parameter('collectionId', optional: false), + new Parameter('documentId', optional: false), + new Parameter('data', optional: false), + new Parameter('permissions', optional: true), + ] + ), + new Method( + namespace: 'documentsDB', + group: $this->getSdkGroup(), + name: 'createDocuments', + desc: 'Create documents', + description: '/docs/references/vectordb/create-documents.md', + auth: [AuthType::ADMIN, AuthType::KEY], + responses: [ + new SDKResponse( + code: SwooleResponse::STATUS_CODE_CREATED, + model: $this->getBulkResponseModel(), + ) + ], + contentType: ContentType::JSON, + parameters: [ + new Parameter('databaseId', optional: false), + new Parameter('collectionId', optional: false), + new Parameter('documents', optional: false), + ] + ) + ]) + ->param('databaseId', '', new UID(), 'Database ID.') + ->param('documentId', '', new CustomId(), 'Document ID. Choose a custom ID or generate a random ID with `ID.unique()`. Valid chars are a-z, A-Z, 0-9, period, hyphen, and underscore. Can\'t start with a special char. Max length is 36 chars.', true) + ->param('collectionId', '', new UID(), 'Collection ID. You can create a new collection using the Database service [server integration](https://appwrite.io/docs/server/databases#databasesCreateCollection). Make sure to define attributes before creating documents.') + ->param('data', [], new JSON(), 'Document data as JSON object.', true, example: '{"username":"walter.obrien","email":"walter.obrien@example.com","fullName":"Walter O\'Brien","age":30,"isAdmin":false}') + ->param('permissions', null, new Permissions(APP_LIMIT_ARRAY_PARAMS_SIZE, [Database::PERMISSION_READ, Database::PERMISSION_UPDATE, Database::PERMISSION_DELETE, Database::PERMISSION_WRITE]), 'An array of permissions strings. By default, only the current user is granted all permissions. [Learn more about permissions](https://appwrite.io/docs/permissions).', true) + ->param('documents', [], fn (array $plan) => new ArrayList(new JSON(), $plan['databasesBatchSize'] ?? APP_LIMIT_DATABASE_BATCH), 'Array of documents data as JSON objects.', true, ['plan']) + ->param('transactionId', null, new UID(), 'Transaction ID for staging the operation.', true) + ->inject('response') + ->inject('dbForProject') + ->inject('getDatabasesDB') + ->inject('user') + ->inject('queueForEvents') + ->inject('queueForStatsUsage') + ->inject('queueForRealtime') + ->inject('queueForFunctions') + ->inject('queueForWebhooks') + ->inject('plan') + ->callback($this->action(...)); + } +} diff --git a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Delete.php b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Delete.php new file mode 100644 index 0000000000..004d10578d --- /dev/null +++ b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Delete.php @@ -0,0 +1,75 @@ +setHttpMethod(self::HTTP_REQUEST_METHOD_DELETE) + ->setHttpPath('/v1/vectordb/:databaseId/collections/:collectionId/documents/:documentId') + ->desc('Delete document') + ->groups(['api', 'database']) + ->label('scope', 'documents.write') + ->label('resourceType', RESOURCE_TYPE_DATABASES) + ->label('event', 'databases.[databaseId].collections.[collectionId].documents.[documentId].delete') + ->label('audits.event', 'document.delete') + ->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}/document/{request.documentId}') + ->label('abuse-key', 'ip:{ip},method:{method},url:{url},userId:{userId}') + ->label('abuse-limit', APP_LIMIT_WRITE_RATE_DEFAULT) + ->label('abuse-time', APP_LIMIT_WRITE_RATE_PERIOD_DEFAULT) + ->label('sdk', new Method( + namespace: 'documentsDB', + group: $this->getSdkGroup(), + name: 'deleteDocument', + description: '/docs/references/vectordb/delete-document.md', + auth: [AuthType::SESSION, AuthType::KEY, AuthType::JWT], + responses: [ + new SDKResponse( + code: SwooleResponse::STATUS_CODE_NOCONTENT, + model: UtopiaResponse::MODEL_NONE, + ) + ], + contentType: ContentType::NONE + )) + ->param('databaseId', '', new UID(), 'Database ID.') + ->param('collectionId', '', new UID(), 'Collection ID. You can create a new collection using the Database service [server integration](https://appwrite.io/docs/server/databases#databasesCreateCollection).') + ->param('documentId', '', new UID(), 'Document ID.') + ->param('transactionId', null, new UID(), 'Transaction ID for staging the operation.', true) + ->inject('requestTimestamp') + ->inject('response') + ->inject('dbForProject') + ->inject('getDatabasesDB') + ->inject('queueForEvents') + ->inject('queueForStatsUsage') + ->inject('transactionState') + ->inject('plan') + ->callback($this->action(...)); + } +} diff --git a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Get.php b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Get.php new file mode 100644 index 0000000000..d512a06de2 --- /dev/null +++ b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Get.php @@ -0,0 +1,63 @@ +setHttpMethod(self::HTTP_REQUEST_METHOD_GET) + ->setHttpPath('/v1/vectordb/:databaseId/collections/:collectionId/documents/:documentId') + ->desc('Get document') + ->groups(['api', 'database']) + ->label('scope', 'documents.read') + ->label('resourceType', RESOURCE_TYPE_DATABASES) + ->label('sdk', new Method( + namespace: 'documentsDB', + group: $this->getSdkGroup(), + name: 'getDocument', + description: '/docs/references/vectordb/get-document.md', + auth: [AuthType::SESSION, AuthType::KEY, AuthType::JWT], + responses: [ + new SDKResponse( + code: SwooleResponse::STATUS_CODE_OK, + model: $this->getResponseModel(), + ) + ], + contentType: ContentType::JSON + )) + ->param('databaseId', '', new UID(), 'Database ID.') + ->param('collectionId', '', new UID(), 'Collection ID. You can create a new collection using the Database service [server integration](https://appwrite.io/docs/server/databases#databasesCreateCollection).') + ->param('documentId', '', new UID(), 'Document ID.') + ->param('queries', [], new ArrayList(new Text(APP_LIMIT_ARRAY_ELEMENT_SIZE), APP_LIMIT_ARRAY_PARAMS_SIZE), 'Array of query strings generated using the Query class provided by the SDK. [Learn more about queries](https://appwrite.io/docs/queries). Maximum of ' . APP_LIMIT_ARRAY_PARAMS_SIZE . ' queries are allowed, each ' . APP_LIMIT_ARRAY_ELEMENT_SIZE . ' characters long.', true) + ->param('transactionId', null, new UID(), 'Transaction ID to read uncommitted changes within the transaction.', true) + ->inject('response') + ->inject('dbForProject') + ->inject('getDatabasesDB') + ->inject('queueForStatsUsage') + ->inject('transactionState') + ->callback($this->action(...)); + } +} diff --git a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Logs/XList.php b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Logs/XList.php new file mode 100644 index 0000000000..97b08662c7 --- /dev/null +++ b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Logs/XList.php @@ -0,0 +1,56 @@ +setHttpMethod(self::HTTP_REQUEST_METHOD_GET) + ->setHttpPath('/v1/vectordb/:databaseId/collections/:collectionId/documents/:documentId/logs') + ->desc('List document logs') + ->groups(['api', 'database']) + ->label('scope', 'documents.read') + ->label('resourceType', RESOURCE_TYPE_DATABASES) + ->label('sdk', new Method( + namespace: 'documentsDB', + group: 'logs', + name: 'listDocumentLogs', + description: '/docs/references/vectordb/get-document-logs.md', + auth: [AuthType::ADMIN], + responses: [ + new SDKResponse( + code: SwooleResponse::STATUS_CODE_OK, + model: $this->getResponseModel(), + ) + ], + contentType: ContentType::JSON, + )) + ->param('databaseId', '', new UID(), 'Database ID.') + ->param('collectionId', '', new UID(), 'Collection ID.') + ->param('documentId', '', new UID(), 'Document ID.') + ->param('queries', [], new Queries([new Limit(), new Offset()]), 'Array of query strings generated using the Query class provided by the SDK. [Learn more about queries](https://appwrite.io/docs/queries). Only supported methods are limit and offset', true) + ->inject('response') + ->inject('dbForProject') + ->inject('locale') + ->inject('geodb') + ->callback($this->action(...)); + } +} diff --git a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Update.php b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Update.php new file mode 100644 index 0000000000..709e20262b --- /dev/null +++ b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Update.php @@ -0,0 +1,74 @@ +setHttpMethod(self::HTTP_REQUEST_METHOD_PATCH) + ->setHttpPath('/v1/vectordb/:databaseId/collections/:collectionId/documents/:documentId') + ->desc('Update document') + ->groups(['api', 'database']) + ->label('event', 'databases.[databaseId].collections.[collectionId].documents.[documentId].update') + ->label('scope', 'documents.write') + ->label('resourceType', RESOURCE_TYPE_DATABASES) + ->label('audits.event', 'document.update') + ->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}/document/{response.$id}') + ->label('abuse-key', 'ip:{ip},method:{method},url:{url},userId:{userId}') + ->label('abuse-limit', APP_LIMIT_WRITE_RATE_DEFAULT * 2) + ->label('abuse-time', APP_LIMIT_WRITE_RATE_PERIOD_DEFAULT) + ->label('sdk', new Method( + namespace: 'documentsDB', + group: $this->getSdkGroup(), + name: 'updateDocument', + description: '/docs/references/vectordb/update-document.md', + auth: [AuthType::SESSION, AuthType::KEY, AuthType::JWT], + responses: [ + new SDKResponse( + code: SwooleResponse::STATUS_CODE_OK, + model: $this->getResponseModel(), + ) + ], + contentType: ContentType::JSON + )) + ->param('databaseId', '', new UID(), 'Database ID.') + ->param('collectionId', '', new UID(), 'Collection ID.') + ->param('documentId', '', new UID(), 'Document ID.') + ->param('data', [], new JSON(), 'Document data as JSON object. Include only fields and value pairs to be updated.', true) + ->param('permissions', null, new Permissions(APP_LIMIT_ARRAY_PARAMS_SIZE, [Database::PERMISSION_READ, Database::PERMISSION_UPDATE, Database::PERMISSION_DELETE, Database::PERMISSION_WRITE]), 'An array of permissions strings. By default, the current permissions are inherited. [Learn more about permissions](https://appwrite.io/docs/permissions).', true) + ->param('transactionId', null, new UID(), 'Transaction ID for staging the operation.', true) + ->inject('requestTimestamp') + ->inject('response') + ->inject('dbForProject') + ->inject('getDatabasesDB') + ->inject('queueForEvents') + ->inject('queueForStatsUsage') + ->inject('transactionState') + ->inject('plan') + ->callback($this->action(...)); + } +} diff --git a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Upsert.php b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Upsert.php new file mode 100644 index 0000000000..45c5a2b202 --- /dev/null +++ b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Upsert.php @@ -0,0 +1,77 @@ +setHttpMethod(self::HTTP_REQUEST_METHOD_PUT) + ->setHttpPath('/v1/vectordb/:databaseId/collections/:collectionId/documents/:documentId') + ->desc('Upsert a document') + ->groups(['api', 'database']) + ->label('event', 'databases.[databaseId].collections.[collectionId].documents.[documentId].upsert') + ->label('scope', 'documents.write') + ->label('resourceType', RESOURCE_TYPE_DATABASES) + ->label('audits.event', 'document.upsert') + ->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}/document/{response.$id}') + ->label('abuse-key', 'ip:{ip},method:{method},url:{url},userId:{userId}') + ->label('abuse-limit', APP_LIMIT_WRITE_RATE_DEFAULT * 2) + ->label('abuse-time', APP_LIMIT_WRITE_RATE_PERIOD_DEFAULT) + ->label('sdk', [ + new Method( + namespace: 'documentsDB', + group: $this->getSdkGroup(), + name: 'upsertDocument', + description: '/docs/references/vectordb/upsert-document.md', + auth: [AuthType::SESSION, AuthType::KEY, AuthType::JWT], + responses: [ + new SDKResponse( + code: SwooleResponse::STATUS_CODE_CREATED, + model: $this->getResponseModel(), + ) + ], + contentType: ContentType::JSON + ), + ]) + ->param('databaseId', '', new UID(), 'Database ID.') + ->param('collectionId', '', new UID(), 'Collection ID.') + ->param('documentId', '', new UID(), 'Document ID.') + ->param('data', [], new JSON(), 'Document data as JSON object. Include all required fields of the document to be created or updated.', true) + ->param('permissions', null, new Permissions(APP_LIMIT_ARRAY_PARAMS_SIZE, [Database::PERMISSION_READ, Database::PERMISSION_UPDATE, Database::PERMISSION_DELETE, Database::PERMISSION_WRITE]), 'An array of permissions strings. By default, the current permissions are inherited. [Learn more about permissions](https://appwrite.io/docs/permissions).', true) + ->param('transactionId', null, new UID(), 'Transaction ID for staging the operation.', true) + ->inject('requestTimestamp') + ->inject('response') + ->inject('user') + ->inject('dbForProject') + ->inject('getDatabasesDB') + ->inject('queueForEvents') + ->inject('queueForStatsUsage') + ->inject('transactionState') + ->inject('plan') + ->callback($this->action(...)); + } +} diff --git a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/XList.php b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/XList.php new file mode 100644 index 0000000000..061fc71fea --- /dev/null +++ b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/XList.php @@ -0,0 +1,62 @@ +setHttpMethod(self::HTTP_REQUEST_METHOD_GET) + ->setHttpPath('/v1/vectordb/:databaseId/collections/:collectionId/documents') + ->desc('List documents') + ->groups(['api', 'database']) + ->label('scope', 'documents.read') + ->label('resourceType', RESOURCE_TYPE_DATABASES) + ->label('sdk', new Method( + namespace: 'documentsDB', + group: $this->getSdkGroup(), + name: 'listDocuments', + description: '/docs/references/vectordb/list-documents.md', + auth: [AuthType::SESSION, AuthType::KEY, AuthType::JWT], + responses: [ + new SDKResponse( + code: SwooleResponse::STATUS_CODE_OK, + model: $this->getResponseModel(), + ) + ], + contentType: ContentType::JSON + )) + ->param('databaseId', '', new UID(), 'Database ID.') + ->param('collectionId', '', new UID(), 'Collection ID. You can create a new collection using the Database service [server integration](https://appwrite.io/docs/server/databases#databasesCreateCollection).') + ->param('queries', [], new ArrayList(new Text(APP_LIMIT_ARRAY_ELEMENT_SIZE), APP_LIMIT_ARRAY_PARAMS_SIZE), 'Array of query strings generated using the Query class provided by the SDK. [Learn more about queries](https://appwrite.io/docs/queries). Maximum of ' . APP_LIMIT_ARRAY_PARAMS_SIZE . ' queries are allowed, each ' . APP_LIMIT_ARRAY_ELEMENT_SIZE . ' characters long.', true) + ->param('transactionId', null, new UID(), 'Transaction ID to read uncommitted changes within the transaction.', true) + ->inject('response') + ->inject('dbForProject') + ->inject('getDatabasesDB') + ->inject('queueForStatsUsage') + ->inject('transactionState') + ->callback($this->action(...)); + } +} diff --git a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Get.php b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Get.php new file mode 100644 index 0000000000..fd3c1e4d57 --- /dev/null +++ b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Get.php @@ -0,0 +1,55 @@ +setHttpMethod(self::HTTP_REQUEST_METHOD_GET) + ->setHttpPath('/v1/vectordb/:databaseId/collections/:collectionId') + ->desc('Get collection') + ->groups(['api', 'database']) + ->label('scope', 'collections.read') + ->label('resourceType', RESOURCE_TYPE_DATABASES) + ->label('sdk', new Method( + namespace: 'documentsDB', + group: 'collections', + name: 'getCollection', + description: '/docs/references/vectordb/get-collection.md', + auth: [AuthType::KEY], + responses: [ + new SDKResponse( + code: SwooleResponse::STATUS_CODE_OK, + model: $this->getResponseModel(), + ) + ], + contentType: ContentType::JSON + )) + ->param('databaseId', '', new UID(), 'Database ID.') + ->param('collectionId', '', new UID(), 'Collection ID.') + ->inject('response') + ->inject('dbForProject') + ->callback($this->action(...)); + } +} diff --git a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Indexes/Create.php b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Indexes/Create.php new file mode 100644 index 0000000000..8928fa73af --- /dev/null +++ b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Indexes/Create.php @@ -0,0 +1,72 @@ +setHttpMethod(self::HTTP_REQUEST_METHOD_POST) + ->setHttpPath('/v1/vectordb/:databaseId/collections/:collectionId/indexes') + ->desc('Create index') + ->groups(['api', 'database']) + ->label('event', 'databases.[databaseId].tables.[tableId].indexes.[indexId].create') + ->label('scope', 'collections.write') + ->label('resourceType', RESOURCE_TYPE_DATABASES) + ->label('audits.event', 'index.create') + ->label('audits.resource', 'database/{request.databaseId}/collection/{request.tableId}') + ->label('sdk', new Method( + namespace: 'documentsDB', + group: $this->getSdkGroup(), + name: 'createIndex', + description: '/docs/references/vectordb/create-index.md', + auth: [AuthType::KEY], + responses: [ + new SDKResponse( + code: SwooleResponse::STATUS_CODE_ACCEPTED, + model: $this->getResponseModel(), + ) + ], + contentType: ContentType::JSON + )) + ->param('databaseId', '', new UID(), 'Database ID.') + ->param('collectionId', '', new UID(), 'Collection ID. You can create a new collection using the Database service [server integration](https://appwrite.io/docs/server/databases#databasesCreateCollection).') + ->param('key', null, new Key(), 'Index Key.') + ->param('type', null, new WhiteList([Database::INDEX_KEY, Database::INDEX_FULLTEXT, Database::INDEX_UNIQUE]), 'Index type.') + ->param('attributes', null, new ArrayList(new Key(true), APP_LIMIT_ARRAY_PARAMS_SIZE), 'Array of attributes to index. Maximum of ' . APP_LIMIT_ARRAY_PARAMS_SIZE . ' attributes are allowed, each 32 characters long.') + ->param('orders', [], new ArrayList(new WhiteList(['ASC', 'DESC'], false, Database::VAR_STRING), APP_LIMIT_ARRAY_PARAMS_SIZE), 'Array of index orders. Maximum of ' . APP_LIMIT_ARRAY_PARAMS_SIZE . ' orders are allowed.', true) + ->param('lengths', [], new ArrayList(new Nullable(new Integer()), APP_LIMIT_ARRAY_PARAMS_SIZE), 'Length of index. Maximum of ' . APP_LIMIT_ARRAY_PARAMS_SIZE, optional: true) + ->inject('response') + ->inject('dbForProject') + ->inject('getDatabasesDB') + ->inject('queueForDatabase') + ->inject('queueForEvents') + ->callback($this->action(...)); + } +} diff --git a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Indexes/Delete.php b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Indexes/Delete.php new file mode 100644 index 0000000000..3b8fb71de8 --- /dev/null +++ b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Indexes/Delete.php @@ -0,0 +1,66 @@ +setHttpMethod(self::HTTP_REQUEST_METHOD_DELETE) + ->setHttpPath('/v1/vectordb/:databaseId/collections/:collectionId/indexes/:key') + ->desc('Delete index') + ->groups(['api', 'database']) + ->label('scope', 'collections.write') + ->label('resourceType', RESOURCE_TYPE_DATABASES) + ->label('event', 'databases.[databaseId].collections.[collectionId].indexes.[indexId].update') + ->label('audits.event', 'index.delete') + ->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}') + ->label('sdk', new Method( + namespace: 'documentsDB', + group: $this->getSdkGroup(), + name: 'deleteIndex', // getName needs to be different from parent action to avoid conflict in path name + description: '/docs/references/vectordb/delete-index.md', + auth: [AuthType::KEY], + responses: [ + new SDKResponse( + code: SwooleResponse::STATUS_CODE_NOCONTENT, + model: UtopiaResponse::MODEL_NONE, + ) + ], + contentType: ContentType::NONE + )) + ->param('databaseId', '', new UID(), 'Database ID.') + ->param('collectionId', '', new UID(), 'Collection ID. You can create a new collection using the Database service [server integration](https://appwrite.io/docs/server/databases#databasesCreateCollection).') + ->param('key', '', new Key(), 'Index Key.') + ->inject('response') + ->inject('dbForProject') + ->inject('queueForDatabase') + ->inject('queueForEvents') + ->callback($this->action(...)); + } +} diff --git a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Indexes/Get.php b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Indexes/Get.php new file mode 100644 index 0000000000..bf90b8598f --- /dev/null +++ b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Indexes/Get.php @@ -0,0 +1,57 @@ +setHttpMethod(self::HTTP_REQUEST_METHOD_GET) + ->setHttpPath('/v1/vectordb/:databaseId/collections/:collectionId/indexes/:key') + ->desc('Get index') + ->groups(['api', 'database']) + ->label('scope', 'collections.read') + ->label('resourceType', RESOURCE_TYPE_DATABASES) + ->label('sdk', new Method( + namespace: 'documentsDB', + group: $this->getSdkGroup(), + name: 'getIndex', // getName needs to be different from parent action to avoid conflict in path name + description: '/docs/references/vectordb/get-index.md', + auth: [AuthType::KEY], + responses: [ + new SDKResponse( + code: SwooleResponse::STATUS_CODE_OK, + model: $this->getResponseModel(), + ) + ], + contentType: ContentType::JSON + )) + ->param('databaseId', '', new UID(), 'Database ID.') + ->param('collectionId', '', new UID(), 'Collection ID. You can create a new collection using the Database service [server integration](https://appwrite.io/docs/server/databases#databasesCreateCollection).') + ->param('key', null, new Key(), 'Index Key.') + ->inject('response') + ->inject('dbForProject') + ->callback($this->action(...)); + } +} diff --git a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Indexes/XList.php b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Indexes/XList.php new file mode 100644 index 0000000000..2c239fdaef --- /dev/null +++ b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Indexes/XList.php @@ -0,0 +1,57 @@ +setHttpMethod(self::HTTP_REQUEST_METHOD_GET) + ->setHttpPath('/v1/vectordb/:databaseId/collections/:collectionId/indexes') + ->desc('List indexes') + ->groups(['api', 'database']) + ->label('scope', 'collections.read') + ->label('resourceType', RESOURCE_TYPE_DATABASES) + ->label('sdk', new Method( + namespace: 'documentsDB', + group: $this->getSdkGroup(), + name: 'listIndexes', // getName needs to be different from parent action to avoid conflict in path name + description: '/docs/references/vectordb/list-indexes.md', + auth: [AuthType::KEY], + responses: [ + new SDKResponse( + code: SwooleResponse::STATUS_CODE_OK, + model: $this->getResponseModel(), + ) + ], + contentType: ContentType::JSON + )) + ->param('databaseId', '', new UID(), 'Database ID.') + ->param('collectionId', '', new UID(), 'Collection ID. You can create a new collection using the Database service [server integration](https://appwrite.io/docs/server/databases#databasesCreateCollection).') + ->param('queries', [], new Indexes(), 'Array of query strings generated using the Query class provided by the SDK. [Learn more about queries](https://appwrite.io/docs/queries). Maximum of ' . APP_LIMIT_ARRAY_PARAMS_SIZE . ' queries are allowed, each ' . APP_LIMIT_ARRAY_ELEMENT_SIZE . ' characters long. You may filter on the following attributes: ' . implode(', ', Indexes::ALLOWED_ATTRIBUTES), true) + ->inject('response') + ->inject('dbForProject') + ->callback($this->action(...)); + } +} diff --git a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Logs/XList.php b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Logs/XList.php new file mode 100644 index 0000000000..e0e1babb5c --- /dev/null +++ b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Logs/XList.php @@ -0,0 +1,55 @@ +setHttpMethod(self::HTTP_REQUEST_METHOD_GET) + ->setHttpPath('/v1/vectordb/:databaseId/collections/:collectionId/logs') + ->desc('List collection logs') + ->groups(['api', 'database']) + ->label('scope', 'collections.read') + ->label('resourceType', RESOURCE_TYPE_DATABASES) + ->label('sdk', new Method( + namespace: 'documentsDB', + group: $this->getSdkGroup(), + name: 'listCollectionLogs', + description: '/docs/references/vectordb/get-collection-logs.md', + auth: [AuthType::ADMIN], + responses: [ + new SDKResponse( + code: SwooleResponse::STATUS_CODE_OK, + model: $this->getResponseModel(), + ) + ], + contentType: ContentType::JSON + )) + ->param('databaseId', '', new UID(), 'Database ID.') + ->param('collectionId', '', new UID(), 'Collection ID.') + ->param('queries', [], new Queries([new Limit(), new Offset()]), 'Array of query strings generated using the Query class provided by the SDK. [Learn more about queries](https://appwrite.io/docs/queries). Only supported methods are limit and offset', true) + ->inject('response') + ->inject('dbForProject') + ->inject('locale') + ->inject('geodb') + ->callback($this->action(...)); + } +} diff --git a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Update.php b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Update.php new file mode 100644 index 0000000000..ef3417fdbb --- /dev/null +++ b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Update.php @@ -0,0 +1,67 @@ +setHttpMethod(self::HTTP_REQUEST_METHOD_PUT) + ->setHttpPath('/v1/vectordb/:databaseId/collections/:collectionId') + ->desc('Update collection') + ->groups(['api', 'database', 'schema']) + ->label('scope', 'collections.write') + ->label('resourceType', RESOURCE_TYPE_DATABASES) + ->label('event', 'databases.[databaseId].collections.[collectionId].update') + ->label('audits.event', 'collection.update') + ->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}') + ->label('sdk', new Method( + namespace: 'documentsDB', + group: 'collections', + name: 'updateCollection', + description: '/docs/references/vectordb/update-collection.md', + auth: [AuthType::KEY], + responses: [ + new SDKResponse( + code: SwooleResponse::STATUS_CODE_OK, + model: UtopiaResponse::MODEL_COLLECTION, + ) + ], + contentType: ContentType::JSON + )) + ->param('databaseId', '', new UID(), 'Database ID.') + ->param('collectionId', '', new UID(), 'Collection ID.') + ->param('name', null, new Text(128), 'Collection name. Max length: 128 chars.') + ->param('permissions', null, new Permissions(APP_LIMIT_ARRAY_PARAMS_SIZE), 'An array of permission strings. By default, the current permissions are inherited. [Learn more about permissions](https://appwrite.io/docs/permissions).', true) + ->param('documentSecurity', false, new Boolean(true), 'Enables configuring permissions for individual documents. A user needs one of document or collection level permissions to access a document. [Learn more about permissions](https://appwrite.io/docs/permissions).', true) + ->param('enabled', true, new Boolean(), 'Is collection enabled? When set to \'disabled\', users cannot access the collection but Server SDKs with and API key can still read and write to the collection. No data is lost when this is toggled.', true) + ->inject('response') + ->inject('dbForProject') + ->inject('getDatabasesDB') + ->inject('queueForEvents') + ->callback($this->action(...)); + } +} diff --git a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Usage/Get.php b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Usage/Get.php new file mode 100644 index 0000000000..dbbad02f6f --- /dev/null +++ b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Usage/Get.php @@ -0,0 +1,58 @@ +setHttpMethod(self::HTTP_REQUEST_METHOD_GET) + ->setHttpPath('/v1/vectordb/:databaseId/collections/:collectionId/usage') + ->desc('Get collection usage stats') + ->groups(['api', 'database', 'usage']) + ->label('scope', 'collections.read') + ->label('resourceType', RESOURCE_TYPE_DATABASES) + ->label('sdk', new Method( + namespace: 'documentsDB', + group: null, + name: 'getCollectionUsage', + description: '/docs/references/vectordb/get-collection-usage.md', + auth: [AuthType::ADMIN], + responses: [ + new SDKResponse( + code: SwooleResponse::STATUS_CODE_OK, + model: $this->getResponseModel(), + ) + ], + contentType: ContentType::JSON, + )) + ->param('databaseId', '', new UID(), 'Database ID.') + ->param('range', '30d', new WhiteList(['24h', '30d', '90d'], true), 'Date range.', true) + ->param('collectionId', '', new UID(), 'Collection ID.') + ->inject('response') + ->inject('dbForProject') + ->inject('getDatabasesDB') + ->callback($this->action(...)); + } +} diff --git a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/XList.php b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/XList.php new file mode 100644 index 0000000000..7ca34d50b0 --- /dev/null +++ b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/XList.php @@ -0,0 +1,58 @@ +setHttpMethod(self::HTTP_REQUEST_METHOD_GET) + ->setHttpPath('/v1/vectordb/:databaseId/collections') + ->desc('List collections') + ->groups(['api', 'database']) + ->label('scope', 'collections.read') + ->label('resourceType', RESOURCE_TYPE_DATABASES) + ->label('sdk', new Method( + namespace: 'documentsDB', + group: 'collections', + name: 'listCollections', + description: '/docs/references/vectordb/list-collections.md', + auth: [AuthType::KEY], + responses: [ + new SDKResponse( + code: SwooleResponse::STATUS_CODE_OK, + model: $this->getResponseModel(), + ) + ], + contentType: ContentType::JSON + )) + ->param('databaseId', '', new UID(), 'Database ID.') + ->param('queries', [], new Collections(), 'Array of query strings generated using the Query class provided by the SDK. [Learn more about queries](https://appwrite.io/docs/queries). Maximum of ' . APP_LIMIT_ARRAY_PARAMS_SIZE . ' queries are allowed, each ' . APP_LIMIT_ARRAY_ELEMENT_SIZE . ' characters long. You may filter on the following attributes: ' . implode(', ', Collections::ALLOWED_ATTRIBUTES), true) + ->param('search', '', new Text(256), 'Search term to filter your list results. Max length: 256 chars.', true) + ->inject('response') + ->inject('dbForProject') + ->callback($this->action(...)); + } +} diff --git a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Create.php b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Create.php new file mode 100644 index 0000000000..3e47ffa69e --- /dev/null +++ b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Create.php @@ -0,0 +1,58 @@ +setHttpMethod(self::HTTP_REQUEST_METHOD_POST) + ->setHttpPath('/v1/vectordb') + ->desc('Create database') + ->groups(['api', 'database']) + ->label('event', 'databases.[databaseId].create') + ->label('scope', 'databases.write') + ->label('resourceType', RESOURCE_TYPE_DATABASES) + ->label('audits.event', 'database.create') + ->label('audits.resource', 'database/{response.$id}') + ->label('sdk', new Method( + namespace: 'documentsDB', + group: 'vectordb', + name: 'create', + description: '/docs/references/vectordb/create.md', + auth: [AuthType::KEY], + responses: [ + new SDKResponse( + code: SwooleResponse::STATUS_CODE_CREATED, + model: UtopiaResponse::MODEL_DATABASE, + ) + ], + contentType: ContentType::JSON + )) + ->param('databaseId', '', new CustomId(), 'Unique Id. Choose a custom ID or generate a random ID with `ID.unique()`. Valid chars are a-z, A-Z, 0-9, period, hyphen, and underscore. Can\'t start with a special char. Max length is 36 chars.') + ->param('name', '', new Text(128), 'Database name. Max length: 128 chars.') + ->param('enabled', true, new Boolean(), 'Is the database enabled? When set to \'disabled\', users cannot access the database but Server SDKs with an API key can still read and write to the database. No data is lost when this is toggled.', true) + ->inject('project') + ->inject('response') + ->inject('dbForProject') + ->inject('queueForEvents') + ->callback($this->action(...)); + } +} diff --git a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Delete.php b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Delete.php new file mode 100644 index 0000000000..717728c599 --- /dev/null +++ b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Delete.php @@ -0,0 +1,55 @@ +setHttpMethod(self::HTTP_REQUEST_METHOD_DELETE) + ->setHttpPath('/v1/vectordb/:databaseId') + ->desc('Delete database') + ->groups(['api', 'database', 'schema']) + ->label('scope', 'databases.write') + ->label('resourceType', RESOURCE_TYPE_DATABASES) + ->label('event', 'databases.[databaseId].delete') + ->label('audits.event', 'database.delete') + ->label('audits.resource', 'database/{request.databaseId}') + ->label('sdk', new Method( + namespace: 'documentsDB', + group: 'vectordb', + name: 'delete', + description: '/docs/references/vectordb/delete.md', + auth: [AuthType::KEY], + responses: [ + new SDKResponse( + code: SwooleResponse::STATUS_CODE_NOCONTENT, + model: UtopiaResponse::MODEL_NONE, + ) + ], + contentType: ContentType::NONE + )) + ->param('databaseId', '', new UID(), 'Database ID.') + ->inject('response') + ->inject('dbForProject') + ->inject('queueForDatabase') + ->inject('queueForEvents') + ->inject('queueForStatsUsage') + ->callback($this->action(...)); + } +} diff --git a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Get.php b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Get.php new file mode 100644 index 0000000000..3b49c34e75 --- /dev/null +++ b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Get.php @@ -0,0 +1,49 @@ +setHttpMethod(self::HTTP_REQUEST_METHOD_GET) + ->setHttpPath('/v1/vectordb/:databaseId') + ->desc('Get database') + ->groups(['api', 'database']) + ->label('scope', 'databases.read') + ->label('resourceType', RESOURCE_TYPE_DATABASES) + ->label('sdk', new Method( + namespace: 'documentsDB', + group: 'vectordb', + name: 'get', + description: '/docs/references/vectordb/get.md', + auth: [AuthType::KEY], + responses: [ + new SDKResponse( + code: SwooleResponse::STATUS_CODE_OK, + model: UtopiaResponse::MODEL_DATABASE, + ) + ], + contentType: ContentType::JSON + )) + ->param('databaseId', '', new UID(), 'Database ID.') + ->inject('response') + ->inject('dbForProject') + ->callback($this->action(...)); + } +} diff --git a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Logs/XList.php b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Logs/XList.php new file mode 100644 index 0000000000..f38e0ad39c --- /dev/null +++ b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Logs/XList.php @@ -0,0 +1,57 @@ +setHttpMethod(self::HTTP_REQUEST_METHOD_GET) + ->setHttpPath('/v1/vectordb/:databaseId/logs') + ->desc('List database logs') + ->groups(['api', 'database']) + ->label('scope', 'databases.read') + ->label('resourceType', RESOURCE_TYPE_DATABASES) + ->label('sdk', [ + new Method( + namespace: 'documentsDB', + group: 'logs', + name: 'listDatabaseLogs', + description: '/docs/references/vectordb/get-logs.md', + auth: [AuthType::ADMIN], + responses: [ + new SDKResponse( + code: SwooleResponse::STATUS_CODE_OK, + model: UtopiaResponse::MODEL_LOG_LIST, + ) + ], + contentType: ContentType::JSON + ), + ]) + ->param('databaseId', '', new UID(), 'Database ID.') + ->param('queries', [], new Queries([new Limit(), new Offset()]), 'Array of query strings generated using the Query class provided by the SDK. [Learn more about queries](https://appwrite.io/docs/queries). Only supported methods are limit and offset', true) + ->inject('response') + ->inject('dbForProject') + ->inject('locale') + ->inject('geodb') + ->callback($this->action(...)); + } +} diff --git a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Update.php b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Update.php new file mode 100644 index 0000000000..7863e98043 --- /dev/null +++ b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Update.php @@ -0,0 +1,57 @@ +setHttpMethod(self::HTTP_REQUEST_METHOD_PUT) + ->setHttpPath('/v1/vectordb/:databaseId') + ->desc('Update database') + ->groups(['api', 'database', 'schema']) + ->label('scope', 'databases.write') + ->label('resourceType', RESOURCE_TYPE_DATABASES) + ->label('event', 'databases.[databaseId].update') + ->label('audits.event', 'database.update') + ->label('audits.resource', 'database/{response.$id}') + ->label('sdk', new Method( + namespace: 'documentsDB', + group: 'vectordb', + name: 'update', + description: '/docs/references/vectordb/update.md', + auth: [AuthType::KEY], + responses: [ + new SDKResponse( + code: SwooleResponse::STATUS_CODE_OK, + model: UtopiaResponse::MODEL_DATABASE, + ) + ], + contentType: ContentType::JSON + )) + ->param('databaseId', '', new UID(), 'Database ID.') + ->param('name', null, new Text(128), 'Database name. Max length: 128 chars.') + ->param('enabled', true, new Boolean(), 'Is database enabled? When set to \'disabled\', users cannot access the database but Server SDKs with an API key can still read and write to the database. No data is lost when this is toggled.', true) + ->inject('response') + ->inject('dbForProject') + ->inject('queueForEvents') + ->callback($this->action(...)); + } +} diff --git a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Usage/Get.php b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Usage/Get.php new file mode 100644 index 0000000000..4590405aad --- /dev/null +++ b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Usage/Get.php @@ -0,0 +1,53 @@ +setHttpMethod(self::HTTP_REQUEST_METHOD_GET) + ->setHttpPath('/v1/vectordb/:databaseId/usage') + ->desc('Get VectorDB usage stats') + ->groups(['api', 'database', 'usage']) + ->label('scope', 'collections.read') + ->label('resourceType', RESOURCE_TYPE_DATABASES) + ->label('sdk', [ + new Method( + namespace: 'documentsDB', + group: null, + name: 'getUsage', + description: '/docs/references/vectordb/get-database-usage.md', + auth: [AuthType::ADMIN], + responses: [ + new SDKResponse( + code: SwooleResponse::STATUS_CODE_OK, + model: UtopiaResponse::MODEL_USAGE_DATABASE, + ) + ], + contentType: ContentType::JSON, + ), + ]) + ->param('databaseId', '', new UID(), 'Database ID.') + ->param('range', '30d', new WhiteList(['24h', '30d', '90d'], true), 'Date range.', true) + ->inject('response') + ->inject('dbForProject') + ->callback($this->action(...)); + } +} diff --git a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Usage/XList.php b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Usage/XList.php new file mode 100644 index 0000000000..4ba298129c --- /dev/null +++ b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Usage/XList.php @@ -0,0 +1,51 @@ +setHttpMethod(self::HTTP_REQUEST_METHOD_GET) + ->setHttpPath('/v1/vectordb/usage') + ->desc('Get VectorDB usage stats') + ->groups(['api', 'database', 'usage']) + ->label('scope', 'collections.read') + ->label('resourceType', RESOURCE_TYPE_DATABASES) + ->label('sdk', [ + new Method( + namespace: 'documentsDB', + group: null, + name: 'listUsage', + description: '/docs/references/vectordb/list-usage.md', + auth: [AuthType::ADMIN], + responses: [ + new SDKResponse( + code: SwooleResponse::STATUS_CODE_OK, + model: UtopiaResponse::MODEL_USAGE_DATABASES, + ) + ], + contentType: ContentType::JSON + ), + ]) + ->param('range', '30d', new WhiteList(['24h', '30d', '90d'], true), 'Date range.', true) + ->inject('response') + ->inject('dbForProject') + ->callback($this->action(...)); + } +} diff --git a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/XList.php b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/XList.php new file mode 100644 index 0000000000..e37c05ad88 --- /dev/null +++ b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/XList.php @@ -0,0 +1,51 @@ +setHttpMethod(self::HTTP_REQUEST_METHOD_GET) + ->setHttpPath('/v1/vectordb') + ->desc('List databases') + ->groups(['api', 'database']) + ->label('scope', 'databases.read') + ->label('resourceType', RESOURCE_TYPE_DATABASES) + ->label('sdk', new Method( + namespace: 'documentsDB', + group: 'vectordb', + name: 'list', + description: '/docs/references/vectordb/list.md', + auth: [AuthType::KEY], + responses: [ + new SDKResponse( + code: SwooleResponse::STATUS_CODE_OK, + model: UtopiaResponse::MODEL_DATABASE_LIST, + ) + ], + contentType: ContentType::JSON + )) + ->param('queries', [], new Databases(), 'Array of query strings generated using the Query class provided by the SDK. [Learn more about queries](https://appwrite.io/docs/queries). Maximum of ' . APP_LIMIT_ARRAY_PARAMS_SIZE . ' queries are allowed, each ' . APP_LIMIT_ARRAY_ELEMENT_SIZE . ' characters long. You may filter on the following columns: ' . implode(', ', Databases::ALLOWED_ATTRIBUTES), true) + ->param('search', '', new Text(256), 'Search term to filter your list results. Max length: 256 chars.', true) + ->inject('response') + ->inject('dbForProject') + ->callback($this->action(...)); + } +} From 60e95458a764ce3a1ce6a4481a1dc1888c86fc7b Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Fri, 31 Oct 2025 18:01:47 +0530 Subject: [PATCH 02/27] vector db endpoints * configs and default attributes + indexes * separate collection creation setup * fix -> database metadata fix on collection creation --- .env | 1 + app/config/collections.php | 2 + app/config/collections/platform.php | 11 + app/config/collections/vectordb.php | 49 ++-- app/controllers/api/projects.php | 9 +- composer.json | 7 + composer.lock | 102 +++++++- docker-compose.yml | 155 ++++++++---- ollama.dockerfile | 21 ++ .../Platform/Modules/Databases/Constants.php | 1 + .../Http/Databases/Collections/Action.php | 13 + .../Databases/Http/Databases/Create.php | 5 +- .../Http/Databases/Transactions/Update.php | 1 + .../Http/VectorDB/Collections/Create.php | 108 ++++++++- .../Http/VectorDB/Collections/Get.php | 2 +- .../Http/VectorDB/Collections/Update.php | 59 ++++- .../Http/VectorDB/Collections/XList.php | 2 +- .../Databases/Http/VectorDB/Create.php | 1 + .../Modules/Databases/Services/Http.php | 4 +- .../Databases/Services/Registry/VectorDB.php | 53 ++++ src/Appwrite/Utopia/Response.php | 4 + .../Response/Model/VectorDBCollection.php | 120 +++++++++ .../Databases/VectorDB/DatabasesBase.php | 229 ++++++++++++++++++ .../VectorDB/DatabasesCustomClientTest.php | 14 ++ 24 files changed, 879 insertions(+), 94 deletions(-) create mode 100644 ollama.dockerfile create mode 100644 src/Appwrite/Platform/Modules/Databases/Services/Registry/VectorDB.php create mode 100644 src/Appwrite/Utopia/Response/Model/VectorDBCollection.php create mode 100644 tests/e2e/Services/Databases/VectorDB/DatabasesBase.php create mode 100644 tests/e2e/Services/Databases/VectorDB/DatabasesCustomClientTest.php diff --git a/.env b/.env index c757e4731a..3eef0189d0 100644 --- a/.env +++ b/.env @@ -48,6 +48,7 @@ _APP_DB_PORT_DOCUMENTSDB=27017 _APP_DB_ADAPTER_VECTORDB=postgresql _APP_DB_HOST_VECTORDB=postgresql _APP_DB_PORT_VECTORDB=5432 +OLLAMA_MODELS=embeddinggemma _APP_STORAGE_DEVICE=Local _APP_STORAGE_S3_ACCESS_KEY= _APP_STORAGE_S3_SECRET= diff --git a/app/config/collections.php b/app/config/collections.php index 533dee57a8..887df8d773 100644 --- a/app/config/collections.php +++ b/app/config/collections.php @@ -4,6 +4,7 @@ $common = include __DIR__ . '/collections/common.php'; $projects = include __DIR__ . '/collections/projects.php'; $databases = include __DIR__ . '/collections/databases.php'; +$vectordb = include __DIR__ . '/collections/vectordb.php'; $platform = include __DIR__ . '/collections/platform.php'; $logs = include __DIR__ . '/collections/logs.php'; @@ -26,6 +27,7 @@ unset($common['files']); $collections = [ 'buckets' => $buckets, 'databases' => $databases, + 'vectordb' => $vectordb, 'projects' => array_merge($projects, $common), 'console' => array_merge($platform, $common), 'logs' => $logs, diff --git a/app/config/collections/platform.php b/app/config/collections/platform.php index 09ad724e1f..7db26694c6 100644 --- a/app/config/collections/platform.php +++ b/app/config/collections/platform.php @@ -89,6 +89,17 @@ return [ 'array' => false, 'filters' => [], ], + [ + '$id' => ID::custom('vectorDatabase'), + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => 256, + 'signed' => true, + 'required' => true, + 'default' => null, + 'array' => false, + 'filters' => [], + ], [ '$id' => ID::custom('logo'), 'type' => Database::VAR_STRING, diff --git a/app/config/collections/vectordb.php b/app/config/collections/vectordb.php index decde40639..ff091d74c2 100644 --- a/app/config/collections/vectordb.php +++ b/app/config/collections/vectordb.php @@ -40,15 +40,6 @@ return [ 'array' => false, 'filters' => [], ], - [ - '$id' => ID::custom('description'), - 'type' => Database::VAR_STRING, - 'size' => 1024, - 'required' => true, - 'signed' => true, - 'array' => false, - 'filters' => [], - ], [ '$id' => ID::custom('dimensions'), 'type' => Database::VAR_INTEGER, @@ -58,15 +49,6 @@ return [ 'array' => false, 'filters' => [], ], - [ - '$id' => ID::custom('metric'), - 'type' => Database::VAR_STRING, - 'size' => 256, - 'default' => 'cosine', - 'signed' => true, - 'array' => false, - 'filters' => [], - ], [ '$id' => ID::custom('embeddingModel'), 'type' => Database::VAR_STRING, @@ -128,6 +110,27 @@ return [ 'filters' => [], ], ], + 'defaultAttributes' => [ + [ + '$id' => ID::custom('embeddings'), + 'type' => Database::VAR_VECTOR, + 'required' => true, + 'size' => 128, + 'signed' => false, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => ID::custom('metadata'), + 'type' => Database::VAR_OBJECT, + 'default' => [], + 'required' => false, + 'size' => 0, + 'signed' => false, + 'array' => false, + 'filters' => [], + ], + ], 'indexes' => [ [ '$id' => ID::custom('_fulltext_search'), @@ -158,5 +161,15 @@ return [ 'orders' => [Database::ORDER_ASC], ], ], + 'defaultIndexes' => [ + // not creating default indexes on the embeddings as it depends on the type of query users using the most + [ + '$id' => ID::custom('_key_metadata'), + 'type' => Database::INDEX_OBJECT, + 'attributes' => ['metadata'], + 'lengths' => [], + 'orders' => [], + ], + ] ] ]; diff --git a/app/controllers/api/projects.php b/app/controllers/api/projects.php index 02c1465c13..32a988d477 100644 --- a/app/controllers/api/projects.php +++ b/app/controllers/api/projects.php @@ -67,6 +67,12 @@ function getDatabaseDSN(string $databasetype, $region): string $databaseOverride = System::getEnv('_APP_DATABASE_DOCUMENTSDB_OVERRIDE'); $dbScheme = System::getEnv('_APP_DB_HOST_DOCUMENTSDB', 'mongodb'); break; + case 'vectorDatabase': + $databases = Config::getParam('pools-vectordb', []); + $databaseKeys = System::getEnv('_APP_DATABASE_VECTORDB_KEYS', ''); + $databaseOverride = System::getEnv('_APP_DATABASE_VECTORDB_OVERRIDE'); + $dbScheme = System::getEnv('_APP_DB_HOST_VECTORDB', 'postgresql'); + break; default: // legacy/tablesdb $databases = Config::getParam('pools-database', []); @@ -232,7 +238,8 @@ App::post('/v1/projects') 'accessedAt' => DateTime::now(), 'search' => implode(' ', [$projectId, $name]), 'database' => $dsn, - 'documentsDatabase' => getDatabaseDSN('documentsDatabase', $region) + 'documentsDatabase' => getDatabaseDSN('documentsDatabase', $region), + 'vectorDatabase' => getDatabaseDSN('vectorDatabase', $region) ])); } catch (Duplicate) { throw new Exception(Exception::PROJECT_ALREADY_EXISTS); diff --git a/composer.json b/composer.json index cd5e326418..badbe4972f 100644 --- a/composer.json +++ b/composer.json @@ -30,6 +30,12 @@ "Appwrite\\Tests\\": "tests/extensions" } }, + "repositories": [ + { + "type": "vcs", + "url": "https://github.com/utopia-php/database" + } + ], "minimum-stability": "dev", "prefer-stable": true, "require": { @@ -55,6 +61,7 @@ "utopia-php/cli": "0.15.*", "utopia-php/config": "0.2.*", "utopia-php/database": "dev-var_object as 1.4.5", + "utopia-php/agents": "dev-feat-embeddings", "utopia-php/detector": "0.2.*", "utopia-php/domains": "0.9.*", "utopia-php/emails": "0.6.*", diff --git a/composer.lock b/composer.lock index 0f055f13da..3c1ffb73f7 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "7b51994d7c4a92e761235dff86b068e3", + "content-hash": "ac89d83b4f8d0d03065a9cd569432546", "packages": [ { "name": "adhocore/jwt", @@ -3494,6 +3494,60 @@ }, "time": "2025-10-20T07:18:33+00:00" }, + { + "name": "utopia-php/agents", + "version": "dev-feat-embeddings", + "source": { + "type": "git", + "url": "https://github.com/utopia-php/agents.git", + "reference": "d59a69bde505670c8434921659d116ea45b2acd1" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/utopia-php/agents/zipball/d59a69bde505670c8434921659d116ea45b2acd1", + "reference": "d59a69bde505670c8434921659d116ea45b2acd1", + "shasum": "" + }, + "require": { + "php": ">=8.3", + "utopia-php/fetch": "0.4.*" + }, + "require-dev": { + "laravel/pint": "1.2.*", + "phpstan/phpstan": "1.9.x-dev", + "phpunit/phpunit": "^9.3", + "vimeo/psalm": "4.0.1" + }, + "type": "library", + "autoload": { + "psr-4": { + "Utopia\\Agents\\": "src/Agents" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Utopia PHP", + "email": "team@appwrite.io" + } + ], + "description": "A simple PHP AI agents library", + "keywords": [ + "agents", + "ai", + "automation", + "machine-learning", + "php" + ], + "support": { + "issues": "https://github.com/utopia-php/agents/issues", + "source": "https://github.com/utopia-php/agents/tree/feat-embeddings" + }, + "time": "2025-10-30T12:38:42+00:00" + }, { "name": "utopia-php/analytics", "version": "0.10.2", @@ -3839,12 +3893,12 @@ "source": { "type": "git", "url": "https://github.com/utopia-php/database.git", - "reference": "f62ff5183e4baf61746007c3b23548b85f33c71b" + "reference": "f5c0cfdc6d8993613ae03d22ebcd3b899fda63aa" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/database/zipball/f62ff5183e4baf61746007c3b23548b85f33c71b", - "reference": "f62ff5183e4baf61746007c3b23548b85f33c71b", + "url": "https://api.github.com/repos/utopia-php/database/zipball/f5c0cfdc6d8993613ae03d22ebcd3b899fda63aa", + "reference": "f5c0cfdc6d8993613ae03d22ebcd3b899fda63aa", "shasum": "" }, "require": { @@ -3873,7 +3927,38 @@ "Utopia\\Database\\": "src/Database" } }, - "notification-url": "https://packagist.org/downloads/", + "autoload-dev": { + "psr-4": { + "Tests\\E2E\\": "tests/e2e", + "Tests\\Unit\\": "tests/unit" + } + }, + "scripts": { + "build": [ + "Composer\\Config::disableProcessTimeout", + "docker compose build" + ], + "start": [ + "Composer\\Config::disableProcessTimeout", + "docker compose up -d" + ], + "test": [ + "Composer\\Config::disableProcessTimeout", + "docker compose exec tests vendor/bin/phpunit --configuration phpunit.xml" + ], + "lint": [ + "php -d memory_limit=2G ./vendor/bin/pint --test" + ], + "format": [ + "php -d memory_limit=2G ./vendor/bin/pint" + ], + "check": [ + "./vendor/bin/phpstan analyse --level 7 src tests --memory-limit 2G" + ], + "coverage": [ + "./vendor/bin/coverage-check ./tmp/clover.xml 90" + ] + }, "license": [ "MIT" ], @@ -3886,10 +3971,10 @@ "utopia" ], "support": { - "issues": "https://github.com/utopia-php/database/issues", - "source": "https://github.com/utopia-php/database/tree/var_object" + "source": "https://github.com/utopia-php/database/tree/var_object", + "issues": "https://github.com/utopia-php/database/issues" }, - "time": "2025-10-28T13:22:33+00:00" + "time": "2025-10-31T06:11:11+00:00" }, { "name": "utopia-php/detector", @@ -8901,6 +8986,7 @@ ], "minimum-stability": "dev", "stability-flags": { + "utopia-php/agents": 20, "utopia-php/database": 20, "utopia-php/migration": 20 }, diff --git a/docker-compose.yml b/docker-compose.yml index 0e30d7e44f..8c1b889d9b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -89,11 +89,13 @@ services: - ./public:/usr/src/code/public - ./src:/usr/src/code/src - ./dev:/usr/src/code/dev + - .vendor:/usr/src/code/dev depends_on: - ${_APP_DB_HOST:-mongodb} - postgresql - redis + - ollama # - clamav entrypoint: - php @@ -138,6 +140,12 @@ services: - _APP_DB_SCHEMA - _APP_DB_USER - _APP_DB_PASS + - _APP_DB_ADAPTER_VECTORDB + - _APP_DB_HOST_VECTORDB + - _APP_DB_PORT_VECTORDB + - _APP_DB_SCHEMA_VECTORDB + - _APP_DB_USER_VECTORDB + - _APP_DB_PASS_VECTORDB - _APP_SMTP_HOST - _APP_SMTP_PORT - _APP_SMTP_SECURE @@ -272,6 +280,7 @@ services: - ${_APP_DB_HOST:-mongodb} - postgresql - redis + - ollama environment: - _APP_ENV - _APP_WORKER_PER_CORE @@ -288,6 +297,12 @@ services: - _APP_DB_SCHEMA - _APP_DB_USER - _APP_DB_PASS + - _APP_DB_ADAPTER_VECTORDB + - _APP_DB_HOST_VECTORDB + - _APP_DB_PORT_VECTORDB + - _APP_DB_SCHEMA_VECTORDB + - _APP_DB_USER_VECTORDB + - _APP_DB_PASS_VECTORDB - _APP_USAGE_STATS - _APP_LOGGING_CONFIG - _APP_DATABASE_SHARED_TABLES @@ -306,6 +321,7 @@ services: - redis - ${_APP_DB_HOST:-mongodb} - postgresql + - ollama environment: - _APP_ENV - _APP_WORKER_PER_CORE @@ -339,6 +355,7 @@ services: - postgresql - request-catcher-sms - request-catcher-webhook + - ollama environment: - _APP_ENV - _APP_WORKER_PER_CORE @@ -369,6 +386,7 @@ services: - redis - ${_APP_DB_HOST:-mongodb} - postgresql + - ollama volumes: - appwrite-uploads:/storage/uploads:rw - appwrite-cache:/storage/cache:rw @@ -437,6 +455,7 @@ services: - redis - ${_APP_DB_HOST:-mongodb} - postgresql + - ollama environment: - _APP_ENV - _APP_WORKER_PER_CORE @@ -451,6 +470,12 @@ services: - _APP_DB_SCHEMA - _APP_DB_USER - _APP_DB_PASS + - _APP_DB_ADAPTER_VECTORDB + - _APP_DB_HOST_VECTORDB + - _APP_DB_PORT_VECTORDB + - _APP_DB_SCHEMA_VECTORDB + - _APP_DB_USER_VECTORDB + - _APP_DB_PASS_VECTORDB - _APP_LOGGING_CONFIG - _APP_WORKERS_NUM - _APP_QUEUE_NAME @@ -474,6 +499,7 @@ services: - redis - ${_APP_DB_HOST:-mongodb} - postgresql + - ollama environment: - _APP_BROWSER_HOST - _APP_ENV @@ -543,6 +569,7 @@ services: - redis - ${_APP_DB_HOST:-mongodb} - postgresql + - ollama volumes: - appwrite-config:/storage/config:rw - appwrite-certificates:/storage/certificates:rw @@ -588,6 +615,7 @@ services: - ${_APP_DB_HOST:-mongodb} - postgresql - openruntimes-executor + - ollama environment: - _APP_ENV - _APP_WORKER_PER_CORE @@ -725,6 +753,7 @@ services: depends_on: - ${_APP_DB_HOST:-mongodb} - postgresql + - ollama environment: - _APP_ENV - _APP_WORKER_PER_CORE @@ -809,6 +838,7 @@ services: - redis - ${_APP_DB_HOST:-mongodb} - postgresql + - ollama environment: - _APP_ENV - _APP_WORKER_PER_CORE @@ -842,6 +872,7 @@ services: - redis - ${_APP_DB_HOST:-mongodb} - postgresql + - ollama environment: - _APP_ENV - _APP_WORKER_PER_CORE @@ -875,6 +906,7 @@ services: - redis - ${_APP_DB_HOST:-mongodb} - postgresql + - ollama environment: - _APP_ENV - _APP_WORKER_PER_CORE @@ -908,6 +940,7 @@ services: - ${_APP_DB_HOST:-mongodb} - postgresql - redis + - ollama environment: - _APP_ENV - _APP_WORKER_PER_CORE @@ -938,6 +971,7 @@ services: - ${_APP_DB_HOST:-mongodb} - postgresql - redis + - ollama environment: - _APP_ENV - _APP_WORKER_PER_CORE @@ -967,6 +1001,7 @@ services: - ${_APP_DB_HOST:-mongodb} - postgresql - redis + - ollama environment: - _APP_ENV - _APP_WORKER_PER_CORE @@ -1089,55 +1124,55 @@ services: - MARIADB_AUTO_UPGRADE=1 command: "mysqld --innodb-flush-method=fsync" - mongodb: - image: mongo:8.0.10 - container_name: appwrite-mongodb - <<: *x-logging - networks: - - appwrite - volumes: - - appwrite-mongodb:/data/db - - appwrite-mongodb-keyfile:/data/keyfile - - ./mongo-init.js:/docker-entrypoint-initdb.d/mongo-init.js:ro - - ./mongo-entrypoint.sh:/mongo-entrypoint.sh:ro - ports: - - "27017:27017" - environment: - - MONGO_INITDB_ROOT_USERNAME=root - - MONGO_INITDB_ROOT_PASSWORD=${_APP_DB_ROOT_PASS} - - MONGO_INITDB_DATABASE=${_APP_DB_SCHEMA} - - MONGO_INITDB_USERNAME=${_APP_DB_USER} - - MONGO_INITDB_PASSWORD=${_APP_DB_PASS} - entrypoint: ["/bin/bash", "/mongo-entrypoint.sh"] - healthcheck: - test: | - bash -c " - if mongosh -u root -p ${_APP_DB_ROOT_PASS} --authenticationDatabase admin --quiet --eval 'rs.status().ok' 2>/dev/null | grep -q 1; then - exit 0 - else - mongosh -u root -p ${_APP_DB_ROOT_PASS} --authenticationDatabase admin --quiet --eval \" - rs.initiate({_id: 'rs0', members: [{_id: 0, host: 'appwrite-mongodb:27017'}]}) - \" 2>/dev/null || exit 1 - fi - " - interval: 10s - timeout: 10s - retries: 10 - start_period: 30s + # mongodb: + # image: mongo:8.0.10 + # container_name: appwrite-mongodb + # <<: *x-logging + # networks: + # - appwrite + # volumes: + # - appwrite-mongodb:/data/db + # - appwrite-mongodb-keyfile:/data/keyfile + # - ./mongo-init.js:/docker-entrypoint-initdb.d/mongo-init.js:ro + # - ./mongo-entrypoint.sh:/mongo-entrypoint.sh:ro + # ports: + # - "27017:27017" + # environment: + # - MONGO_INITDB_ROOT_USERNAME=root + # - MONGO_INITDB_ROOT_PASSWORD=${_APP_DB_ROOT_PASS} + # - MONGO_INITDB_DATABASE=${_APP_DB_SCHEMA} + # - MONGO_INITDB_USERNAME=${_APP_DB_USER} + # - MONGO_INITDB_PASSWORD=${_APP_DB_PASS} + # entrypoint: ["/bin/bash", "/mongo-entrypoint.sh"] + # healthcheck: + # test: | + # bash -c " + # if mongosh -u root -p ${_APP_DB_ROOT_PASS} --authenticationDatabase admin --quiet --eval 'rs.status().ok' 2>/dev/null | grep -q 1; then + # exit 0 + # else + # mongosh -u root -p ${_APP_DB_ROOT_PASS} --authenticationDatabase admin --quiet --eval \" + # rs.initiate({_id: 'rs0', members: [{_id: 0, host: 'appwrite-mongodb:27017'}]}) + # \" 2>/dev/null || exit 1 + # fi + # " + # interval: 10s + # timeout: 10s + # retries: 10 + # start_period: 30s - appwrite-mongo-express: - image: mongo-express - container_name: appwrite-mongo-express - networks: - - appwrite - ports: - - "8082:8081" - environment: - ME_CONFIG_MONGODB_URL: "mongodb://root:${_APP_DB_ROOT_PASS}@appwrite-mongodb:27017/?replicaSet=rs0&directConnection=true" - ME_CONFIG_BASICAUTH_USERNAME: ${_APP_DB_USER} - ME_CONFIG_BASICAUTH_PASSWORD: ${_APP_DB_PASS} - depends_on: - - mongodb + # appwrite-mongo-express: + # image: mongo-express + # container_name: appwrite-mongo-express + # networks: + # - appwrite + # ports: + # - "8082:8081" + # environment: + # ME_CONFIG_MONGODB_URL: "mongodb://root:${_APP_DB_ROOT_PASS}@appwrite-mongodb:27017/?replicaSet=rs0&directConnection=true" + # ME_CONFIG_BASICAUTH_USERNAME: ${_APP_DB_USER} + # ME_CONFIG_BASICAUTH_PASSWORD: ${_APP_DB_PASS} + # depends_on: + # - mongodb postgresql: @@ -1158,6 +1193,22 @@ services: - POSTGRES_PASSWORD=${_APP_DB_PASS} command: "postgres" + ollama: + build: + context: . + dockerfile: ollama.dockerfile + container_name: ollama + ports: + - "11434:11434" + restart: unless-stopped + # persistent for caching models across restarts and preloading + volumes: + - ollama_models:/root/.ollama + environment: + - MODELS=${OLLAMA_MODELS:-embeddinggemma} + # duration to keep model in memory + - OLLAMA_KEEP_ALIVE=24h + redis: image: redis:7.2.4-alpine <<: *x-logging @@ -1256,8 +1307,9 @@ networks: volumes: appwrite-mariadb: - appwrite-mongodb: - appwrite-mongodb-keyfile: + # appwrite-mongodb: + # appwrite-mongodb-keyfile: + appwrite-postgresql: appwrite-redis: appwrite-cache: appwrite-uploads: @@ -1266,4 +1318,5 @@ volumes: appwrite-functions: appwrite-sites: appwrite-builds: - appwrite-config: \ No newline at end of file + appwrite-config: + ollama_models: \ No newline at end of file diff --git a/ollama.dockerfile b/ollama.dockerfile new file mode 100644 index 0000000000..1d4d29a22b --- /dev/null +++ b/ollama.dockerfile @@ -0,0 +1,21 @@ +FROM ollama/ollama:latest + +# Preload specific models +ENV MODELS="embeddinggemma" +ENV OLLAMA_KEEP_ALIVE=24h + +# Pre-pull models at build time for Docker layer caching +RUN ollama serve & \ + sleep 5 && \ + for m in $MODELS; do \ + echo "Pulling model $m..."; \ + ollama pull $m || exit 1; \ + done && \ + pkill ollama + +# Expose Ollama default port +EXPOSE 11434 + +# On container start, quickly ensure models exist (no re-download unless missing) +ENTRYPOINT ["/bin/bash", "-c", "(sleep 2; for m in $MODELS; do ollama list | grep -q $m || ollama pull $m; done) & exec ollama $0"] +CMD ["serve"] diff --git a/src/Appwrite/Platform/Modules/Databases/Constants.php b/src/Appwrite/Platform/Modules/Databases/Constants.php index c8fe6e79bb..8735f4b981 100644 --- a/src/Appwrite/Platform/Modules/Databases/Constants.php +++ b/src/Appwrite/Platform/Modules/Databases/Constants.php @@ -25,3 +25,4 @@ const COLLECTIONS = 'collection'; const TABLESDB = 'tablesdb'; const DOCUMENTSDB = 'documentsdb'; +const VECTORDB = 'vectordb'; diff --git a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Action.php b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Action.php index b22119fad1..4bcc9bd9ab 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Action.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Action.php @@ -13,6 +13,8 @@ abstract class Action extends UtopiaAction */ private ?string $context = COLLECTIONS; + private ?string $databaseType = TABLESDB; + /** * Get the response model used in the SDK and HTTP responses. */ @@ -22,6 +24,9 @@ abstract class Action extends UtopiaAction { if (\str_contains($path, '/tablesdb')) { $this->context = TABLES; + $this->databaseType = TABLESDB; + } elseif (\str_contains($path, '/vectordb')) { + $this->databaseType = VECTORDB; } return parent::setHttpPath($path); } @@ -34,6 +39,14 @@ abstract class Action extends UtopiaAction return $this->context; } + /** + * Get the current API database type. + */ + protected function getDatabaseType(): string + { + return $this->databaseType; + } + /** * Get the key used in event parameters (e.g., 'collectionId' or 'tableId'). */ diff --git a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Create.php b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Create.php index 28f85d04f4..fb0ee989af 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Create.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Create.php @@ -35,6 +35,7 @@ class Create extends Action { return match ($this->getDatabaseType()) { DOCUMENTSDB => $project->getAttribute('documentsDatabase'), + VECTORDB => $project->getAttribute('vectorDatabase'), default => $project->getAttribute('database'), }; } @@ -127,11 +128,11 @@ class Create extends Action try { $dbForProject->createCollection('database_' . $database->getSequence(), $attributes, $indexes); - $dbFordatabases = $getDatabasesDB($database); // legacy and tablesdb are created on the same project database // need to check for multitype databases if ($this->getDatabaseType() !== TABLESDB && $this->getDatabaseType() !== 'legacy') { - $dbFordatabases->create(); + $dbFordatabases = $getDatabasesDB(new Document(['database' => $this->getDatabaseDSN($project)])); + // $dbFordatabases->createCollection(Database::METADATA,$attributes); } } catch (DuplicateException) { throw new Exception(Exception::DATABASE_ALREADY_EXISTS); diff --git a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Transactions/Update.php b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Transactions/Update.php index c837b8547f..8b580d991d 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Transactions/Update.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Transactions/Update.php @@ -410,6 +410,7 @@ class Update extends Action { return match ($this->getDatabaseType()) { DOCUMENTSDB => $project->getAttribute('documentsDatabase'), + VECTORDB => $project->getAttribute('vectorDatabase'), default => $project->getAttribute('database'), }; } diff --git a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Create.php b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Create.php index 18e8c63996..86a5c18490 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Create.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Create.php @@ -2,29 +2,42 @@ namespace Appwrite\Platform\Modules\Databases\Http\VectorDB\Collections; -use Appwrite\Platform\Modules\Databases\Http\Databases\Collections\Create as CollectionCreate; +use Appwrite\Event\Event; +use Appwrite\Extend\Exception; +use Appwrite\Platform\Modules\Databases\Http\Databases\Collections\Action as CollectionAction; use Appwrite\SDK\AuthType; use Appwrite\SDK\ContentType; use Appwrite\SDK\Method; use Appwrite\SDK\Response as SDKResponse; use Appwrite\Utopia\Database\Validator\CustomId; use Appwrite\Utopia\Response as UtopiaResponse; +use Utopia\Config\Config; +use Utopia\Database\Database; +use Utopia\Database\Document; +use Utopia\Database\Exception\Duplicate as DuplicateException; +use Utopia\Database\Exception\Index as IndexException; +use Utopia\Database\Exception\Limit as LimitException; +use Utopia\Database\Exception\NotFound as NotFoundException; +use Utopia\Database\Helpers\ID; +use Utopia\Database\Helpers\Permission; +use Utopia\Database\Validator\Authorization; use Utopia\Database\Validator\Permissions; use Utopia\Database\Validator\UID; use Utopia\Swoole\Response as SwooleResponse; use Utopia\Validator\Boolean; +use Utopia\Validator\Integer; use Utopia\Validator\Text; -class Create extends CollectionCreate +class Create extends CollectionAction { public static function getName(): string { - return 'createDocumentsDBCollection'; + return 'createVectorDBCollection'; } protected function getResponseModel(): string { - return UtopiaResponse::MODEL_COLLECTION; + return UtopiaResponse::MODEL_VECTORDB_COLLECTION; } public function __construct() @@ -37,7 +50,7 @@ class Create extends CollectionCreate ->label('event', 'databases.[databaseId].collections.[collectionId].create') ->label('scope', 'collections.write') ->label('resourceType', RESOURCE_TYPE_DATABASES) - ->label('audits.event', 'collections.create') + ->label('audits.event', 'collection.create') ->label('audits.resource', 'database/{request.databaseId}/collection/{response.$id}') ->label('sdk', new Method( namespace: 'documentsDB', @@ -53,9 +66,11 @@ class Create extends CollectionCreate ], contentType: ContentType::JSON )) - ->param('databaseId', '', new UID(), 'Database ID.') - ->param('collectionId', '', new CustomId(), 'Unique Id. Choose a custom ID or generate a random ID with `ID.unique()`. Valid chars are a-z, A-Z, 0-9, period, hyphen, and underscore. Can\'t start with a special char. Max length is 36 chars.') + ->param('databaseId', '', fn (Database $dbForProject) => new UID($dbForProject->getAdapter()->getMaxUIDLength()), 'Database ID.', false, ['dbForProject']) + ->param('collectionId', '', fn (Database $dbForProject) => new CustomId(false, $dbForProject->getAdapter()->getMaxUIDLength()), 'Unique Id. Choose a custom ID or generate a random ID with `ID.unique()`. Valid chars are a-z, A-Z, 0-9, period, hyphen, and underscore. Can\'t start with a special char. Max length is 36 chars.', false, ['dbForProject']) ->param('name', '', new Text(128), 'Collection name. Max length: 128 chars.') + ->param('dimensions', 0, new Integer(), 'Embedding dimensions.') + ->param('embeddingModel', '', new Text(256), 'Embedding model identifier.') ->param('permissions', null, new Permissions(APP_LIMIT_ARRAY_PARAMS_SIZE), 'An array of permissions strings. By default, no user is granted with any permissions. [Learn more about permissions](https://appwrite.io/docs/permissions).', true) ->param('documentSecurity', false, new Boolean(true), 'Enables configuring permissions for individual documents. A user needs one of document or collection level permissions to access a document. [Learn more about permissions](https://appwrite.io/docs/permissions).', true) ->param('enabled', true, new Boolean(), 'Is collection enabled? When set to \'disabled\', users cannot access the collection but Server SDKs with and API key can still read and write to the collection. No data is lost when this is toggled.', true) @@ -65,4 +80,83 @@ class Create extends CollectionCreate ->inject('queueForEvents') ->callback($this->action(...)); } + + public function action(string $databaseId, string $collectionId, string $name, int $dimensions, string $embeddingModel, ?array $permissions, bool $documentSecurity, bool $enabled, UtopiaResponse $response, Database $dbForProject, callable $getDatabasesDB, Event $queueForEvents): void + { + $database = Authorization::skip(fn () => $dbForProject->getDocument('databases', $databaseId)); + + if ($database->isEmpty()) { + throw new Exception(Exception::DATABASE_NOT_FOUND); + } + + $collectionId = $collectionId === 'unique()' ? ID::unique() : $collectionId; + + // Map aggregate permissions into the multiple permissions they represent. + $permissions = Permission::aggregate($permissions) ?? []; + + try { + $collection = $dbForProject->createDocument('database_' . $database->getSequence(), new Document([ + '$id' => $collectionId, + 'databaseInternalId' => $database->getSequence(), + 'databaseId' => $databaseId, + '$permissions' => $permissions, + 'documentSecurity' => $documentSecurity, + 'enabled' => $enabled, + 'name' => $name, + 'dimensions' => $dimensions, + 'embeddingModel' => $embeddingModel, + 'search' => \implode(' ', [$collectionId, $name]), + ])); + + } catch (DuplicateException) { + throw new Exception($this->getDuplicateException()); + } catch (LimitException) { + throw new Exception($this->getLimitException()); + } catch (NotFoundException) { + throw new Exception(Exception::DATABASE_NOT_FOUND); + } + /** @var Database $dbForDatabases */ + $dbForDatabases = $getDatabasesDB($database); + + $attributes = []; + $indexes = []; + $collections = (Config::getParam('collections', [])['vectordb'] ?? [])['collections'] ?? []; + foreach ($collections['defaultAttributes'] as $attribute) { + if ($attribute['$id'] === 'embeddings') { + $attribute['size'] = $dimensions; + } + $attributes[] = new Document($attribute); + } + foreach ($collections['defaultIndexes'] as $index) { + $indexes[] = new Document($index); + } + try { + // passing null in creates only creates the metadata collection + if (!$dbForDatabases->exists(null, Database::METADATA)) { + $dbForDatabases->create(); + } + $dbForDatabases->createCollection( + id: 'database_' . $database->getSequence() . '_collection_' . $collection->getSequence(), + permissions: $permissions, + documentSecurity: $documentSecurity, + attributes:$attributes, + indexes:$indexes + ); + } catch (DuplicateException) { + throw new Exception($this->getDuplicateException()); + } catch (IndexException) { + throw new Exception($this->getInvalidIndexException()); + } catch (LimitException) { + throw new Exception($this->getLimitException()); + } + + $queueForEvents + ->setContext('database', $database) + ->setParam('databaseId', $databaseId) + ->setParam($this->getEventsParamKey(), $collection->getId()); + + $response + ->setStatusCode(SwooleResponse::STATUS_CODE_CREATED) + ->dynamic($collection, $this->getResponseModel()); + } } diff --git a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Get.php b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Get.php index fd3c1e4d57..0ee2ba0183 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Get.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Get.php @@ -20,7 +20,7 @@ class Get extends CollectionGet protected function getResponseModel(): string { - return UtopiaResponse::MODEL_COLLECTION; + return UtopiaResponse::MODEL_VECTORDB_COLLECTION; } public function __construct() diff --git a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Update.php b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Update.php index ef3417fdbb..e833da21ea 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Update.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Update.php @@ -2,19 +2,23 @@ namespace Appwrite\Platform\Modules\Databases\Http\VectorDB\Collections; -use Appwrite\Platform\Modules\Databases\Http\Databases\Collections\Update as CollectionUpdate; +use Appwrite\Extend\Exception; +use Appwrite\Platform\Modules\Databases\Http\Databases\Collections\Action as CollectionAction; use Appwrite\SDK\AuthType; use Appwrite\SDK\ContentType; use Appwrite\SDK\Method; use Appwrite\SDK\Response as SDKResponse; use Appwrite\Utopia\Response as UtopiaResponse; +use Utopia\Database\Helpers\Permission; +use Utopia\Database\Validator\Authorization; use Utopia\Database\Validator\Permissions; use Utopia\Database\Validator\UID; use Utopia\Swoole\Response as SwooleResponse; use Utopia\Validator\Boolean; +use Utopia\Validator\Integer; use Utopia\Validator\Text; -class Update extends CollectionUpdate +class Update extends CollectionAction { public static function getName(): string { @@ -23,7 +27,7 @@ class Update extends CollectionUpdate protected function getResponseModel(): string { - return UtopiaResponse::MODEL_COLLECTION; + return UtopiaResponse::MODEL_VECTORDB_COLLECTION; } public function __construct() @@ -47,7 +51,7 @@ class Update extends CollectionUpdate responses: [ new SDKResponse( code: SwooleResponse::STATUS_CODE_OK, - model: UtopiaResponse::MODEL_COLLECTION, + model: UtopiaResponse::MODEL_VECTORDB_COLLECTION, ) ], contentType: ContentType::JSON @@ -55,6 +59,9 @@ class Update extends CollectionUpdate ->param('databaseId', '', new UID(), 'Database ID.') ->param('collectionId', '', new UID(), 'Collection ID.') ->param('name', null, new Text(128), 'Collection name. Max length: 128 chars.') + ->param('description', null, new Text(1024), 'Collection description. Max length: 1024 chars.', true) + ->param('dimensions', null, new Integer(), 'Embedding dimensions.', true) + ->param('embeddingModel', null, new Text(256), 'Embedding model identifier.', true) ->param('permissions', null, new Permissions(APP_LIMIT_ARRAY_PARAMS_SIZE), 'An array of permission strings. By default, the current permissions are inherited. [Learn more about permissions](https://appwrite.io/docs/permissions).', true) ->param('documentSecurity', false, new Boolean(true), 'Enables configuring permissions for individual documents. A user needs one of document or collection level permissions to access a document. [Learn more about permissions](https://appwrite.io/docs/permissions).', true) ->param('enabled', true, new Boolean(), 'Is collection enabled? When set to \'disabled\', users cannot access the collection but Server SDKs with and API key can still read and write to the collection. No data is lost when this is toggled.', true) @@ -64,4 +71,48 @@ class Update extends CollectionUpdate ->inject('queueForEvents') ->callback($this->action(...)); } + + public function action(string $databaseId, string $collectionId, ?string $name, ?string $description, ?int $dimensions, ?string $embeddingModel, ?array $permissions, bool $documentSecurity, bool $enabled, UtopiaResponse $response, \Utopia\Database\Database $dbForProject, callable $getDatabasesDB, \Appwrite\Event\Event $queueForEvents): void + { + $database = Authorization::skip(fn () => $dbForProject->getDocument('databases', $databaseId)); + if ($database->isEmpty()) { + throw new Exception(Exception::DATABASE_NOT_FOUND); + } + + $collection = $dbForProject->getDocument('database_' . $database->getSequence(), $collectionId); + if ($collection->isEmpty()) { + throw new Exception($this->getNotFoundException()); + } + + $permissions ??= $collection->getPermissions(); + + // Map aggregate permissions into the multiple permissions they represent. + $permissions = Permission::aggregate($permissions); + + $enabled ??= $collection->getAttribute('enabled', true); + + $updated = $dbForProject->updateDocument( + 'database_' . $database->getSequence(), + $collectionId, + $collection + ->setAttribute('name', $name ?? $collection->getAttribute('name')) + ->setAttribute('description', $description ?? $collection->getAttribute('description')) + ->setAttribute('dimensions', $dimensions ?? $collection->getAttribute('dimensions')) + ->setAttribute('embeddingModel', $embeddingModel ?? $collection->getAttribute('embeddingModel')) + ->setAttribute('$permissions', $permissions) + ->setAttribute('documentSecurity', $documentSecurity) + ->setAttribute('enabled', $enabled) + ->setAttribute('search', \implode(' ', [$collectionId, $name ?? $collection->getAttribute('name')])) + ); + + $dbForDatabases = $getDatabasesDB($database); + $dbForDatabases->updateCollection('database_' . $database->getSequence() . '_collection_' . $updated->getSequence(), $permissions, $documentSecurity); + + $queueForEvents + ->setContext('database', $database) + ->setParam('databaseId', $databaseId) + ->setParam($this->getEventsParamKey(), $updated->getId()); + + $response->dynamic($updated, $this->getResponseModel()); + } } diff --git a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/XList.php b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/XList.php index 7ca34d50b0..9c7ea55c27 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/XList.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/XList.php @@ -22,7 +22,7 @@ class XList extends CollectionXList protected function getResponseModel(): string { - return UtopiaResponse::MODEL_COLLECTION_LIST; + return UtopiaResponse::MODEL_VECTORDB_COLLECTION_LIST; } public function __construct() diff --git a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Create.php b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Create.php index 3e47ffa69e..4ee993a864 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Create.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Create.php @@ -52,6 +52,7 @@ class Create extends DatabaseCreate ->inject('project') ->inject('response') ->inject('dbForProject') + ->inject('getDatabasesDB') ->inject('queueForEvents') ->callback($this->action(...)); } diff --git a/src/Appwrite/Platform/Modules/Databases/Services/Http.php b/src/Appwrite/Platform/Modules/Databases/Services/Http.php index c746249114..682472128e 100644 --- a/src/Appwrite/Platform/Modules/Databases/Services/Http.php +++ b/src/Appwrite/Platform/Modules/Databases/Services/Http.php @@ -6,6 +6,7 @@ use Appwrite\Platform\Modules\Databases\Http\Init\Timeout; use Appwrite\Platform\Modules\Databases\Services\Registry\DocumentsDB as DocumentsDBRegistry; use Appwrite\Platform\Modules\Databases\Services\Registry\Legacy as LegacyRegistry; use Appwrite\Platform\Modules\Databases\Services\Registry\TablesDB as TablesDBDBRegistry; +use Appwrite\Platform\Modules\Databases\Services\Registry\VectorDB as VectorDBRegistry; use Utopia\Platform\Service; class Http extends Service @@ -19,7 +20,8 @@ class Http extends Service foreach ([ LegacyRegistry::class, TablesDBDBRegistry::class, - DocumentsDBRegistry::class + DocumentsDBRegistry::class, + VectorDBRegistry::class ] as $registrar) { new $registrar($this); } diff --git a/src/Appwrite/Platform/Modules/Databases/Services/Registry/VectorDB.php b/src/Appwrite/Platform/Modules/Databases/Services/Registry/VectorDB.php new file mode 100644 index 0000000000..9b526f2ca2 --- /dev/null +++ b/src/Appwrite/Platform/Modules/Databases/Services/Registry/VectorDB.php @@ -0,0 +1,53 @@ +registerDatabaseActions($service); + $this->registerCollectionActions($service); + // $this->registerIndexActions($service); + // $this->registerRowActions($service); + // $this->registerTransactionActions($service); + } + + private function registerDatabaseActions(Service $service): void + { + $service->addAction(CreateVectorDatabase::getName(), new CreateVectorDatabase()); + $service->addAction(GetVectorDatabase::getName(), new GetVectorDatabase()); + $service->addAction(UpdateVectorDatabase::getName(), new UpdateVectorDatabase()); + $service->addAction(DeleteVectorDatabase::getName(), new DeleteVectorDatabase()); + $service->addAction(ListVectorDatabases::getName(), new ListVectorDatabases()); + $service->addAction(GetVectorDatabaseUsage::getName(), new GetVectorDatabaseUsage()); + $service->addAction(ListVectorDatabaseUsage::getName(), new ListVectorDatabaseUsage()); + } + + private function registerCollectionActions(Service $service): void + { + $service->addAction(CreateCollection::getName(), new CreateCollection()); + $service->addAction(GetCollection::getName(), new GetCollection()); + $service->addAction(UpdateCollection::getName(), new UpdateCollection()); + $service->addAction(DeleteCollection::getName(), new DeleteCollection()); + $service->addAction(ListCollections::getName(), new ListCollections()); + $service->addAction(ListCollectionLogs::getName(), new ListCollectionLogs()); + $service->addAction(GetCollectionUsage::getName(), new GetCollectionUsage()); + } +} diff --git a/src/Appwrite/Utopia/Response.php b/src/Appwrite/Utopia/Response.php index b391f0ea72..533eb273db 100644 --- a/src/Appwrite/Utopia/Response.php +++ b/src/Appwrite/Utopia/Response.php @@ -186,6 +186,8 @@ class Response extends SwooleResponse public const MODEL_DATABASE_LIST = 'databaseList'; public const MODEL_COLLECTION = 'collection'; public const MODEL_COLLECTION_LIST = 'collectionList'; + public const MODEL_VECTORDB_COLLECTION = 'vectordbCollection'; + public const MODEL_VECTORDB_COLLECTION_LIST = 'vectordbCollectionList'; public const MODEL_TABLE = 'table'; public const MODEL_TABLE_LIST = 'tableList'; public const MODEL_INDEX = 'index'; @@ -438,6 +440,7 @@ class Response extends SwooleResponse ->setModel(new BaseList('Documents List', self::MODEL_DOCUMENT_LIST, 'documents', self::MODEL_DOCUMENT)) ->setModel(new BaseList('Tables List', self::MODEL_TABLE_LIST, 'tables', self::MODEL_TABLE)) ->setModel(new BaseList('Collections List', self::MODEL_COLLECTION_LIST, 'collections', self::MODEL_COLLECTION)) + ->setModel(new BaseList('VectorDB Collections List', self::MODEL_VECTORDB_COLLECTION_LIST, 'collections', self::MODEL_VECTORDB_COLLECTION)) ->setModel(new BaseList('Databases List', self::MODEL_DATABASE_LIST, 'databases', self::MODEL_DATABASE)) ->setModel(new BaseList('Indexes List', self::MODEL_INDEX_LIST, 'indexes', self::MODEL_INDEX)) ->setModel(new BaseList('Column Indexes List', self::MODEL_COLUMN_INDEX_LIST, 'indexes', self::MODEL_COLUMN_INDEX)) @@ -492,6 +495,7 @@ class Response extends SwooleResponse ->setModel(new Database()) // Collection API Models ->setModel(new Collection()) + ->setModel(new \Appwrite\Utopia\Response\Model\VectorDBCollection()) ->setModel(new Attribute()) ->setModel(new AttributeList()) ->setModel(new AttributeString()) diff --git a/src/Appwrite/Utopia/Response/Model/VectorDBCollection.php b/src/Appwrite/Utopia/Response/Model/VectorDBCollection.php new file mode 100644 index 0000000000..b0cab1f4c7 --- /dev/null +++ b/src/Appwrite/Utopia/Response/Model/VectorDBCollection.php @@ -0,0 +1,120 @@ +addRule('$id', [ + 'type' => self::TYPE_STRING, + 'description' => 'Collection ID.', + 'default' => '', + 'example' => '5e5ea5c16897e', + ]) + ->addRule('$createdAt', [ + 'type' => self::TYPE_DATETIME, + 'description' => 'Collection creation date in ISO 8601 format.', + 'default' => '', + 'example' => self::TYPE_DATETIME_EXAMPLE, + ]) + ->addRule('$updatedAt', [ + 'type' => self::TYPE_DATETIME, + 'description' => 'Collection update date in ISO 8601 format.', + 'default' => '', + 'example' => self::TYPE_DATETIME_EXAMPLE, + ]) + ->addRule('$permissions', [ + 'type' => self::TYPE_STRING, + 'description' => 'Collection permissions. [Learn more about permissions](https://appwrite.io/docs/permissions).', + 'default' => '', + 'example' => ['read("any")'], + 'array' => true + ]) + ->addRule('databaseId', [ + 'type' => self::TYPE_STRING, + 'description' => 'Database ID.', + 'default' => '', + 'example' => '5e5ea5c16897e', + ]) + ->addRule('name', [ + 'type' => self::TYPE_STRING, + 'description' => 'Collection name.', + 'default' => '', + 'example' => 'My Collection', + ]) + ->addRule('enabled', [ + 'type' => self::TYPE_BOOLEAN, + 'description' => 'Collection enabled. Can be \'enabled\' or \'disabled\'. When disabled, the collection is inaccessible to users, but remains accessible to Server SDKs using API keys.', + 'default' => true, + 'example' => false, + ]) + ->addRule('documentSecurity', [ + 'type' => self::TYPE_BOOLEAN, + 'description' => 'Whether document-level permissions are enabled. [Learn more about permissions](https://appwrite.io/docs/permissions).', + 'default' => '', + 'example' => true, + ]) + ->addRule('dimensions', [ + 'type' => self::TYPE_INTEGER, + 'description' => 'Embedding dimensions.', + 'default' => 0, + 'example' => 1536, + ]) + ->addRule('embeddingModel', [ + 'type' => self::TYPE_STRING, + 'description' => 'Embedding model identifier.', + 'default' => '', + 'example' => 'text-embedding-3-large', + ]) + ->addRule('search', [ + 'type' => self::TYPE_STRING, + 'description' => 'Search text.', + 'default' => '', + 'example' => 'collectionId name', + ]) + ->addRule('attributes', [ + 'type' => [ + Response::MODEL_ATTRIBUTE_BOOLEAN, + Response::MODEL_ATTRIBUTE_INTEGER, + Response::MODEL_ATTRIBUTE_FLOAT, + Response::MODEL_ATTRIBUTE_EMAIL, + Response::MODEL_ATTRIBUTE_ENUM, + Response::MODEL_ATTRIBUTE_URL, + Response::MODEL_ATTRIBUTE_IP, + Response::MODEL_ATTRIBUTE_DATETIME, + Response::MODEL_ATTRIBUTE_RELATIONSHIP, + Response::MODEL_ATTRIBUTE_POINT, + Response::MODEL_ATTRIBUTE_LINE, + Response::MODEL_ATTRIBUTE_POLYGON, + Response::MODEL_ATTRIBUTE_STRING, + ], + 'description' => 'Collection attributes.', + 'default' => [], + 'example' => new \stdClass(), + 'array' => true, + ]) + ->addRule('indexes', [ + 'type' => Response::MODEL_INDEX, + 'description' => 'Collection indexes.', + 'default' => [], + 'example' => new \stdClass(), + 'array' => true + ]) + ; + } + + public function getName(): string + { + return 'VectorDB Collection'; + } + + public function getType(): string + { + return Response::MODEL_VECTORDB_COLLECTION; + } +} diff --git a/tests/e2e/Services/Databases/VectorDB/DatabasesBase.php b/tests/e2e/Services/Databases/VectorDB/DatabasesBase.php new file mode 100644 index 0000000000..44efbabcbe --- /dev/null +++ b/tests/e2e/Services/Databases/VectorDB/DatabasesBase.php @@ -0,0 +1,229 @@ +client->call(Client::METHOD_POST, '/vectordb', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'databaseId' => ID::unique(), + 'name' => 'Test Database' + ]); + + $this->assertNotEmpty($database['body']['$id']); + $this->assertEquals(201, $database['headers']['status-code']); + $this->assertEquals('Test Database', $database['body']['name']); + $this->assertEquals('vectordb', $database['body']['type']); + + return ['databaseId' => $database['body']['$id']]; + } + + /** + * @depends testCreateDatabase + */ + public function testCreateCollection(array $data): array + { + $databaseId = $data['databaseId']; + /** + * Test for SUCCESS + */ + $movies = $this->client->call(Client::METHOD_POST, '/vectordb/' . $databaseId . '/collections', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'collectionId' => ID::unique(), + 'name' => 'Movies', + 'documentSecurity' => true, + 'dimensions' => 1536, + 'embeddingModel' => 'gemma', + 'permissions' => [ + Permission::create(Role::user($this->getUser()['$id'])), + ], + ]); + + $this->assertEquals(201, $movies['headers']['status-code']); + $this->assertEquals($movies['body']['name'], 'Movies'); + + $actors = $this->client->call(Client::METHOD_POST, '/vectordb/' . $databaseId . '/collections', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'collectionId' => ID::unique(), + 'name' => 'Actors', + 'documentSecurity' => true, + 'dimensions' => 1536, + 'embeddingModel' => 'gemma', + 'permissions' => [ + Permission::create(Role::user($this->getUser()['$id'])), + ], + ]); + + $this->assertEquals(201, $actors['headers']['status-code']); + $this->assertEquals($actors['body']['name'], 'Actors'); + + return [ + 'databaseId' => $databaseId, + 'moviesId' => $movies['body']['$id'], + 'actorsId' => $actors['body']['$id'], + ]; + } + + public function testCreateDatabaseSample(): array + { + $database = $this->client->call(Client::METHOD_POST, '/vectordb', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'databaseId' => ID::unique(), + 'name' => 'Sample VectorDB' + ]); + + $this->assertNotEmpty($database['body']['$id']); + $this->assertEquals(201, $database['headers']['status-code']); + $this->assertEquals('Sample VectorDB', $database['body']['name']); + $this->assertEquals('vectordb', $database['body']['type']); + + return ['databaseId' => $database['body']['$id']]; + } + + /** + * @depends testCreateDatabaseSample + */ + public function testCreateCollectionSample(array $data): array + { + $databaseId = $data['databaseId']; + + $collection = $this->client->call(Client::METHOD_POST, '/vectordb/' . $databaseId . '/collections', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'collectionId' => ID::unique(), + 'name' => 'Sample Collection', + 'dimensions' => 1536, + 'embeddingModel' => 'text-embedding-3-large', + 'documentSecurity' => true, + 'permissions' => [ + Permission::create(Role::user($this->getUser()['$id'])), + ], + ]); + + $this->assertEquals(201, $collection['headers']['status-code']); + $this->assertEquals('Sample Collection', $collection['body']['name']); + $this->assertEquals(1536, $collection['body']['dimensions']); + $this->assertEquals('text-embedding-3-large', $collection['body']['embeddingModel']); + + return [ + 'databaseId' => $databaseId, + 'collectionId' => $collection['body']['$id'], + ]; + } + + public function testCreateMultipleDatabasesWithCollections(): array + { + $projectId = $this->getProject()['$id']; + $apiKey = $this->getProject()['apiKey']; + $userId = $this->getUser()['$id']; + + /** + * Helper to create a database + */ + $createDatabase = function (string $name) use ($projectId, $apiKey) { + $db = $this->client->call(Client::METHOD_POST, '/vectordb', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $projectId, + 'x-appwrite-key' => $apiKey + ], [ + 'databaseId' => ID::unique(), + 'name' => $name + ]); + + $this->assertEquals(201, $db['headers']['status-code']); + $this->assertEquals('vectordb', $db['body']['type']); + $this->assertEquals($name, $db['body']['name']); + $this->assertNotEmpty($db['body']['$id']); + + return $db['body']['$id']; + }; + + /** + * Helper to create a collection + */ + $createCollection = function (string $databaseId, string $name) use ($projectId, $apiKey, $userId) { + $res = $this->client->call(Client::METHOD_POST, '/vectordb/' . $databaseId . '/collections', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $projectId, + 'x-appwrite-key' => $apiKey + ], [ + 'collectionId' => ID::unique(), + 'name' => $name, + 'documentSecurity' => true, + 'dimensions' => 1536, + 'embeddingModel' => 'gemma', + 'permissions' => [ + Permission::create(Role::user($userId)), + ], + ]); + + $this->assertEquals(201, $res['headers']['status-code']); + $this->assertEquals($name, $res['body']['name']); + + return $res['body']['$id']; + }; + + /** + * === Database 1: MediaDB === + */ + $mediaDbId = $createDatabase('MediaDB'); + + $mediaCollections = ['Movies', 'Actors', 'Directors']; + $mediaCollectionIds = []; + + foreach ($mediaCollections as $col) { + $mediaCollectionIds[$col] = $createCollection($mediaDbId, $col); + } + + /** + * === Database 2: ContentDB === + */ + $contentDbId = $createDatabase('ContentDB'); + + $contentCollections = ['Articles', 'Authors']; + $contentCollectionIds = []; + + foreach ($contentCollections as $col) { + $contentCollectionIds[$col] = $createCollection($contentDbId, $col); + } + + return [ + 'databases' => [ + 'MediaDB' => [ + 'id' => $mediaDbId, + 'collections' => $mediaCollectionIds, + ], + 'ContentDB' => [ + 'id' => $contentDbId, + 'collections' => $contentCollectionIds, + ], + ] + ]; + } + +} diff --git a/tests/e2e/Services/Databases/VectorDB/DatabasesCustomClientTest.php b/tests/e2e/Services/Databases/VectorDB/DatabasesCustomClientTest.php new file mode 100644 index 0000000000..1d8f282a4d --- /dev/null +++ b/tests/e2e/Services/Databases/VectorDB/DatabasesCustomClientTest.php @@ -0,0 +1,14 @@ + Date: Fri, 31 Oct 2025 19:07:34 +0530 Subject: [PATCH 03/27] added vector embedding creation + update endpoint --- app/init/resources.php | 8 ++ .../Http/VectorDB/Collections/Create.php | 4 +- .../Documents/Attribute/Increment.php | 72 ----------- .../Documents/Embedding/Create.php | 114 ++++++++++++++++++ .../Decrement.php => Embedding/Update.php} | 38 +++--- .../Documents/{ => Text}/Create.php | 6 +- .../Documents/{ => Text}/Update.php | 0 .../Http/VectorDB/Collections/Update.php | 4 +- .../Databases/Services/Registry/VectorDB.php | 9 +- src/Appwrite/Utopia/Response.php | 6 + .../Utopia/Response/Model/AttributeObject.php | 29 +++++ .../Utopia/Response/Model/AttributeVector.php | 37 ++++++ .../Response/Model/VectorDBCollection.php | 15 +-- .../Databases/VectorDB/DatabasesBase.php | 47 +++++++- 14 files changed, 274 insertions(+), 115 deletions(-) delete mode 100644 src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Attribute/Increment.php create mode 100644 src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Embedding/Create.php rename src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/{Attribute/Decrement.php => Embedding/Update.php} (60%) rename src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/{ => Text}/Create.php (98%) rename src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/{ => Text}/Update.php (100%) create mode 100644 src/Appwrite/Utopia/Response/Model/AttributeObject.php create mode 100644 src/Appwrite/Utopia/Response/Model/AttributeVector.php diff --git a/app/init/resources.php b/app/init/resources.php index 2a563ddeae..7ff41a996c 100644 --- a/app/init/resources.php +++ b/app/init/resources.php @@ -60,6 +60,8 @@ use Utopia\Telemetry\Adapter\None as NoTelemetry; use Utopia\Validator\Hostname; use Utopia\Validator\WhiteList; use Utopia\VCS\Adapter\Git\GitHub as VcsGitHub; +use Utopia\Agents\Agent; +use Utopia\Agents\Adapters\Ollama; // Runtime Execution App::setResource('log', fn () => new Log()); @@ -1167,3 +1169,9 @@ App::setResource('httpReferrerSafe', function (Request $request, string $httpRef App::setResource('transactionState', function (Database $dbForProject, callable $getDatabasesDB) { return new TransactionState($dbForProject, $getDatabasesDB); }, ['dbForProject', 'getDatabasesDB']); + +App::setResource('embeddingAgent', function () { + // TODO: ollama endpoint should be taken from env in cloud(for autoscaling) + $adapter = new Ollama(); + return new Agent($adapter); +}); \ No newline at end of file diff --git a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Create.php b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Create.php index 86a5c18490..1515ff9421 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Create.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Create.php @@ -27,6 +27,8 @@ use Utopia\Swoole\Response as SwooleResponse; use Utopia\Validator\Boolean; use Utopia\Validator\Integer; use Utopia\Validator\Text; +use Utopia\Validator\WhiteList; +use Utopia\Agents\Adapters\Ollama; class Create extends CollectionAction { @@ -70,7 +72,7 @@ class Create extends CollectionAction ->param('collectionId', '', fn (Database $dbForProject) => new CustomId(false, $dbForProject->getAdapter()->getMaxUIDLength()), 'Unique Id. Choose a custom ID or generate a random ID with `ID.unique()`. Valid chars are a-z, A-Z, 0-9, period, hyphen, and underscore. Can\'t start with a special char. Max length is 36 chars.', false, ['dbForProject']) ->param('name', '', new Text(128), 'Collection name. Max length: 128 chars.') ->param('dimensions', 0, new Integer(), 'Embedding dimensions.') - ->param('embeddingModel', '', new Text(256), 'Embedding model identifier.') + ->param('embeddingModel', '', new WhiteList((new Ollama())->getModels()), 'Embedding model identifier.') ->param('permissions', null, new Permissions(APP_LIMIT_ARRAY_PARAMS_SIZE), 'An array of permissions strings. By default, no user is granted with any permissions. [Learn more about permissions](https://appwrite.io/docs/permissions).', true) ->param('documentSecurity', false, new Boolean(true), 'Enables configuring permissions for individual documents. A user needs one of document or collection level permissions to access a document. [Learn more about permissions](https://appwrite.io/docs/permissions).', true) ->param('enabled', true, new Boolean(), 'Is collection enabled? When set to \'disabled\', users cannot access the collection but Server SDKs with and API key can still read and write to the collection. No data is lost when this is toggled.', true) diff --git a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Attribute/Increment.php b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Attribute/Increment.php deleted file mode 100644 index 297ec03338..0000000000 --- a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Attribute/Increment.php +++ /dev/null @@ -1,72 +0,0 @@ -setHttpMethod(self::HTTP_REQUEST_METHOD_PATCH) - ->setHttpPath('/v1/vectordb/:databaseId/collections/:collectionId/documents/:documentId/:attribute/increment') - ->desc('Increment document attribute') - ->groups(['api', 'database']) - ->label('event', 'vectordb.[databaseId].collections.[collectionId].documents.[documentId].update') - ->label('scope', 'documents.write') - ->label('resourceType', RESOURCE_TYPE_DATABASES) - ->label('audits.event', 'documents.update') - ->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}') - ->label('abuse-key', 'ip:{ip},method:{method},url:{url},userId:{userId}') - ->label('abuse-limit', APP_LIMIT_WRITE_RATE_DEFAULT * 2) - ->label('abuse-time', APP_LIMIT_WRITE_RATE_PERIOD_DEFAULT) - ->label('sdk', new Method( - namespace: 'documentsDB', - group: $this->getSdkGroup(), - name: 'incrementDocumentAttribute', - description: '/docs/references/vectordb/increment-document-attribute.md', - auth: [AuthType::SESSION, AuthType::JWT, AuthType::ADMIN, AuthType::KEY], - responses: [ - new SDKResponse( - code: SwooleResponse::STATUS_CODE_OK, - model: $this->getResponseModel(), - ) - ], - contentType: ContentType::JSON - )) - ->param('databaseId', '', new UID(), 'Database ID.') - ->param('collectionId', '', new UID(), 'Collection ID.') - ->param('documentId', '', new UID(), 'Document ID.') - ->param('attribute', '', new Key(), 'Attribute key.') - ->param('value', 1, new Numeric(), 'Value to increment the attribute by. The value must be a number.', true) - ->param('max', null, new Numeric(), 'Maximum value for the attribute. If the current value is greater than this value, an error will be thrown.', true) - ->param('transactionId', null, new UID(), 'Transaction ID for staging the operation.', true) - ->inject('response') - ->inject('dbForProject') - ->inject('getDatabasesDB') - ->inject('queueForEvents') - ->inject('queueForStatsUsage') - ->inject('plan') - ->callback($this->action(...)); - } -} diff --git a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Embedding/Create.php b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Embedding/Create.php new file mode 100644 index 0000000000..10ae664299 --- /dev/null +++ b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Embedding/Create.php @@ -0,0 +1,114 @@ +setHttpMethod(self::HTTP_REQUEST_METHOD_POST) + ->setHttpPath('/v1/vectordb/:databaseId/collections/:collectionId/documents') + ->desc('Create document') + ->groups(['api', 'database']) + ->label('scope', 'documents.write') + ->label('resourceType', RESOURCE_TYPE_DATABASES) + ->label('audits.event', 'document.create') + ->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}') + ->label('abuse-key', 'ip:{ip},method:{method},url:{url},userId:{userId}') + ->label('abuse-limit', APP_LIMIT_WRITE_RATE_DEFAULT * 2) + ->label('abuse-time', APP_LIMIT_WRITE_RATE_PERIOD_DEFAULT) + ->label('sdk', [ + new Method( + namespace: 'vectorDB', + group: $this->getSdkGroup(), + name: 'createEmbeddingDocument', + desc: 'Create document', + description: '/docs/references/vectordb/create-document.md', + auth: [AuthType::SESSION, AuthType::KEY, AuthType::JWT], + responses: [ + new SDKResponse( + code: SwooleResponse::STATUS_CODE_CREATED, + model: $this->getResponseModel(), + ) + ], + contentType: ContentType::JSON, + parameters: [ + new Parameter('databaseId', optional: false), + new Parameter('collectionId', optional: false), + new Parameter('documentId', optional: false), + new Parameter('data', optional: false), + new Parameter('permissions', optional: true), + ] + ), + new Method( + namespace: 'documentsDB', + group: $this->getSdkGroup(), + name: 'createDocuments', + desc: 'Create documents', + description: '/docs/references/vectordb/create-documents.md', + auth: [AuthType::ADMIN, AuthType::KEY], + responses: [ + new SDKResponse( + code: SwooleResponse::STATUS_CODE_CREATED, + model: $this->getBulkResponseModel(), + ) + ], + contentType: ContentType::JSON, + parameters: [ + new Parameter('databaseId', optional: false), + new Parameter('collectionId', optional: false), + new Parameter('documents', optional: false), + ] + ) + ]) + ->param('databaseId', '', new UID(), 'Database ID.') + ->param('documentId', '', new CustomId(), 'Document ID. Choose a custom ID or generate a random ID with `ID.unique()`. Valid chars are a-z, A-Z, 0-9, period, hyphen, and underscore. Can\'t start with a special char. Max length is 36 chars.', true) + ->param('collectionId', '', new UID(), 'Collection ID. You can create a new collection using the Database service [server integration](https://appwrite.io/docs/server/databases#databasesCreateCollection). Make sure to define attributes before creating documents.') + ->param('data', [], new JSON(), 'Document data as JSON object.', true, example: '{"username":"walter.obrien","email":"walter.obrien@example.com","fullName":"Walter O\'Brien","age":30,"isAdmin":false}') + ->param('permissions', null, new Permissions(APP_LIMIT_ARRAY_PARAMS_SIZE, [Database::PERMISSION_READ, Database::PERMISSION_UPDATE, Database::PERMISSION_DELETE, Database::PERMISSION_WRITE]), 'An array of permissions strings. By default, only the current user is granted all permissions. [Learn more about permissions](https://appwrite.io/docs/permissions).', true) + ->param('documents', [], fn (array $plan) => new ArrayList(new JSON(), $plan['databasesBatchSize'] ?? APP_LIMIT_DATABASE_BATCH), 'Array of documents data as JSON objects.', true, ['plan']) + ->param('transactionId', null, new UID(), 'Transaction ID for staging the operation.', true) + ->inject('response') + ->inject('dbForProject') + ->inject('getDatabasesDB') + ->inject('user') + ->inject('queueForEvents') + ->inject('queueForStatsUsage') + ->inject('queueForRealtime') + ->inject('queueForFunctions') + ->inject('queueForWebhooks') + ->inject('plan') + ->callback($this->action(...)); + } +} diff --git a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Attribute/Decrement.php b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Embedding/Update.php similarity index 60% rename from src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Attribute/Decrement.php rename to src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Embedding/Update.php index 7c429483e8..23f874d387 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Attribute/Decrement.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Embedding/Update.php @@ -1,23 +1,24 @@ setHttpMethod(self::HTTP_REQUEST_METHOD_PATCH) - ->setHttpPath('/v1/vectordb/:databaseId/collections/:collectionId/documents/:documentId/:attribute/decrement') - ->desc('Decrement document attribute') + ->setHttpPath('/v1/vectordb/:databaseId/collections/:collectionId/documents/:documentId') + ->desc('Update document') ->groups(['api', 'database']) - ->label('event', 'vectordb.[databaseId].collections.[collectionId].documents.[documentId].update') + ->label('event', 'databases.[databaseId].collections.[collectionId].documents.[documentId].update') ->label('scope', 'documents.write') ->label('resourceType', RESOURCE_TYPE_DATABASES) - ->label('audits.event', 'documents.update') - ->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}') + ->label('audits.event', 'document.update') + ->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}/document/{response.$id}') ->label('abuse-key', 'ip:{ip},method:{method},url:{url},userId:{userId}') ->label('abuse-limit', APP_LIMIT_WRITE_RATE_DEFAULT * 2) ->label('abuse-time', APP_LIMIT_WRITE_RATE_PERIOD_DEFAULT) ->label('sdk', new Method( - namespace: 'documentsDB', + namespace: 'vectorDB', group: $this->getSdkGroup(), - name: 'decrementDocumentAttribute', - description: '/docs/references/vectordb/decrement-document-attribute.md', - auth: [AuthType::SESSION, AuthType::JWT, AuthType::ADMIN, AuthType::KEY], + name: 'updateEmbeddingDocument', + description: '/docs/references/vectordb/update-document.md', + auth: [AuthType::SESSION, AuthType::KEY, AuthType::JWT], responses: [ new SDKResponse( code: SwooleResponse::STATUS_CODE_OK, @@ -57,15 +58,16 @@ class Decrement extends DecrementDocumentAttribute ->param('databaseId', '', new UID(), 'Database ID.') ->param('collectionId', '', new UID(), 'Collection ID.') ->param('documentId', '', new UID(), 'Document ID.') - ->param('attribute', '', new Key(), 'Attribute key.') - ->param('value', 1, new Numeric(), 'Value to decrement the attribute by. The value must be a number.', true) - ->param('min', null, new Numeric(), 'Minimum value for the attribute. If the current value is lesser than this value, an exception will be thrown.', true) + ->param('data', [], new JSON(), 'Document data as JSON object. Include only fields and value pairs to be updated.', true) + ->param('permissions', null, new Permissions(APP_LIMIT_ARRAY_PARAMS_SIZE, [Database::PERMISSION_READ, Database::PERMISSION_UPDATE, Database::PERMISSION_DELETE, Database::PERMISSION_WRITE]), 'An array of permissions strings. By default, the current permissions are inherited. [Learn more about permissions](https://appwrite.io/docs/permissions).', true) ->param('transactionId', null, new UID(), 'Transaction ID for staging the operation.', true) + ->inject('requestTimestamp') ->inject('response') ->inject('dbForProject') ->inject('getDatabasesDB') ->inject('queueForEvents') ->inject('queueForStatsUsage') + ->inject('transactionState') ->inject('plan') ->callback($this->action(...)); } diff --git a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Create.php b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Text/Create.php similarity index 98% rename from src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Create.php rename to src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Text/Create.php index d65266690f..4ea9d30849 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Create.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Text/Create.php @@ -1,6 +1,6 @@ setHttpMethod(self::HTTP_REQUEST_METHOD_POST) - ->setHttpPath('/v1/vectordb/:databaseId/collections/:collectionId/documents') + ->setHttpPath('/v1/vectordb/:databaseId/collections/:collectionId/text') ->desc('Create document') ->groups(['api', 'database']) ->label('scope', 'documents.write') @@ -52,7 +52,7 @@ class Create extends DocumentCreate new Method( namespace: 'documentsDB', group: $this->getSdkGroup(), - name: 'createDocument', + name: 'createTextDocument', desc: 'Create document', description: '/docs/references/vectordb/create-document.md', auth: [AuthType::SESSION, AuthType::KEY, AuthType::JWT], diff --git a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Update.php b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Text/Update.php similarity index 100% rename from src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Update.php rename to src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Text/Update.php diff --git a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Update.php b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Update.php index e833da21ea..c236b976ab 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Update.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Update.php @@ -16,7 +16,9 @@ use Utopia\Database\Validator\UID; use Utopia\Swoole\Response as SwooleResponse; use Utopia\Validator\Boolean; use Utopia\Validator\Integer; +use Utopia\Validator\WhiteList; use Utopia\Validator\Text; +use Utopia\Agents\Adapters\Ollama; class Update extends CollectionAction { @@ -61,7 +63,7 @@ class Update extends CollectionAction ->param('name', null, new Text(128), 'Collection name. Max length: 128 chars.') ->param('description', null, new Text(1024), 'Collection description. Max length: 1024 chars.', true) ->param('dimensions', null, new Integer(), 'Embedding dimensions.', true) - ->param('embeddingModel', null, new Text(256), 'Embedding model identifier.', true) + ->param('embeddingModel', '', new WhiteList((new Ollama())->getModels()), 'Embedding model identifier.') ->param('permissions', null, new Permissions(APP_LIMIT_ARRAY_PARAMS_SIZE), 'An array of permission strings. By default, the current permissions are inherited. [Learn more about permissions](https://appwrite.io/docs/permissions).', true) ->param('documentSecurity', false, new Boolean(true), 'Enables configuring permissions for individual documents. A user needs one of document or collection level permissions to access a document. [Learn more about permissions](https://appwrite.io/docs/permissions).', true) ->param('enabled', true, new Boolean(), 'Is collection enabled? When set to \'disabled\', users cannot access the collection but Server SDKs with and API key can still read and write to the collection. No data is lost when this is toggled.', true) diff --git a/src/Appwrite/Platform/Modules/Databases/Services/Registry/VectorDB.php b/src/Appwrite/Platform/Modules/Databases/Services/Registry/VectorDB.php index 9b526f2ca2..f570dbdbc5 100644 --- a/src/Appwrite/Platform/Modules/Databases/Services/Registry/VectorDB.php +++ b/src/Appwrite/Platform/Modules/Databases/Services/Registry/VectorDB.php @@ -16,6 +16,8 @@ use Appwrite\Platform\Modules\Databases\Http\VectorDB\Update as UpdateVectorData use Appwrite\Platform\Modules\Databases\Http\VectorDB\Usage\Get as GetVectorDatabaseUsage; use Appwrite\Platform\Modules\Databases\Http\VectorDB\Usage\XList as ListVectorDatabaseUsage; use Appwrite\Platform\Modules\Databases\Http\VectorDB\XList as ListVectorDatabases; +use Appwrite\Platform\Modules\Databases\Http\VectorDB\Collections\Documents\Embedding\Create as CreateEmbeddingDocument; +use Appwrite\Platform\Modules\Databases\Http\VectorDB\Collections\Documents\Embedding\Update as UpdateEmbeddingDocument; use Utopia\Platform\Service; class VectorDB extends Base @@ -25,7 +27,7 @@ class VectorDB extends Base $this->registerDatabaseActions($service); $this->registerCollectionActions($service); // $this->registerIndexActions($service); - // $this->registerRowActions($service); + $this->registerDocumentActions($service); // $this->registerTransactionActions($service); } @@ -50,4 +52,9 @@ class VectorDB extends Base $service->addAction(ListCollectionLogs::getName(), new ListCollectionLogs()); $service->addAction(GetCollectionUsage::getName(), new GetCollectionUsage()); } + + private function registerDocumentActions(Service $service):void{ + $service->addAction(CreateEmbeddingDocument::getName(), new CreateEmbeddingDocument()); + $service->addAction(UpdateEmbeddingDocument::getName(), new UpdateEmbeddingDocument()); + } } diff --git a/src/Appwrite/Utopia/Response.php b/src/Appwrite/Utopia/Response.php index 533eb273db..6c5c8e2f4c 100644 --- a/src/Appwrite/Utopia/Response.php +++ b/src/Appwrite/Utopia/Response.php @@ -27,6 +27,8 @@ use Appwrite\Utopia\Response\Model\AttributeLine; use Appwrite\Utopia\Response\Model\AttributeList; use Appwrite\Utopia\Response\Model\AttributePoint; use Appwrite\Utopia\Response\Model\AttributePolygon; +use Appwrite\Utopia\Response\Model\AttributeObject; +use Appwrite\Utopia\Response\Model\AttributeVector; use Appwrite\Utopia\Response\Model\AttributeRelationship; use Appwrite\Utopia\Response\Model\AttributeString; use Appwrite\Utopia\Response\Model\AttributeURL; @@ -215,6 +217,8 @@ class Response extends SwooleResponse public const MODEL_ATTRIBUTE_POINT = 'attributePoint'; public const MODEL_ATTRIBUTE_LINE = 'attributeLine'; public const MODEL_ATTRIBUTE_POLYGON = 'attributePolygon'; + public const MODEL_ATTRIBUTE_OBJECT = 'attributeObject'; + public const MODEL_ATTRIBUTE_VECTOR = 'attributeVector'; // Database Columns public const MODEL_COLUMN = 'column'; @@ -511,6 +515,8 @@ class Response extends SwooleResponse ->setModel(new AttributePoint()) ->setModel(new AttributeLine()) ->setModel(new AttributePolygon()) + ->setModel(new AttributeObject()) + ->setModel(new AttributeVector()) // Table API Models ->setModel(new Table()) ->setModel(new Column()) diff --git a/src/Appwrite/Utopia/Response/Model/AttributeObject.php b/src/Appwrite/Utopia/Response/Model/AttributeObject.php new file mode 100644 index 0000000000..14ff818b3c --- /dev/null +++ b/src/Appwrite/Utopia/Response/Model/AttributeObject.php @@ -0,0 +1,29 @@ + 'object', + ]; + + public function getName(): string + { + return 'AttributeObject'; + } + + public function getType(): string + { + return Response::MODEL_ATTRIBUTE_OBJECT; + } +} + + diff --git a/src/Appwrite/Utopia/Response/Model/AttributeVector.php b/src/Appwrite/Utopia/Response/Model/AttributeVector.php new file mode 100644 index 0000000000..56c3317cd0 --- /dev/null +++ b/src/Appwrite/Utopia/Response/Model/AttributeVector.php @@ -0,0 +1,37 @@ +addRule('size', [ + 'type' => self::TYPE_INTEGER, + 'description' => 'Vector dimensions.', + 'default' => 0, + 'example' => 1536, + ]); + } + + public array $conditions = [ + 'type' => 'vector', + ]; + + public function getName(): string + { + return 'AttributeVector'; + } + + public function getType(): string + { + return Response::MODEL_ATTRIBUTE_VECTOR; + } +} + + diff --git a/src/Appwrite/Utopia/Response/Model/VectorDBCollection.php b/src/Appwrite/Utopia/Response/Model/VectorDBCollection.php index b0cab1f4c7..97b2e75224 100644 --- a/src/Appwrite/Utopia/Response/Model/VectorDBCollection.php +++ b/src/Appwrite/Utopia/Response/Model/VectorDBCollection.php @@ -79,19 +79,8 @@ class VectorDBCollection extends Model ]) ->addRule('attributes', [ 'type' => [ - Response::MODEL_ATTRIBUTE_BOOLEAN, - Response::MODEL_ATTRIBUTE_INTEGER, - Response::MODEL_ATTRIBUTE_FLOAT, - Response::MODEL_ATTRIBUTE_EMAIL, - Response::MODEL_ATTRIBUTE_ENUM, - Response::MODEL_ATTRIBUTE_URL, - Response::MODEL_ATTRIBUTE_IP, - Response::MODEL_ATTRIBUTE_DATETIME, - Response::MODEL_ATTRIBUTE_RELATIONSHIP, - Response::MODEL_ATTRIBUTE_POINT, - Response::MODEL_ATTRIBUTE_LINE, - Response::MODEL_ATTRIBUTE_POLYGON, - Response::MODEL_ATTRIBUTE_STRING, + Response::MODEL_ATTRIBUTE_OBJECT, + Response::MODEL_ATTRIBUTE_VECTOR, ], 'description' => 'Collection attributes.', 'default' => [], diff --git a/tests/e2e/Services/Databases/VectorDB/DatabasesBase.php b/tests/e2e/Services/Databases/VectorDB/DatabasesBase.php index 44efbabcbe..fede56e078 100644 --- a/tests/e2e/Services/Databases/VectorDB/DatabasesBase.php +++ b/tests/e2e/Services/Databases/VectorDB/DatabasesBase.php @@ -7,6 +7,7 @@ use Utopia\Database\Database; use Utopia\Database\Helpers\ID; use Utopia\Database\Helpers\Permission; use Utopia\Database\Helpers\Role; +use Utopia\Agents\Adapters\Ollama; trait DatabasesBase { @@ -50,7 +51,7 @@ trait DatabasesBase 'name' => 'Movies', 'documentSecurity' => true, 'dimensions' => 1536, - 'embeddingModel' => 'gemma', + 'embeddingModel' => Ollama::MODEL_EMBEDDING_GEMMA, 'permissions' => [ Permission::create(Role::user($this->getUser()['$id'])), ], @@ -68,7 +69,7 @@ trait DatabasesBase 'name' => 'Actors', 'documentSecurity' => true, 'dimensions' => 1536, - 'embeddingModel' => 'gemma', + 'embeddingModel' => Ollama::MODEL_EMBEDDING_GEMMA, 'permissions' => [ Permission::create(Role::user($this->getUser()['$id'])), ], @@ -166,7 +167,7 @@ trait DatabasesBase /** * Helper to create a collection */ - $createCollection = function (string $databaseId, string $name) use ($projectId, $apiKey, $userId) { + $createCollection = function (string $databaseId, string $name, int $dimensions = 1536, string $embeddingModel = Ollama::MODEL_EMBEDDING_GEMMA) use ($projectId, $apiKey, $userId) { $res = $this->client->call(Client::METHOD_POST, '/vectordb/' . $databaseId . '/collections', [ 'content-type' => 'application/json', 'x-appwrite-project' => $projectId, @@ -175,8 +176,8 @@ trait DatabasesBase 'collectionId' => ID::unique(), 'name' => $name, 'documentSecurity' => true, - 'dimensions' => 1536, - 'embeddingModel' => 'gemma', + 'dimensions' => $dimensions, + 'embeddingModel' => $embeddingModel, 'permissions' => [ Permission::create(Role::user($userId)), ], @@ -212,11 +213,45 @@ trait DatabasesBase $contentCollectionIds[$col] = $createCollection($contentDbId, $col); } + // Create a tiny-dimension collection and insert a document to validate vector and object attributes + $tinyCollectionName = 'VectorsTiny'; + $tinyDimensions = 8; + $tinyCollectionId = $createCollection($mediaDbId, $tinyCollectionName, $tinyDimensions, Ollama::MODEL_EMBEDDING_GEMMA); + + // Build embeddings vector of correct length and metadata object + $embeddings = []; + for ($i = 0; $i < $tinyDimensions; $i++) { + $embeddings[] = (float)($i + 1) / 10.0; + } + $metadata = [ + 'genre' => 'drama', + 'score' => 9, + 'tags' => ['award', 'festival'] + ]; + + $docRes = $this->client->call(Client::METHOD_POST, '/vectordb/' . $mediaDbId . '/collections/' . $tinyCollectionId . '/documents', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $projectId, + 'x-appwrite-key' => $apiKey + ], [ + 'documentId' => ID::unique(), + 'data' => [ + 'embeddings' => $embeddings, + 'metadata' => $metadata, + ], + 'permissions' => [ + Permission::read(Role::user($userId)), + Permission::write(Role::user($userId)), + ], + ]); + + $this->assertEquals(201, $docRes['headers']['status-code']); + return [ 'databases' => [ 'MediaDB' => [ 'id' => $mediaDbId, - 'collections' => $mediaCollectionIds, + 'collections' => $mediaCollectionIds + ['VectorsTiny' => $tinyCollectionId], ], 'ContentDB' => [ 'id' => $contentDbId, From 4988881568c3260d24ba2e485f037b43049d3d97 Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Mon, 3 Nov 2025 11:17:57 +0530 Subject: [PATCH 04/27] updated method names and sdk namespaces for vectordb --- composer.lock | 8 ++++---- .../Databases/Http/VectorDB/Collections/Create.php | 2 +- .../Databases/Http/VectorDB/Collections/Delete.php | 4 ++-- .../Http/VectorDB/Collections/Documents/Bulk/Delete.php | 4 ++-- .../Http/VectorDB/Collections/Documents/Bulk/Update.php | 4 ++-- .../Http/VectorDB/Collections/Documents/Bulk/Upsert.php | 4 ++-- .../Http/VectorDB/Collections/Documents/Delete.php | 4 ++-- .../VectorDB/Collections/Documents/Embedding/Create.php | 4 ++-- .../VectorDB/Collections/Documents/Embedding/Update.php | 2 +- .../Databases/Http/VectorDB/Collections/Documents/Get.php | 4 ++-- .../Http/VectorDB/Collections/Documents/Logs/XList.php | 4 ++-- .../Http/VectorDB/Collections/Documents/Text/Create.php | 6 +++--- .../Http/VectorDB/Collections/Documents/Text/Update.php | 4 ++-- .../Http/VectorDB/Collections/Documents/Upsert.php | 4 ++-- .../Http/VectorDB/Collections/Documents/XList.php | 4 ++-- .../Modules/Databases/Http/VectorDB/Collections/Get.php | 4 ++-- .../Http/VectorDB/Collections/Indexes/Create.php | 4 ++-- .../Http/VectorDB/Collections/Indexes/Delete.php | 4 ++-- .../Databases/Http/VectorDB/Collections/Indexes/Get.php | 4 ++-- .../Databases/Http/VectorDB/Collections/Indexes/XList.php | 4 ++-- .../Databases/Http/VectorDB/Collections/Logs/XList.php | 4 ++-- .../Databases/Http/VectorDB/Collections/Update.php | 4 ++-- .../Databases/Http/VectorDB/Collections/Usage/Get.php | 4 ++-- .../Modules/Databases/Http/VectorDB/Collections/XList.php | 4 ++-- .../Platform/Modules/Databases/Http/VectorDB/Create.php | 4 ++-- .../Platform/Modules/Databases/Http/VectorDB/Delete.php | 4 ++-- .../Platform/Modules/Databases/Http/VectorDB/Get.php | 4 ++-- .../Modules/Databases/Http/VectorDB/Logs/XList.php | 4 ++-- .../Platform/Modules/Databases/Http/VectorDB/Update.php | 4 ++-- .../Modules/Databases/Http/VectorDB/Usage/Get.php | 4 ++-- .../Modules/Databases/Http/VectorDB/Usage/XList.php | 4 ++-- .../Platform/Modules/Databases/Http/VectorDB/XList.php | 4 ++-- 32 files changed, 65 insertions(+), 65 deletions(-) diff --git a/composer.lock b/composer.lock index 3c1ffb73f7..3f077630af 100644 --- a/composer.lock +++ b/composer.lock @@ -3893,12 +3893,12 @@ "source": { "type": "git", "url": "https://github.com/utopia-php/database.git", - "reference": "f5c0cfdc6d8993613ae03d22ebcd3b899fda63aa" + "reference": "9a96110abbec4bd841f3eacc473471a37bebf9be" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/database/zipball/f5c0cfdc6d8993613ae03d22ebcd3b899fda63aa", - "reference": "f5c0cfdc6d8993613ae03d22ebcd3b899fda63aa", + "url": "https://api.github.com/repos/utopia-php/database/zipball/9a96110abbec4bd841f3eacc473471a37bebf9be", + "reference": "9a96110abbec4bd841f3eacc473471a37bebf9be", "shasum": "" }, "require": { @@ -3974,7 +3974,7 @@ "source": "https://github.com/utopia-php/database/tree/var_object", "issues": "https://github.com/utopia-php/database/issues" }, - "time": "2025-10-31T06:11:11+00:00" + "time": "2025-11-03T05:12:58+00:00" }, { "name": "utopia-php/detector", diff --git a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Create.php b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Create.php index 1515ff9421..b9f5b58701 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Create.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Create.php @@ -55,7 +55,7 @@ class Create extends CollectionAction ->label('audits.event', 'collection.create') ->label('audits.resource', 'database/{request.databaseId}/collection/{response.$id}') ->label('sdk', new Method( - namespace: 'documentsDB', + namespace: 'vectorDB', group: 'collections', name: 'createCollection', description: '/docs/references/vectordb/create-collection.md', diff --git a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Delete.php b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Delete.php index 4ba64769b1..5545b8867e 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Delete.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Delete.php @@ -15,7 +15,7 @@ class Delete extends CollectionDelete { public static function getName(): string { - return 'deleteDocumentsDBCollection'; + return 'deleteVectorDBCollection'; } protected function getResponseModel(): string @@ -36,7 +36,7 @@ class Delete extends CollectionDelete ->label('audits.event', 'collection.delete') ->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}') ->label('sdk', new Method( - namespace: 'documentsDB', + namespace: 'vectorDB', group: 'collections', name: 'deleteCollection', description: '/docs/references/vectordb/delete-collection.md', diff --git a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Bulk/Delete.php b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Bulk/Delete.php index acbbe74f5c..54ff25066c 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Bulk/Delete.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Bulk/Delete.php @@ -17,7 +17,7 @@ class Delete extends DocumentsDelete { public static function getName(): string { - return 'deleteDocumentsDBDocuments'; + return 'deleteVectorDBDocuments'; } protected function getResponseModel(): string @@ -40,7 +40,7 @@ class Delete extends DocumentsDelete ->label('abuse-limit', APP_LIMIT_WRITE_RATE_DEFAULT) ->label('abuse-time', APP_LIMIT_WRITE_RATE_PERIOD_DEFAULT) ->label('sdk', new Method( - namespace: 'documentsDB', + namespace: 'vectorDB', group: $this->getSdkGroup(), name: 'deleteDocuments', description: '/docs/references/vectordb/delete-documents.md', diff --git a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Bulk/Update.php b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Bulk/Update.php index 55dc18e84a..fad7adbf34 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Bulk/Update.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Bulk/Update.php @@ -18,7 +18,7 @@ class Update extends DocumentsUpdate { public static function getName(): string { - return 'updateDocumentsDBDocuments'; + return 'updateVectorDBDocuments'; } protected function getResponseModel(): string @@ -41,7 +41,7 @@ class Update extends DocumentsUpdate ->label('abuse-limit', APP_LIMIT_WRITE_RATE_DEFAULT * 2) ->label('abuse-time', APP_LIMIT_WRITE_RATE_PERIOD_DEFAULT) ->label('sdk', new Method( - namespace: 'documentsDB', + namespace: 'vectorDB', group: $this->getSdkGroup(), name: 'updateDocuments', description: '/docs/references/vectordb/update-documents.md', diff --git a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Bulk/Upsert.php b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Bulk/Upsert.php index d2206de4da..f7d16e4d60 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Bulk/Upsert.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Bulk/Upsert.php @@ -17,7 +17,7 @@ class Upsert extends DocumentsUpsert { public static function getName(): string { - return 'upsertDocumentsDBDocuments'; + return 'upsertVectorDBDocuments'; } protected function getResponseModel(): string @@ -41,7 +41,7 @@ class Upsert extends DocumentsUpsert ->label('abuse-time', APP_LIMIT_WRITE_RATE_PERIOD_DEFAULT) ->label('sdk', [ new Method( - namespace: 'documentsDB', + namespace: 'vectorDB', group: $this->getSdkGroup(), name: 'upsertDocuments', description: '/docs/references/vectordb/upsert-documents.md', diff --git a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Delete.php b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Delete.php index 004d10578d..2464f26f91 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Delete.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Delete.php @@ -15,7 +15,7 @@ class Delete extends DocumentDelete { public static function getName(): string { - return 'deleteDocumentsDBDocument'; + return 'deleteVectorDBDocument'; } /** @@ -45,7 +45,7 @@ class Delete extends DocumentDelete ->label('abuse-limit', APP_LIMIT_WRITE_RATE_DEFAULT) ->label('abuse-time', APP_LIMIT_WRITE_RATE_PERIOD_DEFAULT) ->label('sdk', new Method( - namespace: 'documentsDB', + namespace: 'vectorDB', group: $this->getSdkGroup(), name: 'deleteDocument', description: '/docs/references/vectordb/delete-document.md', diff --git a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Embedding/Create.php b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Embedding/Create.php index 10ae664299..2e99a402e6 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Embedding/Create.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Embedding/Create.php @@ -52,7 +52,7 @@ class Create extends DocumentCreate new Method( namespace: 'vectorDB', group: $this->getSdkGroup(), - name: 'createEmbeddingDocument', + name: 'createDocument', desc: 'Create document', description: '/docs/references/vectordb/create-document.md', auth: [AuthType::SESSION, AuthType::KEY, AuthType::JWT], @@ -72,7 +72,7 @@ class Create extends DocumentCreate ] ), new Method( - namespace: 'documentsDB', + namespace: 'vectorDB', group: $this->getSdkGroup(), name: 'createDocuments', desc: 'Create documents', diff --git a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Embedding/Update.php b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Embedding/Update.php index 23f874d387..50afe8077c 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Embedding/Update.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Embedding/Update.php @@ -44,7 +44,7 @@ class Update extends DocumentUpdate ->label('sdk', new Method( namespace: 'vectorDB', group: $this->getSdkGroup(), - name: 'updateEmbeddingDocument', + name: 'updateDocument', description: '/docs/references/vectordb/update-document.md', auth: [AuthType::SESSION, AuthType::KEY, AuthType::JWT], responses: [ diff --git a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Get.php b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Get.php index d512a06de2..8a1de28148 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Get.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Get.php @@ -17,7 +17,7 @@ class Get extends DocumentGet { public static function getName(): string { - return 'getDocumentsDBDocument'; + return 'getVectorDBDocument'; } protected function getResponseModel(): string @@ -35,7 +35,7 @@ class Get extends DocumentGet ->label('scope', 'documents.read') ->label('resourceType', RESOURCE_TYPE_DATABASES) ->label('sdk', new Method( - namespace: 'documentsDB', + namespace: 'vectorDB', group: $this->getSdkGroup(), name: 'getDocument', description: '/docs/references/vectordb/get-document.md', diff --git a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Logs/XList.php b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Logs/XList.php index 97b08662c7..56ae0d68ab 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Logs/XList.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Logs/XList.php @@ -17,7 +17,7 @@ class XList extends DocumentLogXList { public static function getName(): string { - return 'listDocumentsDBDocumentLogs'; + return 'listVectorDBDocumentLogs'; } public function __construct() @@ -30,7 +30,7 @@ class XList extends DocumentLogXList ->label('scope', 'documents.read') ->label('resourceType', RESOURCE_TYPE_DATABASES) ->label('sdk', new Method( - namespace: 'documentsDB', + namespace: 'vectorDB', group: 'logs', name: 'listDocumentLogs', description: '/docs/references/vectordb/get-document-logs.md', diff --git a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Text/Create.php b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Text/Create.php index 4ea9d30849..b5a8eec650 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Text/Create.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Text/Create.php @@ -21,7 +21,7 @@ class Create extends DocumentCreate { public static function getName(): string { - return 'createDocumentsDBDocument'; + return 'createVectorDBDocument'; } protected function getResponseModel(): string @@ -50,7 +50,7 @@ class Create extends DocumentCreate ->label('abuse-time', APP_LIMIT_WRITE_RATE_PERIOD_DEFAULT) ->label('sdk', [ new Method( - namespace: 'documentsDB', + namespace: 'vectorDB', group: $this->getSdkGroup(), name: 'createTextDocument', desc: 'Create document', @@ -72,7 +72,7 @@ class Create extends DocumentCreate ] ), new Method( - namespace: 'documentsDB', + namespace: 'vectorDB', group: $this->getSdkGroup(), name: 'createDocuments', desc: 'Create documents', diff --git a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Text/Update.php b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Text/Update.php index 709e20262b..b7b06168ff 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Text/Update.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Text/Update.php @@ -18,7 +18,7 @@ class Update extends DocumentUpdate { public static function getName(): string { - return 'updateDocumentsDBDocument'; + return 'updateVectorDBDocument'; } protected function getResponseModel(): string @@ -42,7 +42,7 @@ class Update extends DocumentUpdate ->label('abuse-limit', APP_LIMIT_WRITE_RATE_DEFAULT * 2) ->label('abuse-time', APP_LIMIT_WRITE_RATE_PERIOD_DEFAULT) ->label('sdk', new Method( - namespace: 'documentsDB', + namespace: 'vectorDB', group: $this->getSdkGroup(), name: 'updateDocument', description: '/docs/references/vectordb/update-document.md', diff --git a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Upsert.php b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Upsert.php index 45c5a2b202..6cbd86cba6 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Upsert.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Upsert.php @@ -18,7 +18,7 @@ class Upsert extends DocumentUpsert { public static function getName(): string { - return 'upsertDocumentsDBDocument'; + return 'upsertVectorDBDocument'; } protected function getResponseModel(): string @@ -43,7 +43,7 @@ class Upsert extends DocumentUpsert ->label('abuse-time', APP_LIMIT_WRITE_RATE_PERIOD_DEFAULT) ->label('sdk', [ new Method( - namespace: 'documentsDB', + namespace: 'vectorDB', group: $this->getSdkGroup(), name: 'upsertDocument', description: '/docs/references/vectordb/upsert-document.md', diff --git a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/XList.php b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/XList.php index 061fc71fea..dfc20a860f 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/XList.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/XList.php @@ -17,7 +17,7 @@ class XList extends DocumentXList { public static function getName(): string { - return 'listDocumentsDBDocuments'; + return 'listVectorDBDocuments'; } protected function getResponseModel(): string @@ -35,7 +35,7 @@ class XList extends DocumentXList ->label('scope', 'documents.read') ->label('resourceType', RESOURCE_TYPE_DATABASES) ->label('sdk', new Method( - namespace: 'documentsDB', + namespace: 'vectorDB', group: $this->getSdkGroup(), name: 'listDocuments', description: '/docs/references/vectordb/list-documents.md', diff --git a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Get.php b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Get.php index 0ee2ba0183..83bde1bf5f 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Get.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Get.php @@ -15,7 +15,7 @@ class Get extends CollectionGet { public static function getName(): string { - return 'getDocumentsDBCollection'; + return 'getVectorDBCollection'; } protected function getResponseModel(): string @@ -33,7 +33,7 @@ class Get extends CollectionGet ->label('scope', 'collections.read') ->label('resourceType', RESOURCE_TYPE_DATABASES) ->label('sdk', new Method( - namespace: 'documentsDB', + namespace: 'vectorDB', group: 'collections', name: 'getCollection', description: '/docs/references/vectordb/get-collection.md', diff --git a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Indexes/Create.php b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Indexes/Create.php index 8928fa73af..e68ad1fe24 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Indexes/Create.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Indexes/Create.php @@ -21,7 +21,7 @@ class Create extends IndexCreate { public static function getName(): string { - return 'createDocumentsDBIndex'; + return 'createVectorDBIndex'; } protected function getResponseModel(): string @@ -42,7 +42,7 @@ class Create extends IndexCreate ->label('audits.event', 'index.create') ->label('audits.resource', 'database/{request.databaseId}/collection/{request.tableId}') ->label('sdk', new Method( - namespace: 'documentsDB', + namespace: 'vectorDB', group: $this->getSdkGroup(), name: 'createIndex', description: '/docs/references/vectordb/create-index.md', diff --git a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Indexes/Delete.php b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Indexes/Delete.php index 3b8fb71de8..c6ea3da7fe 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Indexes/Delete.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Indexes/Delete.php @@ -16,7 +16,7 @@ class Delete extends IndexDelete { public static function getName(): string { - return 'deleteDocumentsDBIndex'; + return 'deleteVectorDBIndex'; } /** @@ -41,7 +41,7 @@ class Delete extends IndexDelete ->label('audits.event', 'index.delete') ->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}') ->label('sdk', new Method( - namespace: 'documentsDB', + namespace: 'vectorDB', group: $this->getSdkGroup(), name: 'deleteIndex', // getName needs to be different from parent action to avoid conflict in path name description: '/docs/references/vectordb/delete-index.md', diff --git a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Indexes/Get.php b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Indexes/Get.php index bf90b8598f..b1dedefe2e 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Indexes/Get.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Indexes/Get.php @@ -16,7 +16,7 @@ class Get extends IndexGet { public static function getName(): string { - return 'getDocumentsDBIndex'; + return 'getVectorDBIndex'; } protected function getResponseModel(): string @@ -34,7 +34,7 @@ class Get extends IndexGet ->label('scope', 'collections.read') ->label('resourceType', RESOURCE_TYPE_DATABASES) ->label('sdk', new Method( - namespace: 'documentsDB', + namespace: 'vectorDB', group: $this->getSdkGroup(), name: 'getIndex', // getName needs to be different from parent action to avoid conflict in path name description: '/docs/references/vectordb/get-index.md', diff --git a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Indexes/XList.php b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Indexes/XList.php index 2c239fdaef..f081fe23ee 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Indexes/XList.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Indexes/XList.php @@ -16,7 +16,7 @@ class XList extends IndexXList { public static function getName(): string { - return 'listDocumentsDBIndexes'; + return 'listVectorDBIndexes'; } protected function getResponseModel(): string @@ -34,7 +34,7 @@ class XList extends IndexXList ->label('scope', 'collections.read') ->label('resourceType', RESOURCE_TYPE_DATABASES) ->label('sdk', new Method( - namespace: 'documentsDB', + namespace: 'vectorDB', group: $this->getSdkGroup(), name: 'listIndexes', // getName needs to be different from parent action to avoid conflict in path name description: '/docs/references/vectordb/list-indexes.md', diff --git a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Logs/XList.php b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Logs/XList.php index e0e1babb5c..931620edb8 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Logs/XList.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Logs/XList.php @@ -17,7 +17,7 @@ class XList extends CollectionLogXList { public static function getName(): string { - return 'listDocumentsDBCollectionLogs'; + return 'listVectorDBCollectionLogs'; } public function __construct() @@ -30,7 +30,7 @@ class XList extends CollectionLogXList ->label('scope', 'collections.read') ->label('resourceType', RESOURCE_TYPE_DATABASES) ->label('sdk', new Method( - namespace: 'documentsDB', + namespace: 'vectorDB', group: $this->getSdkGroup(), name: 'listCollectionLogs', description: '/docs/references/vectordb/get-collection-logs.md', diff --git a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Update.php b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Update.php index c236b976ab..222c5bcaa4 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Update.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Update.php @@ -24,7 +24,7 @@ class Update extends CollectionAction { public static function getName(): string { - return 'updateDocumentsDBCollection'; + return 'updateVectorDBCollection'; } protected function getResponseModel(): string @@ -45,7 +45,7 @@ class Update extends CollectionAction ->label('audits.event', 'collection.update') ->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}') ->label('sdk', new Method( - namespace: 'documentsDB', + namespace: 'vectorDB', group: 'collections', name: 'updateCollection', description: '/docs/references/vectordb/update-collection.md', diff --git a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Usage/Get.php b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Usage/Get.php index dbbad02f6f..2e26e9d6b5 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Usage/Get.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Usage/Get.php @@ -16,7 +16,7 @@ class Get extends CollectionUsageGet { public static function getName(): string { - return 'getDocumentsDBCollectionUsage'; + return 'getVectorDBCollectionUsage'; } protected function getResponseModel(): string @@ -34,7 +34,7 @@ class Get extends CollectionUsageGet ->label('scope', 'collections.read') ->label('resourceType', RESOURCE_TYPE_DATABASES) ->label('sdk', new Method( - namespace: 'documentsDB', + namespace: 'vectorDB', group: null, name: 'getCollectionUsage', description: '/docs/references/vectordb/get-collection-usage.md', diff --git a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/XList.php b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/XList.php index 9c7ea55c27..dc0e73f7f1 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/XList.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/XList.php @@ -17,7 +17,7 @@ class XList extends CollectionXList { public static function getName(): string { - return 'listDocumentsDBCollections'; + return 'listVectorDBCollections'; } protected function getResponseModel(): string @@ -35,7 +35,7 @@ class XList extends CollectionXList ->label('scope', 'collections.read') ->label('resourceType', RESOURCE_TYPE_DATABASES) ->label('sdk', new Method( - namespace: 'documentsDB', + namespace: 'vectorDB', group: 'collections', name: 'listCollections', description: '/docs/references/vectordb/list-collections.md', diff --git a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Create.php b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Create.php index 4ee993a864..978f565cec 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Create.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Create.php @@ -17,7 +17,7 @@ class Create extends DatabaseCreate { public static function getName(): string { - return 'createDocumentsDBDatabase'; + return 'createVectorDBDatabase'; } public function __construct() @@ -33,7 +33,7 @@ class Create extends DatabaseCreate ->label('audits.event', 'database.create') ->label('audits.resource', 'database/{response.$id}') ->label('sdk', new Method( - namespace: 'documentsDB', + namespace: 'vectorDB', group: 'vectordb', name: 'create', description: '/docs/references/vectordb/create.md', diff --git a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Delete.php b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Delete.php index 717728c599..24b0b6b67b 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Delete.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Delete.php @@ -15,7 +15,7 @@ class Delete extends DatabaseDelete { public static function getName(): string { - return 'deleteDocumentsDBDatabase'; + return 'deleteVectorDBDatabase'; } public function __construct() @@ -31,7 +31,7 @@ class Delete extends DatabaseDelete ->label('audits.event', 'database.delete') ->label('audits.resource', 'database/{request.databaseId}') ->label('sdk', new Method( - namespace: 'documentsDB', + namespace: 'vectorDB', group: 'vectordb', name: 'delete', description: '/docs/references/vectordb/delete.md', diff --git a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Get.php b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Get.php index 3b49c34e75..18c55b7fe8 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Get.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Get.php @@ -15,7 +15,7 @@ class Get extends DatabaseGet { public static function getName(): string { - return 'getDocumentsDBDatabase'; + return 'getVectorDBDatabase'; } public function __construct() @@ -28,7 +28,7 @@ class Get extends DatabaseGet ->label('scope', 'databases.read') ->label('resourceType', RESOURCE_TYPE_DATABASES) ->label('sdk', new Method( - namespace: 'documentsDB', + namespace: 'vectorDB', group: 'vectordb', name: 'get', description: '/docs/references/vectordb/get.md', diff --git a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Logs/XList.php b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Logs/XList.php index f38e0ad39c..6232c4892b 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Logs/XList.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Logs/XList.php @@ -18,7 +18,7 @@ class XList extends DatabaseLogs { public static function getName(): string { - return 'listDocumentsDBLogs'; + return 'listVectorDBLogs'; } public function __construct() @@ -32,7 +32,7 @@ class XList extends DatabaseLogs ->label('resourceType', RESOURCE_TYPE_DATABASES) ->label('sdk', [ new Method( - namespace: 'documentsDB', + namespace: 'vectorDB', group: 'logs', name: 'listDatabaseLogs', description: '/docs/references/vectordb/get-logs.md', diff --git a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Update.php b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Update.php index 7863e98043..71be301a4d 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Update.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Update.php @@ -17,7 +17,7 @@ class Update extends DatabaseUpdate { public static function getName(): string { - return 'updateDocumentsDBDatabase'; + return 'updateVectorDBDatabase'; } public function __construct() @@ -33,7 +33,7 @@ class Update extends DatabaseUpdate ->label('audits.event', 'database.update') ->label('audits.resource', 'database/{response.$id}') ->label('sdk', new Method( - namespace: 'documentsDB', + namespace: 'vectorDB', group: 'vectordb', name: 'update', description: '/docs/references/vectordb/update.md', diff --git a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Usage/Get.php b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Usage/Get.php index 4590405aad..075837b9cb 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Usage/Get.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Usage/Get.php @@ -16,7 +16,7 @@ class Get extends DatabaseUsageGet { public static function getName(): string { - return 'getDocumentsDBUsage'; + return 'getVectorDBUsage'; } public function __construct() @@ -30,7 +30,7 @@ class Get extends DatabaseUsageGet ->label('resourceType', RESOURCE_TYPE_DATABASES) ->label('sdk', [ new Method( - namespace: 'documentsDB', + namespace: 'vectorDB', group: null, name: 'getUsage', description: '/docs/references/vectordb/get-database-usage.md', diff --git a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Usage/XList.php b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Usage/XList.php index 4ba298129c..fe6ed3622b 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Usage/XList.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Usage/XList.php @@ -15,7 +15,7 @@ class XList extends DatabaseUsageXList { public static function getName(): string { - return 'listDocumentsDBUsage'; + return 'listVectorDBUsage'; } public function __construct() @@ -29,7 +29,7 @@ class XList extends DatabaseUsageXList ->label('resourceType', RESOURCE_TYPE_DATABASES) ->label('sdk', [ new Method( - namespace: 'documentsDB', + namespace: 'vectorDB', group: null, name: 'listUsage', description: '/docs/references/vectordb/list-usage.md', diff --git a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/XList.php b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/XList.php index e37c05ad88..1826c26b74 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/XList.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/XList.php @@ -16,7 +16,7 @@ class XList extends DatabaseXList { public static function getName(): string { - return 'listDocumentsDBDatabases'; + return 'listVectorDBDatabases'; } public function __construct() @@ -29,7 +29,7 @@ class XList extends DatabaseXList ->label('scope', 'databases.read') ->label('resourceType', RESOURCE_TYPE_DATABASES) ->label('sdk', new Method( - namespace: 'documentsDB', + namespace: 'vectorDB', group: 'vectordb', name: 'list', description: '/docs/references/vectordb/list.md', From d2275c05fbd63a1ab544d50dffc79c8a041b5adc Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Wed, 5 Nov 2025 20:12:37 +0530 Subject: [PATCH 05/27] updated tests --- app/config/collections/vectordb.php | 9 - app/controllers/general.php | 9 + app/init/resources.php | 7 +- composer.json | 4 + docker-compose.yml | 5 +- .../Http/VectorDB/Collections/Create.php | 43 +- .../Http/VectorDB/Collections/Delete.php | 2 +- .../Documents/{Embedding => }/Create.php | 2 +- .../Documents/Embedding/Update.php | 74 -- .../Collections/Documents/Text/Create.php | 131 ++- .../Documents/{Text => }/Update.php | 0 .../VectorDB/Collections/Documents/Upsert.php | 3 +- .../VectorDB/Collections/Documents/XList.php | 2 + .../VectorDB/Collections/Indexes/Create.php | 2 +- .../VectorDB/Collections/Indexes/XList.php | 2 + .../Http/VectorDB/Collections/Update.php | 12 +- .../Http/VectorDB/Collections/XList.php | 2 + .../Modules/Databases/Http/VectorDB/XList.php | 2 + .../Databases/Services/Registry/VectorDB.php | 45 +- src/Appwrite/Utopia/Response.php | 14 +- .../Utopia/Response/Model/AttributeObject.php | 2 - .../Utopia/Response/Model/AttributeVector.php | 2 - .../Utopia/Response/Model/Embedding.php | 39 + .../Response/Model/VectorDBCollection.php | 6 - .../Databases/VectorDB/DatabasesBase.php | 967 +++++++++++++++++- .../VectorDB/DatabasesConsoleClientTest.php | 336 ++++++ .../VectorDB/DatabasesCustomClientTest.php | 191 ++++ .../VectorDB/DatabasesCustomServerTest.php | 886 ++++++++++++++++ .../DatabasesPermissionsGuestTest.php | 268 +++++ .../DatabasesPermissionsMemberTest.php | 254 +++++ .../Permissions/DatabasesPermissionsScope.php | 87 ++ .../DatabasesPermissionsTeamTest.php | 199 ++++ 32 files changed, 3399 insertions(+), 208 deletions(-) rename src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/{Embedding => }/Create.php (99%) delete mode 100644 src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Embedding/Update.php rename src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/{Text => }/Update.php (100%) create mode 100644 src/Appwrite/Utopia/Response/Model/Embedding.php create mode 100644 tests/e2e/Services/Databases/VectorDB/DatabasesConsoleClientTest.php create mode 100644 tests/e2e/Services/Databases/VectorDB/DatabasesCustomServerTest.php create mode 100644 tests/e2e/Services/Databases/VectorDB/Permissions/DatabasesPermissionsGuestTest.php create mode 100644 tests/e2e/Services/Databases/VectorDB/Permissions/DatabasesPermissionsMemberTest.php create mode 100644 tests/e2e/Services/Databases/VectorDB/Permissions/DatabasesPermissionsScope.php create mode 100644 tests/e2e/Services/Databases/VectorDB/Permissions/DatabasesPermissionsTeamTest.php diff --git a/app/config/collections/vectordb.php b/app/config/collections/vectordb.php index ff091d74c2..c7d87b3509 100644 --- a/app/config/collections/vectordb.php +++ b/app/config/collections/vectordb.php @@ -49,15 +49,6 @@ return [ 'array' => false, 'filters' => [], ], - [ - '$id' => ID::custom('embeddingModel'), - 'type' => Database::VAR_STRING, - 'size' => 256, - 'required' => true, - 'signed' => true, - 'array' => false, - 'filters' => [], - ], [ '$id' => ID::custom('enabled'), 'type' => Database::VAR_BOOLEAN, diff --git a/app/controllers/general.php b/app/controllers/general.php index 07de95a38f..a293b65a31 100644 --- a/app/controllers/general.php +++ b/app/controllers/general.php @@ -1183,6 +1183,15 @@ App::error() $file = $error->getFile(); $line = $error->getLine(); $trace = $error->getTrace(); + $trace = $error->getTrace(); + + foreach (array_slice($trace, 0, 100) as $index => $traceEntry) { + $file = isset($traceEntry['file']) ? $traceEntry['file'] : '[internal function]'; + $line = isset($traceEntry['line']) ? $traceEntry['line'] : ''; + $function = isset($traceEntry['function']) ? $traceEntry['function'] : ''; + Console::error("[$index] $file : $line -> $function()"); + } + if (php_sapi_name() === 'cli') { Console::error('[Error] Timestamp: ' . date('c', time())); diff --git a/app/init/resources.php b/app/init/resources.php index 7ff41a996c..adf6ea9620 100644 --- a/app/init/resources.php +++ b/app/init/resources.php @@ -26,6 +26,8 @@ use Appwrite\Network\Validator\Origin; use Appwrite\Utopia\Request; use Executor\Executor; use Utopia\Abuse\Adapters\TimeLimit\Redis as TimeLimitRedis; +use Utopia\Agents\Adapters\Ollama; +use Utopia\Agents\Agent; use Utopia\App; use Utopia\Cache\Adapter\Pool as CachePool; use Utopia\Cache\Adapter\Sharding; @@ -60,8 +62,6 @@ use Utopia\Telemetry\Adapter\None as NoTelemetry; use Utopia\Validator\Hostname; use Utopia\Validator\WhiteList; use Utopia\VCS\Adapter\Git\GitHub as VcsGitHub; -use Utopia\Agents\Agent; -use Utopia\Agents\Adapters\Ollama; // Runtime Execution App::setResource('log', fn () => new Log()); @@ -1173,5 +1173,6 @@ App::setResource('transactionState', function (Database $dbForProject, callable App::setResource('embeddingAgent', function () { // TODO: ollama endpoint should be taken from env in cloud(for autoscaling) $adapter = new Ollama(); + $adapter->setEndpoint('http://ollama:11434/api/embed'); return new Agent($adapter); -}); \ No newline at end of file +}); diff --git a/composer.json b/composer.json index badbe4972f..c1e712f37f 100644 --- a/composer.json +++ b/composer.json @@ -34,6 +34,10 @@ { "type": "vcs", "url": "https://github.com/utopia-php/database" + }, + { + "type": "vcs", + "url": "https://github.com/utopia-php/agents" } ], "minimum-stability": "dev", diff --git a/docker-compose.yml b/docker-compose.yml index 8c1b889d9b..4e8b58086e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -89,7 +89,7 @@ services: - ./public:/usr/src/code/public - ./src:/usr/src/code/src - ./dev:/usr/src/code/dev - - .vendor:/usr/src/code/dev + - ./vendor:/usr/src/code/vendor depends_on: - ${_APP_DB_HOST:-mongodb} @@ -451,6 +451,7 @@ services: volumes: - ./app:/usr/src/code/app - ./src:/usr/src/code/src + - ./vendor:/usr/src/code/vendor depends_on: - redis - ${_APP_DB_HOST:-mongodb} @@ -1208,6 +1209,8 @@ services: - MODELS=${OLLAMA_MODELS:-embeddinggemma} # duration to keep model in memory - OLLAMA_KEEP_ALIVE=24h + networks: + - appwrite redis: image: redis:7.2.4-alpine diff --git a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Create.php b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Create.php index b9f5b58701..b319c28c4a 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Create.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Create.php @@ -25,10 +25,8 @@ use Utopia\Database\Validator\Permissions; use Utopia\Database\Validator\UID; use Utopia\Swoole\Response as SwooleResponse; use Utopia\Validator\Boolean; -use Utopia\Validator\Integer; +use Utopia\Validator\Range; use Utopia\Validator\Text; -use Utopia\Validator\WhiteList; -use Utopia\Agents\Adapters\Ollama; class Create extends CollectionAction { @@ -71,8 +69,7 @@ class Create extends CollectionAction ->param('databaseId', '', fn (Database $dbForProject) => new UID($dbForProject->getAdapter()->getMaxUIDLength()), 'Database ID.', false, ['dbForProject']) ->param('collectionId', '', fn (Database $dbForProject) => new CustomId(false, $dbForProject->getAdapter()->getMaxUIDLength()), 'Unique Id. Choose a custom ID or generate a random ID with `ID.unique()`. Valid chars are a-z, A-Z, 0-9, period, hyphen, and underscore. Can\'t start with a special char. Max length is 36 chars.', false, ['dbForProject']) ->param('name', '', new Text(128), 'Collection name. Max length: 128 chars.') - ->param('dimensions', 0, new Integer(), 'Embedding dimensions.') - ->param('embeddingModel', '', new WhiteList((new Ollama())->getModels()), 'Embedding model identifier.') + ->param('dimensions', null, new Range(1, 16000), 'Embedding dimensions.') ->param('permissions', null, new Permissions(APP_LIMIT_ARRAY_PARAMS_SIZE), 'An array of permissions strings. By default, no user is granted with any permissions. [Learn more about permissions](https://appwrite.io/docs/permissions).', true) ->param('documentSecurity', false, new Boolean(true), 'Enables configuring permissions for individual documents. A user needs one of document or collection level permissions to access a document. [Learn more about permissions](https://appwrite.io/docs/permissions).', true) ->param('enabled', true, new Boolean(), 'Is collection enabled? When set to \'disabled\', users cannot access the collection but Server SDKs with and API key can still read and write to the collection. No data is lost when this is toggled.', true) @@ -83,7 +80,7 @@ class Create extends CollectionAction ->callback($this->action(...)); } - public function action(string $databaseId, string $collectionId, string $name, int $dimensions, string $embeddingModel, ?array $permissions, bool $documentSecurity, bool $enabled, UtopiaResponse $response, Database $dbForProject, callable $getDatabasesDB, Event $queueForEvents): void + public function action(string $databaseId, string $collectionId, string $name, int $dimensions, ?array $permissions, bool $documentSecurity, bool $enabled, UtopiaResponse $response, Database $dbForProject, callable $getDatabasesDB, Event $queueForEvents): void { $database = Authorization::skip(fn () => $dbForProject->getDocument('databases', $databaseId)); @@ -106,7 +103,6 @@ class Create extends CollectionAction 'enabled' => $enabled, 'name' => $name, 'dimensions' => $dimensions, - 'embeddingModel' => $embeddingModel, 'search' => \implode(' ', [$collectionId, $name]), ])); @@ -152,6 +148,39 @@ class Create extends CollectionAction throw new Exception($this->getLimitException()); } + // Create attribute metadata documents in the attributes table + // This is necessary so that indexes can find the attributes when they're created + foreach ($collections['defaultAttributes'] as $attributeConfig) { + $key = \is_string($attributeConfig['$id']) ? $attributeConfig['$id'] : (string)$attributeConfig['$id']; + $size = $key === 'embeddings' ? $dimensions : ($attributeConfig['size'] ?? 0); + + try { + $attributeDoc = new Document([ + '$id' => ID::custom($database->getSequence() . '_' . $collection->getSequence() . '_' . $key), + 'key' => $key, + 'databaseInternalId' => $database->getSequence(), + 'databaseId' => $databaseId, + 'collectionInternalId' => $collection->getSequence(), + 'collectionId' => $collectionId, + 'type' => $attributeConfig['type'], + 'status' => 'available', + 'size' => $size, + 'required' => $attributeConfig['required'] ?? false, + 'signed' => $attributeConfig['signed'] ?? false, + 'default' => $attributeConfig['default'] ?? null, + 'array' => $attributeConfig['array'] ?? false, + 'format' => $attributeConfig['format'] ?? '', + 'formatOptions' => $attributeConfig['formatOptions'] ?? [], + 'filters' => $attributeConfig['filters'] ?? [], + 'options' => $attributeConfig['options'] ?? [], + ]); + + $dbForProject->createDocument('attributes', $attributeDoc); + } catch (DuplicateException) { + // Attribute already exists, skip + } + } + $queueForEvents ->setContext('database', $database) ->setParam('databaseId', $databaseId) diff --git a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Delete.php b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Delete.php index 5545b8867e..1d89cb0f19 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Delete.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Delete.php @@ -20,7 +20,7 @@ class Delete extends CollectionDelete protected function getResponseModel(): string { - return UtopiaResponse::MODEL_COLLECTION; + return UtopiaResponse::MODEL_VECTORDB_COLLECTION; } public function __construct() diff --git a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Embedding/Create.php b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Create.php similarity index 99% rename from src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Embedding/Create.php rename to src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Create.php index 2e99a402e6..6c331e40dd 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Embedding/Create.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Create.php @@ -1,6 +1,6 @@ setHttpMethod(self::HTTP_REQUEST_METHOD_PATCH) - ->setHttpPath('/v1/vectordb/:databaseId/collections/:collectionId/documents/:documentId') - ->desc('Update document') - ->groups(['api', 'database']) - ->label('event', 'databases.[databaseId].collections.[collectionId].documents.[documentId].update') - ->label('scope', 'documents.write') - ->label('resourceType', RESOURCE_TYPE_DATABASES) - ->label('audits.event', 'document.update') - ->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}/document/{response.$id}') - ->label('abuse-key', 'ip:{ip},method:{method},url:{url},userId:{userId}') - ->label('abuse-limit', APP_LIMIT_WRITE_RATE_DEFAULT * 2) - ->label('abuse-time', APP_LIMIT_WRITE_RATE_PERIOD_DEFAULT) - ->label('sdk', new Method( - namespace: 'vectorDB', - group: $this->getSdkGroup(), - name: 'updateDocument', - description: '/docs/references/vectordb/update-document.md', - auth: [AuthType::SESSION, AuthType::KEY, AuthType::JWT], - responses: [ - new SDKResponse( - code: SwooleResponse::STATUS_CODE_OK, - model: $this->getResponseModel(), - ) - ], - contentType: ContentType::JSON - )) - ->param('databaseId', '', new UID(), 'Database ID.') - ->param('collectionId', '', new UID(), 'Collection ID.') - ->param('documentId', '', new UID(), 'Document ID.') - ->param('data', [], new JSON(), 'Document data as JSON object. Include only fields and value pairs to be updated.', true) - ->param('permissions', null, new Permissions(APP_LIMIT_ARRAY_PARAMS_SIZE, [Database::PERMISSION_READ, Database::PERMISSION_UPDATE, Database::PERMISSION_DELETE, Database::PERMISSION_WRITE]), 'An array of permissions strings. By default, the current permissions are inherited. [Learn more about permissions](https://appwrite.io/docs/permissions).', true) - ->param('transactionId', null, new UID(), 'Transaction ID for staging the operation.', true) - ->inject('requestTimestamp') - ->inject('response') - ->inject('dbForProject') - ->inject('getDatabasesDB') - ->inject('queueForEvents') - ->inject('queueForStatsUsage') - ->inject('transactionState') - ->inject('plan') - ->callback($this->action(...)); - } -} diff --git a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Text/Create.php b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Text/Create.php index b5a8eec650..1896e91b8d 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Text/Create.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Text/Create.php @@ -2,22 +2,26 @@ namespace Appwrite\Platform\Modules\Databases\Http\VectorDB\Collections\Documents\Text; -use Appwrite\Platform\Modules\Databases\Http\Databases\Collections\Documents\Create as DocumentCreate; +use Appwrite\Auth\Auth; +use Appwrite\Extend\Exception; +use Appwrite\Platform\Modules\Databases\Http\Databases\Collections\Documents\Action as CreateDocumentAction; use Appwrite\SDK\AuthType; use Appwrite\SDK\ContentType; use Appwrite\SDK\Method; use Appwrite\SDK\Parameter; use Appwrite\SDK\Response as SDKResponse; -use Appwrite\Utopia\Database\Validator\CustomId; use Appwrite\Utopia\Response as UtopiaResponse; use Utopia\Database\Database; -use Utopia\Database\Validator\Permissions; +use Utopia\Database\Document; +use Utopia\Database\Validator\Authorization; use Utopia\Database\Validator\UID; use Utopia\Swoole\Response as SwooleResponse; use Utopia\Validator\ArrayList; use Utopia\Validator\JSON; -class Create extends DocumentCreate +// Agent is dynamically provided via container; avoid strict type to pass lints + +class Create extends CreateDocumentAction { public static function getName(): string { @@ -26,12 +30,12 @@ class Create extends DocumentCreate protected function getResponseModel(): string { - return UtopiaResponse::MODEL_DOCUMENT; + return UtopiaResponse::MODEL_EMBEDDING; } protected function getBulkResponseModel(): string { - return UtopiaResponse::MODEL_DOCUMENT_LIST; + return UtopiaResponse::MODEL_EMBEDDING_LIST; } public function __construct() @@ -39,11 +43,11 @@ class Create extends DocumentCreate $this ->setHttpMethod(self::HTTP_REQUEST_METHOD_POST) ->setHttpPath('/v1/vectordb/:databaseId/collections/:collectionId/text') - ->desc('Create document') + ->desc('Create Text Embeddings') ->groups(['api', 'database']) - ->label('scope', 'documents.write') + ->label('scope', 'documents.read') ->label('resourceType', RESOURCE_TYPE_DATABASES) - ->label('audits.event', 'document.create') + ->label('audits.event', 'embedding.create') ->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}') ->label('abuse-key', 'ip:{ip},method:{method},url:{url},userId:{userId}') ->label('abuse-limit', APP_LIMIT_WRITE_RATE_DEFAULT * 2) @@ -52,35 +56,13 @@ class Create extends DocumentCreate new Method( namespace: 'vectorDB', group: $this->getSdkGroup(), - name: 'createTextDocument', - desc: 'Create document', + name: 'createTextEmbeddings', + desc: 'Create Text Embedding', description: '/docs/references/vectordb/create-document.md', auth: [AuthType::SESSION, AuthType::KEY, AuthType::JWT], responses: [ new SDKResponse( - code: SwooleResponse::STATUS_CODE_CREATED, - model: $this->getResponseModel(), - ) - ], - contentType: ContentType::JSON, - parameters: [ - new Parameter('databaseId', optional: false), - new Parameter('collectionId', optional: false), - new Parameter('documentId', optional: false), - new Parameter('data', optional: false), - new Parameter('permissions', optional: true), - ] - ), - new Method( - namespace: 'vectorDB', - group: $this->getSdkGroup(), - name: 'createDocuments', - desc: 'Create documents', - description: '/docs/references/vectordb/create-documents.md', - auth: [AuthType::ADMIN, AuthType::KEY], - responses: [ - new SDKResponse( - code: SwooleResponse::STATUS_CODE_CREATED, + code: SwooleResponse::STATUS_CODE_OK, model: $this->getBulkResponseModel(), ) ], @@ -93,22 +75,81 @@ class Create extends DocumentCreate ) ]) ->param('databaseId', '', new UID(), 'Database ID.') - ->param('documentId', '', new CustomId(), 'Document ID. Choose a custom ID or generate a random ID with `ID.unique()`. Valid chars are a-z, A-Z, 0-9, period, hyphen, and underscore. Can\'t start with a special char. Max length is 36 chars.', true) ->param('collectionId', '', new UID(), 'Collection ID. You can create a new collection using the Database service [server integration](https://appwrite.io/docs/server/databases#databasesCreateCollection). Make sure to define attributes before creating documents.') - ->param('data', [], new JSON(), 'Document data as JSON object.', true, example: '{"username":"walter.obrien","email":"walter.obrien@example.com","fullName":"Walter O\'Brien","age":30,"isAdmin":false}') - ->param('permissions', null, new Permissions(APP_LIMIT_ARRAY_PARAMS_SIZE, [Database::PERMISSION_READ, Database::PERMISSION_UPDATE, Database::PERMISSION_DELETE, Database::PERMISSION_WRITE]), 'An array of permissions strings. By default, only the current user is granted all permissions. [Learn more about permissions](https://appwrite.io/docs/permissions).', true) ->param('documents', [], fn (array $plan) => new ArrayList(new JSON(), $plan['databasesBatchSize'] ?? APP_LIMIT_DATABASE_BATCH), 'Array of documents data as JSON objects.', true, ['plan']) - ->param('transactionId', null, new UID(), 'Transaction ID for staging the operation.', true) ->inject('response') + ->inject('embeddingAgent') ->inject('dbForProject') ->inject('getDatabasesDB') - ->inject('user') - ->inject('queueForEvents') - ->inject('queueForStatsUsage') - ->inject('queueForRealtime') - ->inject('queueForFunctions') - ->inject('queueForWebhooks') - ->inject('plan') ->callback($this->action(...)); } + + public function action(string $databaseId, string $collectionId, array $documents, UtopiaResponse $response, $embeddingAgent, Database $dbForProject, callable $getDatabasesDB): void + { + $isAPIKey = Auth::isAppUser(Authorization::getRoles()); + $isPrivilegedUser = Auth::isPrivilegedUser(Authorization::getRoles()); + + $database = Authorization::skip(fn () => $dbForProject->getDocument('databases', $databaseId)); + if ($database->isEmpty() || (!$database->getAttribute('enabled', false) && !$isAPIKey && !$isPrivilegedUser)) { + throw new Exception(Exception::DATABASE_NOT_FOUND); + } + + $collection = Authorization::skip(fn () => $dbForProject->getDocument('database_' . $database->getSequence(), $collectionId)); + if ($collection->isEmpty() || (!$collection->getAttribute('enabled', false) && !$isAPIKey && !$isPrivilegedUser)) { + throw new Exception($this->getParentNotFoundException()); + } + + if (empty($documents)) { + throw new Exception(Exception::DOCUMENT_MISSING_DATA); + } + + // Validate and process each document + $availableModels = []; + if (method_exists($embeddingAgent, 'getAdapter') && method_exists($embeddingAgent->getAdapter(), 'getModels')) { + $availableModels = $embeddingAgent->getAdapter()->getModels(); + } + + $results = []; + foreach ($documents as $index => $item) { + if (!\is_array($item)) { + throw new Exception(Exception::DOCUMENT_INVALID_STRUCTURE, 'Invalid item at index ' . $index); + } + + $text = $item['text'] ?? ''; + $model = $item['embeddingModel'] ?? ($item['embeddigModel'] ?? null); + + if (!\is_string($text) || $text === '') { + throw new Exception(Exception::GENERAL_ARGUMENT_INVALID, 'Missing or invalid "text" at index ' . $index); + } + if (!\is_string($model) || $model === '') { + throw new Exception(Exception::GENERAL_ARGUMENT_INVALID, 'Missing or invalid "embeddingModel" at index ' . $index); + } + if (!empty($availableModels) && !\in_array($model, $availableModels, true)) { + throw new Exception(Exception::GENERAL_BAD_REQUEST, 'Unknown embedding model: ' . $model); + } + + if (method_exists($embeddingAgent, 'getAdapter') && method_exists($embeddingAgent->getAdapter(), 'setModel')) { + $embeddingAgent->getAdapter()->setModel($model); + } + + $embedResult = $embeddingAgent->embed($text); + $vector = $embedResult['embedding'] ?? []; + $dimensions = \is_array($vector) ? \count($vector) : 0; + + $results[] = new Document([ + 'model' => $model, + 'dimensions' => $dimensions, + 'embeddings' => $vector, + ]); + } + + $list = new Document([ + 'embeddings' => array_map(fn($d) => $d, $results), + 'total' => \count($results), + ]); + + $response + ->setStatusCode(SwooleResponse::STATUS_CODE_OK) + ->dynamic($list, $this->getBulkResponseModel()); + } } diff --git a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Text/Update.php b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Update.php similarity index 100% rename from src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Text/Update.php rename to src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Update.php diff --git a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Upsert.php b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Upsert.php index 6cbd86cba6..02c2700e8b 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Upsert.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Upsert.php @@ -7,6 +7,7 @@ use Appwrite\SDK\AuthType; use Appwrite\SDK\ContentType; use Appwrite\SDK\Method; use Appwrite\SDK\Response as SDKResponse; +use Appwrite\Utopia\Database\Validator\CustomId; use Appwrite\Utopia\Response as UtopiaResponse; use Utopia\Database\Database; use Utopia\Database\Validator\Permissions; @@ -59,7 +60,7 @@ class Upsert extends DocumentUpsert ]) ->param('databaseId', '', new UID(), 'Database ID.') ->param('collectionId', '', new UID(), 'Collection ID.') - ->param('documentId', '', new UID(), 'Document ID.') + ->param('documentId', '', fn (Database $dbForProject) => new CustomId(false, $dbForProject->getAdapter()->getMaxUIDLength()), 'Document ID.', false, ['dbForProject']) ->param('data', [], new JSON(), 'Document data as JSON object. Include all required fields of the document to be created or updated.', true) ->param('permissions', null, new Permissions(APP_LIMIT_ARRAY_PARAMS_SIZE, [Database::PERMISSION_READ, Database::PERMISSION_UPDATE, Database::PERMISSION_DELETE, Database::PERMISSION_WRITE]), 'An array of permissions strings. By default, the current permissions are inherited. [Learn more about permissions](https://appwrite.io/docs/permissions).', true) ->param('transactionId', null, new UID(), 'Transaction ID for staging the operation.', true) diff --git a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/XList.php b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/XList.php index dfc20a860f..a55db09a65 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/XList.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/XList.php @@ -11,6 +11,7 @@ use Appwrite\Utopia\Response as UtopiaResponse; use Utopia\Database\Validator\UID; use Utopia\Swoole\Response as SwooleResponse; use Utopia\Validator\ArrayList; +use Utopia\Validator\Boolean; use Utopia\Validator\Text; class XList extends DocumentXList @@ -52,6 +53,7 @@ class XList extends DocumentXList ->param('collectionId', '', new UID(), 'Collection ID. You can create a new collection using the Database service [server integration](https://appwrite.io/docs/server/databases#databasesCreateCollection).') ->param('queries', [], new ArrayList(new Text(APP_LIMIT_ARRAY_ELEMENT_SIZE), APP_LIMIT_ARRAY_PARAMS_SIZE), 'Array of query strings generated using the Query class provided by the SDK. [Learn more about queries](https://appwrite.io/docs/queries). Maximum of ' . APP_LIMIT_ARRAY_PARAMS_SIZE . ' queries are allowed, each ' . APP_LIMIT_ARRAY_ELEMENT_SIZE . ' characters long.', true) ->param('transactionId', null, new UID(), 'Transaction ID to read uncommitted changes within the transaction.', true) + ->param('total', true, new Boolean(true), 'When set to false, the total count returned will be 0 and will not be calculated.', true) ->inject('response') ->inject('dbForProject') ->inject('getDatabasesDB') diff --git a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Indexes/Create.php b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Indexes/Create.php index e68ad1fe24..aecab100e8 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Indexes/Create.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Indexes/Create.php @@ -58,7 +58,7 @@ class Create extends IndexCreate ->param('databaseId', '', new UID(), 'Database ID.') ->param('collectionId', '', new UID(), 'Collection ID. You can create a new collection using the Database service [server integration](https://appwrite.io/docs/server/databases#databasesCreateCollection).') ->param('key', null, new Key(), 'Index Key.') - ->param('type', null, new WhiteList([Database::INDEX_KEY, Database::INDEX_FULLTEXT, Database::INDEX_UNIQUE]), 'Index type.') + ->param('type', null, new WhiteList([Database::INDEX_HNSW_EUCLIDEAN,Database::INDEX_HNSW_DOT, Database::INDEX_HNSW_COSINE]), 'Index type.') ->param('attributes', null, new ArrayList(new Key(true), APP_LIMIT_ARRAY_PARAMS_SIZE), 'Array of attributes to index. Maximum of ' . APP_LIMIT_ARRAY_PARAMS_SIZE . ' attributes are allowed, each 32 characters long.') ->param('orders', [], new ArrayList(new WhiteList(['ASC', 'DESC'], false, Database::VAR_STRING), APP_LIMIT_ARRAY_PARAMS_SIZE), 'Array of index orders. Maximum of ' . APP_LIMIT_ARRAY_PARAMS_SIZE . ' orders are allowed.', true) ->param('lengths', [], new ArrayList(new Nullable(new Integer()), APP_LIMIT_ARRAY_PARAMS_SIZE), 'Length of index. Maximum of ' . APP_LIMIT_ARRAY_PARAMS_SIZE, optional: true) diff --git a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Indexes/XList.php b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Indexes/XList.php index f081fe23ee..c5a1dc48e3 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Indexes/XList.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Indexes/XList.php @@ -11,6 +11,7 @@ use Appwrite\Utopia\Database\Validator\Queries\Indexes; use Appwrite\Utopia\Response as UtopiaResponse; use Utopia\Database\Validator\UID; use Utopia\Swoole\Response as SwooleResponse; +use Utopia\Validator\Boolean; class XList extends IndexXList { @@ -50,6 +51,7 @@ class XList extends IndexXList ->param('databaseId', '', new UID(), 'Database ID.') ->param('collectionId', '', new UID(), 'Collection ID. You can create a new collection using the Database service [server integration](https://appwrite.io/docs/server/databases#databasesCreateCollection).') ->param('queries', [], new Indexes(), 'Array of query strings generated using the Query class provided by the SDK. [Learn more about queries](https://appwrite.io/docs/queries). Maximum of ' . APP_LIMIT_ARRAY_PARAMS_SIZE . ' queries are allowed, each ' . APP_LIMIT_ARRAY_ELEMENT_SIZE . ' characters long. You may filter on the following attributes: ' . implode(', ', Indexes::ALLOWED_ATTRIBUTES), true) + ->param('total', true, new Boolean(true), 'When set to false, the total count returned will be 0 and will not be calculated.', true) ->inject('response') ->inject('dbForProject') ->callback($this->action(...)); diff --git a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Update.php b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Update.php index 222c5bcaa4..b2b0d8e57b 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Update.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Update.php @@ -15,10 +15,8 @@ use Utopia\Database\Validator\Permissions; use Utopia\Database\Validator\UID; use Utopia\Swoole\Response as SwooleResponse; use Utopia\Validator\Boolean; -use Utopia\Validator\Integer; -use Utopia\Validator\WhiteList; +use Utopia\Validator\Range; use Utopia\Validator\Text; -use Utopia\Agents\Adapters\Ollama; class Update extends CollectionAction { @@ -61,9 +59,7 @@ class Update extends CollectionAction ->param('databaseId', '', new UID(), 'Database ID.') ->param('collectionId', '', new UID(), 'Collection ID.') ->param('name', null, new Text(128), 'Collection name. Max length: 128 chars.') - ->param('description', null, new Text(1024), 'Collection description. Max length: 1024 chars.', true) - ->param('dimensions', null, new Integer(), 'Embedding dimensions.', true) - ->param('embeddingModel', '', new WhiteList((new Ollama())->getModels()), 'Embedding model identifier.') + ->param('dimensions', null, new Range(1, 16000), 'Embedding dimensions.', true) ->param('permissions', null, new Permissions(APP_LIMIT_ARRAY_PARAMS_SIZE), 'An array of permission strings. By default, the current permissions are inherited. [Learn more about permissions](https://appwrite.io/docs/permissions).', true) ->param('documentSecurity', false, new Boolean(true), 'Enables configuring permissions for individual documents. A user needs one of document or collection level permissions to access a document. [Learn more about permissions](https://appwrite.io/docs/permissions).', true) ->param('enabled', true, new Boolean(), 'Is collection enabled? When set to \'disabled\', users cannot access the collection but Server SDKs with and API key can still read and write to the collection. No data is lost when this is toggled.', true) @@ -74,7 +70,7 @@ class Update extends CollectionAction ->callback($this->action(...)); } - public function action(string $databaseId, string $collectionId, ?string $name, ?string $description, ?int $dimensions, ?string $embeddingModel, ?array $permissions, bool $documentSecurity, bool $enabled, UtopiaResponse $response, \Utopia\Database\Database $dbForProject, callable $getDatabasesDB, \Appwrite\Event\Event $queueForEvents): void + public function action(string $databaseId, string $collectionId, ?string $name, ?int $dimensions, ?array $permissions, bool $documentSecurity, bool $enabled, UtopiaResponse $response, \Utopia\Database\Database $dbForProject, callable $getDatabasesDB, \Appwrite\Event\Event $queueForEvents): void { $database = Authorization::skip(fn () => $dbForProject->getDocument('databases', $databaseId)); if ($database->isEmpty()) { @@ -98,9 +94,7 @@ class Update extends CollectionAction $collectionId, $collection ->setAttribute('name', $name ?? $collection->getAttribute('name')) - ->setAttribute('description', $description ?? $collection->getAttribute('description')) ->setAttribute('dimensions', $dimensions ?? $collection->getAttribute('dimensions')) - ->setAttribute('embeddingModel', $embeddingModel ?? $collection->getAttribute('embeddingModel')) ->setAttribute('$permissions', $permissions) ->setAttribute('documentSecurity', $documentSecurity) ->setAttribute('enabled', $enabled) diff --git a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/XList.php b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/XList.php index dc0e73f7f1..1095e5ad25 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/XList.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/XList.php @@ -11,6 +11,7 @@ use Appwrite\Utopia\Database\Validator\Queries\Collections; use Appwrite\Utopia\Response as UtopiaResponse; use Utopia\Database\Validator\UID; use Utopia\Swoole\Response as SwooleResponse; +use Utopia\Validator\Boolean; use Utopia\Validator\Text; class XList extends CollectionXList @@ -51,6 +52,7 @@ class XList extends CollectionXList ->param('databaseId', '', new UID(), 'Database ID.') ->param('queries', [], new Collections(), 'Array of query strings generated using the Query class provided by the SDK. [Learn more about queries](https://appwrite.io/docs/queries). Maximum of ' . APP_LIMIT_ARRAY_PARAMS_SIZE . ' queries are allowed, each ' . APP_LIMIT_ARRAY_ELEMENT_SIZE . ' characters long. You may filter on the following attributes: ' . implode(', ', Collections::ALLOWED_ATTRIBUTES), true) ->param('search', '', new Text(256), 'Search term to filter your list results. Max length: 256 chars.', true) + ->param('total', true, new Boolean(true), 'When set to false, the total count returned will be 0 and will not be calculated.', true) ->inject('response') ->inject('dbForProject') ->callback($this->action(...)); diff --git a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/XList.php b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/XList.php index 1826c26b74..9f902964f4 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/XList.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/XList.php @@ -10,6 +10,7 @@ use Appwrite\SDK\Response as SDKResponse; use Appwrite\Utopia\Database\Validator\Queries\Databases; use Appwrite\Utopia\Response as UtopiaResponse; use Utopia\Swoole\Response as SwooleResponse; +use Utopia\Validator\Boolean; use Utopia\Validator\Text; class XList extends DatabaseXList @@ -44,6 +45,7 @@ class XList extends DatabaseXList )) ->param('queries', [], new Databases(), 'Array of query strings generated using the Query class provided by the SDK. [Learn more about queries](https://appwrite.io/docs/queries). Maximum of ' . APP_LIMIT_ARRAY_PARAMS_SIZE . ' queries are allowed, each ' . APP_LIMIT_ARRAY_ELEMENT_SIZE . ' characters long. You may filter on the following columns: ' . implode(', ', Databases::ALLOWED_ATTRIBUTES), true) ->param('search', '', new Text(256), 'Search term to filter your list results. Max length: 256 chars.', true) + ->param('total', true, new Boolean(true), 'When set to false, the total count returned will be 0 and will not be calculated.', true) ->inject('response') ->inject('dbForProject') ->callback($this->action(...)); diff --git a/src/Appwrite/Platform/Modules/Databases/Services/Registry/VectorDB.php b/src/Appwrite/Platform/Modules/Databases/Services/Registry/VectorDB.php index f570dbdbc5..f05e339ac0 100644 --- a/src/Appwrite/Platform/Modules/Databases/Services/Registry/VectorDB.php +++ b/src/Appwrite/Platform/Modules/Databases/Services/Registry/VectorDB.php @@ -4,7 +4,20 @@ namespace Appwrite\Platform\Modules\Databases\Services\Registry; use Appwrite\Platform\Modules\Databases\Http\VectorDB\Collections\Create as CreateCollection; use Appwrite\Platform\Modules\Databases\Http\VectorDB\Collections\Delete as DeleteCollection; +use Appwrite\Platform\Modules\Databases\Http\VectorDB\Collections\Documents\Bulk\Delete as DeleteDocuments; +use Appwrite\Platform\Modules\Databases\Http\VectorDB\Collections\Documents\Bulk\Update as UpdateDocuments; +use Appwrite\Platform\Modules\Databases\Http\VectorDB\Collections\Documents\Bulk\Upsert as UpsertDocuments; +use Appwrite\Platform\Modules\Databases\Http\VectorDB\Collections\Documents\Create as CreateDocument; +use Appwrite\Platform\Modules\Databases\Http\VectorDB\Collections\Documents\Delete as DeleteDocument; +use Appwrite\Platform\Modules\Databases\Http\VectorDB\Collections\Documents\Get as GetDocument; +use Appwrite\Platform\Modules\Databases\Http\VectorDB\Collections\Documents\Update as UpdateDocument; +use Appwrite\Platform\Modules\Databases\Http\VectorDB\Collections\Documents\Upsert as UpsertDocument; +use Appwrite\Platform\Modules\Databases\Http\VectorDB\Collections\Documents\XList as ListDocuments; use Appwrite\Platform\Modules\Databases\Http\VectorDB\Collections\Get as GetCollection; +use Appwrite\Platform\Modules\Databases\Http\VectorDB\Collections\Indexes\Create as CreateIndex; +use Appwrite\Platform\Modules\Databases\Http\VectorDB\Collections\Indexes\Delete as DeleteIndex; +use Appwrite\Platform\Modules\Databases\Http\VectorDB\Collections\Indexes\Get as GetIndex; +use Appwrite\Platform\Modules\Databases\Http\VectorDB\Collections\Indexes\XList as ListIndexes; use Appwrite\Platform\Modules\Databases\Http\VectorDB\Collections\Logs\XList as ListCollectionLogs; use Appwrite\Platform\Modules\Databases\Http\VectorDB\Collections\Update as UpdateCollection; use Appwrite\Platform\Modules\Databases\Http\VectorDB\Collections\Usage\Get as GetCollectionUsage; @@ -16,8 +29,7 @@ use Appwrite\Platform\Modules\Databases\Http\VectorDB\Update as UpdateVectorData use Appwrite\Platform\Modules\Databases\Http\VectorDB\Usage\Get as GetVectorDatabaseUsage; use Appwrite\Platform\Modules\Databases\Http\VectorDB\Usage\XList as ListVectorDatabaseUsage; use Appwrite\Platform\Modules\Databases\Http\VectorDB\XList as ListVectorDatabases; -use Appwrite\Platform\Modules\Databases\Http\VectorDB\Collections\Documents\Embedding\Create as CreateEmbeddingDocument; -use Appwrite\Platform\Modules\Databases\Http\VectorDB\Collections\Documents\Embedding\Update as UpdateEmbeddingDocument; +use Appwrite\Platform\Modules\Databases\Http\VectorDB\Collections\Documents\Text\Create as CreateTextEmbeddings; use Utopia\Platform\Service; class VectorDB extends Base @@ -26,8 +38,9 @@ class VectorDB extends Base { $this->registerDatabaseActions($service); $this->registerCollectionActions($service); - // $this->registerIndexActions($service); + $this->registerIndexActions($service); $this->registerDocumentActions($service); + $this->registerEmbeddingActions($service); // $this->registerTransactionActions($service); } @@ -53,8 +66,28 @@ class VectorDB extends Base $service->addAction(GetCollectionUsage::getName(), new GetCollectionUsage()); } - private function registerDocumentActions(Service $service):void{ - $service->addAction(CreateEmbeddingDocument::getName(), new CreateEmbeddingDocument()); - $service->addAction(UpdateEmbeddingDocument::getName(), new UpdateEmbeddingDocument()); + private function registerIndexActions(Service $service): void + { + $service->addAction(CreateIndex::getName(), new CreateIndex()); + $service->addAction(GetIndex::getName(), new GetIndex()); + $service->addAction(DeleteIndex::getName(), new DeleteIndex()); + $service->addAction(ListIndexes::getName(), new ListIndexes()); + } + + private function registerDocumentActions(Service $service): void + { + $service->addAction(CreateDocument::getName(), new CreateDocument()); + $service->addAction(UpdateDocument::getName(), new UpdateDocument()); + $service->addAction(UpsertDocument::getName(), new UpsertDocument()); + $service->addAction(GetDocument::getName(), new GetDocument()); + $service->addAction(ListDocuments::getName(), new ListDocuments()); + $service->addAction(DeleteDocument::getName(), new DeleteDocument()); + $service->addAction(UpdateDocuments::getName(), new UpdateDocuments()); + $service->addAction(UpsertDocuments::getName(), new UpsertDocuments()); + $service->addAction(DeleteDocuments::getName(), new DeleteDocuments()); + } + + private function registerEmbeddingActions(Service $service):void{ + $service->addAction(CreateTextEmbeddings::getName(),new CreateTextEmbeddings()); } } diff --git a/src/Appwrite/Utopia/Response.php b/src/Appwrite/Utopia/Response.php index 6c5c8e2f4c..cb14aae7da 100644 --- a/src/Appwrite/Utopia/Response.php +++ b/src/Appwrite/Utopia/Response.php @@ -25,13 +25,13 @@ use Appwrite\Utopia\Response\Model\AttributeInteger; use Appwrite\Utopia\Response\Model\AttributeIP; use Appwrite\Utopia\Response\Model\AttributeLine; use Appwrite\Utopia\Response\Model\AttributeList; +use Appwrite\Utopia\Response\Model\AttributeObject; use Appwrite\Utopia\Response\Model\AttributePoint; use Appwrite\Utopia\Response\Model\AttributePolygon; -use Appwrite\Utopia\Response\Model\AttributeObject; -use Appwrite\Utopia\Response\Model\AttributeVector; use Appwrite\Utopia\Response\Model\AttributeRelationship; use Appwrite\Utopia\Response\Model\AttributeString; use Appwrite\Utopia\Response\Model\AttributeURL; +use Appwrite\Utopia\Response\Model\AttributeVector; use Appwrite\Utopia\Response\Model\AuthProvider; use Appwrite\Utopia\Response\Model\BaseList; use Appwrite\Utopia\Response\Model\Branch; @@ -190,6 +190,8 @@ class Response extends SwooleResponse public const MODEL_COLLECTION_LIST = 'collectionList'; public const MODEL_VECTORDB_COLLECTION = 'vectordbCollection'; public const MODEL_VECTORDB_COLLECTION_LIST = 'vectordbCollectionList'; + public const MODEL_EMBEDDING = 'embedding'; + public const MODEL_EMBEDDING_LIST = 'embeddingList'; public const MODEL_TABLE = 'table'; public const MODEL_TABLE_LIST = 'tableList'; public const MODEL_INDEX = 'index'; @@ -495,8 +497,10 @@ class Response extends SwooleResponse ->setModel(new BaseList('Migrations Firebase Projects List', self::MODEL_MIGRATION_FIREBASE_PROJECT_LIST, 'projects', self::MODEL_MIGRATION_FIREBASE_PROJECT)) ->setModel(new BaseList('Specifications List', self::MODEL_SPECIFICATION_LIST, 'specifications', self::MODEL_SPECIFICATION)) ->setModel(new BaseList('VCS Content List', self::MODEL_VCS_CONTENT_LIST, 'contents', self::MODEL_VCS_CONTENT)) + ->setModel(new BaseList('Embedding list', self::MODEL_EMBEDDING_LIST, 'embeddings', self::MODEL_EMBEDDING)) // Entities ->setModel(new Database()) + ->setModel(new \Appwrite\Utopia\Response\Model\Embedding()) // Collection API Models ->setModel(new Collection()) ->setModel(new \Appwrite\Utopia\Response\Model\VectorDBCollection()) @@ -875,7 +879,11 @@ class Response extends SwooleResponse $this ->setContentType(Response::CONTENT_TYPE_YAML) - ->send(\yaml_emit($data, YAML_UTF8_ENCODING)); + ->send((function (array $payload) { + $func = 'yaml_emit'; + $encoding = \defined('YAML_UTF8_ENCODING') ? \constant('YAML_UTF8_ENCODING') : 0; + return \call_user_func($func, $payload, $encoding); + })($data)); } /** diff --git a/src/Appwrite/Utopia/Response/Model/AttributeObject.php b/src/Appwrite/Utopia/Response/Model/AttributeObject.php index 14ff818b3c..542f7f744c 100644 --- a/src/Appwrite/Utopia/Response/Model/AttributeObject.php +++ b/src/Appwrite/Utopia/Response/Model/AttributeObject.php @@ -25,5 +25,3 @@ class AttributeObject extends Attribute return Response::MODEL_ATTRIBUTE_OBJECT; } } - - diff --git a/src/Appwrite/Utopia/Response/Model/AttributeVector.php b/src/Appwrite/Utopia/Response/Model/AttributeVector.php index 56c3317cd0..4b58b979ee 100644 --- a/src/Appwrite/Utopia/Response/Model/AttributeVector.php +++ b/src/Appwrite/Utopia/Response/Model/AttributeVector.php @@ -33,5 +33,3 @@ class AttributeVector extends Attribute return Response::MODEL_ATTRIBUTE_VECTOR; } } - - diff --git a/src/Appwrite/Utopia/Response/Model/Embedding.php b/src/Appwrite/Utopia/Response/Model/Embedding.php new file mode 100644 index 0000000000..7c881ca1ac --- /dev/null +++ b/src/Appwrite/Utopia/Response/Model/Embedding.php @@ -0,0 +1,39 @@ +addRule('model', [ + 'type' => self::TYPE_STRING, + 'description' => 'Embedding model used to generate embeddings.', + 'example' => 'embeddinggemma' + ]) + ->addRule('dimensions', [ + 'type' => self::TYPE_INTEGER, + 'description' => 'Number of dimensions for each embedding vector.', + 'example' => 768 + ]) + ->addRule('embeddings', [ + 'type' => self::TYPE_FLOAT, + 'array' => true, + 'description' => 'Embedding vector values.', + 'example' => [0.01, 0.02, 0.03] + ]); + } +} diff --git a/src/Appwrite/Utopia/Response/Model/VectorDBCollection.php b/src/Appwrite/Utopia/Response/Model/VectorDBCollection.php index 97b2e75224..017780173b 100644 --- a/src/Appwrite/Utopia/Response/Model/VectorDBCollection.php +++ b/src/Appwrite/Utopia/Response/Model/VectorDBCollection.php @@ -65,12 +65,6 @@ class VectorDBCollection extends Model 'default' => 0, 'example' => 1536, ]) - ->addRule('embeddingModel', [ - 'type' => self::TYPE_STRING, - 'description' => 'Embedding model identifier.', - 'default' => '', - 'example' => 'text-embedding-3-large', - ]) ->addRule('search', [ 'type' => self::TYPE_STRING, 'description' => 'Search text.', diff --git a/tests/e2e/Services/Databases/VectorDB/DatabasesBase.php b/tests/e2e/Services/Databases/VectorDB/DatabasesBase.php index fede56e078..e55be44407 100644 --- a/tests/e2e/Services/Databases/VectorDB/DatabasesBase.php +++ b/tests/e2e/Services/Databases/VectorDB/DatabasesBase.php @@ -7,7 +7,7 @@ use Utopia\Database\Database; use Utopia\Database\Helpers\ID; use Utopia\Database\Helpers\Permission; use Utopia\Database\Helpers\Role; -use Utopia\Agents\Adapters\Ollama; +use Utopia\Database\Query; trait DatabasesBase { @@ -33,6 +33,544 @@ trait DatabasesBase return ['databaseId' => $database['body']['$id']]; } + /** + * @depends testCreateCollectionSample + */ + public function testCreateDocument(array $data): array + { + $databaseId = $data['databaseId']; + $collectionId = $data['collectionId']; + + // Build embedding vector matching collection dimensions (1536) + $vector = array_fill(0, 1536, 0.1); + $vector[0] = 1.0; + + $res = $this->client->call(Client::METHOD_POST, "/vectordb/{$databaseId}/collections/{$collectionId}/documents", [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'documentId' => ID::unique(), + 'data' => [ + 'embeddings' => $vector, + 'metadata' => ['type' => 'sample', 'rank' => 1] + ], + 'permissions' => [ + Permission::read(Role::any()), + Permission::update(Role::any()), + Permission::delete(Role::any()), + ] + ]); + + $this->assertEquals(201, $res['headers']['status-code']); + $this->assertNotEmpty($res['body']['$id']); + $documentId = $res['body']['$id']; + + // createdAt/updatedAt should be present and equal on initial create + $this->assertArrayHasKey('$createdAt', $res['body']); + $this->assertArrayHasKey('$updatedAt', $res['body']); + $this->assertNotEmpty($res['body']['$createdAt']); + $this->assertNotEmpty($res['body']['$updatedAt']); + $this->assertEquals($res['body']['$createdAt'], $res['body']['$updatedAt']); + + // Edge: invalid dimensions (vector too short) → expect 4xx + $badVec = [1.0, 0.0]; + $bad = $this->client->call(Client::METHOD_POST, "/vectordb/{$databaseId}/collections/{$collectionId}/documents", [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'documentId' => ID::unique(), + 'data' => [ + 'embeddings' => $badVec, + 'metadata' => ['type' => 'bad'] + ], + ]); + $this->assertGreaterThanOrEqual(400, $bad['headers']['status-code']); + $this->assertLessThan(500, $bad['headers']['status-code']); + + // Edge: invalid type values (strings) → expect 4xx + $strVec = ['1.0', '0.0', '0.0']; + $bad2 = $this->client->call(Client::METHOD_POST, "/vectordb/{$databaseId}/collections/{$collectionId}/documents", [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'documentId' => ID::unique(), + 'data' => [ + 'embeddings' => $strVec, + 'metadata' => ['type' => 'bad-strings'] + ], + ]); + $this->assertGreaterThanOrEqual(400, $bad2['headers']['status-code']); + $this->assertLessThan(500, $bad2['headers']['status-code']); + + // Create another valid doc to verify list totals later + $res2 = $this->client->call(Client::METHOD_POST, "/vectordb/{$databaseId}/collections/{$collectionId}/documents", [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'documentId' => ID::unique(), + 'data' => [ + 'embeddings' => $vector, + 'metadata' => ['type' => 'sample', 'rank' => 99] + ], + 'permissions' => [Permission::read(Role::any())] + ]); + $this->assertEquals(201, $res2['headers']['status-code']); + $documentId2 = $res2['body']['$id']; + + return [ + 'databaseId' => $databaseId, + 'collectionId' => $collectionId, + 'documentId' => $documentId, + 'documentId2' => $documentId2, + 'createdAt' => $res['body']['$createdAt'], + 'updatedAt' => $res['body']['$updatedAt'], + ]; + } + + /** + * @depends testCreateDocument + */ + public function testGetDocument(array $data): array + { + $databaseId = $data['databaseId']; + $collectionId = $data['collectionId']; + $documentId = $data['documentId']; + + $res = $this->client->call(Client::METHOD_GET, "/vectordb/{$databaseId}/collections/{$collectionId}/documents/{$documentId}", [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]); + + $this->assertEquals(200, $res['headers']['status-code']); + $this->assertEquals($documentId, $res['body']['$id']); + + // Edge: missing document should return 404 + $missing = $this->client->call(Client::METHOD_GET, "/vectordb/{$databaseId}/collections/{$collectionId}/documents/" . ID::unique(), [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]); + $this->assertEquals(404, $missing['headers']['status-code']); + + return $data; + } + + /** + * @depends testCreateDocument + */ + public function testListDocuments(array $data): array + { + $databaseId = $data['databaseId']; + $collectionId = $data['collectionId']; + + $list = $this->client->call(Client::METHOD_GET, "/vectordb/{$databaseId}/collections/{$collectionId}/documents", [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'queries' => [Query::limit(5)->toString()] + ]); + + $this->assertEquals(200, $list['headers']['status-code']); + $this->assertIsInt($list['body']['total']); + $this->assertGreaterThanOrEqual(1, $list['body']['total']); + + // Pagination: limit 1, then offset 1 + $page1 = $this->client->call(Client::METHOD_GET, "/vectordb/{$databaseId}/collections/{$collectionId}/documents", [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'queries' => [ + Query::limit(1)->toString(), + Query::orderAsc('$id')->toString() + ] + ]); + $this->assertEquals(200, $page1['headers']['status-code']); + $this->assertEquals(1, \count($page1['body']['documents'] ?? [])); + + $page2 = $this->client->call(Client::METHOD_GET, "/vectordb/{$databaseId}/collections/{$collectionId}/documents", [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'queries' => [ + Query::limit(1)->toString(), + Query::offset(1)->toString(), + Query::orderAsc('$id')->toString() + ] + ]); + $this->assertEquals(200, $page2['headers']['status-code']); + $this->assertEquals(1, \count($page2['body']['documents'] ?? [])); + + return $data; + } + + /** + * @depends testCreateDocument + */ + public function testUpsertDocument(array $data): array + { + $databaseId = $data['databaseId']; + $collectionId = $data['collectionId']; + $documentId = $data['documentId']; + + $vector = array_fill(0, 1536, 0.0); + // $vector[1] = 1.0; + + $upd = $this->client->call(Client::METHOD_PUT, "/vectordb/{$databaseId}/collections/{$collectionId}/documents/{$documentId}", [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'data' => [ + 'embeddings' => $vector, + 'metadata' => ['type' => 'sample', 'rank' => 2] + ] + ]); + + $this->assertEquals(200, $upd['headers']['status-code']); + + // Verify update took effect + $get = $this->client->call(Client::METHOD_GET, "/vectordb/{$databaseId}/collections/{$collectionId}/documents/{$documentId}", [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]); + $this->assertEquals(200, $get['headers']['status-code']); + $this->assertEquals(2, $get['body']['metadata']['rank']); + // updatedAt should be greater or changed from earlier + $this->assertArrayHasKey('$updatedAt', $get['body']); + + return $data; + } + + /** + * @depends testUpsertDocument + */ + public function testUpdateDocument(array $data): array + { + // Upsert is used for update semantics + $databaseId = $data['databaseId']; + $collectionId = $data['collectionId']; + $documentId = $data['documentId']; + + $vector = array_fill(0, 1536, 0.0); + $vector[2] = 1.0; + + $upd = $this->client->call(Client::METHOD_PUT, "/vectordb/{$databaseId}/collections/{$collectionId}/documents/{$documentId}", [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'data' => [ + 'embeddings' => $vector, + 'metadata' => ['type' => 'sample', 'rank' => 3] + ] + ]); + + $this->assertEquals(200, $upd['headers']['status-code']); + + // Re-update to check idempotence and metadata replacement + $vector2 = array_fill(0, 1536, 0.0); + $vector2[3] = 1.0; + $upd2 = $this->client->call(Client::METHOD_PUT, "/vectordb/{$databaseId}/collections/{$collectionId}/documents/{$documentId}", [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'data' => [ + 'embeddings' => $vector2, + 'metadata' => ['type' => 'sample', 'rank' => 4] + ] + ]); + $this->assertEquals(200, $upd2['headers']['status-code']); + + // Verify updatedAt changed again + $get2 = $this->client->call(Client::METHOD_GET, "/vectordb/{$databaseId}/collections/{$collectionId}/documents/{$documentId}", [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]); + $this->assertEquals(200, $get2['headers']['status-code']); + $this->assertArrayHasKey('$updatedAt', $get2['body']); + + return $data; + } + + /** + * @depends testUpdateDocument + */ + public function testTimestampsMutation(array $data): array + { + $databaseId = $data['databaseId']; + $collectionId = $data['collectionId']; + $documentId = $data['documentId']; + + // Fetch current document to get latest timestamps + $curr = $this->client->call(Client::METHOD_GET, "/vectordb/{$databaseId}/collections/{$collectionId}/documents/{$documentId}", [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]); + $this->assertEquals(200, $curr['headers']['status-code']); + + $initialCreatedAt = $data['createdAt']; + $initialUpdatedAt = $data['updatedAt']; + $afterUpdatesCreatedAt = $curr['body']['$createdAt']; + $afterUpdatesUpdatedAt = $curr['body']['$updatedAt']; + + // createdAt must remain stable + $this->assertEquals($initialCreatedAt, $afterUpdatesCreatedAt); + // updatedAt must change after updates + $this->assertNotEquals($initialUpdatedAt, $afterUpdatesUpdatedAt); + + // Try to forcibly set $createdAt and $updatedAt via API + $oldTs = '1970-01-01T00:00:00.000+00:00'; + $vectorForce = array_fill(0, 1536, 0.0); + $vectorForce[9] = 1.0; + $force = $this->client->call(Client::METHOD_PUT, "/vectordb/{$databaseId}/collections/{$collectionId}/documents/{$documentId}", [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'data' => [ + '$createdAt' => $oldTs, + '$updatedAt' => $oldTs, + 'embeddings' => $vectorForce, + 'metadata' => ['type' => 'sample', 'rank' => 6] + ] + ]); + $this->assertEquals(200, $force['headers']['status-code']); + + $forcedGet = $this->client->call(Client::METHOD_GET, "/vectordb/{$databaseId}/collections/{$collectionId}/documents/{$documentId}", [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]); + $this->assertEquals(200, $forcedGet['headers']['status-code']); + // Both timestamps should be updated to client-provided value + $this->assertEquals($oldTs, $forcedGet['body']['$createdAt']); + $this->assertEquals($oldTs, $forcedGet['body']['$updatedAt']); + + // Perform another update to ensure updatedAt changes again + $vector = array_fill(0, 1536, 0.0); + $vector[10] = 1.0; + $upd = $this->client->call(Client::METHOD_PUT, "/vectordb/{$databaseId}/collections/{$collectionId}/documents/{$documentId}", [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'data' => [ + 'embeddings' => $vector, + 'metadata' => ['type' => 'sample', 'rank' => 5] + ] + ]); + $this->assertEquals(200, $upd['headers']['status-code']); + + $final = $this->client->call(Client::METHOD_GET, "/vectordb/{$databaseId}/collections/{$collectionId}/documents/{$documentId}", [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]); + $this->assertEquals(200, $final['headers']['status-code']); + // createdAt should persist the client-forced value + $this->assertEquals($oldTs, $final['body']['$createdAt']); + // updatedAt should change from the previous value + $this->assertNotEquals($afterUpdatesUpdatedAt, $final['body']['$updatedAt']); + + return $data; + } + + /** + * @depends testUpdateDocument + */ + public function testDocumentsVectorQueries(array $data): array + { + $databaseId = $data['databaseId']; + $collectionId = $data['collectionId']; + + // Create two more documents with distinct embeddings + $mk = function (array $vec, string $name) use ($databaseId, $collectionId) { + $this->client->call(Client::METHOD_POST, "/vectordb/{$databaseId}/collections/{$collectionId}/documents", [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'documentId' => ID::unique(), + 'data' => [ + 'embeddings' => $vec, + 'metadata' => ['name' => $name] + ], + 'permissions' => [Permission::read(Role::any())] + ]); + }; + + $vA = array_fill(0, 1536, 0.0); + $vA[0] = 1.0; // close to [1,0,0,...] + $vB = array_fill(0, 1536, 0.0); + $vB[1] = 1.0; // close to [0,1,0,...] + + $mk($vA, 'A'); + $mk($vB, 'B'); + + // Dot product + $dot = $this->client->call(Client::METHOD_GET, "/vectordb/{$databaseId}/collections/{$collectionId}/documents", [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'queries' => [ + Query::vectorDot('embeddings', $vA)->toString(), + Query::limit(2)->toString() + ] + ]); + $this->assertEquals(200, $dot['headers']['status-code']); + $this->assertGreaterThanOrEqual(1, $dot['body']['total']); + + // Cosine + $cos = $this->client->call(Client::METHOD_GET, "/vectordb/{$databaseId}/collections/{$collectionId}/documents", [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'queries' => [ + Query::vectorCosine('embeddings', $vB)->toString(), + Query::limit(2)->toString() + ] + ]); + $this->assertEquals(200, $cos['headers']['status-code']); + $this->assertGreaterThanOrEqual(1, $cos['body']['total']); + + // Euclidean + $eu = $this->client->call(Client::METHOD_GET, "/vectordb/{$databaseId}/collections/{$collectionId}/documents", [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'queries' => [ + Query::vectorEuclidean('embeddings', $vA)->toString(), + Query::limit(2)->toString() + ] + ]); + $this->assertEquals(200, $eu['headers']['status-code']); + $this->assertGreaterThanOrEqual(1, $eu['body']['total']); + + // Combined vector + metadata filters + $combo = $this->client->call(Client::METHOD_GET, "/vectordb/{$databaseId}/collections/{$collectionId}/documents", [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'queries' => [ + Query::vectorCosine('embeddings', $vA)->toString(), + Query::notEqual('metadata', [['name' => 'B']])->toString(), + Query::limit(2)->toString() + ] + ]); + $this->assertEquals(200, $combo['headers']['status-code']); + $this->assertGreaterThanOrEqual(1, $combo['body']['total']); + + // Ordering with $id ascending combined with vector + $ordered = $this->client->call(Client::METHOD_GET, "/vectordb/{$databaseId}/collections/{$collectionId}/documents", [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'queries' => [ + Query::vectorDot('embeddings', $vA)->toString(), + Query::orderAsc('$id')->toString(), + Query::limit(3)->toString() + ] + ]); + $this->assertEquals(200, $ordered['headers']['status-code']); + + return $data; + } + + /** + * @depends testDocumentsVectorQueries + */ + public function testDeleteDocument(array $data): void + { + $databaseId = $data['databaseId']; + $collectionId = $data['collectionId']; + $documentId = $data['documentId']; + + $del = $this->client->call(Client::METHOD_DELETE, "/vectordb/{$databaseId}/collections/{$collectionId}/documents/{$documentId}", [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]); + $this->assertEquals(204, $del['headers']['status-code']); + + // GET after delete should be 404 + $getMissing = $this->client->call(Client::METHOD_GET, "/vectordb/{$databaseId}/collections/{$collectionId}/documents/{$documentId}", [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]); + $this->assertEquals(404, $getMissing['headers']['status-code']); + + // List should still work and reflect at least one less document compared to earlier pages (best-effort) + $list = $this->client->call(Client::METHOD_GET, "/vectordb/{$databaseId}/collections/{$collectionId}/documents", [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'queries' => [Query::limit(5)->toString()] + ]); + $this->assertEquals(200, $list['headers']['status-code']); + } + + /** + * @depends testCreateCollectionSample + */ + public function testDocumentPermissions(array $data): void + { + $databaseId = $data['databaseId']; + $collectionId = $data['collectionId']; + + // Create doc readable only by a specific user + $docId = ID::unique(); + $vector = array_fill(0, 1536, 0.0); + $vector[0] = 1.0; + $create = $this->client->call(Client::METHOD_POST, "/vectordb/{$databaseId}/collections/{$collectionId}/documents", [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'documentId' => $docId, + 'data' => [ + 'embeddings' => $vector, + 'metadata' => ['scope' => 'private'] + ], + 'permissions' => [ + Permission::read(Role::user($this->getUser()['$id'])) + ] + ]); + $this->assertEquals(201, $create['headers']['status-code']); + + $guest = $this->client->call(Client::METHOD_GET, "/vectordb/{$databaseId}/collections/{$collectionId}/documents/{$docId}", [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'] + ]); + $this->assertEquals(404, $guest['headers']['status-code']); + + // GET with key should succeed regardless of document user-level permission + $withKey = $this->client->call(Client::METHOD_GET, "/vectordb/{$databaseId}/collections/{$collectionId}/documents/{$docId}", [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]); + $this->assertEquals(200, $withKey['headers']['status-code']); + } + /** * @depends testCreateDatabase */ @@ -51,7 +589,6 @@ trait DatabasesBase 'name' => 'Movies', 'documentSecurity' => true, 'dimensions' => 1536, - 'embeddingModel' => Ollama::MODEL_EMBEDDING_GEMMA, 'permissions' => [ Permission::create(Role::user($this->getUser()['$id'])), ], @@ -69,7 +606,6 @@ trait DatabasesBase 'name' => 'Actors', 'documentSecurity' => true, 'dimensions' => 1536, - 'embeddingModel' => Ollama::MODEL_EMBEDDING_GEMMA, 'permissions' => [ Permission::create(Role::user($this->getUser()['$id'])), ], @@ -119,7 +655,6 @@ trait DatabasesBase 'collectionId' => ID::unique(), 'name' => 'Sample Collection', 'dimensions' => 1536, - 'embeddingModel' => 'text-embedding-3-large', 'documentSecurity' => true, 'permissions' => [ Permission::create(Role::user($this->getUser()['$id'])), @@ -129,7 +664,6 @@ trait DatabasesBase $this->assertEquals(201, $collection['headers']['status-code']); $this->assertEquals('Sample Collection', $collection['body']['name']); $this->assertEquals(1536, $collection['body']['dimensions']); - $this->assertEquals('text-embedding-3-large', $collection['body']['embeddingModel']); return [ 'databaseId' => $databaseId, @@ -167,7 +701,7 @@ trait DatabasesBase /** * Helper to create a collection */ - $createCollection = function (string $databaseId, string $name, int $dimensions = 1536, string $embeddingModel = Ollama::MODEL_EMBEDDING_GEMMA) use ($projectId, $apiKey, $userId) { + $createCollection = function (string $databaseId, string $name, int $dimensions = 1536) use ($projectId, $apiKey, $userId) { $res = $this->client->call(Client::METHOD_POST, '/vectordb/' . $databaseId . '/collections', [ 'content-type' => 'application/json', 'x-appwrite-project' => $projectId, @@ -177,7 +711,6 @@ trait DatabasesBase 'name' => $name, 'documentSecurity' => true, 'dimensions' => $dimensions, - 'embeddingModel' => $embeddingModel, 'permissions' => [ Permission::create(Role::user($userId)), ], @@ -216,36 +749,7 @@ trait DatabasesBase // Create a tiny-dimension collection and insert a document to validate vector and object attributes $tinyCollectionName = 'VectorsTiny'; $tinyDimensions = 8; - $tinyCollectionId = $createCollection($mediaDbId, $tinyCollectionName, $tinyDimensions, Ollama::MODEL_EMBEDDING_GEMMA); - - // Build embeddings vector of correct length and metadata object - $embeddings = []; - for ($i = 0; $i < $tinyDimensions; $i++) { - $embeddings[] = (float)($i + 1) / 10.0; - } - $metadata = [ - 'genre' => 'drama', - 'score' => 9, - 'tags' => ['award', 'festival'] - ]; - - $docRes = $this->client->call(Client::METHOD_POST, '/vectordb/' . $mediaDbId . '/collections/' . $tinyCollectionId . '/documents', [ - 'content-type' => 'application/json', - 'x-appwrite-project' => $projectId, - 'x-appwrite-key' => $apiKey - ], [ - 'documentId' => ID::unique(), - 'data' => [ - 'embeddings' => $embeddings, - 'metadata' => $metadata, - ], - 'permissions' => [ - Permission::read(Role::user($userId)), - Permission::write(Role::user($userId)), - ], - ]); - - $this->assertEquals(201, $docRes['headers']['status-code']); + $tinyCollectionId = $createCollection($mediaDbId, $tinyCollectionName, $tinyDimensions); return [ 'databases' => [ @@ -261,4 +765,393 @@ trait DatabasesBase ]; } + public function testInvalidCollectionDimensions(): void + { + // dimensions = 0 -> expect 4xx + $bad0 = $this->client->call(Client::METHOD_POST, '/vectordb', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'databaseId' => ID::unique(), + 'name' => 'BadDims0' + ]); + $this->assertEquals(201, $bad0['headers']['status-code']); + $dbId = $bad0['body']['$id']; + $col = $this->client->call(Client::METHOD_POST, '/vectordb/' . $dbId . '/collections', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'collectionId' => ID::unique(), + 'name' => 'ZeroDims', + 'documentSecurity' => true, + 'dimensions' => 0, + 'permissions' => [Permission::create(Role::user($this->getUser()['$id']))], + ]); + $this->assertGreaterThanOrEqual(400, $col['headers']['status-code']); + $this->assertLessThan(500, $col['headers']['status-code']); + + // dimensions too large -> expect 4xx + $col2 = $this->client->call(Client::METHOD_POST, '/vectordb/' . $dbId . '/collections', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'collectionId' => ID::unique(), + 'name' => 'HugeDims', + 'documentSecurity' => true, + 'dimensions' => 16001, + 'permissions' => [Permission::create(Role::user($this->getUser()['$id']))], + ]); + $this->assertGreaterThanOrEqual(400, $col2['headers']['status-code']); + $this->assertLessThan(500, $col2['headers']['status-code']); + } + + public function testSingleDimensionVectorCollection(): void + { + $db = $this->client->call(Client::METHOD_POST, '/vectordb', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'databaseId' => ID::unique(), + 'name' => 'SingleDim' + ]); + $this->assertEquals(201, $db['headers']['status-code']); + $databaseId = $db['body']['$id']; + + $col = $this->client->call(Client::METHOD_POST, '/vectordb/' . $databaseId . '/collections', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'collectionId' => ID::unique(), + 'name' => 'OneDim', + 'documentSecurity' => true, + 'dimensions' => 1, + 'permissions' => [Permission::create(Role::user($this->getUser()['$id']))], + ]); + $this->assertEquals(201, $col['headers']['status-code']); + $collectionId = $col['body']['$id']; + + // Create two docs with 1D embeddings + $id1 = ID::unique(); + $this->client->call(Client::METHOD_PUT, "/vectordb/{$databaseId}/collections/{$collectionId}/documents/{$id1}", [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'data' => ['embeddings' => [1.0]] + ]); + $id2 = ID::unique(); + $this->client->call(Client::METHOD_PUT, "/vectordb/{$databaseId}/collections/{$collectionId}/documents/{$id2}", [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'data' => ['embeddings' => [0.5]] + ]); + + // Query with vectorCosine + $res = $this->client->call(Client::METHOD_GET, "/vectordb/{$databaseId}/collections/{$collectionId}/documents", [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'queries' => [Query::vectorCosine('embeddings', [1.0])->toString(), Query::limit(2)->toString()] + ]); + $this->assertEquals(200, $res['headers']['status-code']); + $this->assertGreaterThanOrEqual(1, $res['body']['total']); + } + + public function testVectorInvalidValues(): void + { + $db = $this->client->call(Client::METHOD_POST, '/vectordb', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'databaseId' => ID::unique(), + 'name' => 'InvalidVals' + ]); + $this->assertEquals(201, $db['headers']['status-code']); + $databaseId = $db['body']['$id']; + + $col = $this->client->call(Client::METHOD_POST, '/vectordb/' . $databaseId . '/collections', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'collectionId' => ID::unique(), + 'name' => 'Docs', + 'documentSecurity' => true, + 'dimensions' => 3, + 'permissions' => [Permission::create(Role::user($this->getUser()['$id']))], + ]); + $this->assertEquals(201, $col['headers']['status-code']); + $collectionId = $col['body']['$id']; + + $badPayloads = [ + ['embeddings' => [INF, 0.0, 0.0]], + ['embeddings' => [-INF, 0.0, 0.0]], + ['embeddings' => [NAN, 0.0, 0.0]], + ['embeddings' => ['x' => 1.0, 'y' => 0.0, 'z' => 0.0]], + ['embeddings' => [1.0, null, 0.0]], + ['embeddings' => [[1.0], [0.0], [0.0]]], + ['embeddings' => [true, false, true]], + ['embeddings' => [1.0, '2.0', 3.0]], + (function () { + $v = []; + $v[0] = 1.0; + $v[2] = 1.0; + return ['embeddings' => $v]; + })(), + ]; + + foreach ($badPayloads as $payload) { + $resp = $this->client->call(Client::METHOD_PUT, "/vectordb/{$databaseId}/collections/{$collectionId}/documents/" . ID::unique(), [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'data' => $payload + ]); + $this->assertGreaterThanOrEqual(400, $resp['headers']['status-code']); + $this->assertLessThan(500, $resp['headers']['status-code']); + } + } + + public function testVectorAllZerosAndQuery(): void + { + $db = $this->client->call(Client::METHOD_POST, '/vectordb', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'databaseId' => ID::unique(), + 'name' => 'ZerosDB' + ]); + $this->assertEquals(201, $db['headers']['status-code']); + $databaseId = $db['body']['$id']; + + $col = $this->client->call(Client::METHOD_POST, '/vectordb/' . $databaseId . '/collections', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'collectionId' => ID::unique(), + 'name' => 'Zeros', + 'documentSecurity' => true, + 'dimensions' => 3, + 'permissions' => [Permission::create(Role::user($this->getUser()['$id']))], + ]); + $this->assertEquals(201, $col['headers']['status-code']); + $collectionId = $col['body']['$id']; + + $this->client->call(Client::METHOD_PUT, "/vectordb/{$databaseId}/collections/{$collectionId}/documents/" . ID::unique(), [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ 'data' => ['embeddings' => [0.0, 0.0, 0.0]] ]); + + $this->client->call(Client::METHOD_PUT, "/vectordb/{$databaseId}/collections/{$collectionId}/documents/" . ID::unique(), [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ 'data' => ['embeddings' => [1.0, 0.0, 0.0]] ]); + + $results = $this->client->call(Client::METHOD_GET, "/vectordb/{$databaseId}/collections/{$collectionId}/documents", [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ 'queries' => [Query::vectorCosine('embeddings', [1.0, 0.0, 0.0])->toString()] ]); + $this->assertEquals(200, $results['headers']['status-code']); + $this->assertGreaterThan(0, $results['body']['total']); + } + + public function testVectorMultipleQueriesRejection(): void + { + // Create a simple DB and collection + $db = $this->client->call(Client::METHOD_POST, '/vectordb', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ 'databaseId' => ID::unique(), 'name' => 'MultiQueryDB' ]); + $this->assertEquals(201, $db['headers']['status-code']); + $databaseId = $db['body']['$id']; + $col = $this->client->call(Client::METHOD_POST, '/vectordb/' . $databaseId . '/collections', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ 'collectionId' => ID::unique(), 'name' => 'Docs', 'documentSecurity' => true, 'dimensions' => 3, 'permissions' => [Permission::create(Role::user($this->getUser()['$id']))] ]); + $this->assertEquals(201, $col['headers']['status-code']); + $collectionId = $col['body']['$id']; + + // Two vector queries simultaneously should fail + $fail = $this->client->call(Client::METHOD_GET, "/vectordb/{$databaseId}/collections/{$collectionId}/documents", [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'queries' => [ + Query::vectorCosine('embeddings', [1.0, 0.0, 0.0])->toString(), + Query::vectorEuclidean('embeddings', [1.0, 0.0, 0.0])->toString() + ] + ]); + $this->assertGreaterThanOrEqual(400, $fail['headers']['status-code']); + $this->assertLessThan(500, $fail['headers']['status-code']); + } + + public function testVectorQueryOnNonVectorAttribute(): void + { + $db = $this->client->call(Client::METHOD_POST, '/vectordb', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ 'databaseId' => ID::unique(), 'name' => 'NonVec' ]); + $this->assertEquals(201, $db['headers']['status-code']); + $databaseId = $db['body']['$id']; + $col = $this->client->call(Client::METHOD_POST, '/vectordb/' . $databaseId . '/collections', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ 'collectionId' => ID::unique(), 'name' => 'Docs', 'documentSecurity' => true, 'dimensions' => 3, 'permissions' => [Permission::create(Role::user($this->getUser()['$id']))] ]); + $this->assertEquals(201, $col['headers']['status-code']); + $collectionId = $col['body']['$id']; + + // Query on non-vector attribute 'metadata' should fail + $fail = $this->client->call(Client::METHOD_GET, "/vectordb/{$databaseId}/collections/{$collectionId}/documents", [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ 'queries' => [Query::vectorCosine('metadata', [1.0, 0.0, 0.0])->toString()] ]); + $this->assertGreaterThanOrEqual(400, $fail['headers']['status-code']); + $this->assertLessThan(500, $fail['headers']['status-code']); + } + + public function testVectorEmptyQueryCollection(): void + { + $db = $this->client->call(Client::METHOD_POST, '/vectordb', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ 'databaseId' => ID::unique(), 'name' => 'EmptyQ' ]); + $this->assertEquals(201, $db['headers']['status-code']); + $databaseId = $db['body']['$id']; + $col = $this->client->call(Client::METHOD_POST, '/vectordb/' . $databaseId . '/collections', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ 'collectionId' => ID::unique(), 'name' => 'Docs', 'documentSecurity' => true, 'dimensions' => 3, 'permissions' => [Permission::create(Role::user($this->getUser()['$id']))] ]); + $this->assertEquals(201, $col['headers']['status-code']); + $collectionId = $col['body']['$id']; + + $res = $this->client->call(Client::METHOD_GET, "/vectordb/{$databaseId}/collections/{$collectionId}/documents", [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ 'queries' => [Query::vectorCosine('embeddings', [1.0, 0.0, 0.0])->toString()] ]); + $this->assertEquals(200, $res['headers']['status-code']); + $this->assertEquals(0, $res['body']['total']); + } + + /** + * @depends testCreateCollection + */ + public function testCreateIndexes(array $data): array + { + $databaseId = $data['databaseId']; + $collectionId = $data['moviesId']; + + // HNSW Euclidean + $idxEuclidean = $this->client->call(Client::METHOD_POST, "/vectordb/{$databaseId}/collections/{$collectionId}/indexes", [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'key' => 'embedding_euclidean', + 'type' => Database::INDEX_HNSW_EUCLIDEAN, + 'attributes' => ['embeddings'] + ]); + $this->assertEquals(202, $idxEuclidean['headers']['status-code']); + + // HNSW Dot (Inner Product) + $idxDot = $this->client->call(Client::METHOD_POST, "/vectordb/{$databaseId}/collections/{$collectionId}/indexes", [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'key' => 'embedding_dot', + 'type' => Database::INDEX_HNSW_DOT, + 'attributes' => ['embeddings'] + ]); + $this->assertEquals(202, $idxDot['headers']['status-code']); + + // HNSW Cosine + $idxCosine = $this->client->call(Client::METHOD_POST, "/vectordb/{$databaseId}/collections/{$collectionId}/indexes", [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'key' => 'embedding_cosine', + 'type' => Database::INDEX_HNSW_COSINE, + 'attributes' => ['embeddings'] + ]); + $this->assertEquals(202, $idxCosine['headers']['status-code']); + + return [ + 'databaseId' => $databaseId, + 'collectionId' => $collectionId, + 'indexes' => ['embedding_euclidean', 'embedding_dot', 'embedding_cosine'] + ]; + } + + /** + * @depends testCreateIndexes + */ + public function testListIndexes(array $data): void + { + $databaseId = $data['databaseId']; + $collectionId = $data['collectionId']; + + $list = $this->client->call(Client::METHOD_GET, "/vectordb/{$databaseId}/collections/{$collectionId}/indexes", [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]); + + $this->assertEquals(200, $list['headers']['status-code']); + $keys = array_map(fn ($i) => $i['key'], $list['body']['indexes'] ?? []); + foreach ($data['indexes'] as $expectedKey) { + $this->assertContains($expectedKey, $keys); + } + } + + /** + * @depends testCreateIndexes + */ + public function testGetIndexByKey(array $data): void + { + $databaseId = $data['databaseId']; + $collectionId = $data['collectionId']; + + $keysToTypes = [ + 'embedding_euclidean' => Database::INDEX_HNSW_EUCLIDEAN, + 'embedding_dot' => Database::INDEX_HNSW_DOT, + 'embedding_cosine' => Database::INDEX_HNSW_COSINE, + ]; + + foreach ($keysToTypes as $key => $type) { + $res = $this->client->call(Client::METHOD_GET, "/vectordb/{$databaseId}/collections/{$collectionId}/indexes/{$key}", [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]); + $this->assertEquals(200, $res['headers']['status-code']); + $this->assertEquals($key, $res['body']['key']); + $this->assertEquals($type, $res['body']['type']); + } + } + } diff --git a/tests/e2e/Services/Databases/VectorDB/DatabasesConsoleClientTest.php b/tests/e2e/Services/Databases/VectorDB/DatabasesConsoleClientTest.php new file mode 100644 index 0000000000..7f33777b07 --- /dev/null +++ b/tests/e2e/Services/Databases/VectorDB/DatabasesConsoleClientTest.php @@ -0,0 +1,336 @@ +client->call(Client::METHOD_POST, '/vectordb', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'databaseId' => ID::unique(), + 'name' => 'Vector Console DB', + ]); + $this->assertEquals(201, $database['headers']['status-code']); + $this->assertEquals('Vector Console DB', $database['body']['name']); + $this->assertTrue($database['body']['enabled']); + + $databaseId = $database['body']['$id']; + + /** + * Test for SUCCESS + */ + $movies = $this->client->call(Client::METHOD_POST, '/vectordb/' . $databaseId . '/collections', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'collectionId' => ID::unique(), + 'name' => 'Movies', + 'dimensions' => 3, + 'permissions' => [ + Permission::read(Role::any()), + Permission::create(Role::any()), + Permission::update(Role::any()), + Permission::delete(Role::any()), + ], + 'documentSecurity' => true, + ]); + + $this->assertEquals(201, $movies['headers']['status-code']); + $this->assertEquals($movies['body']['name'], 'Movies'); + + /** + * Test when database is disabled but can still create collections + */ + $database = $this->client->call(Client::METHOD_PUT, '/vectordb/' . $databaseId, array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'name' => 'Vector Console DB Updated', + 'enabled' => false, + ]); + + $this->assertFalse($database['body']['enabled']); + + $tvShows = $this->client->call(Client::METHOD_POST, '/vectordb/' . $databaseId . '/collections', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'collectionId' => ID::unique(), + 'name' => 'TvShows', + 'dimensions' => 3, + 'permissions' => [ + Permission::read(Role::any()), + Permission::create(Role::any()), + Permission::update(Role::any()), + Permission::delete(Role::any()), + ], + 'documentSecurity' => true, + ]); + + /** + * Test when collection is disabled but can still modify collections + */ + $database = $this->client->call(Client::METHOD_PUT, '/vectordb/' . $databaseId . '/collections/' . $movies['body']['$id'], array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'name' => 'Movies', + 'enabled' => false, + ]); + + $this->assertEquals(201, $tvShows['headers']['status-code']); + $this->assertEquals($tvShows['body']['name'], 'TvShows'); + + return ['moviesId' => $movies['body']['$id'], 'databaseId' => $databaseId, 'tvShowsId' => $tvShows['body']['$id']]; + } + + /** + * @depends testCreateCollection + * @param array $data + * @throws \Exception + */ + public function testListCollection(array $data) + { + /** + * Test when database is disabled but can still call list collections + */ + $databaseId = $data['databaseId']; + + $collections = $this->client->call(Client::METHOD_GET, '/vectordb/' . $databaseId . '/collections', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'] + ], $this->getHeaders())); + + $this->assertEquals(200, $collections['headers']['status-code']); + $this->assertEquals(2, $collections['body']['total']); + } + + /** + * @depends testCreateCollection + * @param array $data + * @throws \Exception + */ + public function testGetCollection(array $data) + { + $databaseId = $data['databaseId']; + $moviesCollectionId = $data['moviesId']; + + /** + * Test when database and collection are disabled but can still call get collection + */ + $collection = $this->client->call(Client::METHOD_GET, '/vectordb/' . $databaseId . '/collections/' . $moviesCollectionId, array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders())); + + $this->assertEquals(200, $collection['headers']['status-code']); + $this->assertEquals('Movies', $collection['body']['name']); + $this->assertEquals($moviesCollectionId, $collection['body']['$id']); + $this->assertFalse($collection['body']['enabled']); + } + + /** + * @depends testCreateCollection + * @param array $data + * @throws \Exception + * @throws \Exception + */ + public function testUpdateCollection(array $data) + { + $databaseId = $data['databaseId']; + $moviesCollectionId = $data['moviesId']; + + /** + * Test When database and collection are disabled but can still call update collection + */ + $collection = $this->client->call(Client::METHOD_PUT, '/vectordb/' . $databaseId . '/collections/' . $moviesCollectionId, array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'name' => 'Movies Updated', + 'enabled' => false + ]); + + $this->assertEquals(200, $collection['headers']['status-code']); + $this->assertEquals('Movies Updated', $collection['body']['name']); + $this->assertEquals($moviesCollectionId, $collection['body']['$id']); + $this->assertFalse($collection['body']['enabled']); + } + + /** + * @depends testCreateCollection + * @param array $data + * @throws \Exception + * @throws \Exception + */ + public function testDeleteCollection(array $data) + { + $databaseId = $data['databaseId']; + $tvShowsId = $data['tvShowsId']; + + /** + * Test when database and collection are disabled but can still call delete collection + */ + $response = $this->client->call(Client::METHOD_DELETE, '/vectordb/' . $databaseId . '/collections/' . $tvShowsId, array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders())); + + $this->assertEquals(204, $response['headers']['status-code']); + $this->assertEquals($response['body'], ""); + } + + /** + * @depends testCreateCollection + */ + public function testGetDatabaseUsage(array $data) + { + $databaseId = $data['databaseId']; + /** + * Test for FAILURE + */ + + $response = $this->client->call(Client::METHOD_GET, '/vectordb/' . $databaseId . '/usage', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'] + ], $this->getHeaders()), [ + 'range' => '32h' + ]); + + $this->assertEquals(400, $response['headers']['status-code']); + + /** + * Test for SUCCESS + */ + + $response = $this->client->call(Client::METHOD_GET, '/vectordb/' . $databaseId . '/usage', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'] + ], $this->getHeaders()), [ + 'range' => '24h' + ]); + + $this->assertEquals(200, $response['headers']['status-code']); + $this->assertEquals(15, count($response['body'])); + $this->assertEquals('24h', $response['body']['range']); + $this->assertIsNumeric($response['body']['documentsTotal']); + $this->assertIsNumeric($response['body']['collectionsTotal']); + $this->assertIsArray($response['body']['collections']); + $this->assertIsArray($response['body']['documents']); + } + + + /** + * @depends testCreateCollection + */ + public function testGetCollectionUsage(array $data) + { + $databaseId = $data['databaseId']; + /** + * Test for FAILURE + */ + + $response = $this->client->call(Client::METHOD_GET, '/vectordb/' . $databaseId . '/collections/' . $data['moviesId'] . '/usage', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'] + ], $this->getHeaders()), [ + 'range' => '32h' + ]); + + $this->assertEquals(400, $response['headers']['status-code']); + + $response = $this->client->call(Client::METHOD_GET, '/vectordb/' . $databaseId . '/collections/randomCollectionId/usage', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'] + ], $this->getHeaders()), [ + 'range' => '24h' + ]); + + $this->assertEquals(404, $response['headers']['status-code']); + + /** + * Test for SUCCESS + */ + $response = $this->client->call(Client::METHOD_GET, '/vectordb/' . $databaseId . '/collections/' . $data['moviesId'] . '/usage', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'] + ], $this->getHeaders()), [ + 'range' => '24h' + ]); + $this->assertEquals(200, $response['headers']['status-code']); + $this->assertEquals(3, count($response['body'])); + $this->assertEquals('24h', $response['body']['range']); + $this->assertIsNumeric($response['body']['documentsTotal']); + $this->assertIsArray($response['body']['documents']); + } + + /** + * @depends testCreateCollection + * @throws \Utopia\Database\Exception\Query + */ + public function testGetCollectionLogs(array $data) + { + $databaseId = $data['databaseId']; + /** + * Test for SUCCESS + */ + $logs = $this->client->call(Client::METHOD_GET, '/vectordb/' . $databaseId . '/collections/' . $data['moviesId'] . '/logs', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders())); + + $this->assertEquals(200, $logs['headers']['status-code']); + $this->assertIsArray($logs['body']['logs']); + $this->assertIsNumeric($logs['body']['total']); + + $logs = $this->client->call(Client::METHOD_GET, '/vectordb/' . $databaseId . '/collections/' . $data['moviesId'] . '/logs', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'queries' => [Query::limit(1)->toString()] + ]); + + $this->assertEquals(200, $logs['headers']['status-code']); + $this->assertIsArray($logs['body']['logs']); + $this->assertLessThanOrEqual(1, count($logs['body']['logs'])); + $this->assertIsNumeric($logs['body']['total']); + + $logs = $this->client->call(Client::METHOD_GET, '/vectordb/' . $databaseId . '/collections/' . $data['moviesId'] . '/logs', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'queries' => [Query::offset(1)->toString()] + ]); + + $this->assertEquals(200, $logs['headers']['status-code']); + $this->assertIsArray($logs['body']['logs']); + $this->assertIsNumeric($logs['body']['total']); + + $logs = $this->client->call(Client::METHOD_GET, '/vectordb/' . $databaseId . '/collections/' . $data['moviesId'] . '/logs', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'queries' => [Query::offset(1)->toString(), Query::limit(1)->toString()] + ]); + + $this->assertEquals(200, $logs['headers']['status-code']); + $this->assertIsArray($logs['body']['logs']); + $this->assertLessThanOrEqual(1, count($logs['body']['logs'])); + $this->assertIsNumeric($logs['body']['total']); + } +} diff --git a/tests/e2e/Services/Databases/VectorDB/DatabasesCustomClientTest.php b/tests/e2e/Services/Databases/VectorDB/DatabasesCustomClientTest.php index 1d8f282a4d..fafc87e654 100644 --- a/tests/e2e/Services/Databases/VectorDB/DatabasesCustomClientTest.php +++ b/tests/e2e/Services/Databases/VectorDB/DatabasesCustomClientTest.php @@ -2,13 +2,204 @@ namespace Tests\E2E\Services\Databases\VectorDB; +use Tests\E2E\Client; use Tests\E2E\Scopes\ProjectCustom; use Tests\E2E\Scopes\Scope; use Tests\E2E\Scopes\SideClient; +use Utopia\Database\Database; +use Utopia\Database\Helpers\ID; +use Utopia\Database\Helpers\Permission; +use Utopia\Database\Helpers\Role; class DatabasesCustomClientTest extends Scope { use DatabasesBase; use ProjectCustom; use SideClient; + + public function testAllowedPermissions(): void + { + /** + * Test for SUCCESS + */ + + $database = $this->client->call(Client::METHOD_POST, '/vectordb', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'databaseId' => ID::unique(), + 'name' => 'Test Database' + ]); + + $databaseId = $database['body']['$id']; + + // Collection aliases write to create, update, delete + $movies = $this->client->call(Client::METHOD_POST, '/vectordb/' . $databaseId . '/collections', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'collectionId' => ID::unique(), + 'name' => 'Movies', + 'dimensions' => 3, + 'documentSecurity' => true, + 'permissions' => [ + Permission::write(Role::user($this->getUser()['$id'])), + ], + ]); + + $moviesId = $movies['body']['$id']; + + $this->assertContains(Permission::create(Role::user($this->getUser()['$id'])), $movies['body']['$permissions']); + $this->assertContains(Permission::update(Role::user($this->getUser()['$id'])), $movies['body']['$permissions']); + $this->assertContains(Permission::delete(Role::user($this->getUser()['$id'])), $movies['body']['$permissions']); + + // VectorDB uses fixed schema (embeddings, metadata). No attribute creation needed. + + // Document aliases write to update, delete + $document1 = $this->client->call(Client::METHOD_POST, '/vectordb/' . $databaseId . '/collections/' . $moviesId . '/documents', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'documentId' => ID::unique(), + 'data' => [ + 'embeddings' => [1.0, 0.0, 0.0], + 'metadata' => ['k' => 'v'], + ], + 'permissions' => [ + Permission::write(Role::user($this->getUser()['$id'])), + ] + ]); + + $this->assertNotContains(Permission::create(Role::user($this->getUser()['$id'])), $document1['body']['$permissions']); + $this->assertContains(Permission::update(Role::user($this->getUser()['$id'])), $document1['body']['$permissions']); + $this->assertContains(Permission::delete(Role::user($this->getUser()['$id'])), $document1['body']['$permissions']); + + /** + * Test for FAILURE + */ + + // Document does not allow create permission + $document2 = $this->client->call(Client::METHOD_POST, '/vectordb/' . $databaseId . '/collections/' . $moviesId . '/documents', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'documentId' => ID::unique(), + 'data' => [ + 'embeddings' => [0.0, 1.0, 0.0], + 'metadata' => ['k' => 'v'], + ], + 'permissions' => [ + Permission::create(Role::user($this->getUser()['$id'])), + ] + ]); + + $this->assertEquals(400, $document2['headers']['status-code']); + } + + public function testUpdateWithoutPermission(): array + { + // As a part of preparation, we get ID of currently logged-in user + $response = $this->client->call(Client::METHOD_GET, '/account', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders())); + $this->assertEquals(200, $response['headers']['status-code']); + + $userId = $response['body']['$id']; + + $database = $this->client->call(Client::METHOD_POST, '/vectordb', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'databaseId' => ID::custom('permissionCheckDatabase'), + 'name' => 'Test Database', + ]); + $this->assertEquals(201, $database['headers']['status-code']); + $this->assertEquals('Test Database', $database['body']['name']); + + $databaseId = $database['body']['$id']; + // Create collection + $response = $this->client->call(Client::METHOD_POST, '/vectordb/' . $databaseId . '/collections', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'collectionId' => ID::custom('permissionCheck'), + 'name' => 'permissionCheck', + 'dimensions' => 3, + 'permissions' => [], + 'documentSecurity' => true, + ]); + $this->assertEquals(201, $response['headers']['status-code']); + + // Creating document by server, give read permission to our user + some other user + $response = $this->client->call(Client::METHOD_POST, '/vectordb/' . $databaseId . '/collections/permissionCheck/documents', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'documentId' => ID::custom('permissionCheckDocument'), + 'data' => [ + 'embeddings' => [1.0, 0.0, 0.0], + 'metadata' => ['name' => 'AppwriteBeginner'], + ], + 'permissions' => [ + Permission::read(Role::user(ID::custom('user2'))), + Permission::read(Role::user($userId)), + Permission::update(Role::user($userId)), + Permission::delete(Role::user($userId)), + ], + ]); + + $this->assertEquals(201, $response['headers']['status-code']); + + // Update document + // This is the point of this test. We should be allowed to do this action, and it should not fail on permission check + $response = $this->client->call(Client::METHOD_PATCH, '/vectordb/' . $databaseId . '/collections/permissionCheck/documents/permissionCheckDocument', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'] + ], $this->getHeaders()), [ + 'data' => [ + 'embeddings' => [0.0, 1.0, 0.0], + 'metadata' => ['name' => 'AppwriteExpert'], + ] + ]); + + $this->assertEquals(200, $response['headers']['status-code']); + + // Get name of the document, should be the new one + $response = $this->client->call(Client::METHOD_GET, '/vectordb/' . $databaseId . '/collections/permissionCheck/documents/permissionCheckDocument', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders())); + $this->assertEquals(200, $response['headers']['status-code']); + $this->assertEquals("AppwriteExpert", $response['body']['metadata']['name']); + + // Cleanup to prevent collision with other tests + // Delete collection + $response = $this->client->call(Client::METHOD_DELETE, '/vectordb/' . $databaseId . '/collections/permissionCheck', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ])); + + $this->assertEquals(204, $response['headers']['status-code']); + + + // Wait for database worker to finish deleting collection + sleep(2); + + // Make sure collection has been deleted + $response = $this->client->call(Client::METHOD_GET, '/vectordb/' . $databaseId . '/collections/permissionCheck', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ])); + $this->assertEquals(404, $response['headers']['status-code']); + + return []; + } } diff --git a/tests/e2e/Services/Databases/VectorDB/DatabasesCustomServerTest.php b/tests/e2e/Services/Databases/VectorDB/DatabasesCustomServerTest.php new file mode 100644 index 0000000000..b97911359a --- /dev/null +++ b/tests/e2e/Services/Databases/VectorDB/DatabasesCustomServerTest.php @@ -0,0 +1,886 @@ +client->call(Client::METHOD_POST, '/vectordb', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'databaseId' => ID::custom('first'), + 'name' => 'Test 1', + ]); + $this->assertEquals(201, $db1['headers']['status-code']); + $this->assertEquals('Test 1', $db1['body']['name']); + $this->assertEquals('vectordb', $db1['body']['type']); + // Validate database response model fields on create + $this->assertArrayHasKey('$id', $db1['body']); + $this->assertArrayHasKey('$createdAt', $db1['body']); + $this->assertArrayHasKey('$updatedAt', $db1['body']); + $this->assertArrayHasKey('enabled', $db1['body']); + + $db2 = $this->client->call(Client::METHOD_POST, '/vectordb', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'databaseId' => ID::custom('second'), + 'name' => 'Test 2', + ]); + $this->assertEquals(201, $db2['headers']['status-code']); + $this->assertEquals('Test 2', $db2['body']['name']); + $this->assertEquals('vectordb', $db2['body']['type']); + + $list = $this->client->call(Client::METHOD_GET, '/vectordb', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]); + $this->assertEquals(200, $list['headers']['status-code']); + $this->assertIsInt($list['body']['total']); + $this->assertGreaterThanOrEqual(2, $list['body']['total']); + $this->assertIsArray($list['body']['databases']); + $this->assertArrayHasKey('$id', $list['body']['databases'][0]); + $this->assertArrayHasKey('name', $list['body']['databases'][0]); + $this->assertArrayHasKey('type', $list['body']['databases'][0]); + + return ['databaseId' => $db1['body']['$id']]; + } + + /** + * @depends testListDatabases + */ + public function testGetDatabase(array $data): array + { + $databaseId = $data['databaseId']; + $res = $this->client->call(Client::METHOD_GET, '/vectordb/' . $databaseId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]); + $this->assertEquals(200, $res['headers']['status-code']); + $this->assertEquals($databaseId, $res['body']['$id']); + $this->assertEquals('Test 1', $res['body']['name']); + $this->assertEquals('vectordb', $res['body']['type']); + return ['databaseId' => $databaseId]; + } + + /** + * @depends testListDatabases + */ + public function testUpdateDatabase(array $data): array + { + $databaseId = $data['databaseId']; + $res = $this->client->call(Client::METHOD_PUT, '/vectordb/' . $databaseId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'name' => 'Test 1 Updated', + ]); + $this->assertEquals(200, $res['headers']['status-code']); + $this->assertEquals('Test 1 Updated', $res['body']['name']); + $this->assertEquals('vectordb', $res['body']['type']); + return ['databaseId' => $databaseId]; + } + + /** + * @depends testListDatabases + */ + public function testDeleteDatabase(array $data): void + { + $databaseId = $data['databaseId']; + $del = $this->client->call(Client::METHOD_DELETE, '/vectordb/' . $databaseId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]); + $this->assertEquals(204, $del['headers']['status-code']); + $this->assertEquals("", $del['body']); + + $get = $this->client->call(Client::METHOD_GET, '/vectordb/' . $databaseId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]); + $this->assertEquals(404, $get['headers']['status-code']); + } + + public function testCollectionsCRUD(): array + { + // Create database for collections tests + $database = $this->client->call(Client::METHOD_POST, '/vectordb', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'databaseId' => ID::unique(), + 'name' => 'Collections DB', + ]); + $this->assertEquals(201, $database['headers']['status-code']); + $databaseId = $database['body']['$id']; + + // Create two collections + $col1 = $this->client->call(Client::METHOD_POST, '/vectordb/' . $databaseId . '/collections', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'name' => 'Test 1', + 'collectionId' => ID::custom('first'), + 'permissions' => [ + Permission::read(Role::any()), + Permission::create(Role::any()), + Permission::update(Role::any()), + Permission::delete(Role::any()), + ], + 'documentSecurity' => true, + 'dimensions' => 3, + ]); + $this->assertEquals(201, $col1['headers']['status-code']); + // Validate collection response model on create + $this->assertArrayHasKey('$id', $col1['body']); + $this->assertArrayHasKey('$createdAt', $col1['body']); + $this->assertArrayHasKey('$updatedAt', $col1['body']); + $this->assertArrayHasKey('enabled', $col1['body']); + $this->assertArrayHasKey('documentSecurity', $col1['body']); + $this->assertArrayHasKey('dimensions', $col1['body']); + + $col2 = $this->client->call(Client::METHOD_POST, '/vectordb/' . $databaseId . '/collections', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'name' => 'Test 2', + 'collectionId' => ID::custom('second'), + 'permissions' => [ + Permission::read(Role::any()), + Permission::create(Role::any()), + Permission::update(Role::any()), + Permission::delete(Role::any()), + ], + 'documentSecurity' => true, + 'dimensions' => 3, + ]); + $this->assertEquals(201, $col2['headers']['status-code']); + $this->assertArrayHasKey('$id', $col2['body']); + $this->assertArrayHasKey('$createdAt', $col2['body']); + $this->assertArrayHasKey('$updatedAt', $col2['body']); + + // List collections + $list = $this->client->call(Client::METHOD_GET, '/vectordb/' . $databaseId . '/collections', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]); + $this->assertEquals(200, $list['headers']['status-code']); + $this->assertIsInt($list['body']['total']); + $this->assertGreaterThanOrEqual(2, $list['body']['total']); + $this->assertIsArray($list['body']['collections']); + $this->assertArrayHasKey('$id', $list['body']['collections'][0]); + $this->assertArrayHasKey('name', $list['body']['collections'][0]); + $this->assertArrayHasKey('dimensions', $list['body']['collections'][0]); + + // Get collection + $get = $this->client->call(Client::METHOD_GET, '/vectordb/' . $databaseId . '/collections/' . $col1['body']['$id'], [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]); + $this->assertEquals(200, $get['headers']['status-code']); + $this->assertEquals($col1['body']['$id'], $get['body']['$id']); + $this->assertEquals('Test 1', $get['body']['name']); + $this->assertEquals(3, $get['body']['dimensions']); + + // Update collection (name only) + $upd = $this->client->call(Client::METHOD_PUT, '/vectordb/' . $databaseId . '/collections/' . $col1['body']['$id'], [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'name' => 'Test 1 Updated', + ]); + $this->assertEquals(200, $upd['headers']['status-code']); + $this->assertEquals('Test 1 Updated', $upd['body']['name']); + $this->assertArrayHasKey('$updatedAt', $upd['body']); + + // Delete collection + $del = $this->client->call(Client::METHOD_DELETE, '/vectordb/' . $databaseId . '/collections/' . $col2['body']['$id'], [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]); + $this->assertEquals(204, $del['headers']['status-code']); + $this->assertEquals("", $del['body']); + + return [ + 'databaseId' => $databaseId, + 'collectionId' => $col1['body']['$id'], + ]; + } + + /** + * @depends testCollectionsCRUD + */ + public function testUpdateCollectionMore(array $data): array + { + $databaseId = $data['databaseId']; + $collectionId = $data['collectionId']; + + // Update collection name and dimensions + $upd = $this->client->call(Client::METHOD_PUT, '/vectordb/' . $databaseId . '/collections/' . $collectionId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'name' => 'Test 1 Renamed', + 'dimensions' => 4, + ]); + $this->assertEquals(200, $upd['headers']['status-code']); + $this->assertEquals('Test 1 Renamed', $upd['body']['name']); + $this->assertEquals(4, $upd['body']['dimensions']); + + // Read back to confirm + $get = $this->client->call(Client::METHOD_GET, '/vectordb/' . $databaseId . '/collections/' . $collectionId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]); + $this->assertEquals(200, $get['headers']['status-code']); + $this->assertEquals('Test 1 Renamed', $get['body']['name']); + $this->assertEquals(4, $get['body']['dimensions']); + + return $data; + } + + /** + * @depends testCollectionsCRUD + */ + public function testUpdateCollectionEnabledFlag(array $data): array + { + $databaseId = $data['databaseId']; + $collectionId = $data['collectionId']; + + // Disable collection + $disable = $this->client->call(Client::METHOD_PUT, '/vectordb/' . $databaseId . '/collections/' . $collectionId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'name' => 'Updated', + 'enabled' => false, + ]); + $this->assertEquals(200, $disable['headers']['status-code']); + $this->assertFalse($disable['body']['enabled']); + + // Re-enable collection + $enable = $this->client->call(Client::METHOD_PUT, '/vectordb/' . $databaseId . '/collections/' . $collectionId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'name' => 'Updated', + 'enabled' => true, + ]); + $this->assertEquals(200, $enable['headers']['status-code']); + $this->assertTrue($enable['body']['enabled']); + + return $data; + } + + public function testUpdateDatabaseNameAndEnabled(): void + { + // Create isolated database for this test to avoid ordering conflicts + $create = $this->client->call(Client::METHOD_POST, '/vectordb', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'databaseId' => ID::unique(), + 'name' => 'Update DB', + ]); + $this->assertEquals(201, $create['headers']['status-code']); + $databaseId = $create['body']['$id']; + + // Update name + $rename = $this->client->call(Client::METHOD_PUT, '/vectordb/' . $databaseId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'name' => 'Test DB Renamed', + ]); + $this->assertEquals(200, $rename['headers']['status-code']); + $this->assertEquals('Test DB Renamed', $rename['body']['name']); + + // Toggle enabled off then on + $disable = $this->client->call(Client::METHOD_PUT, '/vectordb/' . $databaseId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'name' => 'Test DB Renamed', + 'enabled' => false, + ]); + $this->assertEquals(200, $disable['headers']['status-code']); + $this->assertFalse($disable['body']['enabled']); + + $enable = $this->client->call(Client::METHOD_PUT, '/vectordb/' . $databaseId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'name' => 'Test DB Renamed', + 'enabled' => true, + ]); + $this->assertEquals(200, $enable['headers']['status-code']); + $this->assertTrue($enable['body']['enabled']); + + // Cleanup + $del = $this->client->call(Client::METHOD_DELETE, '/vectordb/' . $databaseId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]); + $this->assertEquals(204, $del['headers']['status-code']); + } + + /** + * @depends testCollectionsCRUD + */ + public function testRecreateIndex(array $data): void + { + $databaseId = $data['databaseId']; + $collectionId = $data['collectionId']; + + // Create a new index variant + $create = $this->client->call(Client::METHOD_POST, "/vectordb/{$databaseId}/collections/{$collectionId}/indexes", [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'key' => 'embedding_euclidean_v2', + 'type' => Database::INDEX_HNSW_EUCLIDEAN, + 'attributes' => ['embeddings'] + ]); + $this->assertEquals(202, $create['headers']['status-code']); + + // Ensure it exists + $get = $this->client->call(Client::METHOD_GET, "/vectordb/{$databaseId}/collections/{$collectionId}/indexes/embedding_euclidean_v2", [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]); + $this->assertEquals(200, $get['headers']['status-code']); + $this->assertEquals('embedding_euclidean_v2', $get['body']['key']); + + // Delete it + $del = $this->client->call(Client::METHOD_DELETE, "/vectordb/{$databaseId}/collections/{$collectionId}/indexes/embedding_euclidean_v2", [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]); + $this->assertEquals(204, $del['headers']['status-code']); + } + + /** + * @depends testCollectionsCRUD + */ + public function testIndexesCRUD(array $data): void + { + $databaseId = $data['databaseId']; + $collectionId = $data['collectionId']; + + // Create indexes + $eu = $this->client->call(Client::METHOD_POST, "/vectordb/{$databaseId}/collections/{$collectionId}/indexes", [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'key' => 'embedding_euclidean', + 'type' => Database::INDEX_HNSW_EUCLIDEAN, + 'attributes' => ['embeddings'] + ]); + $this->assertEquals(202, $eu['headers']['status-code']); + + $dot = $this->client->call(Client::METHOD_POST, "/vectordb/{$databaseId}/collections/{$collectionId}/indexes", [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'key' => 'embedding_dot', + 'type' => Database::INDEX_HNSW_DOT, + 'attributes' => ['embeddings'] + ]); + $this->assertEquals(202, $dot['headers']['status-code']); + + $cos = $this->client->call(Client::METHOD_POST, "/vectordb/{$databaseId}/collections/{$collectionId}/indexes", [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'key' => 'embedding_cosine', + 'type' => Database::INDEX_HNSW_COSINE, + 'attributes' => ['embeddings'] + ]); + $this->assertEquals(202, $cos['headers']['status-code']); + + // List indexes + $list = $this->client->call(Client::METHOD_GET, "/vectordb/{$databaseId}/collections/{$collectionId}/indexes", [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]); + $this->assertEquals(200, $list['headers']['status-code']); + $this->assertIsArray($list['body']['indexes']); + $keys = array_map(fn ($i) => $i['key'], $list['body']['indexes']); + $this->assertContains('embedding_euclidean', $keys); + $this->assertContains('embedding_dot', $keys); + $this->assertContains('embedding_cosine', $keys); + + // Get index by key + $get = $this->client->call(Client::METHOD_GET, "/vectordb/{$databaseId}/collections/{$collectionId}/indexes/embedding_euclidean", [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]); + $this->assertEquals(200, $get['headers']['status-code']); + $this->assertEquals('embedding_euclidean', $get['body']['key']); + $this->assertEquals(Database::INDEX_HNSW_EUCLIDEAN, $get['body']['type']); + + // Delete index + $del = $this->client->call(Client::METHOD_DELETE, "/vectordb/{$databaseId}/collections/{$collectionId}/indexes/embedding_dot", [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]); + $this->assertEquals(204, $del['headers']['status-code']); + sleep(4); + // Ensure it's gone + $getMissing = $this->client->call(Client::METHOD_GET, "/vectordb/{$databaseId}/collections/{$collectionId}/indexes/embedding_dot", [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]); + $this->assertEquals(404, $getMissing['headers']['status-code']); + } + + public function testBulkCreate(): array + { + // Setup: create isolated database and collection + $db = $this->client->call(Client::METHOD_POST, '/vectordb', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'databaseId' => ID::unique(), + 'name' => 'BulkDBCreate' + ]); + $this->assertEquals(201, $db['headers']['status-code']); + $databaseId = $db['body']['$id']; + + $col = $this->client->call(Client::METHOD_POST, "/vectordb/{$databaseId}/collections", [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'collectionId' => ID::unique(), + 'name' => 'BulkColCreate', + 'documentSecurity' => true, + 'dimensions' => 3, + 'permissions' => [Permission::read(Role::any())] + ]); + $this->assertEquals(201, $col['headers']['status-code']); + $collectionId = $col['body']['$id']; + + $docs = [ + [ + 'embeddings' => [1.0, 0.0, 0.0], + 'metadata' => ['group' => 'bulkA'], + '$permissions' => [Permission::read(Role::any())] + ], + [ + 'embeddings' => [0.0, 1.0, 0.0], + 'metadata' => ['group' => 'bulkB'], + '$permissions' => [Permission::read(Role::any())] + ], + ]; + + $res = $this->client->call(Client::METHOD_POST, "/vectordb/{$databaseId}/collections/{$collectionId}/documents", [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'documents' => $docs + ]); + + $this->assertEquals(201, $res['headers']['status-code']); + $this->assertIsInt($res['body']['total'] ?? 0); + $this->assertGreaterThanOrEqual(2, $res['body']['total']); + $this->assertIsArray($res['body']['documents']); + $this->assertCount(2, $res['body']['documents']); + + $ids = array_map(fn ($d) => $d['$id'], $res['body']['documents']); + $this->assertNotEmpty($ids[0]); + $this->assertNotEmpty($ids[1]); + + // Fetch and validate persisted data via GET + foreach ($ids as $i => $id) { + $get = $this->client->call(Client::METHOD_GET, "/vectordb/{$databaseId}/collections/{$collectionId}/documents/{$id}", [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]); + $this->assertEquals(200, $get['headers']['status-code']); + $this->assertEquals($id, $get['body']['$id']); + $this->assertIsArray($get['body']['embeddings']); + $this->assertCount(3, $get['body']['embeddings']); + $this->assertArrayHasKey('group', $get['body']['metadata']); + } + + return [ 'databaseId' => $databaseId, 'collectionId' => $collectionId, 'bulkIds' => $ids ]; + } + + public function testCreateTextEmbeddingsSuccessAndErrors(): void + { + // Setup new database and collection + $db = $this->client->call(Client::METHOD_POST, '/vectordb', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'databaseId' => ID::unique(), + 'name' => 'EmbedDB', + ]); + $this->assertEquals(201, $db['headers']['status-code']); + $databaseId = $db['body']['$id']; + + $col = $this->client->call(Client::METHOD_POST, "/vectordb/{$databaseId}/collections", [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'collectionId' => ID::unique(), + 'name' => 'EmbedCol', + 'documentSecurity' => true, + 'dimensions' => 3, + 'permissions' => [Permission::read(Role::any())] + ]); + $this->assertEquals(201, $col['headers']['status-code']); + $collectionId = $col['body']['$id']; + + // Success: two embeddings + $ok = $this->client->call(Client::METHOD_POST, "/vectordb/{$databaseId}/collections/{$collectionId}/text", [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'documents' => [ + ['text' => 'hello world', 'embeddingModel' => 'embeddinggemma'], + ['text' => 'second sentence', 'embeddingModel' => 'embeddinggemma'], + ] + ]); + $this->assertEquals(200, $ok['headers']['status-code']); + $this->assertIsInt($ok['body']['total'] ?? 0); + $this->assertEquals(2, $ok['body']['total']); + $this->assertIsArray($ok['body']['embeddings']); + $this->assertCount(2, $ok['body']['embeddings']); + foreach ($ok['body']['embeddings'] as $embed) { + $this->assertIsString($embed['model']); + $this->assertIsInt($embed['dimensions']); + $this->assertIsArray($embed['embeddings']); + $this->assertGreaterThan(0, count($embed['embeddings'])); + } + + // Error: missing documents + $missingDocs = $this->client->call(Client::METHOD_POST, "/vectordb/{$databaseId}/collections/{$collectionId}/text", [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], []); + $this->assertEquals(400, $missingDocs['headers']['status-code']); + + // Error: invalid item structure (not an object) + $invalidItem = $this->client->call(Client::METHOD_POST, "/vectordb/{$databaseId}/collections/{$collectionId}/text", [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'documents' => [ 'oops' ] + ]); + $this->assertEquals(400, $invalidItem['headers']['status-code']); + + // Error: missing text + $missingText = $this->client->call(Client::METHOD_POST, "/vectordb/{$databaseId}/collections/{$collectionId}/text", [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'documents' => [ ['embeddingModel' => 'embeddinggemma'] ] + ]); + $this->assertEquals(400, $missingText['headers']['status-code']); + + // Error: missing embeddingModel + $missingModel = $this->client->call(Client::METHOD_POST, "/vectordb/{$databaseId}/collections/{$collectionId}/text", [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'documents' => [ ['text' => 'no model'] ] + ]); + $this->assertEquals(400, $missingModel['headers']['status-code']); + + // Error: unknown embedding model + $unknownModel = $this->client->call(Client::METHOD_POST, "/vectordb/{$databaseId}/collections/{$collectionId}/text", [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'documents' => [ ['text' => 'hello', 'embeddingModel' => 'nonexistent-model'] ] + ]); + $this->assertEquals(400, $unknownModel['headers']['status-code']); + } + + public function testBulkUpsert(): void + { + // Setup fresh db/collection + $db = $this->client->call(Client::METHOD_POST, '/vectordb', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ 'databaseId' => ID::unique(), 'name' => 'BulkDBUpsert' ]); + $this->assertEquals(201, $db['headers']['status-code']); + $databaseId = $db['body']['$id']; + + $col = $this->client->call(Client::METHOD_POST, "/vectordb/{$databaseId}/collections", [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'collectionId' => ID::unique(), + 'name' => 'BulkColUpsert', + 'documentSecurity' => true, + 'dimensions' => 3, + 'permissions' => [Permission::read(Role::any())] + ]); + $this->assertEquals(201, $col['headers']['status-code']); + $collectionId = $col['body']['$id']; + + $docs = [ + [ + 'embeddings' => [0.5, 0.5, 0.0], + 'metadata' => ['group' => 'bulkA', 'updated' => true], + '$permissions' => [Permission::read(Role::any())] + ], + [ + 'embeddings' => [0.2, 0.8, 0.0], + 'metadata' => ['group' => 'bulkB', 'updated' => true], + '$permissions' => [Permission::read(Role::any())] + ], + ]; + + $res = $this->client->call(Client::METHOD_PUT, "/vectordb/{$databaseId}/collections/{$collectionId}/documents", [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'documents' => $docs + ]); + + $this->assertEquals(200, $res['headers']['status-code']); + $this->assertIsArray($res['body']['documents']); + $this->assertCount(2, $res['body']['documents']); + $this->assertTrue($res['body']['documents'][0]['metadata']['updated']); + $this->assertTrue($res['body']['documents'][1]['metadata']['updated']); + + // Fetch and validate updated content + $ids = array_map(fn ($d) => $d['$id'], $res['body']['documents']); + foreach ($ids as $id) { + $get = $this->client->call(Client::METHOD_GET, "/vectordb/{$databaseId}/collections/{$collectionId}/documents/{$id}", [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]); + $this->assertEquals(200, $get['headers']['status-code']); + $this->assertTrue($get['body']['metadata']['updated']); + } + + // Perform another bulk upsert to mutate the same documents + $docs2 = [ + [ 'embeddings' => [0.6, 0.4, 0.0], 'metadata' => ['updatedAgain' => true] ], + [ 'embeddings' => [0.3, 0.7, 0.0], 'metadata' => ['updatedAgain' => true] ], + ]; + $res2 = $this->client->call(Client::METHOD_PUT, "/vectordb/{$databaseId}/collections/{$collectionId}/documents", [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'documents' => $docs2 + ]); + $this->assertEquals(200, $res2['headers']['status-code']); + $this->assertIsArray($res2['body']['documents']); + $this->assertCount(2, $res2['body']['documents']); + + // Fetch again and assert second update persisted + $ids2 = array_map(fn ($d) => $d['$id'], $res2['body']['documents']); + foreach ($ids2 as $id) { + $get2 = $this->client->call(Client::METHOD_GET, "/vectordb/{$databaseId}/collections/{$collectionId}/documents/{$id}", [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]); + $this->assertEquals(200, $get2['headers']['status-code']); + $this->assertTrue($get2['body']['metadata']['updatedAgain']); + } + } + + public function testBulkUpdate(): void + { + // Setup: create db/collection and two docs + $db = $this->client->call(Client::METHOD_POST, '/vectordb', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ 'databaseId' => ID::unique(), 'name' => 'BulkDBUpdate' ]); + $this->assertEquals(201, $db['headers']['status-code']); + $databaseId = $db['body']['$id']; + + $col = $this->client->call(Client::METHOD_POST, "/vectordb/{$databaseId}/collections", [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'collectionId' => ID::unique(), + 'name' => 'BulkColUpdate', + 'documentSecurity' => true, + 'dimensions' => 3, + 'permissions' => [Permission::read(Role::any())] + ]); + $this->assertEquals(201, $col['headers']['status-code']); + $collectionId = $col['body']['$id']; + + $seed = $this->client->call(Client::METHOD_PUT, "/vectordb/{$databaseId}/collections/{$collectionId}/documents", [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'documents' => [ + ['embeddings' => [1.0,0.0,0.0], 'metadata' => ['seed' => 1], '$permissions' => [Permission::read(Role::any())]], + ['embeddings' => [0.0,1.0,0.0], 'metadata' => ['seed' => 2], '$permissions' => [Permission::read(Role::any())]] + ] + ]); + $this->assertEquals(200, $seed['headers']['status-code']); + $ids = array_map(fn ($d) => $d['$id'], $seed['body']['documents']); + + $res = $this->client->call(Client::METHOD_PATCH, "/vectordb/{$databaseId}/collections/{$collectionId}/documents", [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'data' => [ 'metadata' => ['bulkUpdated' => true] ], + 'queries' => [ + \Utopia\Database\Query::equal('$id', $ids)->toString() + ] + ]); + + $this->assertEquals(200, $res['headers']['status-code']); + $this->assertIsArray($res['body']['documents']); + $this->assertCount(2, $res['body']['documents']); + foreach ($res['body']['documents'] as $doc) { + $this->assertTrue($doc['metadata']['bulkUpdated']); + } + + // Fetch by IDs and assert update persisted + foreach ($ids as $id) { + $get = $this->client->call(Client::METHOD_GET, "/vectordb/{$databaseId}/collections/{$collectionId}/documents/{$id}", [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]); + $this->assertEquals(200, $get['headers']['status-code']); + $this->assertTrue($get['body']['metadata']['bulkUpdated']); + } + } + + public function testBulkDelete(): void + { + // Setup: create db/collection and two docs + $db = $this->client->call(Client::METHOD_POST, '/vectordb', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ 'databaseId' => ID::unique(), 'name' => 'BulkDBDelete' ]); + $this->assertEquals(201, $db['headers']['status-code']); + $databaseId = $db['body']['$id']; + + $col = $this->client->call(Client::METHOD_POST, "/vectordb/{$databaseId}/collections", [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'collectionId' => ID::unique(), + 'name' => 'BulkColDelete', + 'documentSecurity' => true, + 'dimensions' => 3, + 'permissions' => [Permission::read(Role::any())] + ]); + $this->assertEquals(201, $col['headers']['status-code']); + $collectionId = $col['body']['$id']; + + $seed = $this->client->call(Client::METHOD_PUT, "/vectordb/{$databaseId}/collections/{$collectionId}/documents", [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'documents' => [ + ['embeddings' => [1.0,0.0,0.0], 'metadata' => ['seed' => 1], '$permissions' => [Permission::read(Role::any())]], + ['embeddings' => [0.0,1.0,0.0], 'metadata' => ['seed' => 2], '$permissions' => [Permission::read(Role::any())]] + ] + ]); + $this->assertEquals(200, $seed['headers']['status-code']); + $ids = array_map(fn ($d) => $d['$id'], $seed['body']['documents']); + + $res = $this->client->call(Client::METHOD_DELETE, "/vectordb/{$databaseId}/collections/{$collectionId}/documents", [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'queries' => [ + \Utopia\Database\Query::equal('$id', $ids)->toString() + ] + ]); + $this->assertEquals(200, $res['headers']['status-code']); + $this->assertIsInt($res['body']['total'] ?? 0); + $this->assertGreaterThanOrEqual(2, $res['body']['total']); + + // Ensure they are deleted + foreach ($ids as $id) { + $get = $this->client->call(Client::METHOD_GET, "/vectordb/{$databaseId}/collections/{$collectionId}/documents/{$id}", [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]); + $this->assertEquals(404, $get['headers']['status-code']); + } + } + +} diff --git a/tests/e2e/Services/Databases/VectorDB/Permissions/DatabasesPermissionsGuestTest.php b/tests/e2e/Services/Databases/VectorDB/Permissions/DatabasesPermissionsGuestTest.php new file mode 100644 index 0000000000..6851427a09 --- /dev/null +++ b/tests/e2e/Services/Databases/VectorDB/Permissions/DatabasesPermissionsGuestTest.php @@ -0,0 +1,268 @@ +client->call(Client::METHOD_POST, '/vectordb', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'databaseId' => ID::unique(), + 'name' => 'VectorGuestDB', + ]); + $this->assertEquals(201, $database['headers']['status-code']); + $this->assertEquals('VectorGuestDB', $database['body']['name']); + + $databaseId = $database['body']['$id']; + $publicMovies = $this->client->call(Client::METHOD_POST, '/vectordb/' . $databaseId . '/collections', $this->getServerHeader(), [ + 'collectionId' => ID::unique(), + 'name' => 'Movies', + 'dimensions' => 3, + 'permissions' => [ + Permission::read(Role::any()), + Permission::create(Role::any()), + Permission::update(Role::any()), + Permission::delete(Role::any()), + ], + ]); + $privateMovies = $this->client->call(Client::METHOD_POST, '/vectordb/' . $databaseId . '/collections', $this->getServerHeader(), [ + 'collectionId' => ID::unique(), + 'name' => 'Movies', + 'dimensions' => 3, + 'permissions' => [], + 'documentSecurity' => true, + ]); + + $publicCollection = ['id' => $publicMovies['body']['$id']]; + $privateCollection = ['id' => $privateMovies['body']['$id']]; + + return [ + 'databaseId' => $databaseId, + 'publicCollectionId' => $publicCollection['id'], + 'privateCollectionId' => $privateCollection['id'], + ]; + } + + public function permissionsProvider(): array + { + return [ + [[Permission::read(Role::any())]], + [[Permission::read(Role::users())]], + [[Permission::update(Role::any()), Permission::delete(Role::any())]], + [[Permission::read(Role::any()), Permission::update(Role::any()), Permission::delete(Role::any())]], + [[Permission::read(Role::users()), Permission::update(Role::users()), Permission::delete(Role::users())]], + [[Permission::read(Role::any()), Permission::update(Role::users()), Permission::delete(Role::users())]], + ]; + } + + /** + * @dataProvider permissionsProvider + */ + public function testReadDocuments($permissions) + { + $data = $this->createCollection(); + $publicCollectionId = $data['publicCollectionId']; + $privateCollectionId = $data['privateCollectionId']; + $databaseId = $data['databaseId']; + + $publicResponse = $this->client->call(Client::METHOD_POST, '/vectordb/' . $databaseId . '/collections/' . $publicCollectionId . '/documents', $this->getServerHeader(), [ + 'documentId' => ID::unique(), + 'data' => [ + 'embeddings' => [1.0, 0.0, 0.0], + 'metadata' => ['title' => 'Lorem'], + ], + 'permissions' => $permissions, + ]); + $privateResponse = $this->client->call(Client::METHOD_POST, '/vectordb/' . $databaseId . '/collections/' . $privateCollectionId . '/documents', $this->getServerHeader(), [ + 'documentId' => ID::unique(), + 'data' => [ + 'embeddings' => [0.0, 1.0, 0.0], + 'metadata' => ['title' => 'Lorem'], + ], + 'permissions' => $permissions, + ]); + + $this->assertEquals(201, $publicResponse['headers']['status-code']); + $this->assertEquals(201, $privateResponse['headers']['status-code']); + + $roles = Authorization::getRoles(); + Authorization::cleanRoles(); + + $publicDocuments = $this->client->call(Client::METHOD_GET, '/vectordb/' . $databaseId . '/collections/' . $publicCollectionId . '/documents', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ]); + $privateDocuments = $this->client->call(Client::METHOD_GET, '/vectordb/' . $databaseId . '/collections/' . $privateCollectionId . '/documents', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ]); + + $this->assertEquals(1, $publicDocuments['body']['total']); + $this->assertEquals($permissions, $publicDocuments['body']['documents'][0]['$permissions']); + + if (\in_array(Permission::read(Role::any()), $permissions)) { + $this->assertEquals(1, $privateDocuments['body']['total']); + $this->assertEquals($permissions, $privateDocuments['body']['documents'][0]['$permissions']); + } else { + $this->assertEquals(0, $privateDocuments['body']['total']); + } + + foreach ($roles as $role) { + Authorization::setRole($role); + } + } + + public function testWriteDocument() + { + $data = $this->createCollection(); + $publicCollectionId = $data['publicCollectionId']; + $privateCollectionId = $data['privateCollectionId']; + $databaseId = $data['databaseId']; + + $roles = Authorization::getRoles(); + Authorization::cleanRoles(); + + $publicResponse = $this->client->call(Client::METHOD_POST, '/vectordb/' . $databaseId . '/collections/' . $publicCollectionId . '/documents', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], [ + 'documentId' => ID::unique(), + 'data' => [ + 'embeddings' => [1.0, 0.0, 0.0], + 'metadata' => ['title' => 'Lorem'], + ] + ]); + + $publicDocumentId = $publicResponse['body']['$id']; + $this->assertEquals(201, $publicResponse['headers']['status-code']); + + $privateResponse = $this->client->call(Client::METHOD_POST, '/vectordb/' . $databaseId . '/collections/' . $privateCollectionId . '/documents', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], [ + 'documentId' => ID::unique(), + 'data' => [ + 'embeddings' => [0.0, 1.0, 0.0], + 'metadata' => ['title' => 'Lorem'], + ], + ]); + + $this->assertEquals(401, $privateResponse['headers']['status-code']); + + // Create a document in private collection with API key so we can test that update and delete are also not allowed + $privateResponse = $this->client->call(Client::METHOD_POST, '/vectordb/' . $databaseId . '/collections/' . $privateCollectionId . '/documents', $this->getServerHeader(), [ + 'documentId' => ID::unique(), + 'data' => [ + 'embeddings' => [0.0, 0.0, 1.0], + 'metadata' => ['title' => 'Lorem'], + ], + ]); + + $this->assertEquals(201, $privateResponse['headers']['status-code']); + $privateDocumentId = $privateResponse['body']['$id']; + + $publicDocument = $this->client->call(Client::METHOD_PUT, '/vectordb/' . $databaseId . '/collections/' . $publicCollectionId . '/documents/' . $publicDocumentId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], [ + 'data' => [ + 'embeddings' => [0.5, 0.5, 0.0], + 'metadata' => ['title' => 'Thor: Ragnarok'], + ], + ]); + + $this->assertEquals(200, $publicDocument['headers']['status-code']); + $this->assertEquals('Thor: Ragnarok', $publicDocument['body']['metadata']['title']); + + $privateDocument = $this->client->call(Client::METHOD_PUT, '/vectordb/' . $databaseId . '/collections/' . $privateCollectionId . '/documents/' . $privateDocumentId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], [ + 'data' => [ + 'embeddings' => [0.2, 0.3, 0.5], + 'metadata' => ['title' => 'Thor: Ragnarok'], + ], + ]); + + $this->assertEquals(401, $privateDocument['headers']['status-code']); + + $publicDocument = $this->client->call(Client::METHOD_DELETE, '/vectordb/' . $databaseId . '/collections/' . $publicCollectionId . '/documents/' . $publicDocumentId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ]); + + $this->assertEquals(204, $publicDocument['headers']['status-code']); + + $privateDocument = $this->client->call(Client::METHOD_DELETE, '/vectordb/' . $databaseId . '/collections/' . $privateCollectionId . '/documents/' . $privateDocumentId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ]); + + $this->assertEquals(401, $privateDocument['headers']['status-code']); + + foreach ($roles as $role) { + Authorization::setRole($role); + } + } + + public function testWriteDocumentWithPermissions() + { + $database = $this->client->call(Client::METHOD_POST, '/vectordb', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'databaseId' => ID::unique(), + 'name' => 'VectorGuestPermsWrite', + ]); + $this->assertEquals(201, $database['headers']['status-code']); + $this->assertEquals('VectorGuestPermsWrite', $database['body']['name']); + + $databaseId = $database['body']['$id']; + $movies = $this->client->call(Client::METHOD_POST, '/vectordb/' . $databaseId . '/collections', $this->getServerHeader(), [ + 'collectionId' => ID::unique(), + 'name' => 'Movies', + 'dimensions' => 3, + 'permissions' => [ + Permission::create(Role::any()), + ], + 'documentSecurity' => true + ]); + + $moviesId = $movies['body']['$id']; + + $document = $this->client->call(Client::METHOD_POST, '/vectordb/' . $databaseId . '/collections/' . $moviesId . '/documents', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], [ + 'documentId' => ID::unique(), + 'data' => [ + 'embeddings' => [1.0, 0.0, 0.0], + 'metadata' => ['title' => 'Thor: Ragnarok'], + ], + 'permissions' => [ + Permission::read(Role::any()), + ] + ]); + + $this->assertEquals(201, $document['headers']['status-code']); + $this->assertEquals('Thor: Ragnarok', $document['body']['metadata']['title']); + } +} diff --git a/tests/e2e/Services/Databases/VectorDB/Permissions/DatabasesPermissionsMemberTest.php b/tests/e2e/Services/Databases/VectorDB/Permissions/DatabasesPermissionsMemberTest.php new file mode 100644 index 0000000000..d45037999d --- /dev/null +++ b/tests/e2e/Services/Databases/VectorDB/Permissions/DatabasesPermissionsMemberTest.php @@ -0,0 +1,254 @@ + $this->createUser('user1', 'lorem@ipsum.com'), + 'user2' => $this->createUser('user2', 'dolor@ipsum.com'), + ]; + } + + public function permissionsProvider(): array + { + return [ + [ + 'permissions' => [Permission::read(Role::any())], + 'any' => 1, + 'users' => 1, + 'doconly' => 1, + ], + [ + 'permissions' => [Permission::read(Role::users())], + 'any' => 2, + 'users' => 2, + 'doconly' => 2, + ], + [ + 'permissions' => [Permission::read(Role::user(ID::custom('random')))], + 'any' => 3, + 'users' => 3, + 'doconly' => 2, + ], + [ + 'permissions' => [Permission::read(Role::user(ID::custom('lorem'))), Permission::update(Role::user('lorem')), Permission::delete(Role::user('lorem'))], + 'any' => 4, + 'users' => 4, + 'doconly' => 2, + ], + [ + 'permissions' => [Permission::read(Role::user(ID::custom('dolor'))), Permission::update(Role::user('dolor')), Permission::delete(Role::user('dolor'))], + 'any' => 5, + 'users' => 5, + 'doconly' => 2, + ], + [ + 'permissions' => [Permission::read(Role::user(ID::custom('dolor'))), Permission::read(Role::user('lorem')), Permission::update(Role::user('dolor')), Permission::delete(Role::user('dolor'))], + 'any' => 6, + 'users' => 6, + 'doconly' => 2, + ], + [ + 'permissions' => [Permission::update(Role::any()), Permission::delete(Role::any())], + 'any' => 7, + 'users' => 7, + 'doconly' => 2, + ], + [ + 'permissions' => [Permission::read(Role::any()), Permission::update(Role::any()), Permission::delete(Role::any())], + 'any' => 8, + 'users' => 8, + 'doconly' => 3, + ], + [ + 'permissions' => [Permission::read(Role::any()), Permission::update(Role::users()), Permission::delete(Role::users())], + 'any' => 9, + 'users' => 9, + 'doconly' => 4, + ], + [ + 'permissions' => [Permission::read(Role::user(ID::custom('user1')))], + 'any' => 10, + 'users' => 10, + 'doconly' => 5, + ], + [ + 'permissions' => [Permission::read(Role::user(ID::custom('user1'))), Permission::read(Role::user(ID::custom('user1')))], + 'any' => 11, + 'users' => 11, + 'doconly' => 6, + ], + [ + 'permissions' => [Permission::read(Role::users()), Permission::update(Role::users()), Permission::delete(Role::users())], + 'any' => 12, + 'users' => 12, + 'doconly' => 7, + ], + ]; + } + + /** + * Setup database + * + * Data providers lose object state so explicitly pass [$users, $collections] to each iteration + * + * @return array + * @throws \Exception + */ + public function testSetupDatabase(): array + { + $this->createUsers(); + + $db = $this->client->call(Client::METHOD_POST, '/vectordb', $this->getServerHeader(), [ + 'databaseId' => ID::unique(), + 'name' => 'Test Database', + ]); + $this->assertEquals(201, $db['headers']['status-code']); + + $databaseId = $db['body']['$id']; + + $public = $this->client->call(Client::METHOD_POST, '/vectordb/' . $databaseId . '/collections', $this->getServerHeader(), [ + 'collectionId' => ID::unique(), + 'name' => 'Movies', + 'dimensions' => 3, + 'permissions' => [ + Permission::read(Role::any()), + Permission::create(Role::any()), + Permission::update(Role::any()), + Permission::delete(Role::any()), + ], + 'documentSecurity' => true, + ]); + $this->assertEquals(201, $public['headers']['status-code']); + $this->collections = ['public' => $public['body']['$id']]; + + $private = $this->client->call(Client::METHOD_POST, '/vectordb/' . $databaseId . '/collections', $this->getServerHeader(), [ + 'collectionId' => ID::unique(), + 'name' => 'Private Movies', + 'dimensions' => 3, + 'permissions' => [ + Permission::read(Role::users()), + Permission::create(Role::users()), + Permission::update(Role::users()), + Permission::delete(Role::users()), + ], + 'documentSecurity' => true, + ]); + $this->assertEquals(201, $private['headers']['status-code']); + $this->collections['private'] = $private['body']['$id']; + + $doconly = $this->client->call(Client::METHOD_POST, '/vectordb/' . $databaseId . '/collections', $this->getServerHeader(), [ + 'collectionId' => ID::unique(), + 'name' => 'Document Only Movies', + 'dimensions' => 3, + 'permissions' => [], + 'documentSecurity' => true, + ]); + $this->assertEquals(201, $private['headers']['status-code']); + $this->collections['doconly'] = $doconly['body']['$id']; + + return [ + 'users' => $this->users, + 'collections' => $this->collections, + 'databaseId' => $databaseId + ]; + } + + /** + * Data provider params are passed before test dependencies + * @dataProvider permissionsProvider + * @depends testSetupDatabase + */ + public function testReadDocuments($permissions, $anyCount, $usersCount, $docOnlyCount, $data) + { + $users = $data['users']; + $collections = $data['collections']; + $databaseId = $data['databaseId']; + + $response = $this->client->call(Client::METHOD_POST, '/vectordb/' . $databaseId . '/collections/' . $collections['public'] . '/documents', $this->getServerHeader(), [ + 'documentId' => ID::unique(), + 'data' => [ + 'embeddings' => [1.0, 0.0, 0.0], + 'metadata' => ['title' => 'Lorem'], + ], + 'permissions' => $permissions + ]); + $this->assertEquals(201, $response['headers']['status-code']); + + $response = $this->client->call(Client::METHOD_POST, '/vectordb/' . $databaseId . '/collections/' . $collections['private'] . '/documents', $this->getServerHeader(), [ + 'documentId' => ID::unique(), + 'data' => [ + 'embeddings' => [0.0, 1.0, 0.0], + 'metadata' => ['title' => 'Lorem'], + ], + 'permissions' => $permissions + ]); + $this->assertEquals(201, $response['headers']['status-code']); + + $response = $this->client->call(Client::METHOD_POST, '/vectordb/' . $databaseId . '/collections/' . $collections['doconly'] . '/documents', $this->getServerHeader(), [ + 'documentId' => ID::unique(), + 'data' => [ + 'embeddings' => [0.0, 0.0, 1.0], + 'metadata' => ['title' => 'Lorem'], + ], + 'permissions' => $permissions + ]); + $this->assertEquals(201, $response['headers']['status-code']); + + /** + * Check "any" permission collection + */ + $documents = $this->client->call(Client::METHOD_GET, '/vectordb/' . $databaseId . '/collections/' . $collections['public'] . '/documents', [ + 'origin' => 'http://localhost', + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'cookie' => 'a_session_' . $this->getProject()['$id'] . '=' . $users['user1']['session'], + ]); + + $this->assertEquals(200, $documents['headers']['status-code']); + $this->assertEquals($anyCount, $documents['body']['total']); + + /** + * Check "users" permission collection + */ + $documents = $this->client->call(Client::METHOD_GET, '/vectordb/' . $databaseId . '/collections/' . $collections['private'] . '/documents', [ + 'origin' => 'http://localhost', + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'cookie' => 'a_session_' . $this->getProject()['$id'] . '=' . $users['user1']['session'], + ]); + + $this->assertEquals(200, $documents['headers']['status-code']); + $this->assertEquals($usersCount, $documents['body']['total']); + + /** + * Check "user:user1" document only permission collection + */ + $documents = $this->client->call(Client::METHOD_GET, '/vectordb/' . $databaseId . '/collections/' . $collections['doconly'] . '/documents', [ + 'origin' => 'http://localhost', + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'cookie' => 'a_session_' . $this->getProject()['$id'] . '=' . $users['user1']['session'], + ]); + + $this->assertEquals(200, $documents['headers']['status-code']); + $this->assertEquals($docOnlyCount, $documents['body']['total']); + } +} diff --git a/tests/e2e/Services/Databases/VectorDB/Permissions/DatabasesPermissionsScope.php b/tests/e2e/Services/Databases/VectorDB/Permissions/DatabasesPermissionsScope.php new file mode 100644 index 0000000000..7d1bfb41d6 --- /dev/null +++ b/tests/e2e/Services/Databases/VectorDB/Permissions/DatabasesPermissionsScope.php @@ -0,0 +1,87 @@ +client->call(Client::METHOD_POST, '/account', [ + 'origin' => 'http://localhost', + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-dev-key' => $this->getProject()['devKey'] ?? '', + ], [ + 'userId' => $id, + 'email' => $email, + 'password' => $password + ]); + + $this->assertEquals(201, $user['headers']['status-code']); + + $session = $this->client->call(Client::METHOD_POST, '/account/sessions/email', [ + 'origin' => 'http://localhost', + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], [ + 'email' => $email, + 'password' => $password, + ]); + + $session = $session['cookies']['a_session_' . $this->getProject()['$id']]; + + $user = [ + '$id' => $user['body']['$id'], + 'email' => $user['body']['email'], + 'session' => $session, + ]; + $this->users[$id] = $user; + + return $user; + } + + public function getCreatedUser(string $id): array + { + return $this->users[$id] ?? []; + } + + public function createTeam(string $id, string $name): array + { + $team = $this->client->call(Client::METHOD_POST, '/teams', $this->getServerHeader(), [ + 'teamId' => $id, + 'name' => $name + ]); + $this->teams[$id] = $team['body']; + + return $team['body']; + } + + public function addToTeam(string $user, string $team, array $roles = []): array + { + $membership = $this->client->call(Client::METHOD_POST, '/teams/' . $team . '/memberships', $this->getServerHeader(), [ + 'teamId' => $team, + 'email' => $this->getCreatedUser($user)['email'], + 'roles' => $roles, + 'url' => 'http://localhost:5000/join-us#title' + ]); + + return [ + 'user' => $membership['body']['userId'], + 'membership' => $membership['body']['$id'] + ]; + } + + public function getServerHeader(): array + { + return [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]; + } +} diff --git a/tests/e2e/Services/Databases/VectorDB/Permissions/DatabasesPermissionsTeamTest.php b/tests/e2e/Services/Databases/VectorDB/Permissions/DatabasesPermissionsTeamTest.php new file mode 100644 index 0000000000..4f2a210ef2 --- /dev/null +++ b/tests/e2e/Services/Databases/VectorDB/Permissions/DatabasesPermissionsTeamTest.php @@ -0,0 +1,199 @@ + $this->createTeam('team1', 'Team 1'), + 'team2' => $this->createTeam('team2', 'Team 2'), + ]; + } + + public function createUsers(): array + { + return [ + 'user1' => $this->createUser('user1', 'lorem@ipsum.com'), + 'user2' => $this->createUser('user2', 'dolor@ipsum.com'), + 'user3' => $this->createUser('user3', 'sit@ipsum.com'), + ]; + } + + public function createCollections($teams) + { + $db = $this->client->call(Client::METHOD_POST, '/vectordb', $this->getServerHeader(), [ + 'databaseId' => $this->databaseId, + 'name' => 'Test Database', + ]); + $this->assertEquals(201, $db['headers']['status-code']); + + $collection1 = $this->client->call(Client::METHOD_POST, '/vectordb/' . $this->databaseId . '/collections', $this->getServerHeader(), [ + 'collectionId' => ID::custom('collection1'), + 'name' => 'Collection 1', + 'dimensions' => 3, + 'permissions' => [ + Permission::read(Role::team($teams['team1']['$id'])), + Permission::create(Role::team($teams['team1']['$id'], 'admin')), + Permission::update(Role::team($teams['team1']['$id'], 'admin')), + Permission::delete(Role::team($teams['team1']['$id'], 'admin')), + ], + ]); + + $this->collections['collection1'] = $collection1['body']['$id']; + + $collection2 = $this->client->call(Client::METHOD_POST, '/vectordb/' . $this->databaseId . '/collections', $this->getServerHeader(), [ + 'collectionId' => ID::custom('collection2'), + 'name' => 'Collection 2', + 'dimensions' => 3, + 'permissions' => [ + Permission::read(Role::team($teams['team2']['$id'])), + Permission::create(Role::team($teams['team2']['$id'], 'owner')), + Permission::update(Role::team($teams['team2']['$id'], 'owner')), + Permission::delete(Role::team($teams['team2']['$id'], 'owner')), + ] + ]); + + $this->collections['collection2'] = $collection2['body']['$id']; + + return $this->collections; + } + + /* + * $success = can $user read from $collection + * [$user, $collection, $success] + */ + public function readDocumentsProvider(): array + { + return [ + ['user1', 'collection1', true], + ['user2', 'collection1', false], + ['user3', 'collection1', true], + ['user1', 'collection2', false], + ['user2', 'collection2', true], + ['user3', 'collection2', true], + ]; + } + + /* + * $success = can $user write to $collection + * [$user, $collection, $success] + */ + public function writeDocumentsProvider(): array + { + return [ + ['user1', 'collection1', true], + ['user2', 'collection1', false], + ['user3', 'collection1', false], + ['user1', 'collection2', false], + ['user2', 'collection2', true], + ['user3', 'collection2', false], + ]; + } + + /** + * Setup database + * + * Data providers lose object state + * so explicitly pass $users to each iteration + * @return array $users + */ + public function testSetupDatabase(): array + { + $this->createUsers(); + $this->createTeams(); + + $this->addToTeam('user1', 'team1', ['admin']); + $this->addToTeam('user2', 'team2', ['owner']); + + // user3 in both teams but with no roles + $this->addToTeam('user3', 'team1'); + $this->addToTeam('user3', 'team2'); + + $this->createCollections($this->teams); + + $response = $this->client->call(Client::METHOD_POST, '/vectordb/' . $this->databaseId . '/collections/' . $this->collections['collection1'] . '/documents', $this->getServerHeader(), [ + 'documentId' => ID::unique(), + 'data' => [ + 'embeddings' => [1.0, 0.0, 0.0], + 'metadata' => ['title' => 'Lorem'], + ], + ]); + $this->assertEquals(201, $response['headers']['status-code']); + + $response = $this->client->call(Client::METHOD_POST, '/vectordb/' . $this->databaseId . '/collections/' . $this->collections['collection2'] . '/documents', $this->getServerHeader(), [ + 'documentId' => ID::unique(), + 'data' => [ + 'embeddings' => [0.0, 1.0, 0.0], + 'metadata' => ['title' => 'Ipsum'], + ], + ]); + $this->assertEquals(201, $response['headers']['status-code']); + + return $this->users; + } + + /** + * Data provider params are passed before test dependencies + * @depends testSetupDatabase + * @dataProvider readDocumentsProvider + */ + public function testReadDocuments($user, $collection, $success, $users) + { + $documents = $this->client->call(Client::METHOD_GET, '/vectordb/' . $this->databaseId . '/collections/' . $collection . '/documents', [ + 'origin' => 'http://localhost', + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'cookie' => 'a_session_' . $this->getProject()['$id'] . '=' . $users[$user]['session'], + ]); + + if ($success) { + $this->assertCount(1, $documents['body']['documents']); + } else { + $this->assertEquals(401, $documents['headers']['status-code']); + } + } + + /** + * @depends testSetupDatabase + * @dataProvider writeDocumentsProvider + */ + public function testWriteDocuments($user, $collection, $success, $users) + { + $documents = $this->client->call(Client::METHOD_POST, '/vectordb/' . $this->databaseId . '/collections/' . $collection . '/documents', [ + 'origin' => 'http://localhost', + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'cookie' => 'a_session_' . $this->getProject()['$id'] . '=' . $users[$user]['session'], + ], [ + 'documentId' => ID::unique(), + 'data' => [ + 'embeddings' => [0.2, 0.3, 0.5], + 'metadata' => ['title' => 'Ipsum'], + ], + ]); + + if ($success) { + $this->assertEquals(201, $documents['headers']['status-code']); + } else { + // 401 if user is a part of team, 404 otherwise + $this->assertContains($documents['headers']['status-code'], [401, 404]); + } + } +} From bd6f62dd688cbf5975e1e1832da614f0ff9965f7 Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Thu, 6 Nov 2025 17:48:39 +0530 Subject: [PATCH 06/27] updated realtime, create endpoint , resources, registers --- .env | 3 +- .github/workflows/tests.yml | 1 + app/config/collections/vectordb.php | 1 - app/init/registers.php | 7 +- app/init/resources.php | 7 +- docker-compose.yml | 114 +++++------ ollama.dockerfile | 8 +- src/Appwrite/Event/Event.php | 4 +- src/Appwrite/Messaging/Adapter/Realtime.php | 2 + .../Http/VectorDB/Collections/Create.php | 36 ++-- .../Collections/Documents/Text/Create.php | 4 +- .../Databases/Services/Registry/VectorDB.php | 7 +- src/Appwrite/Utopia/Response.php | 6 +- .../Realtime/RealtimeCustomClientTest.php | 188 ++++++++++++++++++ 14 files changed, 291 insertions(+), 97 deletions(-) diff --git a/.env b/.env index 3eef0189d0..ae597d5bc7 100644 --- a/.env +++ b/.env @@ -48,7 +48,8 @@ _APP_DB_PORT_DOCUMENTSDB=27017 _APP_DB_ADAPTER_VECTORDB=postgresql _APP_DB_HOST_VECTORDB=postgresql _APP_DB_PORT_VECTORDB=5432 -OLLAMA_MODELS=embeddinggemma +_APP_EMBEDDING_MODELS=embeddinggemma +_APP_EMBEDDING_ENDPOINT='http://ollama:11434/api/embed' _APP_STORAGE_DEVICE=Local _APP_STORAGE_S3_ACCESS_KEY= _APP_STORAGE_S3_SECRET= diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 1d98ee5952..4c26dc0c97 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -172,6 +172,7 @@ jobs: Databases/Legacy, Databases/TablesDB, Databases/DocumentsDB, + Databases/VectorDB, Functions, FunctionsSchedule, GraphQL, diff --git a/app/config/collections/vectordb.php b/app/config/collections/vectordb.php index c7d87b3509..6a8a26aabb 100644 --- a/app/config/collections/vectordb.php +++ b/app/config/collections/vectordb.php @@ -106,7 +106,6 @@ return [ '$id' => ID::custom('embeddings'), 'type' => Database::VAR_VECTOR, 'required' => true, - 'size' => 128, 'signed' => false, 'array' => false, 'filters' => [], diff --git a/app/init/registers.php b/app/init/registers.php index fd21b5ce81..cb8a3bb9ed 100644 --- a/app/init/registers.php +++ b/app/init/registers.php @@ -119,7 +119,7 @@ $register->set('pools', function () { 'pass' => System::getEnv('_APP_DB_PASS', ''), 'path' => System::getEnv('_APP_DB_SCHEMA', ''), ]); - $fallbackForVECTORDB = 'db_main=' . AppwriteURL::unparse([ + $fallbackForVectorDB = 'db_main=' . AppwriteURL::unparse([ 'scheme' => System::getEnv('_APP_DB_ADAPTER_VECTORDB', 'postgresql'), 'host' => System::getEnv('_APP_DB_HOST_VECTORDB', 'postgresql'), 'port' => System::getEnv('_APP_DB_PORT_VECTORDB', '5432'), @@ -156,7 +156,7 @@ $register->set('pools', function () { ], 'vectordb' => [ 'type' => 'database', - 'dsns' => System::getEnv('_APP_CONNECTIONS_DATABASE_VECTORDB', $fallbackForVECTORDB), + 'dsns' => System::getEnv('_APP_CONNECTIONS_DATABASE_VECTORDB', $fallbackForVectorDB), 'multiple' => true, 'schemes' => ['postgresql'], ], @@ -400,6 +400,9 @@ $register->set('smtp', function () { $register->set('geodb', function () { return new Reader(__DIR__ . '/../assets/dbip/dbip-country-lite-2024-09.mmdb'); }); +$register->set('embeddingAgent', function () { + return System::getEnv('_APP_EMBEDDING_ENDPOINT', 'http://ollama:11434/api/embed'); +}); $register->set('passwordsDictionary', function () { $content = \file_get_contents(__DIR__ . '/../assets/security/10k-common-passwords'); $content = explode("\n", $content); diff --git a/app/init/resources.php b/app/init/resources.php index adf6ea9620..18d37c3a37 100644 --- a/app/init/resources.php +++ b/app/init/resources.php @@ -1170,9 +1170,8 @@ App::setResource('transactionState', function (Database $dbForProject, callable return new TransactionState($dbForProject, $getDatabasesDB); }, ['dbForProject', 'getDatabasesDB']); -App::setResource('embeddingAgent', function () { - // TODO: ollama endpoint should be taken from env in cloud(for autoscaling) +App::setResource('embeddingAgent', function ($register) { $adapter = new Ollama(); - $adapter->setEndpoint('http://ollama:11434/api/embed'); + $adapter->setEndpoint($register->get('embeddingAgent')); return new Agent($adapter); -}); +}, ['register']); diff --git a/docker-compose.yml b/docker-compose.yml index 4e8b58086e..3d09f324e1 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -276,6 +276,7 @@ services: volumes: - ./app:/usr/src/code/app - ./src:/usr/src/code/src + - ./vendor:/usr/src/code/vendor depends_on: - ${_APP_DB_HOST:-mongodb} - postgresql @@ -751,6 +752,7 @@ services: - ./app:/usr/src/code/app - ./src:/usr/src/code/src - ./tests:/usr/src/code/tests + - ./vendor:/usr/src/code/vendor depends_on: - ${_APP_DB_HOST:-mongodb} - postgresql @@ -1125,55 +1127,55 @@ services: - MARIADB_AUTO_UPGRADE=1 command: "mysqld --innodb-flush-method=fsync" - # mongodb: - # image: mongo:8.0.10 - # container_name: appwrite-mongodb - # <<: *x-logging - # networks: - # - appwrite - # volumes: - # - appwrite-mongodb:/data/db - # - appwrite-mongodb-keyfile:/data/keyfile - # - ./mongo-init.js:/docker-entrypoint-initdb.d/mongo-init.js:ro - # - ./mongo-entrypoint.sh:/mongo-entrypoint.sh:ro - # ports: - # - "27017:27017" - # environment: - # - MONGO_INITDB_ROOT_USERNAME=root - # - MONGO_INITDB_ROOT_PASSWORD=${_APP_DB_ROOT_PASS} - # - MONGO_INITDB_DATABASE=${_APP_DB_SCHEMA} - # - MONGO_INITDB_USERNAME=${_APP_DB_USER} - # - MONGO_INITDB_PASSWORD=${_APP_DB_PASS} - # entrypoint: ["/bin/bash", "/mongo-entrypoint.sh"] - # healthcheck: - # test: | - # bash -c " - # if mongosh -u root -p ${_APP_DB_ROOT_PASS} --authenticationDatabase admin --quiet --eval 'rs.status().ok' 2>/dev/null | grep -q 1; then - # exit 0 - # else - # mongosh -u root -p ${_APP_DB_ROOT_PASS} --authenticationDatabase admin --quiet --eval \" - # rs.initiate({_id: 'rs0', members: [{_id: 0, host: 'appwrite-mongodb:27017'}]}) - # \" 2>/dev/null || exit 1 - # fi - # " - # interval: 10s - # timeout: 10s - # retries: 10 - # start_period: 30s + mongodb: + image: mongo:8.0.10 + container_name: appwrite-mongodb + <<: *x-logging + networks: + - appwrite + volumes: + - appwrite-mongodb:/data/db + - appwrite-mongodb-keyfile:/data/keyfile + - ./mongo-init.js:/docker-entrypoint-initdb.d/mongo-init.js:ro + - ./mongo-entrypoint.sh:/mongo-entrypoint.sh:ro + ports: + - "27017:27017" + environment: + - MONGO_INITDB_ROOT_USERNAME=root + - MONGO_INITDB_ROOT_PASSWORD=${_APP_DB_ROOT_PASS} + - MONGO_INITDB_DATABASE=${_APP_DB_SCHEMA} + - MONGO_INITDB_USERNAME=${_APP_DB_USER} + - MONGO_INITDB_PASSWORD=${_APP_DB_PASS} + entrypoint: ["/bin/bash", "/mongo-entrypoint.sh"] + healthcheck: + test: | + bash -c " + if mongosh -u root -p ${_APP_DB_ROOT_PASS} --authenticationDatabase admin --quiet --eval 'rs.status().ok' 2>/dev/null | grep -q 1; then + exit 0 + else + mongosh -u root -p ${_APP_DB_ROOT_PASS} --authenticationDatabase admin --quiet --eval \" + rs.initiate({_id: 'rs0', members: [{_id: 0, host: 'appwrite-mongodb:27017'}]}) + \" 2>/dev/null || exit 1 + fi + " + interval: 10s + timeout: 10s + retries: 10 + start_period: 30s - # appwrite-mongo-express: - # image: mongo-express - # container_name: appwrite-mongo-express - # networks: - # - appwrite - # ports: - # - "8082:8081" - # environment: - # ME_CONFIG_MONGODB_URL: "mongodb://root:${_APP_DB_ROOT_PASS}@appwrite-mongodb:27017/?replicaSet=rs0&directConnection=true" - # ME_CONFIG_BASICAUTH_USERNAME: ${_APP_DB_USER} - # ME_CONFIG_BASICAUTH_PASSWORD: ${_APP_DB_PASS} - # depends_on: - # - mongodb + appwrite-mongo-express: + image: mongo-express + container_name: appwrite-mongo-express + networks: + - appwrite + ports: + - "8082:8081" + environment: + ME_CONFIG_MONGODB_URL: "mongodb://root:${_APP_DB_ROOT_PASS}@appwrite-mongodb:27017/?replicaSet=rs0&directConnection=true" + ME_CONFIG_BASICAUTH_USERNAME: ${_APP_DB_USER} + ME_CONFIG_BASICAUTH_PASSWORD: ${_APP_DB_PASS} + depends_on: + - mongodb postgresql: @@ -1198,17 +1200,17 @@ services: build: context: . dockerfile: ollama.dockerfile + args: + MODELS: ${_APP_EMBEDDING_MODELS:-embeddinggemma} + # duration to keep model in memory + OLLAMA_KEEP_ALIVE: 24h container_name: ollama ports: - "11434:11434" restart: unless-stopped # persistent for caching models across restarts and preloading volumes: - - ollama_models:/root/.ollama - environment: - - MODELS=${OLLAMA_MODELS:-embeddinggemma} - # duration to keep model in memory - - OLLAMA_KEEP_ALIVE=24h + - _APP_EMBEDDING_MODELS:/root/.ollama networks: - appwrite @@ -1310,8 +1312,8 @@ networks: volumes: appwrite-mariadb: - # appwrite-mongodb: - # appwrite-mongodb-keyfile: + appwrite-mongodb: + appwrite-mongodb-keyfile: appwrite-postgresql: appwrite-redis: appwrite-cache: @@ -1322,4 +1324,4 @@ volumes: appwrite-sites: appwrite-builds: appwrite-config: - ollama_models: \ No newline at end of file + _APP_EMBEDDING_MODELS: \ No newline at end of file diff --git a/ollama.dockerfile b/ollama.dockerfile index 1d4d29a22b..fc538ea4d2 100644 --- a/ollama.dockerfile +++ b/ollama.dockerfile @@ -1,8 +1,10 @@ -FROM ollama/ollama:latest +FROM ollama/ollama:0.12.7 # Preload specific models -ENV MODELS="embeddinggemma" -ENV OLLAMA_KEEP_ALIVE=24h +ARG MODELS +# needed to set in the environment +ARG OLLAMA_KEEP_ALIVE +ENV OLLAMA_KEEP_ALIVE=${OLLAMA_KEEP_ALIVE:-24h} # Pre-pull models at build time for Docker layer caching RUN ollama serve & \ diff --git a/src/Appwrite/Event/Event.php b/src/Appwrite/Event/Event.php index 73957c1155..b72b70cf2d 100644 --- a/src/Appwrite/Event/Event.php +++ b/src/Appwrite/Event/Event.php @@ -684,8 +684,10 @@ class Event ]; break; case 'documentsdb': + case 'vectordb': + // sending the type itself(eg: documentsdb, vectordb) $eventMap = [ - 'databases' => 'documentsdb' + 'databases' => $database->getAttribute('type') ]; break; } diff --git a/src/Appwrite/Messaging/Adapter/Realtime.php b/src/Appwrite/Messaging/Adapter/Realtime.php index 95c3c79155..30e5942185 100644 --- a/src/Appwrite/Messaging/Adapter/Realtime.php +++ b/src/Appwrite/Messaging/Adapter/Realtime.php @@ -299,6 +299,7 @@ class Realtime extends MessagingAdapter case 'databases': case 'tablesdb': case 'documentsdb': + case 'vectordb': $resource = $parts[4] ?? ''; if (in_array($resource, ['columns', 'attributes', 'indexes'])) { $channels[] = 'console'; @@ -434,6 +435,7 @@ class Realtime extends MessagingAdapter break; case 'documentsdb': + case 'vectordb': $channels[] = 'documents'; $channels[] = "{$basePrefix}.{$databaseId}.collections.{$resourceId}.documents"; $channels[] = "{$basePrefix}.{$databaseId}.collections.{$resourceId}.documents.{$payloadId}"; diff --git a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Create.php b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Create.php index b319c28c4a..e1c9998323 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Create.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Create.php @@ -140,22 +140,11 @@ class Create extends CollectionAction attributes:$attributes, indexes:$indexes ); - } catch (DuplicateException) { - throw new Exception($this->getDuplicateException()); - } catch (IndexException) { - throw new Exception($this->getInvalidIndexException()); - } catch (LimitException) { - throw new Exception($this->getLimitException()); - } - - // Create attribute metadata documents in the attributes table - // This is necessary so that indexes can find the attributes when they're created - foreach ($collections['defaultAttributes'] as $attributeConfig) { - $key = \is_string($attributeConfig['$id']) ? $attributeConfig['$id'] : (string)$attributeConfig['$id']; - $size = $key === 'embeddings' ? $dimensions : ($attributeConfig['size'] ?? 0); - - try { - $attributeDoc = new Document([ + // Create attribute metadata documents in the attributes table + // This is necessary so that indexes can find the attributes when they're created + $attributeDocs = array_map(function ($attributeConfig) use ($database, $collection, $databaseId, $collectionId, $dimensions) { + $key = \is_string($attributeConfig['$id']) ? $attributeConfig['$id'] : (string) $attributeConfig['$id']; + return new Document([ '$id' => ID::custom($database->getSequence() . '_' . $collection->getSequence() . '_' . $key), 'key' => $key, 'databaseInternalId' => $database->getSequence(), @@ -164,7 +153,7 @@ class Create extends CollectionAction 'collectionId' => $collectionId, 'type' => $attributeConfig['type'], 'status' => 'available', - 'size' => $size, + 'size' => $dimensions, 'required' => $attributeConfig['required'] ?? false, 'signed' => $attributeConfig['signed'] ?? false, 'default' => $attributeConfig['default'] ?? null, @@ -174,11 +163,14 @@ class Create extends CollectionAction 'filters' => $attributeConfig['filters'] ?? [], 'options' => $attributeConfig['options'] ?? [], ]); - - $dbForProject->createDocument('attributes', $attributeDoc); - } catch (DuplicateException) { - // Attribute already exists, skip - } + }, $collections['defaultAttributes']); + $dbForProject->createDocuments('attributes', $attributeDocs); + } catch (DuplicateException) { + throw new Exception($this->getDuplicateException()); + } catch (IndexException) { + throw new Exception($this->getInvalidIndexException()); + } catch (LimitException) { + throw new Exception($this->getLimitException()); } $queueForEvents diff --git a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Text/Create.php b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Text/Create.php index 1896e91b8d..4bc8281656 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Text/Create.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Text/Create.php @@ -25,7 +25,7 @@ class Create extends CreateDocumentAction { public static function getName(): string { - return 'createVectorDBDocument'; + return 'createTextEmbedding'; } protected function getResponseModel(): string @@ -144,7 +144,7 @@ class Create extends CreateDocumentAction } $list = new Document([ - 'embeddings' => array_map(fn($d) => $d, $results), + 'embeddings' => array_map(fn ($d) => $d, $results), 'total' => \count($results), ]); diff --git a/src/Appwrite/Platform/Modules/Databases/Services/Registry/VectorDB.php b/src/Appwrite/Platform/Modules/Databases/Services/Registry/VectorDB.php index f05e339ac0..ec071d4e6f 100644 --- a/src/Appwrite/Platform/Modules/Databases/Services/Registry/VectorDB.php +++ b/src/Appwrite/Platform/Modules/Databases/Services/Registry/VectorDB.php @@ -10,6 +10,7 @@ use Appwrite\Platform\Modules\Databases\Http\VectorDB\Collections\Documents\Bulk use Appwrite\Platform\Modules\Databases\Http\VectorDB\Collections\Documents\Create as CreateDocument; use Appwrite\Platform\Modules\Databases\Http\VectorDB\Collections\Documents\Delete as DeleteDocument; use Appwrite\Platform\Modules\Databases\Http\VectorDB\Collections\Documents\Get as GetDocument; +use Appwrite\Platform\Modules\Databases\Http\VectorDB\Collections\Documents\Text\Create as CreateTextEmbeddings; use Appwrite\Platform\Modules\Databases\Http\VectorDB\Collections\Documents\Update as UpdateDocument; use Appwrite\Platform\Modules\Databases\Http\VectorDB\Collections\Documents\Upsert as UpsertDocument; use Appwrite\Platform\Modules\Databases\Http\VectorDB\Collections\Documents\XList as ListDocuments; @@ -29,7 +30,6 @@ use Appwrite\Platform\Modules\Databases\Http\VectorDB\Update as UpdateVectorData use Appwrite\Platform\Modules\Databases\Http\VectorDB\Usage\Get as GetVectorDatabaseUsage; use Appwrite\Platform\Modules\Databases\Http\VectorDB\Usage\XList as ListVectorDatabaseUsage; use Appwrite\Platform\Modules\Databases\Http\VectorDB\XList as ListVectorDatabases; -use Appwrite\Platform\Modules\Databases\Http\VectorDB\Collections\Documents\Text\Create as CreateTextEmbeddings; use Utopia\Platform\Service; class VectorDB extends Base @@ -87,7 +87,8 @@ class VectorDB extends Base $service->addAction(DeleteDocuments::getName(), new DeleteDocuments()); } - private function registerEmbeddingActions(Service $service):void{ - $service->addAction(CreateTextEmbeddings::getName(),new CreateTextEmbeddings()); + private function registerEmbeddingActions(Service $service): void + { + $service->addAction(CreateTextEmbeddings::getName(), new CreateTextEmbeddings()); } } diff --git a/src/Appwrite/Utopia/Response.php b/src/Appwrite/Utopia/Response.php index cb14aae7da..6f78c70c0f 100644 --- a/src/Appwrite/Utopia/Response.php +++ b/src/Appwrite/Utopia/Response.php @@ -63,6 +63,7 @@ use Appwrite\Utopia\Response\Model\DetectionFramework; use Appwrite\Utopia\Response\Model\DetectionRuntime; use Appwrite\Utopia\Response\Model\DevKey; use Appwrite\Utopia\Response\Model\Document as ModelDocument; +use Appwrite\Utopia\Response\Model\Embedding; use Appwrite\Utopia\Response\Model\Error; use Appwrite\Utopia\Response\Model\ErrorDev; use Appwrite\Utopia\Response\Model\Execution; @@ -144,6 +145,7 @@ use Appwrite\Utopia\Response\Model\UsageUsers; use Appwrite\Utopia\Response\Model\User; use Appwrite\Utopia\Response\Model\Variable; use Appwrite\Utopia\Response\Model\VcsContent; +use Appwrite\Utopia\Response\Model\VectorDBCollection; use Appwrite\Utopia\Response\Model\Webhook; use Exception; use JsonException; @@ -500,10 +502,10 @@ class Response extends SwooleResponse ->setModel(new BaseList('Embedding list', self::MODEL_EMBEDDING_LIST, 'embeddings', self::MODEL_EMBEDDING)) // Entities ->setModel(new Database()) - ->setModel(new \Appwrite\Utopia\Response\Model\Embedding()) + ->setModel(new Embedding()) // Collection API Models ->setModel(new Collection()) - ->setModel(new \Appwrite\Utopia\Response\Model\VectorDBCollection()) + ->setModel(new VectorDBCollection()) ->setModel(new Attribute()) ->setModel(new AttributeList()) ->setModel(new AttributeString()) diff --git a/tests/e2e/Services/Realtime/RealtimeCustomClientTest.php b/tests/e2e/Services/Realtime/RealtimeCustomClientTest.php index 50430122f8..1121d99302 100644 --- a/tests/e2e/Services/Realtime/RealtimeCustomClientTest.php +++ b/tests/e2e/Services/Realtime/RealtimeCustomClientTest.php @@ -4299,4 +4299,192 @@ class RealtimeCustomClientTest extends Scope $client->close(); } + + public function testChannelVectorDB() + { + $user = $this->getUser(); + $session = $user['session'] ?? ''; + $projectId = $this->getProject()['$id']; + + $client = $this->getWebsocket(['documents', 'collections'], [ + 'origin' => 'http://localhost', + 'cookie' => 'a_session_' . $projectId . '=' . $session + ]); + + $response = json_decode($client->receive(), true); + + $this->assertArrayHasKey('type', $response); + $this->assertArrayHasKey('data', $response); + $this->assertEquals('connected', $response['type']); + $this->assertNotEmpty($response['data']); + $this->assertCount(2, $response['data']['channels']); + $this->assertContains('documents', $response['data']['channels']); + $this->assertContains('collections', $response['data']['channels']); + $this->assertNotEmpty($response['data']['user']); + $this->assertEquals($user['$id'], $response['data']['user']['$id']); + + // Create VectorDB database + $database = $this->client->call(Client::METHOD_POST, '/vectordb', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'databaseId' => ID::unique(), + 'name' => 'Actors VDB', + ]); + + $databaseId = $database['body']['$id']; + + // Create collection in VectorDB + $actors = $this->client->call(Client::METHOD_POST, '/vectordb/' . $databaseId . '/collections', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'collectionId' => ID::unique(), + 'name' => 'Actors', + 'permissions' => [ + Permission::create(Role::user($this->getUser()['$id'])), + ], + 'documentSecurity' => true, + 'dimensions' => 3, + ]); + + $actorsId = $actors['body']['$id']; + + // Create document in VectorDB + $document = $this->client->call(Client::METHOD_POST, '/vectordb/' . $databaseId . '/collections/' . $actorsId . '/documents', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'documentId' => ID::unique(), + 'data' => [ + 'embeddings' => [1.0, 0.0, 0.0], + 'metadata' => ['name' => 'Chris Evans'] + ], + 'permissions' => [ + Permission::read(Role::any()), + Permission::update(Role::any()), + Permission::delete(Role::any()), + ], + ]); + + $response = json_decode($client->receive(), true); + + $documentId = $document['body']['$id']; + $this->assertArrayHasKey('type', $response); + $this->assertArrayHasKey('data', $response); + $this->assertEquals('event', $response['type']); + $this->assertNotEmpty($response['data']); + $this->assertArrayHasKey('timestamp', $response['data']); + // vectordb channels should include 3 items like documentsdb + $this->assertCount(3, $response['data']['channels']); + $this->assertContains('documents', $response['data']['channels']); + $this->assertContains('vectordb.' . $databaseId . '.collections.' . $actorsId . '.documents.' . $documentId, $response['data']['channels']); + $this->assertContains('vectordb.' . $databaseId . '.collections.' . $actorsId . '.documents', $response['data']['channels']); + $this->assertNotEmpty($response['data']['payload']); + $this->assertIsArray($response['data']['payload']['embeddings']); + $this->assertCount(3, $response['data']['payload']['embeddings']); + $this->assertEquals('Chris Evans', $response['data']['payload']['metadata']['name']); + + // Update document + $this->client->call(Client::METHOD_PATCH, '/vectordb/' . $databaseId . '/collections/' . $actorsId . '/documents/' . $documentId, array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'data' => [ + 'embeddings' => [0.0, 1.0, 0.0], + 'metadata' => ['name' => 'Chris Evans 2'] + ], + 'permissions' => [ + Permission::read(Role::any()), + Permission::update(Role::any()), + Permission::delete(Role::any()), + ], + ]); + + $response = json_decode($client->receive(), true); + $this->assertArrayHasKey('type', $response); + $this->assertArrayHasKey('data', $response); + $this->assertEquals('event', $response['type']); + $this->assertNotEmpty($response['data']); + $this->assertArrayHasKey('timestamp', $response['data']); + $this->assertCount(3, $response['data']['channels']); + $this->assertContains('vectordb.' . $databaseId . '.collections.' . $actorsId . '.documents.' . $documentId, $response['data']['channels']); + $this->assertContains('vectordb.' . $databaseId . '.collections.' . $actorsId . '.documents', $response['data']['channels']); + $this->assertNotEmpty($response['data']['payload']); + $this->assertIsArray($response['data']['payload']['embeddings']); + $this->assertEquals('Chris Evans 2', $response['data']['payload']['metadata']['name']); + + // Delete document + $this->client->call(Client::METHOD_DELETE, '/vectordb/' . $databaseId . '/collections/' . $actorsId . '/documents/' . $documentId, array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders())); + + $response = json_decode($client->receive(), true); + $this->assertArrayHasKey('type', $response); + $this->assertArrayHasKey('data', $response); + $this->assertEquals('event', $response['type']); + $this->assertNotEmpty($response['data']); + $this->assertArrayHasKey('timestamp', $response['data']); + $this->assertCount(3, $response['data']['channels']); + $this->assertContains('vectordb.' . $databaseId . '.collections.' . $actorsId . '.documents.' . $documentId, $response['data']['channels']); + $this->assertContains('vectordb.' . $databaseId . '.collections.' . $actorsId . '.documents', $response['data']['channels']); + + // Bulk create two documents + $this->client->call(Client::METHOD_POST, "/vectordb/{$databaseId}/collections/{$actorsId}/documents", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'documents' => [ + [ + 'embeddings' => [1.0, 0.0, 0.0], + 'metadata' => ['name' => 'Robert Downey Jr.'], + '$permissions' => [ + Permission::read(Role::any()), + Permission::update(Role::any()), + Permission::delete(Role::any()), + ], + ], + [ + 'embeddings' => [0.0, 1.0, 0.0], + 'metadata' => ['name' => 'Scarlett Johansson'], + '$permissions' => [ + Permission::read(Role::any()), + Permission::update(Role::any()), + Permission::delete(Role::any()), + ], + ] + ], + ]); + + // Receive first bulk document event + $response = json_decode($client->receive(), true); + $this->assertArrayHasKey('type', $response); + $this->assertArrayHasKey('data', $response); + $this->assertEquals('event', $response['type']); + $this->assertNotEmpty($response['data']); + $this->assertArrayHasKey('timestamp', $response['data']); + $this->assertCount(3, $response['data']['channels']); + $this->assertContains('vectordb.' . $databaseId . '.collections.' . $actorsId . '.documents.' . $response['data']['payload']['$id'] . '.create', $response['data']['events']); + $this->assertContains('vectordb.*.collections.*.documents.*.create', $response['data']['events']); + $this->assertContains('vectordb.' . $databaseId . '.collections.*.documents.*.create', $response['data']['events']); + $this->assertContains('vectordb.*.collections.' . $actorsId . '.documents.*.create', $response['data']['events']); + $this->assertNotEmpty($response['data']['payload']); + $this->assertIsArray($response['data']['payload']); + + // Receive second bulk document event + $response = json_decode($client->receive(), true); + $this->assertArrayHasKey('type', $response); + $this->assertArrayHasKey('data', $response); + $this->assertEquals('event', $response['type']); + $this->assertNotEmpty($response['data']); + $this->assertArrayHasKey('timestamp', $response['data']); + $this->assertCount(3, $response['data']['channels']); + $this->assertContains('vectordb.' . $databaseId . '.collections.' . $actorsId . '.documents.' . $response['data']['payload']['$id'] . '.create', $response['data']['events']); + + $client->close(); + } } From 857ec3b09907bdbc6e993bd9b7b041fa5291c2bd Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Thu, 6 Nov 2025 17:54:21 +0530 Subject: [PATCH 07/27] reverted docker compose --- docker-compose.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 3d09f324e1..99bc0b5287 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -89,7 +89,6 @@ services: - ./public:/usr/src/code/public - ./src:/usr/src/code/src - ./dev:/usr/src/code/dev - - ./vendor:/usr/src/code/vendor depends_on: - ${_APP_DB_HOST:-mongodb} @@ -276,7 +275,6 @@ services: volumes: - ./app:/usr/src/code/app - ./src:/usr/src/code/src - - ./vendor:/usr/src/code/vendor depends_on: - ${_APP_DB_HOST:-mongodb} - postgresql @@ -452,7 +450,6 @@ services: volumes: - ./app:/usr/src/code/app - ./src:/usr/src/code/src - - ./vendor:/usr/src/code/vendor depends_on: - redis - ${_APP_DB_HOST:-mongodb} @@ -752,7 +749,6 @@ services: - ./app:/usr/src/code/app - ./src:/usr/src/code/src - ./tests:/usr/src/code/tests - - ./vendor:/usr/src/code/vendor depends_on: - ${_APP_DB_HOST:-mongodb} - postgresql From 04b169fab80ab2355cab41543b89b1aae48b6d78 Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Thu, 6 Nov 2025 18:01:01 +0530 Subject: [PATCH 08/27] updated composer packages --- composer.json | 10 ------ composer.lock | 95 +++++++++++++++++---------------------------------- 2 files changed, 32 insertions(+), 73 deletions(-) diff --git a/composer.json b/composer.json index c1e712f37f..910cbcb5e4 100644 --- a/composer.json +++ b/composer.json @@ -30,16 +30,6 @@ "Appwrite\\Tests\\": "tests/extensions" } }, - "repositories": [ - { - "type": "vcs", - "url": "https://github.com/utopia-php/database" - }, - { - "type": "vcs", - "url": "https://github.com/utopia-php/agents" - } - ], "minimum-stability": "dev", "prefer-stable": true, "require": { diff --git a/composer.lock b/composer.lock index 3f077630af..62cccc95f7 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "ac89d83b4f8d0d03065a9cd569432546", + "content-hash": "f9b66abe46119ae12f0cf2becc8bc299", "packages": [ { "name": "adhocore/jwt", @@ -3500,12 +3500,12 @@ "source": { "type": "git", "url": "https://github.com/utopia-php/agents.git", - "reference": "d59a69bde505670c8434921659d116ea45b2acd1" + "reference": "b448a4d06374c7a538c9495ac69a1a8f8e526823" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/agents/zipball/d59a69bde505670c8434921659d116ea45b2acd1", - "reference": "d59a69bde505670c8434921659d116ea45b2acd1", + "url": "https://api.github.com/repos/utopia-php/agents/zipball/b448a4d06374c7a538c9495ac69a1a8f8e526823", + "reference": "b448a4d06374c7a538c9495ac69a1a8f8e526823", "shasum": "" }, "require": { @@ -3546,7 +3546,7 @@ "issues": "https://github.com/utopia-php/agents/issues", "source": "https://github.com/utopia-php/agents/tree/feat-embeddings" }, - "time": "2025-10-30T12:38:42+00:00" + "time": "2025-11-06T07:53:14+00:00" }, { "name": "utopia-php/analytics", @@ -3893,12 +3893,12 @@ "source": { "type": "git", "url": "https://github.com/utopia-php/database.git", - "reference": "9a96110abbec4bd841f3eacc473471a37bebf9be" + "reference": "9a0cea65abbfb325506cb1d619db60dac7ee7e55" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/database/zipball/9a96110abbec4bd841f3eacc473471a37bebf9be", - "reference": "9a96110abbec4bd841f3eacc473471a37bebf9be", + "url": "https://api.github.com/repos/utopia-php/database/zipball/9a0cea65abbfb325506cb1d619db60dac7ee7e55", + "reference": "9a0cea65abbfb325506cb1d619db60dac7ee7e55", "shasum": "" }, "require": { @@ -3927,38 +3927,7 @@ "Utopia\\Database\\": "src/Database" } }, - "autoload-dev": { - "psr-4": { - "Tests\\E2E\\": "tests/e2e", - "Tests\\Unit\\": "tests/unit" - } - }, - "scripts": { - "build": [ - "Composer\\Config::disableProcessTimeout", - "docker compose build" - ], - "start": [ - "Composer\\Config::disableProcessTimeout", - "docker compose up -d" - ], - "test": [ - "Composer\\Config::disableProcessTimeout", - "docker compose exec tests vendor/bin/phpunit --configuration phpunit.xml" - ], - "lint": [ - "php -d memory_limit=2G ./vendor/bin/pint --test" - ], - "format": [ - "php -d memory_limit=2G ./vendor/bin/pint" - ], - "check": [ - "./vendor/bin/phpstan analyse --level 7 src tests --memory-limit 2G" - ], - "coverage": [ - "./vendor/bin/coverage-check ./tmp/clover.xml 90" - ] - }, + "notification-url": "https://packagist.org/downloads/", "license": [ "MIT" ], @@ -3971,23 +3940,23 @@ "utopia" ], "support": { - "source": "https://github.com/utopia-php/database/tree/var_object", - "issues": "https://github.com/utopia-php/database/issues" + "issues": "https://github.com/utopia-php/database/issues", + "source": "https://github.com/utopia-php/database/tree/var_object" }, - "time": "2025-11-03T05:12:58+00:00" + "time": "2025-11-06T09:02:12+00:00" }, { "name": "utopia-php/detector", - "version": "0.2.1", + "version": "0.2.2", "source": { "type": "git", "url": "https://github.com/utopia-php/detector.git", - "reference": "89f96e864220de13800cf398a1f1686a85401eaa" + "reference": "9a41be5f21efe2d865de79b08dff94fff85ce5e9" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/detector/zipball/89f96e864220de13800cf398a1f1686a85401eaa", - "reference": "89f96e864220de13800cf398a1f1686a85401eaa", + "url": "https://api.github.com/repos/utopia-php/detector/zipball/9a41be5f21efe2d865de79b08dff94fff85ce5e9", + "reference": "9a41be5f21efe2d865de79b08dff94fff85ce5e9", "shasum": "" }, "require": { @@ -4017,9 +3986,9 @@ ], "support": { "issues": "https://github.com/utopia-php/detector/issues", - "source": "https://github.com/utopia-php/detector/tree/0.2.1" + "source": "https://github.com/utopia-php/detector/tree/0.2.2" }, - "time": "2025-10-27T13:38:33+00:00" + "time": "2025-10-31T12:43:31+00:00" }, { "name": "utopia-php/dns", @@ -5458,16 +5427,16 @@ "packages-dev": [ { "name": "appwrite/sdk-generator", - "version": "1.4.16", + "version": "1.5.1", "source": { "type": "git", "url": "https://github.com/appwrite/sdk-generator.git", - "reference": "08f839443f678208eb56a6c5a7456dd632adfc9a" + "reference": "cd712674e34136f706e9170641ed6f4ce160e772" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/appwrite/sdk-generator/zipball/08f839443f678208eb56a6c5a7456dd632adfc9a", - "reference": "08f839443f678208eb56a6c5a7456dd632adfc9a", + "url": "https://api.github.com/repos/appwrite/sdk-generator/zipball/cd712674e34136f706e9170641ed6f4ce160e772", + "reference": "cd712674e34136f706e9170641ed6f4ce160e772", "shasum": "" }, "require": { @@ -5503,9 +5472,9 @@ "description": "Appwrite PHP library for generating API SDKs for multiple programming languages and platforms", "support": { "issues": "https://github.com/appwrite/sdk-generator/issues", - "source": "https://github.com/appwrite/sdk-generator/tree/1.4.16" + "source": "https://github.com/appwrite/sdk-generator/tree/1.5.1" }, - "time": "2025-10-29T08:39:55+00:00" + "time": "2025-11-04T09:55:47+00:00" }, { "name": "doctrine/annotations", @@ -6158,24 +6127,24 @@ }, { "name": "phpbench/container", - "version": "2.2.2", + "version": "2.2.3", "source": { "type": "git", "url": "https://github.com/phpbench/container.git", - "reference": "a59b929e00b87b532ca6d0edd8eca0967655af33" + "reference": "0c7b2d36c1ea53fe27302fb8873ded7172047196" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/phpbench/container/zipball/a59b929e00b87b532ca6d0edd8eca0967655af33", - "reference": "a59b929e00b87b532ca6d0edd8eca0967655af33", + "url": "https://api.github.com/repos/phpbench/container/zipball/0c7b2d36c1ea53fe27302fb8873ded7172047196", + "reference": "0c7b2d36c1ea53fe27302fb8873ded7172047196", "shasum": "" }, "require": { "psr/container": "^1.0|^2.0", - "symfony/options-resolver": "^4.2 || ^5.0 || ^6.0 || ^7.0" + "symfony/options-resolver": "^4.2 || ^5.0 || ^6.0 || ^7.0 || ^8.0" }, "require-dev": { - "friendsofphp/php-cs-fixer": "^2.16", + "php-cs-fixer/shim": "^3.89", "phpstan/phpstan": "^0.12.52", "phpunit/phpunit": "^8" }, @@ -6203,9 +6172,9 @@ "description": "Simple, configurable, service container.", "support": { "issues": "https://github.com/phpbench/container/issues", - "source": "https://github.com/phpbench/container/tree/2.2.2" + "source": "https://github.com/phpbench/container/tree/2.2.3" }, - "time": "2023-10-30T13:38:26+00:00" + "time": "2025-11-06T09:05:13+00:00" }, { "name": "phpbench/phpbench", From de59f53de3fde2d6a3d35c1fedb83c25367f4541 Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Thu, 6 Nov 2025 18:25:01 +0530 Subject: [PATCH 09/27] reverted error tracing --- app/controllers/general.php | 8 -------- .../Collections/Documents/Text/Create.php | 15 +++++++-------- 2 files changed, 7 insertions(+), 16 deletions(-) diff --git a/app/controllers/general.php b/app/controllers/general.php index a293b65a31..bb09b021f7 100644 --- a/app/controllers/general.php +++ b/app/controllers/general.php @@ -1185,14 +1185,6 @@ App::error() $trace = $error->getTrace(); $trace = $error->getTrace(); - foreach (array_slice($trace, 0, 100) as $index => $traceEntry) { - $file = isset($traceEntry['file']) ? $traceEntry['file'] : '[internal function]'; - $line = isset($traceEntry['line']) ? $traceEntry['line'] : ''; - $function = isset($traceEntry['function']) ? $traceEntry['function'] : ''; - Console::error("[$index] $file : $line -> $function()"); - } - - if (php_sapi_name() === 'cli') { Console::error('[Error] Timestamp: ' . date('c', time())); diff --git a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Text/Create.php b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Text/Create.php index 4bc8281656..aa570ddb2c 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Text/Create.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Text/Create.php @@ -105,11 +105,11 @@ class Create extends CreateDocumentAction // Validate and process each document $availableModels = []; - if (method_exists($embeddingAgent, 'getAdapter') && method_exists($embeddingAgent->getAdapter(), 'getModels')) { - $availableModels = $embeddingAgent->getAdapter()->getModels(); - } + $availableModels = $embeddingAgent->getAdapter()->getModels(); $results = []; + + // validating all documents first foreach ($documents as $index => $item) { if (!\is_array($item)) { throw new Exception(Exception::DOCUMENT_INVALID_STRUCTURE, 'Invalid item at index ' . $index); @@ -127,10 +127,9 @@ class Create extends CreateDocumentAction if (!empty($availableModels) && !\in_array($model, $availableModels, true)) { throw new Exception(Exception::GENERAL_BAD_REQUEST, 'Unknown embedding model: ' . $model); } - - if (method_exists($embeddingAgent, 'getAdapter') && method_exists($embeddingAgent->getAdapter(), 'setModel')) { - $embeddingAgent->getAdapter()->setModel($model); - } + } + foreach ($documents as $index => $item) { + $embeddingAgent->getAdapter()->setModel($model); $embedResult = $embeddingAgent->embed($text); $vector = $embedResult['embedding'] ?? []; @@ -141,8 +140,8 @@ class Create extends CreateDocumentAction 'dimensions' => $dimensions, 'embeddings' => $vector, ]); - } + } $list = new Document([ 'embeddings' => array_map(fn ($d) => $d, $results), 'total' => \count($results), From c3f221f1fbc216974d26132ac26d90864ec225a3 Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Fri, 7 Nov 2025 13:13:13 +0530 Subject: [PATCH 10/27] added timestamp mutation test --- .../Databases/VectorDB/DatabasesBase.php | 84 --------------- .../VectorDB/DatabasesCustomServerTest.php | 102 ++++++++++++++++++ 2 files changed, 102 insertions(+), 84 deletions(-) diff --git a/tests/e2e/Services/Databases/VectorDB/DatabasesBase.php b/tests/e2e/Services/Databases/VectorDB/DatabasesBase.php index e55be44407..443e37a122 100644 --- a/tests/e2e/Services/Databases/VectorDB/DatabasesBase.php +++ b/tests/e2e/Services/Databases/VectorDB/DatabasesBase.php @@ -303,90 +303,6 @@ trait DatabasesBase return $data; } - /** - * @depends testUpdateDocument - */ - public function testTimestampsMutation(array $data): array - { - $databaseId = $data['databaseId']; - $collectionId = $data['collectionId']; - $documentId = $data['documentId']; - - // Fetch current document to get latest timestamps - $curr = $this->client->call(Client::METHOD_GET, "/vectordb/{$databaseId}/collections/{$collectionId}/documents/{$documentId}", [ - 'content-type' => 'application/json', - 'x-appwrite-project' => $this->getProject()['$id'], - 'x-appwrite-key' => $this->getProject()['apiKey'] - ]); - $this->assertEquals(200, $curr['headers']['status-code']); - - $initialCreatedAt = $data['createdAt']; - $initialUpdatedAt = $data['updatedAt']; - $afterUpdatesCreatedAt = $curr['body']['$createdAt']; - $afterUpdatesUpdatedAt = $curr['body']['$updatedAt']; - - // createdAt must remain stable - $this->assertEquals($initialCreatedAt, $afterUpdatesCreatedAt); - // updatedAt must change after updates - $this->assertNotEquals($initialUpdatedAt, $afterUpdatesUpdatedAt); - - // Try to forcibly set $createdAt and $updatedAt via API - $oldTs = '1970-01-01T00:00:00.000+00:00'; - $vectorForce = array_fill(0, 1536, 0.0); - $vectorForce[9] = 1.0; - $force = $this->client->call(Client::METHOD_PUT, "/vectordb/{$databaseId}/collections/{$collectionId}/documents/{$documentId}", [ - 'content-type' => 'application/json', - 'x-appwrite-project' => $this->getProject()['$id'], - 'x-appwrite-key' => $this->getProject()['apiKey'] - ], [ - 'data' => [ - '$createdAt' => $oldTs, - '$updatedAt' => $oldTs, - 'embeddings' => $vectorForce, - 'metadata' => ['type' => 'sample', 'rank' => 6] - ] - ]); - $this->assertEquals(200, $force['headers']['status-code']); - - $forcedGet = $this->client->call(Client::METHOD_GET, "/vectordb/{$databaseId}/collections/{$collectionId}/documents/{$documentId}", [ - 'content-type' => 'application/json', - 'x-appwrite-project' => $this->getProject()['$id'], - 'x-appwrite-key' => $this->getProject()['apiKey'] - ]); - $this->assertEquals(200, $forcedGet['headers']['status-code']); - // Both timestamps should be updated to client-provided value - $this->assertEquals($oldTs, $forcedGet['body']['$createdAt']); - $this->assertEquals($oldTs, $forcedGet['body']['$updatedAt']); - - // Perform another update to ensure updatedAt changes again - $vector = array_fill(0, 1536, 0.0); - $vector[10] = 1.0; - $upd = $this->client->call(Client::METHOD_PUT, "/vectordb/{$databaseId}/collections/{$collectionId}/documents/{$documentId}", [ - 'content-type' => 'application/json', - 'x-appwrite-project' => $this->getProject()['$id'], - 'x-appwrite-key' => $this->getProject()['apiKey'] - ], [ - 'data' => [ - 'embeddings' => $vector, - 'metadata' => ['type' => 'sample', 'rank' => 5] - ] - ]); - $this->assertEquals(200, $upd['headers']['status-code']); - - $final = $this->client->call(Client::METHOD_GET, "/vectordb/{$databaseId}/collections/{$collectionId}/documents/{$documentId}", [ - 'content-type' => 'application/json', - 'x-appwrite-project' => $this->getProject()['$id'], - 'x-appwrite-key' => $this->getProject()['apiKey'] - ]); - $this->assertEquals(200, $final['headers']['status-code']); - // createdAt should persist the client-forced value - $this->assertEquals($oldTs, $final['body']['$createdAt']); - // updatedAt should change from the previous value - $this->assertNotEquals($afterUpdatesUpdatedAt, $final['body']['$updatedAt']); - - return $data; - } - /** * @depends testUpdateDocument */ diff --git a/tests/e2e/Services/Databases/VectorDB/DatabasesCustomServerTest.php b/tests/e2e/Services/Databases/VectorDB/DatabasesCustomServerTest.php index b97911359a..8f9a3fc04f 100644 --- a/tests/e2e/Services/Databases/VectorDB/DatabasesCustomServerTest.php +++ b/tests/e2e/Services/Databases/VectorDB/DatabasesCustomServerTest.php @@ -883,4 +883,106 @@ class DatabasesCustomServerTest extends Scope } } + public function testCustomTimestamps(): void + { + // Setup: create database and collection + $db = $this->client->call(Client::METHOD_POST, '/vectordb', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'databaseId' => ID::unique(), + 'name' => 'TimestampTestDB' + ]); + $this->assertEquals(201, $db['headers']['status-code']); + $databaseId = $db['body']['$id']; + + $col = $this->client->call(Client::METHOD_POST, "/vectordb/{$databaseId}/collections", [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'collectionId' => ID::unique(), + 'name' => 'TimestampTestCollection', + 'documentSecurity' => true, + 'dimensions' => 1536, + 'permissions' => [Permission::read(Role::any())] + ]); + $this->assertEquals(201, $col['headers']['status-code']); + $collectionId = $col['body']['$id']; + + // Test: Create document with custom timestamps using PUT (upsert) + $customCreatedAt = '1970-01-01T00:00:00.000+00:00'; + $customUpdatedAt = '1970-01-01T00:00:00.000+00:00'; + $vector = array_fill(0, 1536, 0.0); + $vector[0] = 1.0; + $documentId = ID::unique(); + + $doc = $this->client->call(Client::METHOD_POST, "/vectordb/{$databaseId}/collections/{$collectionId}/documents", [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'documentId' => $documentId, + 'data' => [ + '$createdAt' => $customCreatedAt, + '$updatedAt' => $customUpdatedAt, + 'embeddings' => $vector, + 'metadata' => ['test' => 'custom_timestamps'] + ] + ]); + + $this->assertEquals(201, $doc['headers']['status-code']); + $documentId = $doc['body']['$id']; + $this->assertNotEmpty($documentId); + + // Verify timestamps were set correctly + $this->assertEquals($customCreatedAt, $doc['body']['$createdAt'], 'CreatedAt should match custom timestamp'); + $this->assertEquals($customUpdatedAt, $doc['body']['$updatedAt'], 'UpdatedAt should match custom timestamp'); + + // Fetch document and verify timestamps persist + $fetched = $this->client->call(Client::METHOD_GET, "/vectordb/{$databaseId}/collections/{$collectionId}/documents/{$documentId}", [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]); + + $this->assertEquals(200, $fetched['headers']['status-code']); + $this->assertEquals($customCreatedAt, $fetched['body']['$createdAt'], 'CreatedAt should persist after fetch'); + $this->assertEquals($customUpdatedAt, $fetched['body']['$updatedAt'], 'UpdatedAt should persist after fetch'); + + // Test: Update document with new custom timestamps + $newCustomUpdatedAt = '2000-01-01T12:00:00.000+00:00'; + $vector2 = array_fill(0, 1536, 0.0); + $vector2[1] = 1.0; + + $updated = $this->client->call(Client::METHOD_PUT, "/vectordb/{$databaseId}/collections/{$collectionId}/documents/{$documentId}", [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'data' => [ + '$createdAt' => $customCreatedAt, // Keep original createdAt + '$updatedAt' => $newCustomUpdatedAt, // Update updatedAt + 'embeddings' => $vector2, + 'metadata' => ['test' => 'updated_timestamps'] + ] + ]); + + $this->assertEquals(200, $updated['headers']['status-code']); + $this->assertEquals($customCreatedAt, $updated['body']['$createdAt'], 'CreatedAt should remain unchanged'); + $this->assertEquals($newCustomUpdatedAt, $updated['body']['$updatedAt'], 'UpdatedAt should be updated to new custom timestamp'); + + // Final verification + $final = $this->client->call(Client::METHOD_GET, "/vectordb/{$databaseId}/collections/{$collectionId}/documents/{$documentId}", [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]); + + $this->assertEquals(200, $final['headers']['status-code']); + $this->assertEquals($customCreatedAt, $final['body']['$createdAt'], 'CreatedAt should persist through updates'); + $this->assertEquals($newCustomUpdatedAt, $final['body']['$updatedAt'], 'UpdatedAt should reflect the latest custom timestamp'); + } + } From 954011b2c4e1d265bb8f70f510e8979459fff1a3 Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Wed, 12 Nov 2025 16:21:27 +0530 Subject: [PATCH 11/27] added transactions for vectordb --- .../Http/Databases/Transactions/Action.php | 5 + .../Http/Databases/Transactions/Update.php | 39 + .../Http/VectorDB/Transactions/Create.php | 55 + .../Http/VectorDB/Transactions/Delete.php | 55 + .../Http/VectorDB/Transactions/Get.php | 54 + .../Transactions/Operations/Create.php | 59 + .../Http/VectorDB/Transactions/Update.php | 67 + .../Http/VectorDB/Transactions/XList.php | 54 + .../Databases/Services/Registry/VectorDB.php | 18 +- .../VectorDB/Transactions/ACIDTest.php | 528 ++++ .../Transactions/TransactionsBase.php | 2371 +++++++++++++++++ .../TransactionsConsoleClientTest.php | 14 + .../TransactionsCustomClientTest.php | 14 + .../TransactionsCustomServerTest.php | 14 + 14 files changed, 3346 insertions(+), 1 deletion(-) create mode 100644 src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Transactions/Create.php create mode 100644 src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Transactions/Delete.php create mode 100644 src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Transactions/Get.php create mode 100644 src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Transactions/Operations/Create.php create mode 100644 src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Transactions/Update.php create mode 100644 src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Transactions/XList.php create mode 100644 tests/e2e/Services/Databases/VectorDB/Transactions/ACIDTest.php create mode 100644 tests/e2e/Services/Databases/VectorDB/Transactions/TransactionsBase.php create mode 100644 tests/e2e/Services/Databases/VectorDB/Transactions/TransactionsConsoleClientTest.php create mode 100644 tests/e2e/Services/Databases/VectorDB/Transactions/TransactionsCustomClientTest.php create mode 100644 tests/e2e/Services/Databases/VectorDB/Transactions/TransactionsCustomServerTest.php diff --git a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Transactions/Action.php b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Transactions/Action.php index 41c1e0f1f0..ca61a6aab1 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Transactions/Action.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Transactions/Action.php @@ -20,6 +20,7 @@ abstract class Action extends UtopiaAction public function setHttpPath(string $path): UtopiaAction { switch (true) { + // TODO: set the getDatabaseType() from each database group instead of path matching case str_contains($path, '/tablesdb'): $this->context = TABLES; $this->databaseType = TABLESDB; @@ -29,6 +30,10 @@ abstract class Action extends UtopiaAction $this->context = COLLECTIONS; $this->databaseType = DOCUMENTSDB; break; + case str_contains($path, '/vectordb'): + $this->context = COLLECTIONS; + $this->databaseType = VECTORDB; + break; } return parent::setHttpPath($path); } diff --git a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Transactions/Update.php b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Transactions/Update.php index 8b580d991d..6cdd8a8373 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Transactions/Update.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Transactions/Update.php @@ -286,6 +286,11 @@ class Update extends Action $data = $data->getArrayCopy(); } + // Decode JSON strings that may have been encoded when storing nested arrays/objects + if (\is_array($data)) { + $data = $this->decodeJsonStrings($data); + } + if (!isset($dbCache[$databaseInternalId])) { $databaseDoc = Authorization::skip(fn () => $dbForProject->findOne('databases', [ Query::equal('$sequence', [$databaseInternalId]) @@ -710,6 +715,8 @@ class Update extends Action array &$state ): int { $count = 0; + // Decode JSON strings in bulk create data + $data = $this->decodeJsonStrings($data); $dbForProject->withRequestTimestamp($createdAt, function () use ($dbForProject, $collectionId, $data, &$state, &$count) { $documents = \array_map(function ($doc) { return $doc instanceof Document ? $doc : new Document($doc); @@ -748,8 +755,11 @@ class Update extends Action \DateTime $createdAt, array &$state ): int { + // extracting query first then decoding the nested transaction log data + // otherwise can result error as queries would get transformed to array instead of string $queries = Query::parseQueries($data['queries'] ?? []); $updateData = new Document($data['data']); + $data = $this->decodeJsonStrings($data); $dependentDocs = []; @@ -820,6 +830,8 @@ class Update extends Action \DateTime $createdAt, array &$state ): int { + // Decode JSON strings in bulk upsert data + $data = $this->decodeJsonStrings($data); $documents = \array_map(function ($doc) { return $doc instanceof Document ? $doc : new Document($doc); }, $data); @@ -895,4 +907,31 @@ class Update extends Action return $count; } + + /** + * Recursively decode JSON strings in data array + * This handles cases where nested arrays/objects are stored as JSON strings in the database + * + * @param array $data + * @return array + */ + private function decodeJsonStrings(array $data): array + { + foreach ($data as $key => $value) { + if (\is_string($value)) { + $decoded = \json_decode($value, true); + if (\json_last_error() === JSON_ERROR_NONE && (\is_array($decoded) || \is_object($decoded))) { + $data[$key] = $decoded; + // Recursively decode nested structures + if (\is_array($decoded)) { + $data[$key] = $this->decodeJsonStrings($decoded); + } + } + } elseif (\is_array($value)) { + // Recursively process nested arrays + $data[$key] = $this->decodeJsonStrings($value); + } + } + return $data; + } } diff --git a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Transactions/Create.php b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Transactions/Create.php new file mode 100644 index 0000000000..5d13c18378 --- /dev/null +++ b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Transactions/Create.php @@ -0,0 +1,55 @@ +setHttpMethod(self::HTTP_REQUEST_METHOD_POST) + ->setHttpPath('/v1/vectordb/transactions') + ->desc('Create transaction') + ->groups(['api', 'database', 'transactions']) + ->label('scope', 'documents.write') + ->label('resourceType', RESOURCE_TYPE_DATABASES) + ->label('sdk', new Method( + namespace: 'vectorDB', + group: 'transactions', + name: 'createTransaction', + description: '/docs/references/vectordb/create-transaction.md', + auth: [AuthType::KEY, AuthType::SESSION, AuthType::JWT], + responses: [ + new SDKResponse( + code: SwooleResponse::STATUS_CODE_CREATED, + model: UtopiaResponse::MODEL_TRANSACTION, + ) + ], + contentType: ContentType::JSON + )) + ->param('ttl', APP_DATABASE_TXN_TTL_DEFAULT, new Range(min: APP_DATABASE_TXN_TTL_MIN, max: APP_DATABASE_TXN_TTL_MAX), 'Seconds before the transaction expires.', true) + ->inject('response') + ->inject('dbForProject') + ->inject('user') + ->callback($this->action(...)); + } +} diff --git a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Transactions/Delete.php b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Transactions/Delete.php new file mode 100644 index 0000000000..b7bdb3db58 --- /dev/null +++ b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Transactions/Delete.php @@ -0,0 +1,55 @@ +setHttpMethod(self::HTTP_REQUEST_METHOD_DELETE) + ->setHttpPath('/v1/vectordb/transactions/:transactionId') + ->desc('Delete transaction') + ->groups(['api', 'database', 'transactions']) + ->label('scope', 'documents.write') + ->label('resourceType', RESOURCE_TYPE_DATABASES) + ->label('sdk', new Method( + namespace: 'vectorDB', + group: 'transactions', + name: 'deleteTransaction', + description: '/docs/references/vectordb/delete-transaction.md', + auth: [AuthType::KEY, AuthType::SESSION, AuthType::JWT], + responses: [ + new SDKResponse( + code: SwooleResponse::STATUS_CODE_NOCONTENT, + model: UtopiaResponse::MODEL_NONE, + ) + ], + contentType: ContentType::NONE + )) + ->param('transactionId', '', new UID(), 'Transaction ID.') + ->inject('response') + ->inject('dbForProject') + ->inject('queueForDeletes') + ->callback($this->action(...)); + } +} diff --git a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Transactions/Get.php b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Transactions/Get.php new file mode 100644 index 0000000000..32a6115f60 --- /dev/null +++ b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Transactions/Get.php @@ -0,0 +1,54 @@ +setHttpMethod(self::HTTP_REQUEST_METHOD_GET) + ->setHttpPath('/v1/vectordb/transactions/:transactionId') + ->desc('Get transaction') + ->groups(['api', 'database', 'transactions']) + ->label('scope', 'documents.read') + ->label('resourceType', RESOURCE_TYPE_DATABASES) + ->label('sdk', new Method( + namespace: 'vectorDB', + group: 'transactions', + name: 'getTransaction', + description: '/docs/references/vectordb/get-transaction.md', + auth: [AuthType::KEY, AuthType::SESSION, AuthType::JWT], + responses: [ + new SDKResponse( + code: SwooleResponse::STATUS_CODE_OK, + model: UtopiaResponse::MODEL_TRANSACTION, + ) + ], + contentType: ContentType::JSON + )) + ->param('transactionId', '', new UID(), 'Transaction ID.') + ->inject('response') + ->inject('dbForProject') + ->callback($this->action(...)); + } +} diff --git a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Transactions/Operations/Create.php b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Transactions/Operations/Create.php new file mode 100644 index 0000000000..eabc4cd825 --- /dev/null +++ b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Transactions/Operations/Create.php @@ -0,0 +1,59 @@ +setHttpMethod(self::HTTP_REQUEST_METHOD_POST) + ->setHttpPath('/v1/vectordb/transactions/:transactionId/operations') + ->desc('Create operations') + ->groups(['api', 'database', 'transactions']) + ->label('scope', 'documents.write') + ->label('resourceType', RESOURCE_TYPE_DATABASES) + ->label('sdk', new Method( + namespace: 'vectorDB', + group: 'transactions', + name: 'createOperations', + description: '/docs/references/vectordb/create-operations.md', + auth: [AuthType::KEY, AuthType::SESSION, AuthType::JWT], + responses: [ + new SDKResponse( + code: SwooleResponse::STATUS_CODE_CREATED, + model: UtopiaResponse::MODEL_TRANSACTION, + ) + ], + contentType: ContentType::JSON + )) + ->param('transactionId', '', new UID(), 'Transaction ID.') + ->param('operations', [], new ArrayList(new Operation(type: 'documentsdb')), 'Array of staged operations.', true) + ->inject('response') + ->inject('dbForProject') + ->inject('transactionState') + ->inject('plan') + ->callback($this->action(...)); + } +} diff --git a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Transactions/Update.php b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Transactions/Update.php new file mode 100644 index 0000000000..61725a65f5 --- /dev/null +++ b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Transactions/Update.php @@ -0,0 +1,67 @@ +setHttpMethod(self::HTTP_REQUEST_METHOD_PATCH) + ->setHttpPath('/v1/vectordb/transactions/:transactionId') + ->desc('Update transaction') + ->groups(['api', 'database', 'transactions']) + ->label('scope', 'documents.write') + ->label('resourceType', RESOURCE_TYPE_DATABASES) + ->label('sdk', new Method( + namespace: 'vectorDB', + group: 'transactions', + name: 'updateTransaction', + description: '/docs/references/vectordb/update-transaction.md', + auth: [AuthType::KEY, AuthType::SESSION, AuthType::JWT], + responses: [ + new SDKResponse( + code: SwooleResponse::STATUS_CODE_OK, + model: UtopiaResponse::MODEL_TRANSACTION, + ) + ], + contentType: ContentType::JSON + )) + ->param('transactionId', '', new UID(), 'Transaction ID.') + ->param('commit', false, new Boolean(), 'Commit transaction?', true) + ->param('rollback', false, new Boolean(), 'Rollback transaction?', true) + ->inject('project') + ->inject('response') + ->inject('dbForProject') + ->inject('getDatabasesDB') + ->inject('user') + ->inject('transactionState') + ->inject('queueForDeletes') + ->inject('queueForEvents') + ->inject('queueForStatsUsage') + ->inject('queueForRealtime') + ->inject('queueForFunctions') + ->inject('queueForWebhooks') + ->callback($this->action(...)); + } +} diff --git a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Transactions/XList.php b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Transactions/XList.php new file mode 100644 index 0000000000..43e4a0b36b --- /dev/null +++ b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Transactions/XList.php @@ -0,0 +1,54 @@ +setHttpMethod(self::HTTP_REQUEST_METHOD_GET) + ->setHttpPath('/v1/vectordb/transactions') + ->desc('List transactions') + ->groups(['api', 'database', 'transactions']) + ->label('scope', 'documents.read') + ->label('resourceType', RESOURCE_TYPE_DATABASES) + ->label('sdk', new Method( + namespace: 'vectorDB', + group: 'transactions', + name: 'listTransactions', + description: '/docs/references/vectordb/list-transactions.md', + auth: [AuthType::KEY, AuthType::SESSION, AuthType::JWT], + responses: [ + new SDKResponse( + code: SwooleResponse::STATUS_CODE_OK, + model: UtopiaResponse::MODEL_TRANSACTION_LIST, + ) + ], + contentType: ContentType::JSON + )) + ->param('queries', [], new Transactions(), 'Array of query strings generated using the Query class provided by the SDK. [Learn more about queries](https://appwrite.io/docs/queries).', true) + ->inject('response') + ->inject('dbForProject') + ->callback($this->action(...)); + } +} diff --git a/src/Appwrite/Platform/Modules/Databases/Services/Registry/VectorDB.php b/src/Appwrite/Platform/Modules/Databases/Services/Registry/VectorDB.php index ec071d4e6f..22f434d4c4 100644 --- a/src/Appwrite/Platform/Modules/Databases/Services/Registry/VectorDB.php +++ b/src/Appwrite/Platform/Modules/Databases/Services/Registry/VectorDB.php @@ -26,6 +26,12 @@ use Appwrite\Platform\Modules\Databases\Http\VectorDB\Collections\XList as ListC use Appwrite\Platform\Modules\Databases\Http\VectorDB\Create as CreateVectorDatabase; use Appwrite\Platform\Modules\Databases\Http\VectorDB\Delete as DeleteVectorDatabase; use Appwrite\Platform\Modules\Databases\Http\VectorDB\Get as GetVectorDatabase; +use Appwrite\Platform\Modules\Databases\Http\VectorDB\Transactions\Create as CreateTransaction; +use Appwrite\Platform\Modules\Databases\Http\VectorDB\Transactions\Delete as DeleteTransaction; +use Appwrite\Platform\Modules\Databases\Http\VectorDB\Transactions\Get as GetTransaction; +use Appwrite\Platform\Modules\Databases\Http\VectorDB\Transactions\Operations\Create as CreateOperations; +use Appwrite\Platform\Modules\Databases\Http\VectorDB\Transactions\Update as UpdateTransaction; +use Appwrite\Platform\Modules\Databases\Http\VectorDB\Transactions\XList as ListTransactions; use Appwrite\Platform\Modules\Databases\Http\VectorDB\Update as UpdateVectorDatabase; use Appwrite\Platform\Modules\Databases\Http\VectorDB\Usage\Get as GetVectorDatabaseUsage; use Appwrite\Platform\Modules\Databases\Http\VectorDB\Usage\XList as ListVectorDatabaseUsage; @@ -41,7 +47,7 @@ class VectorDB extends Base $this->registerIndexActions($service); $this->registerDocumentActions($service); $this->registerEmbeddingActions($service); - // $this->registerTransactionActions($service); + $this->registerTransactionActions($service); } private function registerDatabaseActions(Service $service): void @@ -87,6 +93,16 @@ class VectorDB extends Base $service->addAction(DeleteDocuments::getName(), new DeleteDocuments()); } + private function registerTransactionActions(Service $service): void + { + $service->addAction(CreateTransaction::getName(), new CreateTransaction()); + $service->addAction(GetTransaction::getName(), new GetTransaction()); + $service->addAction(UpdateTransaction::getName(), new UpdateTransaction()); + $service->addAction(DeleteTransaction::getName(), new DeleteTransaction()); + $service->addAction(ListTransactions::getName(), new ListTransactions()); + $service->addAction(CreateOperations::getName(), new CreateOperations()); + } + private function registerEmbeddingActions(Service $service): void { $service->addAction(CreateTextEmbeddings::getName(), new CreateTextEmbeddings()); diff --git a/tests/e2e/Services/Databases/VectorDB/Transactions/ACIDTest.php b/tests/e2e/Services/Databases/VectorDB/Transactions/ACIDTest.php new file mode 100644 index 0000000000..4de783f2f8 --- /dev/null +++ b/tests/e2e/Services/Databases/VectorDB/Transactions/ACIDTest.php @@ -0,0 +1,528 @@ +client->call(Client::METHOD_POST, '/vectordb', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'databaseId' => ID::unique(), + 'name' => 'AtomicityTestDB' + ]); + + $this->assertEquals(201, $database['headers']['status-code']); + $databaseId = $database['body']['$id']; + + // Create collection for the test + $collection = $this->client->call(Client::METHOD_POST, '/vectordb/' . $databaseId . '/collections', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'collectionId' => ID::unique(), + 'name' => 'AtomicityTest', + 'dimensions' => 3, + 'documentSecurity' => false, + 'permissions' => [ + Permission::create(Role::any()), + Permission::read(Role::any()), + ], + ]); + + $this->assertEquals(201, $collection['headers']['status-code']); + $collectionId = $collection['body']['$id']; + + // Create a document outside the transaction + $existingDocumentId = 'existing_doc'; + $doc1 = $this->client->call(Client::METHOD_POST, "/vectordb/{$databaseId}/collections/{$collectionId}/documents", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'documentId' => $existingDocumentId, + 'data' => [ + 'embeddings' => $this->generateEmbeddings(3), + 'metadata' => ['email' => 'existing@example.com'], + ], + ]); + + $this->assertEquals(201, $doc1['headers']['status-code']); + + // Create transaction + $transaction = $this->client->call(Client::METHOD_POST, '/vectordb/transactions', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ])); + + $this->assertEquals(201, $transaction['headers']['status-code'], 'Transaction creation should succeed. Response: ' . json_encode($transaction)); + $this->assertArrayHasKey('$id', $transaction['body'], 'Transaction response should have $id. Response body: ' . json_encode($transaction['body'])); + $transactionId = $transaction['body']['$id']; + + // Add operations - second create reuses an existing documentId and should cause the commit to fail + $response = $this->client->call(Client::METHOD_POST, "/vectordb/{$databaseId}/collections/{$collectionId}/documents", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'documents' => [ + [ + '$id' => 'txn_doc_1', + 'embeddings' => $this->generateEmbeddings(3, 0.2), + 'metadata' => ['email' => 'newuser@example.com'], + ], + [ + '$id' => $existingDocumentId, + 'embeddings' => $this->generateEmbeddings(3, 0.3), + 'metadata' => ['email' => 'duplicate@example.com'], + ], + [ + '$id' => 'txn_doc_2', + 'embeddings' => $this->generateEmbeddings(3, 0.4), + 'metadata' => ['email' => 'should-not-exist@example.com'], + ], + ], + 'transactionId' => $transactionId, + ]); + + $this->assertEquals(200, $response['headers']['status-code'], 'Adding documents via normal route should succeed. Response: ' . json_encode($response['body'])); + + // Attempt to commit - should fail due to duplicate document ID + $response = $this->client->call(Client::METHOD_PATCH, "/vectordb/transactions/{$transactionId}", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'commit' => true + ]); + + $this->assertEquals(409, $response['headers']['status-code']); + + // Verify NO new documents were created (atomicity) + $documents = $this->client->call(Client::METHOD_GET, "/vectordb/{$databaseId}/collections/{$collectionId}/documents", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders())); + + $this->assertEquals(1, $documents['body']['total']); + $this->assertEquals('existing@example.com', $documents['body']['documents'][0]['metadata']['email']); + } + + /** + * Test consistency - schema validation and constraints + */ + public function testConsistency(): void + { + // Create database + $database = $this->client->call(Client::METHOD_POST, '/vectordb', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'databaseId' => ID::unique(), + 'name' => 'ConsistencyTestDB' + ]); + + $this->assertEquals(201, $database['headers']['status-code']); + $databaseId = $database['body']['$id']; + + // Create collection + $collection = $this->client->call(Client::METHOD_POST, '/vectordb/' . $databaseId . '/collections', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'collectionId' => ID::unique(), + 'name' => 'ConsistencyTest', + 'dimensions' => 3, + 'documentSecurity' => false, + 'permissions' => [ + Permission::create(Role::any()), + Permission::read(Role::any()), + ], + ]); + + $this->assertEquals(201, $collection['headers']['status-code']); + $collectionId = $collection['body']['$id']; + + // Create transaction + $transaction = $this->client->call(Client::METHOD_POST, '/vectordb/transactions', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ])); + + $transactionId = $transaction['body']['$id']; + + // Stage operations with valid and invalid data (embedding length mismatch) + $response = $this->client->call(Client::METHOD_POST, "/vectordb/transactions/{$transactionId}/operations", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'operations' => [ + [ + 'databaseId' => $databaseId, + 'collectionId' => $collectionId, + 'action' => 'create', + 'documentId' => ID::unique(), + 'data' => [ + 'embeddings' => $this->generateEmbeddings(3, 0.2), + 'metadata' => ['name' => 'Valid User'], + ], + ], + [ + 'databaseId' => $databaseId, + 'collectionId' => $collectionId, + 'action' => 'create', + 'documentId' => ID::unique(), + 'data' => [ + 'embeddings' => $this->generateEmbeddings(2, 0.5), // Invalid dimensions + 'metadata' => ['name' => 'Invalid User'], + ], + ], + [ + 'databaseId' => $databaseId, + 'collectionId' => $collectionId, + 'action' => 'create', + 'documentId' => ID::unique(), + 'data' => [ + 'embeddings' => $this->generateEmbeddings(3, 0.6), + 'metadata' => ['name' => 'Should Not Persist'], + ], + ], + ], + ]); + + $this->assertEquals(201, $response['headers']['status-code']); + + // Attempt to commit - should fail due to invalid embeddings + $response = $this->client->call(Client::METHOD_PATCH, "/vectordb/transactions/{$transactionId}", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'commit' => true + ]); + + $this->assertContains($response['headers']['status-code'], [400, 409, 500], 'Transaction commit should fail due to validation. Response: ' . json_encode($response['body'])); + + // Verify no documents were created + $documents = $this->client->call(Client::METHOD_GET, "/vectordb/{$databaseId}/collections/{$collectionId}/documents", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders())); + + $this->assertEquals(0, $documents['body']['total']); + } + + /** + * Test isolation - concurrent transactions on same data + */ + public function testIsolation(): void + { + // Create database + $database = $this->client->call(Client::METHOD_POST, '/vectordb', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'databaseId' => ID::unique(), + 'name' => 'IsolationTestDB' + ]); + + $this->assertEquals(201, $database['headers']['status-code']); + $databaseId = $database['body']['$id']; + + // Create collection + $collection = $this->client->call(Client::METHOD_POST, '/vectordb/' . $databaseId . '/collections', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'collectionId' => ID::unique(), + 'name' => 'IsolationTest', + 'dimensions' => 3, + 'documentSecurity' => false, + 'permissions' => [ + Permission::create(Role::any()), + Permission::read(Role::any()), + Permission::update(Role::any()), + ], + ]); + + $collectionId = $collection['body']['$id']; + + // Create initial document with status metadata + $doc = $this->client->call(Client::METHOD_POST, "/vectordb/{$databaseId}/collections/{$collectionId}/documents", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'documentId' => 'shared_doc', + 'data' => [ + 'embeddings' => $this->generateEmbeddings(3), + 'metadata' => ['status' => 'pending'], + ], + ]); + + $this->assertEquals(201, $doc['headers']['status-code']); + + // Create first transaction + $transaction1 = $this->client->call(Client::METHOD_POST, '/vectordb/transactions', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ])); + + $this->assertEquals(201, $transaction1['headers']['status-code'], 'Transaction 1 creation should succeed'); + $this->assertArrayHasKey('$id', $transaction1['body'], 'Transaction 1 response should have $id'); + $transactionId1 = $transaction1['body']['$id']; + + // Transaction 1: update status to approved + $this->client->call(Client::METHOD_POST, "/vectordb/transactions/{$transactionId1}/operations", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'operations' => [ + [ + 'databaseId' => $databaseId, + 'collectionId' => $collectionId, + 'action' => 'update', + 'documentId' => 'shared_doc', + 'data' => [ + 'metadata' => ['status' => 'approved'], + ], + ], + ], + ]); + + // Commit first transaction + $response1 = $this->client->call(Client::METHOD_PATCH, "/vectordb/transactions/{$transactionId1}", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'commit' => true + ]); + $this->assertEquals(200, $response1['headers']['status-code']); + + // Document should reflect the first transaction's update + $document = $this->client->call(Client::METHOD_GET, "/vectordb/{$databaseId}/collections/{$collectionId}/documents/shared_doc", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders())); + $this->assertEquals('approved', $document['body']['metadata']['status']); + + // Create second transaction after first commit + $transaction2 = $this->client->call(Client::METHOD_POST, '/vectordb/transactions', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ])); + + $this->assertEquals(201, $transaction2['headers']['status-code'], 'Transaction 2 creation should succeed'); + $this->assertArrayHasKey('$id', $transaction2['body'], 'Transaction 2 response should have $id'); + $transactionId2 = $transaction2['body']['$id']; + + // Transaction 2: update status to declined + $this->client->call(Client::METHOD_POST, "/vectordb/transactions/{$transactionId2}/operations", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'operations' => [ + [ + 'databaseId' => $databaseId, + 'collectionId' => $collectionId, + 'action' => 'update', + 'documentId' => 'shared_doc', + 'data' => [ + 'metadata' => ['status' => 'declined'], + ], + ], + ], + ]); + + // Commit second transaction and ensure isolation guarantees + $response2 = $this->client->call(Client::METHOD_PATCH, "/vectordb/transactions/{$transactionId2}", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'commit' => true + ]); + + $this->assertEquals(200, $response2['headers']['status-code']); + + // Final document should reflect the second transaction's update + $document = $this->client->call(Client::METHOD_GET, "/vectordb/{$databaseId}/collections/{$collectionId}/documents/shared_doc", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders())); + + $this->assertEquals('declined', $document['body']['metadata']['status']); + } + + /** + * Test durability - committed data persists + */ + public function testDurability(): void + { + // Create database + $database = $this->client->call(Client::METHOD_POST, '/vectordb', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'databaseId' => ID::unique(), + 'name' => 'DurabilityTestDB' + ]); + + $this->assertEquals(201, $database['headers']['status-code']); + $databaseId = $database['body']['$id']; + + // Create collection + $collection = $this->client->call(Client::METHOD_POST, '/vectordb/' . $databaseId . '/collections', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'collectionId' => ID::unique(), + 'name' => 'DurabilityTest', + 'dimensions' => 3, + 'documentSecurity' => false, + 'permissions' => [ + Permission::create(Role::any()), + Permission::read(Role::any()), + Permission::update(Role::any()), + Permission::delete(Role::any()), + ], + ]); + + $this->assertEquals(201, $collection['headers']['status-code']); + $collectionId = $collection['body']['$id']; + + // Create transaction with multiple operations + $transaction = $this->client->call(Client::METHOD_POST, '/vectordb/transactions', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ])); + + $this->assertEquals(201, $transaction['headers']['status-code'], 'Transaction creation should succeed'); + $this->assertArrayHasKey('$id', $transaction['body'], 'Transaction response should have $id'); + $transactionId = $transaction['body']['$id']; + + // Create two documents via normal route inside transaction + $this->client->call(Client::METHOD_POST, "/vectordb/{$databaseId}/collections/{$collectionId}/documents", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'documents' => [ + [ + '$id' => 'durable_doc_1', + 'embeddings' => $this->generateEmbeddings(3, 0.3), + 'metadata' => ['data' => 'Important data 1'], + ], + [ + '$id' => 'durable_doc_2', + 'embeddings' => $this->generateEmbeddings(3, 0.5), + 'metadata' => ['data' => 'Important data 2'], + ], + ], + 'transactionId' => $transactionId, + ]); + + // Update first document inside the same transaction + $this->client->call(Client::METHOD_PATCH, "/vectordb/{$databaseId}/collections/{$collectionId}/documents/durable_doc_1", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'data' => [ + 'metadata' => ['data' => 'Updated important data 1'], + ], + 'transactionId' => $transactionId, + ]); + + // Commit transaction + $response = $this->client->call(Client::METHOD_PATCH, "/vectordb/transactions/{$transactionId}", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'commit' => true + ]); + + $this->assertEquals(200, $response['headers']['status-code'], 'Commit should succeed. Response: ' . json_encode($response['body'])); + $this->assertEquals('committed', $response['body']['status']); + + // Verify documents exist and have correct data + $document1 = $this->client->call(Client::METHOD_GET, "/vectordb/{$databaseId}/collections/{$collectionId}/documents/durable_doc_1", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders())); + $this->assertEquals(200, $document1['headers']['status-code']); + $this->assertEquals('Updated important data 1', $document1['body']['metadata']['data']); + + $document2 = $this->client->call(Client::METHOD_GET, "/vectordb/{$databaseId}/collections/{$collectionId}/documents/durable_doc_2", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders())); + $this->assertEquals(200, $document2['headers']['status-code']); + $this->assertEquals('Important data 2', $document2['body']['metadata']['data']); + + // Further update outside transaction to ensure persistence + $update = $this->client->call(Client::METHOD_PATCH, "/vectordb/{$databaseId}/collections/{$collectionId}/documents/durable_doc_1", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'data' => [ + 'metadata' => ['data' => 'Modified outside transaction'], + ], + ]); + $this->assertEquals(200, $update['headers']['status-code']); + + // Verify the update persisted + $document1 = $this->client->call(Client::METHOD_GET, "/vectordb/{$databaseId}/collections/{$collectionId}/documents/durable_doc_1", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders())); + $this->assertEquals('Modified outside transaction', $document1['body']['metadata']['data']); + + // List all documents to verify total count + $documents = $this->client->call(Client::METHOD_GET, "/vectordb/{$databaseId}/collections/{$collectionId}/documents", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders())); + $this->assertEquals(2, $documents['body']['total']); + } +} diff --git a/tests/e2e/Services/Databases/VectorDB/Transactions/TransactionsBase.php b/tests/e2e/Services/Databases/VectorDB/Transactions/TransactionsBase.php new file mode 100644 index 0000000000..1462921ef1 --- /dev/null +++ b/tests/e2e/Services/Databases/VectorDB/Transactions/TransactionsBase.php @@ -0,0 +1,2371 @@ +client->call(Client::METHOD_POST, '/vectordb', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'databaseId' => ID::unique(), + 'name' => 'TransactionTestDatabase' + ]); + + $this->assertEquals(201, $database['headers']['status-code']); + $databaseId = $database['body']['$id']; + + // Test creating a transaction with default TTL + $response = $this->client->call(Client::METHOD_POST, '/vectordb/transactions', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders())); + + $this->assertEquals(201, $response['headers']['status-code']); + $this->assertArrayHasKey('$id', $response['body']); + $this->assertArrayHasKey('status', $response['body']); + $this->assertArrayHasKey('operations', $response['body']); + $this->assertArrayHasKey('expiresAt', $response['body']); + $this->assertEquals('pending', $response['body']['status']); + $this->assertEquals(0, $response['body']['operations']); + + $transactionId1 = $response['body']['$id']; + + // Test creating a transaction with custom TTL + $response = $this->client->call(Client::METHOD_POST, '/vectordb/transactions', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'ttl' => 900 + ]); + + $this->assertEquals(201, $response['headers']['status-code']); + $this->assertEquals('pending', $response['body']['status']); + + $expiresAt = new \DateTime($response['body']['expiresAt']); + $now = new \DateTime(); + $diff = $expiresAt->getTimestamp() - $now->getTimestamp(); + $this->assertGreaterThan(800, $diff); + $this->assertLessThan(1000, $diff); + + $transactionId2 = $response['body']['$id']; + + // Test invalid TTL values + $response = $this->client->call(Client::METHOD_POST, '/vectordb/transactions', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'ttl' => 30 // Below minimum + ]); + + $this->assertEquals(400, $response['headers']['status-code']); + + $response = $this->client->call(Client::METHOD_POST, '/vectordb/transactions', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'ttl' => 4000 // Above maximum + ]); + + $this->assertEquals(400, $response['headers']['status-code']); + } + + /** + * Test adding operations to a transaction + */ + public function testCreateOperations(): void + { + // Create database first + $database = $this->client->call(Client::METHOD_POST, '/vectordb', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'databaseId' => ID::unique(), + 'name' => 'TransactionOperationsTestDB' + ]); + + $this->assertEquals(201, $database['headers']['status-code']); + $databaseId = $database['body']['$id']; + + // Create transaction + $transaction = $this->client->call(Client::METHOD_POST, '/vectordb/transactions', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders())); + + $this->assertEquals(201, $transaction['headers']['status-code']); + $transactionId = $transaction['body']['$id']; + + // Create a collection for testing + $collection = $this->client->call(Client::METHOD_POST, '/vectordb/' . $databaseId . '/collections', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'collectionId' => ID::unique(), + 'name' => 'TransactionOperationsTest', + 'dimensions' => 3, + 'documentSecurity' => false, + 'permissions' => [ + Permission::create(Role::any()), + Permission::read(Role::any()), + Permission::update(Role::any()), + Permission::delete(Role::any()), + ], + ]); + + $this->assertEquals(201, $collection['headers']['status-code']); + $collectionId = $collection['body']['$id']; + + // Add valid operations + $response = $this->client->call(Client::METHOD_POST, "/vectordb/transactions/{$transactionId}/operations", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'operations' => [ + [ + 'databaseId' => $databaseId, + 'collectionId' => $collectionId, + 'action' => 'create', + 'documentId' => 'doc1', + 'data' => [ + 'embeddings' => $this->generateEmbeddings(3), + 'metadata' => ['name' => 'Test Document 1'] + ] + ], + [ + 'databaseId' => $databaseId, + 'collectionId' => $collectionId, + 'action' => 'create', + 'documentId' => 'doc2', + 'data' => [ + 'embeddings' => $this->generateEmbeddings(3, 0.2), + 'metadata' => ['name' => 'Test Document 2'] + ] + ] + ] + ]); + + $this->assertEquals(201, $response['headers']['status-code']); + $this->assertEquals(2, $response['body']['operations']); + + // Test adding more operations + $response = $this->client->call(Client::METHOD_POST, "/vectordb/transactions/{$transactionId}/operations", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'operations' => [ + [ + 'databaseId' => $databaseId, + 'collectionId' => $collectionId, + 'action' => 'update', + 'documentId' => 'doc1', + 'data' => [ + 'metadata' => ['name' => 'Updated Document 1'] + ] + ] + ] + ]); + + $this->assertEquals(201, $response['headers']['status-code']); + $this->assertEquals(3, $response['body']['operations']); + + // Test invalid database ID + $response = $this->client->call(Client::METHOD_POST, "/vectordb/transactions/{$transactionId}/operations", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'operations' => [ + [ + 'databaseId' => 'invalid_database', + 'collectionId' => $collectionId, + 'action' => 'create', + 'documentId' => ID::unique(), + 'data' => [ + 'embeddings' => $this->generateEmbeddings(3), + 'metadata' => ['name' => 'Test'] + ] + ] + ] + ]); + + $this->assertEquals(404, $response['headers']['status-code'], 'Invalid database should return 404. Got: ' . json_encode($response['body'])); + + // Test invalid collection ID + $response = $this->client->call(Client::METHOD_POST, "/vectordb/transactions/{$transactionId}/operations", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'operations' => [ + [ + 'databaseId' => $databaseId, + 'collectionId' => 'invalid_collection', + 'action' => 'create', + 'documentId' => ID::unique(), + 'data' => [ + 'embeddings' => $this->generateEmbeddings(3), + 'metadata' => ['name' => 'Test'] + ] + ] + ] + ]); + + $this->assertEquals(404, $response['headers']['status-code']); + } + + /** + * Test committing a transaction + */ + public function testCommit(): void + { + // Create database first + $database = $this->client->call(Client::METHOD_POST, '/vectordb', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'databaseId' => ID::unique(), + 'name' => 'TransactionCommitTestDB' + ]); + + $this->assertEquals(201, $database['headers']['status-code']); + $databaseId = $database['body']['$id']; + + // Create collection + $collection = $this->client->call(Client::METHOD_POST, '/vectordb/' . $databaseId . '/collections', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'collectionId' => ID::unique(), + 'name' => 'TransactionCommitTest', + 'dimensions' => 3, + 'documentSecurity' => false, + 'permissions' => [ + Permission::create(Role::any()), + Permission::read(Role::any()), + Permission::update(Role::any()), + Permission::delete(Role::any()), + ], + ]); + + $this->assertEquals(201, $collection['headers']['status-code']); + $collectionId = $collection['body']['$id']; + + // Create transaction + $transaction = $this->client->call(Client::METHOD_POST, '/vectordb/transactions', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders())); + + $this->assertEquals(201, $transaction['headers']['status-code']); + $transactionId = $transaction['body']['$id']; + + // Add operations + $response = $this->client->call(Client::METHOD_POST, "/vectordb/transactions/{$transactionId}/operations", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'operations' => [ + [ + 'databaseId' => $databaseId, + 'collectionId' => $collectionId, + 'action' => 'create', + 'documentId' => 'doc1', + 'data' => [ + 'embeddings' => $this->generateEmbeddings(3), + 'metadata' => ['name' => 'Test Document 1'] + ] + ], + [ + 'databaseId' => $databaseId, + 'collectionId' => $collectionId, + 'action' => 'create', + 'documentId' => 'doc2', + 'data' => [ + 'embeddings' => $this->generateEmbeddings(3, 0.2), + 'metadata' => ['name' => 'Test Document 2'] + ] + ], + [ + 'databaseId' => $databaseId, + 'collectionId' => $collectionId, + 'action' => 'update', + 'documentId' => 'doc1', + 'data' => [ + 'metadata' => ['name' => 'Updated Document 1'] + ] + ] + ] + ]); + + $this->assertEquals(201, $response['headers']['status-code']); + $this->assertEquals(3, $response['body']['operations']); + + // Commit the transaction + $response = $this->client->call(Client::METHOD_PATCH, "/vectordb/transactions/{$transactionId}", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'commit' => true + ]); + + $this->assertEquals(200, $response['headers']['status-code']); + $this->assertEquals('committed', $response['body']['status']); + + // Verify documents were created + $documents = $this->client->call(Client::METHOD_GET, "/vectordb/{$databaseId}/collections/{$collectionId}/documents", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders())); + + $this->assertEquals(200, $documents['headers']['status-code']); + $this->assertEquals(2, $documents['body']['total']); + + // Verify the update was applied + $doc1Found = false; + foreach ($documents['body']['documents'] as $doc) { + if ($doc['$id'] === 'doc1') { + $this->assertEquals('Updated Document 1', $doc['metadata']['name']); + $doc1Found = true; + } + } + $this->assertTrue($doc1Found, 'Document doc1 should exist with updated name'); + + // Test committing already committed transaction + $response = $this->client->call(Client::METHOD_PATCH, "/vectordb/transactions/{$transactionId}", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'commit' => true + ]); + + $this->assertEquals(400, $response['headers']['status-code']); + } + + /** + * Test rolling back a transaction + */ + public function testRollback(): void + { + // Create database first + $database = $this->client->call(Client::METHOD_POST, '/vectordb', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'databaseId' => ID::unique(), + 'name' => 'TransactionRollbackTestDB' + ]); + + $this->assertEquals(201, $database['headers']['status-code']); + $databaseId = $database['body']['$id']; + + // Create transaction + $transaction = $this->client->call(Client::METHOD_POST, '/vectordb/transactions', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders())); + + $this->assertEquals(201, $transaction['headers']['status-code']); + $transactionId = $transaction['body']['$id']; + + // Create a collection for rollback test + $collection = $this->client->call(Client::METHOD_POST, '/vectordb/' . $databaseId . '/collections', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'collectionId' => ID::unique(), + 'name' => 'TransactionRollbackTest', + 'dimensions' => 3, + 'documentSecurity' => false, + 'permissions' => [ + Permission::create(Role::any()), + Permission::read(Role::any()), + Permission::update(Role::any()), + Permission::delete(Role::any()), + ], + ]); + + $collectionId = $collection['body']['$id']; + + // Add operations + $response = $this->client->call(Client::METHOD_POST, "/vectordb/transactions/{$transactionId}/operations", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'operations' => [ + [ + 'databaseId' => $databaseId, + 'collectionId' => $collectionId, + 'action' => 'create', + 'documentId' => 'rollback_doc', + 'data' => [ + 'embeddings' => $this->generateEmbeddings(3), + 'metadata' => ['value' => 'Should not exist'] + ] + ] + ] + ]); + + $this->assertEquals(201, $response['headers']['status-code']); + + // Rollback the transaction + $response = $this->client->call(Client::METHOD_PATCH, "/vectordb/transactions/{$transactionId}", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'rollback' => true + ]); + + $this->assertEquals(200, $response['headers']['status-code']); + $this->assertEquals('failed', $response['body']['status']); + + // Verify no documents were created + $documents = $this->client->call(Client::METHOD_GET, "/vectordb/{$databaseId}/collections/{$collectionId}/documents", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders())); + + $this->assertEquals(200, $documents['headers']['status-code']); + $this->assertEquals(0, $documents['body']['total']); + } + + /** + * Test transaction expiration + */ + public function testTransactionExpiration(): void + { + // Create database and collection + $database = $this->client->call(Client::METHOD_POST, '/vectordb', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'databaseId' => ID::unique(), + 'name' => 'ExpirationTestDB' + ]); + + $databaseId = $database['body']['$id']; + + $collection = $this->client->call(Client::METHOD_POST, "/vectordb/{$databaseId}/collections", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'collectionId' => ID::unique(), + 'name' => 'TestCollection', + 'dimensions' => 3, + 'permissions' => [ + Permission::read(Role::any()), + Permission::create(Role::any()), + Permission::update(Role::any()), + Permission::delete(Role::any()), + ], + ]); + + $collectionId = $collection['body']['$id']; + + // Create transaction with minimum TTL (60 seconds) + $transaction = $this->client->call(Client::METHOD_POST, '/vectordb/transactions', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'ttl' => 60 + ]); + + $this->assertEquals(201, $transaction['headers']['status-code']); + $transactionId = $transaction['body']['$id']; + + // Add operation + $response = $this->client->call(Client::METHOD_POST, "/vectordb/transactions/{$transactionId}/operations", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'operations' => [ + [ + 'databaseId' => $databaseId, + 'collectionId' => $collectionId, + 'action' => 'create', + 'documentId' => ID::unique(), + 'data' => [ + 'embeddings' => $this->generateEmbeddings(3), + 'metadata' => ['data' => 'Should expire'] + ] + ] + ] + ]); + + $this->assertEquals(201, $response['headers']['status-code']); + + // Verify transaction was created with correct expiration + $txnDetails = $this->client->call(Client::METHOD_GET, "/vectordb/transactions/{$transactionId}", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ])); + + $this->assertEquals(200, $txnDetails['headers']['status-code']); + $this->assertEquals('pending', $txnDetails['body']['status']); + + // Verify expiration time is approximately 60 seconds from now + $expiresAt = new \DateTime($txnDetails['body']['expiresAt']); + $now = new \DateTime(); + $diff = $expiresAt->getTimestamp() - $now->getTimestamp(); + $this->assertGreaterThan(55, $diff); + $this->assertLessThan(65, $diff); + } + + /** + * Test maximum operations per transaction + */ + public function testTransactionSizeLimit(): void + { + // Create database and collection + $database = $this->client->call(Client::METHOD_POST, '/vectordb', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'databaseId' => ID::unique(), + 'name' => 'SizeLimitTestDB' + ]); + + $databaseId = $database['body']['$id']; + + $collection = $this->client->call(Client::METHOD_POST, "/vectordb/{$databaseId}/collections", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'collectionId' => ID::unique(), + 'name' => 'TestCollection', + 'dimensions' => 3, + 'permissions' => [Permission::create(Role::any())], + ]); + + $collectionId = $collection['body']['$id']; + + // Create transaction + $transaction = $this->client->call(Client::METHOD_POST, '/vectordb/transactions', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders())); + + $transactionId = $transaction['body']['$id']; + + // Try to add operations exceeding the limit (assuming limit is 100) + // We'll add 50 operations twice to test incremental limit + $operations = []; + for ($i = 0; $i < 50; $i++) { + $operations[] = [ + 'databaseId' => $databaseId, + 'collectionId' => $collectionId, + 'action' => 'create', + 'documentId' => 'doc_' . $i, + 'data' => [ + 'embeddings' => $this->generateEmbeddings(3, 0.1 + ($i * 0.001)), + 'metadata' => ['value' => 'Test ' . $i] + ] + ]; + } + + // First batch should succeed + $response = $this->client->call(Client::METHOD_POST, "/vectordb/transactions/{$transactionId}/operations", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'operations' => $operations + ]); + + $this->assertEquals(201, $response['headers']['status-code']); + $this->assertEquals(50, $response['body']['operations']); + + // Second batch of 50 more operations + $operations = []; + for ($i = 50; $i < 100; $i++) { + $operations[] = [ + 'databaseId' => $databaseId, + 'collectionId' => $collectionId, + 'documentId' => 'doc_' . $i, + 'action' => 'create', + 'data' => [ + 'embeddings' => $this->generateEmbeddings(3, 0.1 + ($i * 0.001)), + 'metadata' => ['value' => 'Test ' . $i] + ] + ]; + } + + $response = $this->client->call(Client::METHOD_POST, "/vectordb/transactions/{$transactionId}/operations", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'operations' => $operations + ]); + + $this->assertEquals(201, $response['headers']['status-code']); + $this->assertEquals(100, $response['body']['operations']); + + // Try to add one more operation - should fail + $response = $this->client->call(Client::METHOD_POST, "/vectordb/transactions/{$transactionId}/operations", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'operations' => [ + [ + 'databaseId' => $databaseId, + 'collectionId' => $collectionId, + 'action' => 'create', + 'documentId' => 'doc_overflow', + 'data' => [ + 'embeddings' => $this->generateEmbeddings(3), + 'metadata' => ['value' => 'This should fail'] + ] + ] + ] + ]); + + $this->assertEquals(400, $response['headers']['status-code']); + } + + /** + * Test concurrent transactions with conflicting operations + */ + public function testConcurrentTransactionConflicts(): void + { + // Create database and collection + $database = $this->client->call(Client::METHOD_POST, '/vectordb', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'databaseId' => ID::unique(), + 'name' => 'ConflictTestDB' + ]); + + $databaseId = $database['body']['$id']; + + $collection = $this->client->call(Client::METHOD_POST, "/vectordb/{$databaseId}/collections", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'collectionId' => ID::unique(), + 'name' => 'TestCollection', + 'dimensions' => 3, + 'permissions' => [ + Permission::read(Role::any()), + Permission::create(Role::any()), + Permission::update(Role::any()), + ], + ]); + + $collectionId = $collection['body']['$id']; + + // Create initial document + $doc = $this->client->call(Client::METHOD_POST, "/vectordb/{$databaseId}/collections/{$collectionId}/documents", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'documentId' => 'shared_doc', + 'data' => [ + 'embeddings' => $this->generateEmbeddings(3), + 'metadata' => ['counter' => 100] + ] + ]); + + $this->assertEquals(201, $doc['headers']['status-code']); + + // Create two transactions + $txn1 = $this->client->call(Client::METHOD_POST, '/vectordb/transactions', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders())); + + $txn2 = $this->client->call(Client::METHOD_POST, '/vectordb/transactions', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders())); + + $transactionId1 = $txn1['body']['$id']; + $transactionId2 = $txn2['body']['$id']; + + // Both transactions try to update the same document + $this->client->call(Client::METHOD_POST, "/vectordb/transactions/{$transactionId1}/operations", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'operations' => [ + [ + 'databaseId' => $databaseId, + 'collectionId' => $collectionId, + 'action' => 'update', + 'documentId' => 'shared_doc', + 'data' => [ + 'metadata' => ['counter' => 200] + ] + ] + ] + ]); + + $this->client->call(Client::METHOD_POST, "/vectordb/transactions/{$transactionId2}/operations", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'operations' => [ + [ + 'databaseId' => $databaseId, + 'collectionId' => $collectionId, + 'action' => 'update', + 'documentId' => 'shared_doc', + 'data' => [ + 'metadata' => ['counter' => 300] + ] + ] + ] + ]); + + // Commit first transaction + $response1 = $this->client->call(Client::METHOD_PATCH, "/vectordb/transactions/{$transactionId1}", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'commit' => true + ]); + + $this->assertEquals(200, $response1['headers']['status-code']); + + // Commit second transaction - should fail with conflict + $response2 = $this->client->call(Client::METHOD_PATCH, "/vectordb/transactions/{$transactionId2}", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'commit' => true + ]); + + $this->assertEquals(409, $response2['headers']['status-code']); // Conflict + + // Verify the document has the value from first transaction + $doc = $this->client->call(Client::METHOD_GET, "/vectordb/{$databaseId}/collections/{$collectionId}/documents/shared_doc", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders())); + + $this->assertEquals(200, $doc['body']['metadata']['counter']); + } + + /** + * Test deleting a document that's being updated in a transaction + */ + public function testDeleteDocumentDuringTransaction(): void + { + // Create database and collection + $database = $this->client->call(Client::METHOD_POST, '/vectordb', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'databaseId' => ID::unique(), + 'name' => 'DeleteConflictDB' + ]); + + $databaseId = $database['body']['$id']; + + $collection = $this->client->call(Client::METHOD_POST, "/vectordb/{$databaseId}/collections", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'collectionId' => ID::unique(), + 'name' => 'TestCollection', + 'dimensions' => 3, + 'permissions' => [ + Permission::read(Role::any()), + Permission::create(Role::any()), + Permission::update(Role::any()), + Permission::delete(Role::any()), + ], + ]); + + $collectionId = $collection['body']['$id']; + + // Create document + $doc = $this->client->call(Client::METHOD_POST, "/vectordb/{$databaseId}/collections/{$collectionId}/documents", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'documentId' => 'target_doc', + 'data' => [ + 'embeddings' => $this->generateEmbeddings(3), + 'metadata' => ['data' => 'Original'] + ] + ]); + + $this->assertEquals(201, $doc['headers']['status-code']); + + // Create transaction + $transaction = $this->client->call(Client::METHOD_POST, '/vectordb/transactions', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders())); + + $transactionId = $transaction['body']['$id']; + + // Add update operation to transaction + $this->client->call(Client::METHOD_POST, "/vectordb/transactions/{$transactionId}/operations", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'operations' => [ + [ + 'databaseId' => $databaseId, + 'collectionId' => $collectionId, + 'action' => 'update', + 'documentId' => 'target_doc', + 'data' => [ + 'metadata' => ['data' => 'Updated in transaction'] + ] + ] + ] + ]); + + // Delete the document outside of transaction + $response = $this->client->call(Client::METHOD_DELETE, "/vectordb/{$databaseId}/collections/{$collectionId}/documents/target_doc", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ])); + + $this->assertEquals(204, $response['headers']['status-code']); + + // Try to commit transaction - should fail because document no longer exists + $response = $this->client->call(Client::METHOD_PATCH, "/vectordb/transactions/{$transactionId}", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'commit' => true + ]); + + $this->assertEquals(404, $response['headers']['status-code']); // Conflict + } + + /** + * Test bulk operations in transactions + */ + public function testBulkOperations(): void + { + // Create database and collection + $database = $this->client->call(Client::METHOD_POST, '/vectordb', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'databaseId' => ID::unique(), + 'name' => 'BulkOpsDB' + ]); + + $databaseId = $database['body']['$id']; + + $collection = $this->client->call(Client::METHOD_POST, "/vectordb/{$databaseId}/collections", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'collectionId' => ID::unique(), + 'name' => 'TestCollection', + 'dimensions' => 3, + 'permissions' => [ + Permission::read(Role::any()), + Permission::create(Role::any()), + Permission::update(Role::any()), + Permission::delete(Role::any()), + ], + ]); + + $collectionId = $collection['body']['$id']; + + // Create some initial documents + for ($i = 1; $i <= 5; $i++) { + $this->client->call(Client::METHOD_POST, "/vectordb/{$databaseId}/collections/{$collectionId}/documents", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'documentId' => 'existing_' . $i, + 'data' => [ + 'embeddings' => $this->generateEmbeddings(3, 0.1 + ($i * 0.01)), + 'metadata' => [ + 'name' => 'Existing ' . $i, + 'category' => 'old' + ] + ] + ]); + } + + // Create transaction + $transaction = $this->client->call(Client::METHOD_POST, '/vectordb/transactions', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders())); + + $transactionId = $transaction['body']['$id']; + + // Add bulk operations + $response = $this->client->call(Client::METHOD_POST, "/vectordb/transactions/{$transactionId}/operations", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'operations' => [ + // Bulk create + [ + 'databaseId' => $databaseId, + 'collectionId' => $collectionId, + 'action' => 'bulkCreate', + 'data' => [ + [ + '$id' => 'bulk_1', + 'embeddings' => $this->generateEmbeddings(3, 0.2), + 'metadata' => ['name' => 'Bulk 1', 'category' => 'new'] + ], + [ + '$id' => 'bulk_2', + 'embeddings' => $this->generateEmbeddings(3, 0.3), + 'metadata' => ['name' => 'Bulk 2', 'category' => 'new'] + ], + [ + '$id' => 'bulk_3', + 'embeddings' => $this->generateEmbeddings(3, 0.4), + 'metadata' => ['name' => 'Bulk 3', 'category' => 'new'] + ], + ] + ], + // Bulk update + [ + 'databaseId' => $databaseId, + 'collectionId' => $collectionId, + 'action' => 'bulkUpdate', + 'data' => [ + 'queries' => [Query::equal('metadata', [['category' => 'old']])->toString()], + 'data' => ['metadata' => ['category' => 'updated']] + ] + ], + // Bulk delete + [ + 'databaseId' => $databaseId, + 'collectionId' => $collectionId, + 'action' => 'bulkDelete', + 'data' => [ + 'queries' => [Query::equal('$id', ['existing_5'])->toString()] + ] + ] + ] + ]); + + $this->assertEquals(201, $response['headers']['status-code']); + + // Commit transaction + $response = $this->client->call(Client::METHOD_PATCH, "/vectordb/transactions/{$transactionId}", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'commit' => true + ]); + + $this->assertEquals(200, $response['headers']['status-code']); + + // Verify results + $documents = $this->client->call(Client::METHOD_GET, "/vectordb/{$databaseId}/collections/{$collectionId}/documents", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders())); + + // Should have 7 documents (5 existing - 1 deleted + 3 new) + $this->assertEquals(7, $documents['body']['total']); + + // Check categories were updated + $oldCategoryCount = 0; + $updatedCategoryCount = 0; + $newCategoryCount = 0; + + foreach ($documents['body']['documents'] as $doc) { + $category = $doc['metadata']['category'] ?? null; + switch ($category) { + case 'old': + $oldCategoryCount++; + break; + case 'updated': + $updatedCategoryCount++; + break; + case 'new': + $newCategoryCount++; + break; + } + } + + $this->assertEquals(0, $oldCategoryCount); + $this->assertEquals(4, $updatedCategoryCount); // 4 existing docs updated + $this->assertEquals(3, $newCategoryCount); // 3 new docs + } + + /** + * Test transaction with mixed success and failure operations + */ + public function testPartialFailureRollback(): void + { + // Create database and collection + $database = $this->client->call(Client::METHOD_POST, '/vectordb', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'databaseId' => ID::unique(), + 'name' => 'PartialFailureDB' + ]); + + $databaseId = $database['body']['$id']; + + $collection = $this->client->call(Client::METHOD_POST, "/vectordb/{$databaseId}/collections", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'collectionId' => ID::unique(), + 'name' => 'TestCollection', + 'dimensions' => 3, + 'permissions' => [ + Permission::read(Role::any()), + Permission::create(Role::any()), + ], + ]); + + $collectionId = $collection['body']['$id']; + + // Create HNSW index on embeddings + $this->client->call(Client::METHOD_POST, "/vectordb/{$databaseId}/collections/{$collectionId}/indexes", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'key' => 'embeddings_index', + 'type' => Database::INDEX_HNSW_EUCLIDEAN, + 'attributes' => ['embeddings'], + ]); + + sleep(2); + + // Create an existing document + $duplicateId = ID::unique(); + $this->client->call(Client::METHOD_POST, "/vectordb/{$databaseId}/collections/{$collectionId}/documents", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'documentId' => $duplicateId, + 'data' => [ + 'embeddings' => $this->generateEmbeddings(3), + 'metadata' => ['email' => 'existing@example.com'] + ] + ]); + + // Create transaction + $transaction = $this->client->call(Client::METHOD_POST, '/vectordb/transactions', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders())); + + $transactionId = $transaction['body']['$id']; + + // Add operations - mix of valid and invalid (duplicate id) + $response = $this->client->call(Client::METHOD_POST, "/vectordb/transactions/{$transactionId}/operations", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'operations' => [ + [ + 'databaseId' => $databaseId, + 'collectionId' => $collectionId, + 'action' => 'create', + 'documentId' => ID::unique(), + 'data' => [ + 'embeddings' => $this->generateEmbeddings(3, 0.2), + 'metadata' => ['email' => 'valid1@example.com'] + ] + ], + [ + 'databaseId' => $databaseId, + 'collectionId' => $collectionId, + 'action' => 'create', + 'documentId' => ID::unique(), + 'data' => [ + 'embeddings' => $this->generateEmbeddings(3, 0.3), + 'metadata' => ['email' => 'valid2@example.com'] + ] + ], + [ + 'databaseId' => $databaseId, + 'collectionId' => $collectionId, + 'action' => 'create', + 'documentId' => $duplicateId, + 'data' => [ + 'embeddings' => $this->generateEmbeddings(3, 0.4), + 'metadata' => ['email' => 'existing@example.com'] + ] + ], + [ + 'databaseId' => $databaseId, + 'collectionId' => $collectionId, + 'action' => 'create', + 'documentId' => ID::unique(), + 'data' => [ + 'embeddings' => $this->generateEmbeddings(3, 0.5), + 'metadata' => ['email' => 'valid3@example.com'] + ] + ], + ] + ]); + + $this->assertEquals(201, $response['headers']['status-code']); + + // Try to commit - should fail and rollback all operations + $response = $this->client->call(Client::METHOD_PATCH, "/vectordb/transactions/{$transactionId}", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'commit' => true + ]); + + $this->assertEquals(409, $response['headers']['status-code']); // Conflict due to duplicate + + // Verify NO new documents were created (atomicity) + $documents = $this->client->call(Client::METHOD_GET, "/vectordb/{$databaseId}/collections/{$collectionId}/documents", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders())); + + $this->assertEquals(1, $documents['body']['total']); // Only the original document + $this->assertEquals('existing@example.com', $documents['body']['documents'][0]['metadata']['email']); + } + + /** + * Test double commit/rollback attempts + */ + public function testDoubleCommitRollback(): void + { + // Create database and collection + $database = $this->client->call(Client::METHOD_POST, '/vectordb', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'databaseId' => ID::unique(), + 'name' => 'DoubleCommitDB' + ]); + + $databaseId = $database['body']['$id']; + + $collection = $this->client->call(Client::METHOD_POST, "/vectordb/{$databaseId}/collections", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'collectionId' => ID::unique(), + 'name' => 'TestCollection', + 'dimensions' => 3, + 'permissions' => [Permission::create(Role::any())], + ]); + + $collectionId = $collection['body']['$id']; + + // Test double commit + $transaction = $this->client->call(Client::METHOD_POST, '/vectordb/transactions', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders())); + + $transactionId = $transaction['body']['$id']; + + // Add operation + $this->client->call(Client::METHOD_POST, "/vectordb/transactions/{$transactionId}/operations", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'operations' => [ + [ + 'databaseId' => $databaseId, + 'collectionId' => $collectionId, + 'action' => 'create', + 'documentId' => ID::unique(), + 'data' => [ + 'embeddings' => $this->generateEmbeddings(3), + 'metadata' => ['data' => 'Test'] + ] + ] + ] + ]); + + // First commit + $response = $this->client->call(Client::METHOD_PATCH, "/vectordb/transactions/{$transactionId}", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'commit' => true + ]); + + $this->assertEquals(200, $response['headers']['status-code']); + + // Second commit attempt - should fail + $response = $this->client->call(Client::METHOD_PATCH, "/vectordb/transactions/{$transactionId}", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'commit' => true + ]); + + $this->assertEquals(400, $response['headers']['status-code']); // Bad request - already committed + + // Test double rollback + $transaction2 = $this->client->call(Client::METHOD_POST, '/vectordb/transactions', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders())); + + $transactionId2 = $transaction2['body']['$id']; + + // First rollback + $response = $this->client->call(Client::METHOD_PATCH, "/vectordb/transactions/{$transactionId2}", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'rollback' => true + ]); + + $this->assertEquals(200, $response['headers']['status-code']); + + // Second rollback attempt - should fail + $response = $this->client->call(Client::METHOD_PATCH, "/vectordb/transactions/{$transactionId2}", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'rollback' => true + ]); + + $this->assertEquals(400, $response['headers']['status-code']); // Bad request - already rolled back + } + + /** + * Test operations on non-existent documents + */ + public function testOperationsOnNonExistentDocuments(): void + { + // Create database and collection + $database = $this->client->call(Client::METHOD_POST, '/vectordb', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'databaseId' => ID::unique(), + 'name' => 'NonExistentDocDB' + ]); + + $databaseId = $database['body']['$id']; + + $collection = $this->client->call(Client::METHOD_POST, "/vectordb/{$databaseId}/collections", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'collectionId' => ID::unique(), + 'name' => 'TestCollection', + 'dimensions' => 3, + 'permissions' => [ + Permission::create(Role::any()), + Permission::update(Role::any()), + Permission::delete(Role::any()), + ], + ]); + + $collectionId = $collection['body']['$id']; + + // Create transaction + $transaction = $this->client->call(Client::METHOD_POST, '/vectordb/transactions', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders())); + + $transactionId = $transaction['body']['$id']; + + // Try to update non-existent document - should fail at staging time with early validation + $response = $this->client->call(Client::METHOD_POST, "/vectordb/transactions/{$transactionId}/operations", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'operations' => [ + [ + 'databaseId' => $databaseId, + 'collectionId' => $collectionId, + 'action' => 'update', + 'documentId' => 'non_existent_doc', + 'data' => [ + 'metadata' => ['data' => 'Should fail'] + ] + ] + ] + ]); + + $this->assertEquals(404, $response['headers']['status-code']); // Document not found at staging time + + // Test delete non-existent document - should also fail at staging time with early validation + $transaction2 = $this->client->call(Client::METHOD_POST, '/vectordb/transactions', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders())); + + $transactionId2 = $transaction2['body']['$id']; + + $response = $this->client->call(Client::METHOD_POST, "/vectordb/transactions/{$transactionId2}/operations", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'operations' => [ + [ + 'databaseId' => $databaseId, + 'collectionId' => $collectionId, + 'action' => 'delete', + 'documentId' => 'non_existent_doc', + 'data' => [] + ] + ] + ]); + + $this->assertEquals(404, $response['headers']['status-code']); // Document not found at staging time + } + + /** + * Test createDocument with transactionId via normal route + */ + public function testCreateDocument(): void + { + // Create database and collection + $database = $this->client->call(Client::METHOD_POST, '/vectordb', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'databaseId' => ID::unique(), + 'name' => 'WriteRoutesTestDB' + ]); + + $databaseId = $database['body']['$id']; + + $collection = $this->client->call(Client::METHOD_POST, "/vectordb/{$databaseId}/collections", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'collectionId' => ID::unique(), + 'name' => 'TestCollection', + 'dimensions' => 3, + 'documentSecurity' => false, + 'permissions' => [ + Permission::create(Role::any()), + Permission::read(Role::any()), + Permission::update(Role::any()), + Permission::delete(Role::any()), + ], + ]); + + $collectionId = $collection['body']['$id']; + + // Create transaction + $transaction = $this->client->call(Client::METHOD_POST, '/vectordb/transactions', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders())); + + $this->assertEquals(201, $transaction['headers']['status-code']); + $transactionId = $transaction['body']['$id']; + + // Create document via normal route with transactionId + $response = $this->client->call(Client::METHOD_POST, "/vectordb/{$databaseId}/collections/{$collectionId}/documents", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'documentId' => 'doc_from_route', + 'data' => [ + 'embeddings' => $this->generateEmbeddings(3), + 'metadata' => [ + 'name' => 'Created via normal route', + 'counter' => 100, + 'category' => 'test' + ] + ], + 'transactionId' => $transactionId + ]); + + $this->assertEquals(201, $response['headers']['status-code']); + + // Document should not exist outside transaction yet + $response = $this->client->call(Client::METHOD_GET, "/vectordb/{$databaseId}/collections/{$collectionId}/documents/doc_from_route", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders())); + + $this->assertEquals(404, $response['headers']['status-code']); + + // Commit transaction + $response = $this->client->call(Client::METHOD_PATCH, "/vectordb/transactions/{$transactionId}", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'commit' => true + ]); + + $this->assertEquals(200, $response['headers']['status-code']); + + // Document should now exist + $response = $this->client->call(Client::METHOD_GET, "/vectordb/{$databaseId}/collections/{$collectionId}/documents/doc_from_route", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders())); + + $this->assertEquals(200, $response['headers']['status-code']); + $this->assertEquals('Created via normal route', $response['body']['metadata']['name']); + } + + /** + * Test updateDocument with transactionId via normal route + */ + public function testUpdateDocument(): void + { + // Create database and collection + $database = $this->client->call(Client::METHOD_POST, '/vectordb', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'databaseId' => ID::unique(), + 'name' => 'UpdateRouteTestDB' + ]); + + $databaseId = $database['body']['$id']; + + $collection = $this->client->call(Client::METHOD_POST, "/vectordb/{$databaseId}/collections", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'collectionId' => ID::unique(), + 'name' => 'TestCollection', + 'dimensions' => 3, + 'permissions' => [ + Permission::create(Role::any()), + Permission::read(Role::any()), + Permission::update(Role::any()), + ], + ]); + + $collectionId = $collection['body']['$id']; + + // Create document outside transaction + $doc = $this->client->call(Client::METHOD_POST, "/vectordb/{$databaseId}/collections/{$collectionId}/documents", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'documentId' => 'doc_to_update', + 'data' => [ + 'embeddings' => $this->generateEmbeddings(3), + 'metadata' => [ + 'name' => 'Original name', + 'counter' => 50, + 'category' => 'original' + ] + ] + ]); + + $this->assertEquals(201, $doc['headers']['status-code']); + + // Create transaction + $transaction = $this->client->call(Client::METHOD_POST, '/vectordb/transactions', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders())); + + $transactionId = $transaction['body']['$id']; + + // Update document via normal route with transactionId + $response = $this->client->call(Client::METHOD_PATCH, "/vectordb/{$databaseId}/collections/{$collectionId}/documents/doc_to_update", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'data' => [ + 'metadata' => [ + 'name' => 'Updated via normal route', + 'counter' => 150, + 'category' => 'updated' + ] + ], + 'transactionId' => $transactionId + ]); + + $this->assertEquals(200, $response['headers']['status-code']); + + // Document should still have original values outside transaction + $response = $this->client->call(Client::METHOD_GET, "/vectordb/{$databaseId}/collections/{$collectionId}/documents/doc_to_update", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders())); + + $this->assertEquals('Original name', $response['body']['metadata']['name']); + $this->assertEquals(50, $response['body']['metadata']['counter']); + + // Commit transaction + $response = $this->client->call(Client::METHOD_PATCH, "/vectordb/transactions/{$transactionId}", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'commit' => true + ]); + + $this->assertEquals(200, $response['headers']['status-code']); + + // Document should now have updated values + $response = $this->client->call(Client::METHOD_GET, "/vectordb/{$databaseId}/collections/{$collectionId}/documents/doc_to_update", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders())); + + $this->assertEquals('Updated via normal route', $response['body']['metadata']['name']); + $this->assertEquals(150, $response['body']['metadata']['counter']); + } + + /** + * Test upsertDocument with transactionId via normal route + */ + public function testUpsertDocument(): void + { + // Create database and collection + $database = $this->client->call(Client::METHOD_POST, '/vectordb', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'databaseId' => ID::unique(), + 'name' => 'UpsertRouteTestDB' + ]); + + $databaseId = $database['body']['$id']; + + $collection = $this->client->call(Client::METHOD_POST, "/vectordb/{$databaseId}/collections", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'collectionId' => ID::unique(), + 'name' => 'TestCollection', + 'dimensions' => 3, + 'permissions' => [ + Permission::create(Role::any()), + Permission::read(Role::any()), + Permission::update(Role::any()), + ], + ]); + + $collectionId = $collection['body']['$id']; + + // Create transaction + $transaction = $this->client->call(Client::METHOD_POST, '/vectordb/transactions', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders())); + + $transactionId = $transaction['body']['$id']; + + // Upsert document (create) via normal route with transactionId + $response = $this->client->call(Client::METHOD_PUT, "/vectordb/{$databaseId}/collections/{$collectionId}/documents/doc_upsert", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'documentId' => 'doc_upsert', + 'data' => [ + 'embeddings' => $this->generateEmbeddings(3), + 'metadata' => [ + 'name' => 'Created by upsert', + 'counter' => 25 + ] + ], + 'transactionId' => $transactionId + ]); + + $this->assertEquals(201, $response['headers']['status-code']); + + // Document should not exist outside transaction yet + $response = $this->client->call(Client::METHOD_GET, "/vectordb/{$databaseId}/collections/{$collectionId}/documents/doc_upsert", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders())); + + $this->assertEquals(404, $response['headers']['status-code']); + + // Upsert same document (update) in same transaction + $response = $this->client->call(Client::METHOD_PUT, "/vectordb/{$databaseId}/collections/{$collectionId}/documents/doc_upsert", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'documentId' => 'doc_upsert', + 'data' => [ + 'metadata' => [ + 'name' => 'Updated by upsert', + 'counter' => 75 + ] + ], + 'transactionId' => $transactionId + ]); + + $this->assertEquals(201, $response['headers']['status-code']); // Upsert in transaction returns 201 + + // Commit transaction + $response = $this->client->call(Client::METHOD_PATCH, "/vectordb/transactions/{$transactionId}", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'commit' => true + ]); + + $this->assertEquals(200, $response['headers']['status-code']); + + // Document should now exist with updated values + $response = $this->client->call(Client::METHOD_GET, "/vectordb/{$databaseId}/collections/{$collectionId}/documents/doc_upsert", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders())); + + $this->assertEquals(200, $response['headers']['status-code']); + $this->assertEquals('Updated by upsert', $response['body']['metadata']['name']); + $this->assertEquals(75, $response['body']['metadata']['counter']); + } + + /** + * Test deleteDocument with transactionId via normal route + */ + public function testDeleteDocument(): void + { + // Create database and collection + $database = $this->client->call(Client::METHOD_POST, '/vectordb', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'databaseId' => ID::unique(), + 'name' => 'DeleteRouteTestDB' + ]); + + $databaseId = $database['body']['$id']; + + $collection = $this->client->call(Client::METHOD_POST, "/vectordb/{$databaseId}/collections", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'collectionId' => ID::unique(), + 'name' => 'TestCollection', + 'dimensions' => 3, + 'permissions' => [ + Permission::create(Role::any()), + Permission::read(Role::any()), + Permission::delete(Role::any()), + ], + ]); + + $collectionId = $collection['body']['$id']; + + // Create document outside transaction + $doc = $this->client->call(Client::METHOD_POST, "/vectordb/{$databaseId}/collections/{$collectionId}/documents", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'documentId' => 'doc_to_delete', + 'data' => [ + 'embeddings' => $this->generateEmbeddings(3), + 'metadata' => ['name' => 'Will be deleted'] + ] + ]); + + $this->assertEquals(201, $doc['headers']['status-code']); + + // Create transaction + $transaction = $this->client->call(Client::METHOD_POST, '/vectordb/transactions', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders())); + + $transactionId = $transaction['body']['$id']; + + // Delete document via normal route with transactionId + $response = $this->client->call(Client::METHOD_DELETE, "/vectordb/{$databaseId}/collections/{$collectionId}/documents/doc_to_delete", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'transactionId' => $transactionId + ]); + + $this->assertEquals(204, $response['headers']['status-code']); + + // Document should still exist outside transaction + $response = $this->client->call(Client::METHOD_GET, "/vectordb/{$databaseId}/collections/{$collectionId}/documents/doc_to_delete", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders())); + + $this->assertEquals(200, $response['headers']['status-code']); + + // Commit transaction + $response = $this->client->call(Client::METHOD_PATCH, "/vectordb/transactions/{$transactionId}", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'commit' => true + ]); + + $this->assertEquals(200, $response['headers']['status-code']); + + // Document should no longer exist + $response = $this->client->call(Client::METHOD_GET, "/vectordb/{$databaseId}/collections/{$collectionId}/documents/doc_to_delete", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders())); + + $this->assertEquals(404, $response['headers']['status-code']); + } + + /** + * Test bulkCreate with transactionId via normal route + */ + public function testBulkCreate(): void + { + // Create database and collection + $database = $this->client->call(Client::METHOD_POST, '/vectordb', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'databaseId' => ID::unique(), + 'name' => 'BulkCreateTestDB' + ]); + + $databaseId = $database['body']['$id']; + + $collection = $this->client->call(Client::METHOD_POST, "/vectordb/{$databaseId}/collections", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'collectionId' => ID::unique(), + 'name' => 'TestCollection', + 'dimensions' => 3, + 'permissions' => [ + Permission::create(Role::any()), + Permission::read(Role::any()), + ], + ]); + + $collectionId = $collection['body']['$id']; + + // Create transaction + $transaction = $this->client->call(Client::METHOD_POST, '/vectordb/transactions', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders())); + + $transactionId = $transaction['body']['$id']; + + // Bulk create via normal route with transactionId + $response = $this->client->call(Client::METHOD_POST, "/vectordb/{$databaseId}/collections/{$collectionId}/documents", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'documents' => [ + [ + '$id' => 'bulk_create_1', + 'embeddings' => $this->generateEmbeddings(3), + 'metadata' => [ + 'name' => 'Bulk created 1', + 'category' => 'bulk_created' + ] + ], + [ + '$id' => 'bulk_create_2', + 'embeddings' => $this->generateEmbeddings(3, 0.2), + 'metadata' => [ + 'name' => 'Bulk created 2', + 'category' => 'bulk_created' + ] + ], + [ + '$id' => 'bulk_create_3', + 'embeddings' => $this->generateEmbeddings(3, 0.3), + 'metadata' => [ + 'name' => 'Bulk created 3', + 'category' => 'bulk_created' + ] + ] + ], + 'transactionId' => $transactionId + ]); + + $this->assertEquals(200, $response['headers']['status-code']); // Bulk operations return 200 + + // Documents should not exist outside transaction yet + $response = $this->client->call(Client::METHOD_GET, "/vectordb/{$databaseId}/collections/{$collectionId}/documents", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'queries' => [Query::equal('metadata', [['metadata' => ['category' => 'bulk_created']]])->toString()] + ]); + + $this->assertEquals(0, $response['body']['total']); + + // Individual document check + $response = $this->client->call(Client::METHOD_GET, "/vectordb/{$databaseId}/collections/{$collectionId}/documents/bulk_create_1", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders())); + + $this->assertEquals(404, $response['headers']['status-code']); + + // Commit transaction + $response = $this->client->call(Client::METHOD_PATCH, "/vectordb/transactions/{$transactionId}", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'commit' => true + ]); + + $this->assertEquals(200, $response['headers']['status-code']); + + // Documents should now exist + $response = $this->client->call(Client::METHOD_GET, "/vectordb/{$databaseId}/collections/{$collectionId}/documents", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'queries' => [Query::equal('metadata', ['metadata' => ['category' => 'bulk_created']])->toString()] + ]); + + $this->assertEquals(3, $response['body']['total']); + + // Verify individual documents + for ($i = 1; $i <= 3; $i++) { + $response = $this->client->call(Client::METHOD_GET, "/vectordb/{$databaseId}/collections/{$collectionId}/documents/bulk_create_{$i}", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders())); + + $this->assertEquals(200, $response['headers']['status-code']); + $this->assertEquals("Bulk created {$i}", $response['body']['metadata']['name']); + $this->assertEquals('bulk_created', $response['body']['metadata']['category']); + } + } + + /** + * Test bulkUpdate with transactionId via normal route + */ + public function testBulkUpdate(): void + { + // Create database and collection + $database = $this->client->call(Client::METHOD_POST, '/vectordb', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'databaseId' => ID::unique(), + 'name' => 'BulkUpdateTestDB' + ]); + + $databaseId = $database['body']['$id']; + + $collection = $this->client->call(Client::METHOD_POST, "/vectordb/{$databaseId}/collections", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'collectionId' => ID::unique(), + 'name' => 'TestCollection', + 'dimensions' => 3, + 'permissions' => [ + Permission::create(Role::any()), + Permission::read(Role::any()), + Permission::update(Role::any()), + ], + ]); + + $collectionId = $collection['body']['$id']; + + // Create documents for bulk testing + for ($i = 1; $i <= 3; $i++) { + $this->client->call(Client::METHOD_POST, "/vectordb/{$databaseId}/collections/{$collectionId}/documents", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'documentId' => 'bulk_update_' . $i, + 'data' => [ + 'embeddings' => $this->generateEmbeddings(3, 0.1 * $i), + 'metadata' => [ + 'name' => 'Bulk doc ' . $i, + 'category' => 'bulk_test' + ] + ] + ]); + } + + // Create transaction + $transaction = $this->client->call(Client::METHOD_POST, '/vectordb/transactions', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders())); + + $transactionId = $transaction['body']['$id']; + + // Bulk update via normal route with transactionId + $response = $this->client->call(Client::METHOD_PATCH, "/vectordb/{$databaseId}/collections/{$collectionId}/documents", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'queries' => [Query::equal('metadata', ['metadata' => ['category' => 'bulk_test']])->toString()], + 'data' => ['metadata' => ['category' => 'bulk_updated']], + 'transactionId' => $transactionId + ]); + + $this->assertEquals(200, $response['headers']['status-code']); + + // Documents should still have original category outside transaction + $response = $this->client->call(Client::METHOD_GET, "/vectordb/{$databaseId}/collections/{$collectionId}/documents", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'queries' => [Query::equal('metadata', ['metadata' => ['category' => 'bulk_test']])->toString()] + ]); + + $this->assertEquals(3, $response['body']['total']); + + // Commit transaction + $response = $this->client->call(Client::METHOD_PATCH, "/vectordb/transactions/{$transactionId}", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'commit' => true + ]); + + $this->assertEquals(200, $response['headers']['status-code']); + + // Documents should now have updated category + $response = $this->client->call(Client::METHOD_GET, "/vectordb/{$databaseId}/collections/{$collectionId}/documents", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'queries' => [Query::equal('metadata', ['metadata' => ['category' => 'bulk_updated']])->toString()] + ]); + + $this->assertEquals(3, $response['body']['total']); + } + + /** + * Test bulkUpsert with transactionId via normal route + */ + public function testBulkUpsert(): void + { + // Create database and collection + $database = $this->client->call(Client::METHOD_POST, '/vectordb', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'databaseId' => ID::unique(), + 'name' => 'BulkUpsertTestDB' + ]); + + $databaseId = $database['body']['$id']; + + $collection = $this->client->call(Client::METHOD_POST, "/vectordb/{$databaseId}/collections", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'collectionId' => ID::unique(), + 'name' => 'TestCollection', + 'dimensions' => 3, + 'permissions' => [ + Permission::create(Role::any()), + Permission::read(Role::any()), + Permission::update(Role::any()), + ], + ]); + + $collectionId = $collection['body']['$id']; + + // Create transaction + $transaction = $this->client->call(Client::METHOD_POST, '/vectordb/transactions', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders())); + + $this->assertEquals(201, $transaction['headers']['status-code']); + $transactionId = $transaction['body']['$id']; + + // Test 1: Invalid action type + $response = $this->client->call(Client::METHOD_POST, "/vectordb/transactions/{$transactionId}/operations", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'operations' => [ + [ + 'action' => 'invalidAction', + 'databaseId' => $databaseId, + 'collectionId' => $collectionId, + 'documentId' => ID::unique(), + 'data' => ['name' => 'Test'] + ] + ] + ]); + + $this->assertEquals(400, $response['headers']['status-code']); + + // Test 2: Missing required action field + $response = $this->client->call(Client::METHOD_POST, "/vectordb/transactions/{$transactionId}/operations", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'operations' => [ + [ + 'databaseId' => $databaseId, + 'collectionId' => $collectionId, + 'documentId' => ID::unique(), + 'data' => ['name' => 'Test'] + ] + ] + ]); + + $this->assertEquals(400, $response['headers']['status-code']); + + // Test 3: Missing required databaseId field + $response = $this->client->call(Client::METHOD_POST, "/vectordb/transactions/{$transactionId}/operations", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'operations' => [ + [ + 'action' => 'create', + 'collectionId' => $collectionId, + 'documentId' => ID::unique(), + 'data' => ['name' => 'Test'] + ] + ] + ]); + + $this->assertEquals(400, $response['headers']['status-code']); + + // Test 4: Missing documentId for create operation + $response = $this->client->call(Client::METHOD_POST, "/vectordb/transactions/{$transactionId}/operations", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'operations' => [ + [ + 'action' => 'create', + 'databaseId' => $databaseId, + 'collectionId' => $collectionId, + 'data' => ['name' => 'Test'] + ] + ] + ]); + + $this->assertEquals(400, $response['headers']['status-code']); + + // Test 5: Missing data for create operation + $response = $this->client->call(Client::METHOD_POST, "/vectordb/transactions/{$transactionId}/operations", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'operations' => [ + [ + 'action' => 'create', + 'databaseId' => $databaseId, + 'collectionId' => $collectionId, + 'documentId' => ID::unique() + ] + ] + ]); + + $this->assertEquals(400, $response['headers']['status-code']); + + // Test 6: BulkCreate with non-array data + $response = $this->client->call(Client::METHOD_POST, "/vectordb/transactions/{$transactionId}/operations", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'operations' => [ + [ + 'action' => 'bulkCreate', + 'databaseId' => $databaseId, + 'collectionId' => $collectionId, + 'data' => 'not an array' + ] + ] + ]); + + $this->assertEquals(400, $response['headers']['status-code']); + + // Test 7: BulkUpdate with missing queries + $response = $this->client->call(Client::METHOD_POST, "/vectordb/transactions/{$transactionId}/operations", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'operations' => [ + [ + 'action' => 'bulkUpdate', + 'databaseId' => $databaseId, + 'collectionId' => $collectionId, + 'data' => [ + 'data' => ['name' => 'Updated'] + ] + ] + ] + ]); + + $this->assertEquals(400, $response['headers']['status-code']); + + // Test 8: Empty operations array + $response = $this->client->call(Client::METHOD_POST, "/vectordb/transactions/{$transactionId}/operations", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'operations' => [] + ]); + + $this->assertEquals(400, $response['headers']['status-code']); + + // Test 9: Operations not an array + $response = $this->client->call(Client::METHOD_POST, "/vectordb/transactions/{$transactionId}/operations", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'operations' => 'not an array' + ]); + + $this->assertEquals(400, $response['headers']['status-code']); + } + + /** + * Test validation for committing/rolling back transactions + */ + public function testCommitRollbackValidation(): void + { + // Create transaction + $transaction = $this->client->call(Client::METHOD_POST, '/vectordb/transactions', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders())); + + $this->assertEquals(201, $transaction['headers']['status-code']); + $transactionId = $transaction['body']['$id']; + + // Test 1: Missing both commit and rollback + $response = $this->client->call(Client::METHOD_PATCH, "/vectordb/transactions/{$transactionId}", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), []); + + $this->assertEquals(400, $response['headers']['status-code']); + + // Test 2: Both commit and rollback set to true + $response = $this->client->call(Client::METHOD_PATCH, "/vectordb/transactions/{$transactionId}", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'commit' => true, + 'rollback' => true + ]); + + $this->assertEquals(400, $response['headers']['status-code']); + + // Test 3: Invalid transaction ID + $response = $this->client->call(Client::METHOD_PATCH, "/vectordb/transactions/invalid_id", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'commit' => true + ]); + + $this->assertEquals(404, $response['headers']['status-code']); + + // Commit the transaction + $response = $this->client->call(Client::METHOD_PATCH, "/vectordb/transactions/{$transactionId}", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'commit' => true + ]); + + $this->assertEquals(200, $response['headers']['status-code']); + + // Test 4: Attempt to commit already committed transaction + $response = $this->client->call(Client::METHOD_PATCH, "/vectordb/transactions/{$transactionId}", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'commit' => true + ]); + + $this->assertEquals(400, $response['headers']['status-code']); + } + + /** + * Test validation for non-existent resources + */ + public function testNonExistentResources(): void + { + // Create database and transaction + $database = $this->client->call(Client::METHOD_POST, '/vectordb', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'databaseId' => ID::unique(), + 'name' => 'ResourceTestDatabase' + ]); + + $this->assertEquals(201, $database['headers']['status-code']); + $databaseId = $database['body']['$id']; + + $transaction = $this->client->call(Client::METHOD_POST, '/vectordb/transactions', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders())); + + $this->assertEquals(201, $transaction['headers']['status-code']); + $transactionId = $transaction['body']['$id']; + + // Test 1: Non-existent database + $response = $this->client->call(Client::METHOD_POST, "/vectordb/transactions/{$transactionId}/operations", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'operations' => [ + [ + 'action' => 'create', + 'databaseId' => 'nonExistentDatabase', + 'collectionId' => 'someCollection', + 'documentId' => ID::unique(), + 'data' => ['name' => 'Test'] + ] + ] + ]); + + $this->assertEquals(404, $response['headers']['status-code']); + + // Test 2: Non-existent collection + $response = $this->client->call(Client::METHOD_POST, "/vectordb/transactions/{$transactionId}/operations", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'operations' => [ + [ + 'action' => 'create', + 'databaseId' => $databaseId, + 'collectionId' => 'nonExistentCollection', + 'documentId' => ID::unique(), + 'data' => ['name' => 'Test'] + ] + ] + ]); + + $this->assertEquals(404, $response['headers']['status-code']); + } +} diff --git a/tests/e2e/Services/Databases/VectorDB/Transactions/TransactionsConsoleClientTest.php b/tests/e2e/Services/Databases/VectorDB/Transactions/TransactionsConsoleClientTest.php new file mode 100644 index 0000000000..02f7c57da8 --- /dev/null +++ b/tests/e2e/Services/Databases/VectorDB/Transactions/TransactionsConsoleClientTest.php @@ -0,0 +1,14 @@ + Date: Wed, 12 Nov 2025 16:23:59 +0530 Subject: [PATCH 12/27] linting --- .../Platform/Modules/Databases/Http/Databases/Action.php | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Action.php b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Action.php index 688540277e..ec03503033 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Action.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Action.php @@ -21,7 +21,6 @@ class Action extends UtopiaAction if (\str_contains($path, '/documentsdb')) { $this->context = 'documentsdb'; } - if (\str_contains($path, '/vectordb')) { $this->context = 'vectordb'; } From 75045ffd58673fad7ec92cac7367056e18d95cfb Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Wed, 12 Nov 2025 16:28:15 +0530 Subject: [PATCH 13/27] added constants for the vector dimension --- src/Appwrite/Platform/Modules/Databases/Constants.php | 3 +++ .../Modules/Databases/Http/VectorDB/Collections/Create.php | 2 +- .../Modules/Databases/Http/VectorDB/Collections/Update.php | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/Appwrite/Platform/Modules/Databases/Constants.php b/src/Appwrite/Platform/Modules/Databases/Constants.php index 8735f4b981..34142b57b0 100644 --- a/src/Appwrite/Platform/Modules/Databases/Constants.php +++ b/src/Appwrite/Platform/Modules/Databases/Constants.php @@ -26,3 +26,6 @@ const COLLECTIONS = 'collection'; const TABLESDB = 'tablesdb'; const DOCUMENTSDB = 'documentsdb'; const VECTORDB = 'vectordb'; + +const MIN_VECTOR_DIMENSION = 1; +const MAX_VECTOR_DIMENSION = 16000; \ No newline at end of file diff --git a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Create.php b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Create.php index e1c9998323..34e81dec3e 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Create.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Create.php @@ -69,7 +69,7 @@ class Create extends CollectionAction ->param('databaseId', '', fn (Database $dbForProject) => new UID($dbForProject->getAdapter()->getMaxUIDLength()), 'Database ID.', false, ['dbForProject']) ->param('collectionId', '', fn (Database $dbForProject) => new CustomId(false, $dbForProject->getAdapter()->getMaxUIDLength()), 'Unique Id. Choose a custom ID or generate a random ID with `ID.unique()`. Valid chars are a-z, A-Z, 0-9, period, hyphen, and underscore. Can\'t start with a special char. Max length is 36 chars.', false, ['dbForProject']) ->param('name', '', new Text(128), 'Collection name. Max length: 128 chars.') - ->param('dimensions', null, new Range(1, 16000), 'Embedding dimensions.') + ->param('dimensions', null, new Range(MIN_VECTOR_DIMENSION, MAX_VECTOR_DIMENSION), 'Embedding dimensions.') ->param('permissions', null, new Permissions(APP_LIMIT_ARRAY_PARAMS_SIZE), 'An array of permissions strings. By default, no user is granted with any permissions. [Learn more about permissions](https://appwrite.io/docs/permissions).', true) ->param('documentSecurity', false, new Boolean(true), 'Enables configuring permissions for individual documents. A user needs one of document or collection level permissions to access a document. [Learn more about permissions](https://appwrite.io/docs/permissions).', true) ->param('enabled', true, new Boolean(), 'Is collection enabled? When set to \'disabled\', users cannot access the collection but Server SDKs with and API key can still read and write to the collection. No data is lost when this is toggled.', true) diff --git a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Update.php b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Update.php index b2b0d8e57b..06ba3fdc2e 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Update.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Update.php @@ -59,7 +59,7 @@ class Update extends CollectionAction ->param('databaseId', '', new UID(), 'Database ID.') ->param('collectionId', '', new UID(), 'Collection ID.') ->param('name', null, new Text(128), 'Collection name. Max length: 128 chars.') - ->param('dimensions', null, new Range(1, 16000), 'Embedding dimensions.', true) + ->param('dimensions', null, new Range(MIN_VECTOR_DIMENSION, MAX_VECTOR_DIMENSION), 'Embedding dimensions.', true) ->param('permissions', null, new Permissions(APP_LIMIT_ARRAY_PARAMS_SIZE), 'An array of permission strings. By default, the current permissions are inherited. [Learn more about permissions](https://appwrite.io/docs/permissions).', true) ->param('documentSecurity', false, new Boolean(true), 'Enables configuring permissions for individual documents. A user needs one of document or collection level permissions to access a document. [Learn more about permissions](https://appwrite.io/docs/permissions).', true) ->param('enabled', true, new Boolean(), 'Is collection enabled? When set to \'disabled\', users cannot access the collection but Server SDKs with and API key can still read and write to the collection. No data is lost when this is toggled.', true) From 93bd6b39a47cc3489e20e174b614e2585ebf4ad6 Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Thu, 13 Nov 2025 16:47:00 +0530 Subject: [PATCH 14/27] updated migrations + index finding issue in vectordb --- composer.json | 4 +- composer.lock | 160 +++-- .../Platform/Modules/Databases/Constants.php | 2 +- .../Http/VectorDB/Collections/Create.php | 26 +- src/Appwrite/Platform/Workers/Migrations.php | 1 + .../Services/Migrations/MigrationsBase.php | 556 +++++++++++++++++- 6 files changed, 651 insertions(+), 98 deletions(-) diff --git a/composer.json b/composer.json index 910cbcb5e4..e16cd450b8 100644 --- a/composer.json +++ b/composer.json @@ -54,8 +54,8 @@ "utopia-php/cache": "0.13.*", "utopia-php/cli": "0.15.*", "utopia-php/config": "0.2.*", - "utopia-php/database": "dev-var_object as 1.4.5", - "utopia-php/agents": "dev-feat-embeddings", + "utopia-php/database": "3.*", + "utopia-php/agents": "0.*", "utopia-php/detector": "0.2.*", "utopia-php/domains": "0.9.*", "utopia-php/emails": "0.6.*", diff --git a/composer.lock b/composer.lock index 62cccc95f7..19724e39da 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "f9b66abe46119ae12f0cf2becc8bc299", + "content-hash": "a2f0965b1d70d23bbfc5a948dafabd1e", "packages": [ { "name": "adhocore/jwt", @@ -161,16 +161,16 @@ }, { "name": "appwrite/php-runtimes", - "version": "0.19.1", + "version": "0.19.2", "source": { "type": "git", "url": "https://github.com/appwrite/runtimes.git", - "reference": "7bd0cc3cb97de625d7b07230bd91b121f88e72ae" + "reference": "e5c142519df5aced37de9c302971c29c079ce3d9" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/appwrite/runtimes/zipball/7bd0cc3cb97de625d7b07230bd91b121f88e72ae", - "reference": "7bd0cc3cb97de625d7b07230bd91b121f88e72ae", + "url": "https://api.github.com/repos/appwrite/runtimes/zipball/e5c142519df5aced37de9c302971c29c079ce3d9", + "reference": "e5c142519df5aced37de9c302971c29c079ce3d9", "shasum": "" }, "require": { @@ -210,9 +210,9 @@ ], "support": { "issues": "https://github.com/appwrite/runtimes/issues", - "source": "https://github.com/appwrite/runtimes/tree/0.19.1" + "source": "https://github.com/appwrite/runtimes/tree/0.19.2" }, - "time": "2025-05-27T07:12:56+00:00" + "time": "2025-11-11T13:44:44+00:00" }, { "name": "beberlei/assert", @@ -754,16 +754,16 @@ }, { "name": "google/protobuf", - "version": "v4.33.0", + "version": "v4.33.1", "source": { "type": "git", "url": "https://github.com/protocolbuffers/protobuf-php.git", - "reference": "b50269e23204e5ae859a326ec3d90f09efe3047d" + "reference": "0cd73ccf0cd26c3e72299cce1ea6144091a57e12" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/protocolbuffers/protobuf-php/zipball/b50269e23204e5ae859a326ec3d90f09efe3047d", - "reference": "b50269e23204e5ae859a326ec3d90f09efe3047d", + "url": "https://api.github.com/repos/protocolbuffers/protobuf-php/zipball/0cd73ccf0cd26c3e72299cce1ea6144091a57e12", + "reference": "0cd73ccf0cd26c3e72299cce1ea6144091a57e12", "shasum": "" }, "require": { @@ -792,9 +792,9 @@ "proto" ], "support": { - "source": "https://github.com/protocolbuffers/protobuf-php/tree/v4.33.0" + "source": "https://github.com/protocolbuffers/protobuf-php/tree/v4.33.1" }, - "time": "2025-10-15T20:10:28+00:00" + "time": "2025-11-12T21:58:05+00:00" }, { "name": "league/csv", @@ -2668,16 +2668,16 @@ }, { "name": "symfony/http-client", - "version": "v7.3.4", + "version": "v7.3.6", "source": { "type": "git", "url": "https://github.com/symfony/http-client.git", - "reference": "4b62871a01c49457cf2a8e560af7ee8a94b87a62" + "reference": "3c0a55a2c8e21e30a37022801c11c7ab5a6cb2de" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/http-client/zipball/4b62871a01c49457cf2a8e560af7ee8a94b87a62", - "reference": "4b62871a01c49457cf2a8e560af7ee8a94b87a62", + "url": "https://api.github.com/repos/symfony/http-client/zipball/3c0a55a2c8e21e30a37022801c11c7ab5a6cb2de", + "reference": "3c0a55a2c8e21e30a37022801c11c7ab5a6cb2de", "shasum": "" }, "require": { @@ -2744,7 +2744,7 @@ "http" ], "support": { - "source": "https://github.com/symfony/http-client/tree/v7.3.4" + "source": "https://github.com/symfony/http-client/tree/v7.3.6" }, "funding": [ { @@ -2764,7 +2764,7 @@ "type": "tidelift" } ], - "time": "2025-09-11T10:12:26+00:00" + "time": "2025-11-05T17:41:46+00:00" }, { "name": "symfony/http-client-contracts", @@ -3171,16 +3171,16 @@ }, { "name": "symfony/service-contracts", - "version": "v3.6.0", + "version": "v3.6.1", "source": { "type": "git", "url": "https://github.com/symfony/service-contracts.git", - "reference": "f021b05a130d35510bd6b25fe9053c2a8a15d5d4" + "reference": "45112560a3ba2d715666a509a0bc9521d10b6c43" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/service-contracts/zipball/f021b05a130d35510bd6b25fe9053c2a8a15d5d4", - "reference": "f021b05a130d35510bd6b25fe9053c2a8a15d5d4", + "url": "https://api.github.com/repos/symfony/service-contracts/zipball/45112560a3ba2d715666a509a0bc9521d10b6c43", + "reference": "45112560a3ba2d715666a509a0bc9521d10b6c43", "shasum": "" }, "require": { @@ -3234,7 +3234,7 @@ "standards" ], "support": { - "source": "https://github.com/symfony/service-contracts/tree/v3.6.0" + "source": "https://github.com/symfony/service-contracts/tree/v3.6.1" }, "funding": [ { @@ -3245,12 +3245,16 @@ "url": "https://github.com/fabpot", "type": "github" }, + { + "url": "https://github.com/nicolas-grekas", + "type": "github" + }, { "url": "https://tidelift.com/funding/github/packagist/symfony/symfony", "type": "tidelift" } ], - "time": "2025-04-25T09:37:31+00:00" + "time": "2025-07-15T11:30:57+00:00" }, { "name": "tbachert/spi", @@ -3496,16 +3500,16 @@ }, { "name": "utopia-php/agents", - "version": "dev-feat-embeddings", + "version": "0.5.0", "source": { "type": "git", "url": "https://github.com/utopia-php/agents.git", - "reference": "b448a4d06374c7a538c9495ac69a1a8f8e526823" + "reference": "7bbc77d4936ee7e87394a239690c9059b5cdaf1b" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/agents/zipball/b448a4d06374c7a538c9495ac69a1a8f8e526823", - "reference": "b448a4d06374c7a538c9495ac69a1a8f8e526823", + "url": "https://api.github.com/repos/utopia-php/agents/zipball/7bbc77d4936ee7e87394a239690c9059b5cdaf1b", + "reference": "7bbc77d4936ee7e87394a239690c9059b5cdaf1b", "shasum": "" }, "require": { @@ -3544,9 +3548,9 @@ ], "support": { "issues": "https://github.com/utopia-php/agents/issues", - "source": "https://github.com/utopia-php/agents/tree/feat-embeddings" + "source": "https://github.com/utopia-php/agents/tree/0.5.0" }, - "time": "2025-11-06T07:53:14+00:00" + "time": "2025-11-12T06:06:41+00:00" }, { "name": "utopia-php/analytics", @@ -3889,16 +3893,16 @@ }, { "name": "utopia-php/database", - "version": "dev-var_object", + "version": "3.4.0", "source": { "type": "git", "url": "https://github.com/utopia-php/database.git", - "reference": "9a0cea65abbfb325506cb1d619db60dac7ee7e55" + "reference": "e10b4faa4f3a3ef30a5f6d76acdb605469924aec" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/database/zipball/9a0cea65abbfb325506cb1d619db60dac7ee7e55", - "reference": "9a0cea65abbfb325506cb1d619db60dac7ee7e55", + "url": "https://api.github.com/repos/utopia-php/database/zipball/e10b4faa4f3a3ef30a5f6d76acdb605469924aec", + "reference": "e10b4faa4f3a3ef30a5f6d76acdb605469924aec", "shasum": "" }, "require": { @@ -3941,9 +3945,9 @@ ], "support": { "issues": "https://github.com/utopia-php/database/issues", - "source": "https://github.com/utopia-php/database/tree/var_object" + "source": "https://github.com/utopia-php/database/tree/3.4.0" }, - "time": "2025-11-06T09:02:12+00:00" + "time": "2025-11-13T06:34:20+00:00" }, { "name": "utopia-php/detector", @@ -4508,12 +4512,12 @@ "source": { "type": "git", "url": "https://github.com/utopia-php/migration.git", - "reference": "573aae2840e0727844b6614e57fd0fffbfc51518" + "reference": "f5e175fa8da4bee6211dbeb4fe693a3eedb87672" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/migration/zipball/573aae2840e0727844b6614e57fd0fffbfc51518", - "reference": "573aae2840e0727844b6614e57fd0fffbfc51518", + "url": "https://api.github.com/repos/utopia-php/migration/zipball/f5e175fa8da4bee6211dbeb4fe693a3eedb87672", + "reference": "f5e175fa8da4bee6211dbeb4fe693a3eedb87672", "shasum": "" }, "require": { @@ -4522,7 +4526,7 @@ "ext-openssl": "*", "php": ">=8.1", "utopia-php/console": "0.0.*", - "utopia-php/database": "dev-var_object as 1.4.5", + "utopia-php/database": "3.*", "utopia-php/dsn": "0.2.*", "utopia-php/storage": "0.18.*" }, @@ -4555,7 +4559,7 @@ "issues": "https://github.com/utopia-php/migration/issues", "source": "https://github.com/utopia-php/migration/tree/feat-documents-db" }, - "time": "2025-10-28T15:06:08+00:00" + "time": "2025-11-13T11:11:30+00:00" }, { "name": "utopia-php/mongo", @@ -5427,16 +5431,16 @@ "packages-dev": [ { "name": "appwrite/sdk-generator", - "version": "1.5.1", + "version": "1.5.4", "source": { "type": "git", "url": "https://github.com/appwrite/sdk-generator.git", - "reference": "cd712674e34136f706e9170641ed6f4ce160e772" + "reference": "958947b6483a79e11c3812f23bb3056199fa4105" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/appwrite/sdk-generator/zipball/cd712674e34136f706e9170641ed6f4ce160e772", - "reference": "cd712674e34136f706e9170641ed6f4ce160e772", + "url": "https://api.github.com/repos/appwrite/sdk-generator/zipball/958947b6483a79e11c3812f23bb3056199fa4105", + "reference": "958947b6483a79e11c3812f23bb3056199fa4105", "shasum": "" }, "require": { @@ -5472,9 +5476,9 @@ "description": "Appwrite PHP library for generating API SDKs for multiple programming languages and platforms", "support": { "issues": "https://github.com/appwrite/sdk-generator/issues", - "source": "https://github.com/appwrite/sdk-generator/tree/1.5.1" + "source": "https://github.com/appwrite/sdk-generator/tree/1.5.4" }, - "time": "2025-11-04T09:55:47+00:00" + "time": "2025-11-12T12:43:42+00:00" }, { "name": "doctrine/annotations", @@ -6178,16 +6182,16 @@ }, { "name": "phpbench/phpbench", - "version": "1.4.2", + "version": "1.4.3", "source": { "type": "git", "url": "https://github.com/phpbench/phpbench.git", - "reference": "bb61ae6c54b3d58642be154eb09f4e73c3511018" + "reference": "b641dde59d969ea42eed70a39f9b51950bc96878" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/phpbench/phpbench/zipball/bb61ae6c54b3d58642be154eb09f4e73c3511018", - "reference": "bb61ae6c54b3d58642be154eb09f4e73c3511018", + "url": "https://api.github.com/repos/phpbench/phpbench/zipball/b641dde59d969ea42eed70a39f9b51950bc96878", + "reference": "b641dde59d969ea42eed70a39f9b51950bc96878", "shasum": "" }, "require": { @@ -6202,26 +6206,26 @@ "phpbench/container": "^2.2", "psr/log": "^1.1 || ^2.0 || ^3.0", "seld/jsonlint": "^1.1", - "symfony/console": "^6.1 || ^7.0", - "symfony/filesystem": "^6.1 || ^7.0", - "symfony/finder": "^6.1 || ^7.0", - "symfony/options-resolver": "^6.1 || ^7.0", - "symfony/process": "^6.1 || ^7.0", + "symfony/console": "^6.1 || ^7.0 || ^8.0", + "symfony/filesystem": "^6.1 || ^7.0 || ^8.0", + "symfony/finder": "^6.1 || ^7.0 || ^8.0", + "symfony/options-resolver": "^6.1 || ^7.0 || ^8.0", + "symfony/process": "^6.1 || ^7.0 || ^8.0", "webmozart/glob": "^4.6" }, "require-dev": { "dantleech/invoke": "^2.0", "ergebnis/composer-normalize": "^2.39", - "friendsofphp/php-cs-fixer": "^3.0", "jangregor/phpstan-prophecy": "^1.0", + "php-cs-fixer/shim": "^3.9", "phpspec/prophecy": "^1.22", "phpstan/extension-installer": "^1.1", "phpstan/phpstan": "^1.0", "phpstan/phpstan-phpunit": "^1.0", "phpunit/phpunit": "^10.4 || ^11.0", "rector/rector": "^1.2", - "symfony/error-handler": "^6.1 || ^7.0", - "symfony/var-dumper": "^6.1 || ^7.0" + "symfony/error-handler": "^6.1 || ^7.0 || ^8.0", + "symfony/var-dumper": "^6.1 || ^7.0 || ^8.0" }, "suggest": { "ext-xdebug": "For Xdebug profiling extension." @@ -6264,7 +6268,7 @@ ], "support": { "issues": "https://github.com/phpbench/phpbench/issues", - "source": "https://github.com/phpbench/phpbench/tree/1.4.2" + "source": "https://github.com/phpbench/phpbench/tree/1.4.3" }, "funding": [ { @@ -6272,7 +6276,7 @@ "type": "github" } ], - "time": "2025-10-26T14:21:59+00:00" + "time": "2025-11-06T19:07:31+00:00" }, { "name": "phpstan/phpstan", @@ -7921,16 +7925,16 @@ }, { "name": "symfony/console", - "version": "v7.3.5", + "version": "v7.3.6", "source": { "type": "git", "url": "https://github.com/symfony/console.git", - "reference": "cdb80fa5869653c83cfe1a9084a673b6daf57ea7" + "reference": "c28ad91448f86c5f6d9d2c70f0cf68bf135f252a" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/console/zipball/cdb80fa5869653c83cfe1a9084a673b6daf57ea7", - "reference": "cdb80fa5869653c83cfe1a9084a673b6daf57ea7", + "url": "https://api.github.com/repos/symfony/console/zipball/c28ad91448f86c5f6d9d2c70f0cf68bf135f252a", + "reference": "c28ad91448f86c5f6d9d2c70f0cf68bf135f252a", "shasum": "" }, "require": { @@ -7995,7 +7999,7 @@ "terminal" ], "support": { - "source": "https://github.com/symfony/console/tree/v7.3.5" + "source": "https://github.com/symfony/console/tree/v7.3.6" }, "funding": [ { @@ -8015,20 +8019,20 @@ "type": "tidelift" } ], - "time": "2025-10-14T15:46:26+00:00" + "time": "2025-11-04T01:21:42+00:00" }, { "name": "symfony/filesystem", - "version": "v7.3.2", + "version": "v7.3.6", "source": { "type": "git", "url": "https://github.com/symfony/filesystem.git", - "reference": "edcbb768a186b5c3f25d0643159a787d3e63b7fd" + "reference": "e9bcfd7837928ab656276fe00464092cc9e1826a" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/filesystem/zipball/edcbb768a186b5c3f25d0643159a787d3e63b7fd", - "reference": "edcbb768a186b5c3f25d0643159a787d3e63b7fd", + "url": "https://api.github.com/repos/symfony/filesystem/zipball/e9bcfd7837928ab656276fe00464092cc9e1826a", + "reference": "e9bcfd7837928ab656276fe00464092cc9e1826a", "shasum": "" }, "require": { @@ -8065,7 +8069,7 @@ "description": "Provides basic utilities for the filesystem", "homepage": "https://symfony.com", "support": { - "source": "https://github.com/symfony/filesystem/tree/v7.3.2" + "source": "https://github.com/symfony/filesystem/tree/v7.3.6" }, "funding": [ { @@ -8085,7 +8089,7 @@ "type": "tidelift" } ], - "time": "2025-07-07T08:17:47+00:00" + "time": "2025-11-05T09:52:27+00:00" }, { "name": "symfony/finder", @@ -8940,12 +8944,6 @@ } ], "aliases": [ - { - "package": "utopia-php/database", - "version": "dev-var_object", - "alias": "1.4.5", - "alias_normalized": "1.4.5.0" - }, { "package": "utopia-php/migration", "version": "dev-feat-documents-db", @@ -8955,8 +8953,6 @@ ], "minimum-stability": "dev", "stability-flags": { - "utopia-php/agents": 20, - "utopia-php/database": 20, "utopia-php/migration": 20 }, "prefer-stable": true, diff --git a/src/Appwrite/Platform/Modules/Databases/Constants.php b/src/Appwrite/Platform/Modules/Databases/Constants.php index 34142b57b0..6ed373ef77 100644 --- a/src/Appwrite/Platform/Modules/Databases/Constants.php +++ b/src/Appwrite/Platform/Modules/Databases/Constants.php @@ -28,4 +28,4 @@ const DOCUMENTSDB = 'documentsdb'; const VECTORDB = 'vectordb'; const MIN_VECTOR_DIMENSION = 1; -const MAX_VECTOR_DIMENSION = 16000; \ No newline at end of file +const MAX_VECTOR_DIMENSION = 16000; diff --git a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Create.php b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Create.php index 34e81dec3e..76a5a5d173 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Create.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Create.php @@ -140,8 +140,8 @@ class Create extends CollectionAction attributes:$attributes, indexes:$indexes ); - // Create attribute metadata documents in the attributes table - // This is necessary so that indexes can find the attributes when they're created + // Create attribute and indexes metadata documents in the attributes and indexes collections + // needed for the get and list calls $attributeDocs = array_map(function ($attributeConfig) use ($database, $collection, $databaseId, $collectionId, $dimensions) { $key = \is_string($attributeConfig['$id']) ? $attributeConfig['$id'] : (string) $attributeConfig['$id']; return new Document([ @@ -165,6 +165,28 @@ class Create extends CollectionAction ]); }, $collections['defaultAttributes']); $dbForProject->createDocuments('attributes', $attributeDocs); + + $indexDocs = array_map(function ($indexConfig) use ($database, $collection, $databaseId, $collectionId) { + $key = \is_string($indexConfig['$id']) ? $indexConfig['$id'] : (string) $indexConfig['$id']; + + return new Document([ + '$id' => ID::custom($database->getSequence() . '_' . $collection->getSequence() . '_' . $key), + 'key' => $key, + 'status' => 'available', + 'databaseInternalId' => $database->getSequence(), + 'databaseId' => $databaseId, + 'collectionInternalId' => $collection->getSequence(), + 'collectionId' => $collectionId, + 'type' => $indexConfig['type'], + 'attributes' => $indexConfig['attributes'] ?? [], + 'lengths' => $indexConfig['lengths'] ?? [], + 'orders' => $indexConfig['orders'] ?? [], + ]); + }, $collections['defaultIndexes']); + + if (!empty($indexDocs)) { + $dbForProject->createDocuments('indexes', $indexDocs); + } } catch (DuplicateException) { throw new Exception($this->getDuplicateException()); } catch (IndexException) { diff --git a/src/Appwrite/Platform/Workers/Migrations.php b/src/Appwrite/Platform/Workers/Migrations.php index c402092cf4..dc5cce9bd7 100644 --- a/src/Appwrite/Platform/Workers/Migrations.php +++ b/src/Appwrite/Platform/Workers/Migrations.php @@ -450,6 +450,7 @@ class Migrations extends Action { return match ($databaseType) { 'documentsdb' => $this->project->getAttribute('documentsDatabase'), + 'vectordb' => $this->project->getAttribute('vectorDatabase'), default => $this->project->getAttribute('database'), }; } diff --git a/tests/e2e/Services/Migrations/MigrationsBase.php b/tests/e2e/Services/Migrations/MigrationsBase.php index b46408f9f9..46594852df 100644 --- a/tests/e2e/Services/Migrations/MigrationsBase.php +++ b/tests/e2e/Services/Migrations/MigrationsBase.php @@ -7,6 +7,7 @@ use Tests\E2E\Client; use Tests\E2E\General\UsageTest; use Tests\E2E\Scopes\ProjectCustom; use Tests\E2E\Services\Functions\FunctionsBase; +use Utopia\Database\Database; use Utopia\Database\Helpers\ID; use Utopia\Database\Helpers\Permission; use Utopia\Database\Helpers\Role; @@ -82,6 +83,26 @@ trait MigrationsBase return $migrationResult; } + /** + * Get migration status by ID (without creating a new migration) + * + * @param string $migrationId + * @return array + */ + public function getMigrationStatus(string $migrationId): array + { + $response = $this->client->call(Client::METHOD_GET, '/migrations/' . $migrationId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getDestinationProject()['$id'], + 'x-appwrite-key' => $this->getDestinationProject()['apiKey'], + ]); + + $this->assertEquals(200, $response['headers']['status-code']); + $this->assertNotEmpty($response['body']); + + return $response['body']; + } + /** * Appwrite E2E Migration Tests */ @@ -1494,6 +1515,200 @@ trait MigrationsBase ]; } + /** + * VectorDB (embeddings collections) + */ + public function testAppwriteMigrationVectorDBDatabase(): array + { + $response = $this->client->call(Client::METHOD_POST, '/vectordb', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ], [ + 'databaseId' => ID::unique(), + 'name' => 'VDB - Migration DB' + ]); + + $this->assertEquals(201, $response['headers']['status-code']); + $this->assertNotEmpty($response['body']); + $this->assertNotEmpty($response['body']['$id']); + + $databaseId = $response['body']['$id']; + + $result = $this->performMigrationSync([ + 'resources' => [ + Resource::TYPE_DATABASE_VECTORDB, + ], + 'endpoint' => 'http://localhost/v1', + 'projectId' => $this->getProject()['$id'], + 'apiKey' => $this->getProject()['apiKey'], + ]); + + $this->assertEquals('completed', $result['status']); + $this->assertEquals([Resource::TYPE_DATABASE_VECTORDB], $result['resources']); + $this->assertArrayHasKey(Resource::TYPE_DATABASE_VECTORDB, $result['statusCounters']); + $this->assertEquals(0, $result['statusCounters'][Resource::TYPE_DATABASE_VECTORDB]['error'] ?? 0); + + $response = $this->client->call(Client::METHOD_GET, '/vectordb/' . $databaseId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getDestinationProject()['$id'], + 'x-appwrite-key' => $this->getDestinationProject()['apiKey'], + ]); + + $this->assertEquals(200, $response['headers']['status-code']); + $this->assertNotEmpty($response['body']); + $this->assertNotEmpty($response['body']['$id']); + $this->assertEquals($databaseId, $response['body']['$id']); + $this->assertEquals('VDB - Migration DB', $response['body']['name']); + + // Cleanup on destination + $this->client->call(Client::METHOD_DELETE, '/vectordb/' . $databaseId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getDestinationProject()['$id'], + 'x-appwrite-key' => $this->getDestinationProject()['apiKey'], + ]); + + return [ + 'databaseId' => $databaseId, + ]; + } + + /** + * @depends testAppwriteMigrationVectorDBDatabase + */ + public function testAppwriteMigrationVectorDBCollection(array $data): array + { + $databaseId = $data['databaseId']; + + $collection = $this->client->call(Client::METHOD_POST, '/vectordb/' . $databaseId . '/collections', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ], [ + 'collectionId' => ID::unique(), + 'name' => 'VDB - Movies', + 'dimensions' => 3, + ]); + + $this->assertEquals(201, $collection['headers']['status-code']); + + $collectionId = $collection['body']['$id']; + + $result = $this->performMigrationSync([ + 'resources' => [ + Resource::TYPE_DATABASE_VECTORDB, + Resource::TYPE_COLLECTION, + Resource::TYPE_ATTRIBUTE, + ], + 'endpoint' => 'http://localhost/v1', + 'projectId' => $this->getProject()['$id'], + 'apiKey' => $this->getProject()['apiKey'], + ]); + $this->assertEquals('completed', $result['status']); + + $response = $this->client->call(Client::METHOD_GET, '/vectordb/' . $databaseId . '/collections/' . $collectionId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getDestinationProject()['$id'], + 'x-appwrite-key' => $this->getDestinationProject()['apiKey'], + ]); + + $this->assertEquals(200, $response['headers']['status-code']); + $this->assertNotEmpty($response['body']); + $this->assertEquals($collectionId, $response['body']['$id']); + $this->assertEquals('VDB - Movies', $response['body']['name']); + // Verify attributes are present (embeddings and metadata are default attributes) + $this->assertArrayHasKey('attributes', $response['body']); + $this->assertIsArray($response['body']['attributes']); + + // Cleanup + $this->client->call(Client::METHOD_DELETE, '/vectordb/' . $databaseId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getDestinationProject()['$id'], + 'x-appwrite-key' => $this->getDestinationProject()['apiKey'], + ]); + + return [ + 'databaseId' => $databaseId, + 'collectionId' => $collectionId, + ]; + } + + /** + * @depends testAppwriteMigrationVectorDBCollection + */ + public function testAppwriteMigrationVectorDBDocument(array $data): void + { + $databaseId = $data['databaseId']; + $collectionId = $data['collectionId']; + + $document = $this->client->call(Client::METHOD_POST, '/vectordb/' . $databaseId . '/collections/' . $collectionId . '/documents', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ], [ + 'documentId' => ID::unique(), + 'data' => [ + 'embeddings' => [1.0, 0.0, 0.0], + 'metadata' => ['title' => 'Migration Test Movie'], + ] + ]); + + $this->assertEquals(201, $document['headers']['status-code']); + $documentId = $document['body']['$id']; + + // Ensure attributes are exported before documents + $result = $this->performMigrationSync([ + 'resources' => [ + Resource::TYPE_DATABASE_VECTORDB, + Resource::TYPE_COLLECTION, + Resource::TYPE_ATTRIBUTE, + Resource::TYPE_DOCUMENT, + ], + 'endpoint' => 'http://localhost/v1', + 'projectId' => $this->getProject()['$id'], + 'apiKey' => $this->getProject()['apiKey'], + ]); + + $this->assertEquals('completed', $result['status']); + // Verify that TYPE_ATTRIBUTE appears in the resources array for VectorDB + $this->assertContains(Resource::TYPE_ATTRIBUTE, $result['resources'], 'TYPE_ATTRIBUTE should be in resources array for VectorDB'); + + // Verify attributes exist on destination before checking document + $collectionResponse = $this->client->call(Client::METHOD_GET, '/vectordb/' . $databaseId . '/collections/' . $collectionId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getDestinationProject()['$id'], + 'x-appwrite-key' => $this->getDestinationProject()['apiKey'], + ]); + + $this->assertEquals(200, $collectionResponse['headers']['status-code']); + $this->assertArrayHasKey('attributes', $collectionResponse['body']); + $this->assertIsArray($collectionResponse['body']['attributes']); + + $response = $this->client->call(Client::METHOD_GET, '/vectordb/' . $databaseId . '/collections/' . $collectionId . '/documents/' . $documentId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getDestinationProject()['$id'], + 'x-appwrite-key' => $this->getDestinationProject()['apiKey'], + ]); + + $this->assertEquals(200, $response['headers']['status-code']); + $this->assertNotEmpty($response['body']); + $this->assertEquals($documentId, $response['body']['$id']); + $this->assertEquals('Migration Test Movie', $response['body']['metadata']['title']); + + // Cleanup + $this->client->call(Client::METHOD_DELETE, '/vectordb/' . $databaseId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getDestinationProject()['$id'], + 'x-appwrite-key' => $this->getDestinationProject()['apiKey'], + ]); + + $this->client->call(Client::METHOD_DELETE, '/vectordb/' . $databaseId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ]); + } + /** * @depends testAppwriteMigrationDocumentsDBDatabase */ @@ -1688,6 +1903,31 @@ trait MigrationsBase $this->assertEquals('available', $response['body']['status']); }, 5000, 500); + $sqlIndexKey = 'product_unique'; + + $sqlIndex = $this->client->call(Client::METHOD_POST, '/tablesdb/' . $sqlDatabaseId . '/tables/' . $tableId . '/indexes', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $sourceProject['$id'], + 'x-appwrite-key' => $sourceProject['apiKey'], + ], [ + 'key' => $sqlIndexKey, + 'type' => Database::INDEX_UNIQUE, + 'columns' => ['productName'], + ]); + + $this->assertEquals(202, $sqlIndex['headers']['status-code']); + + $this->assertEventually(function () use ($sqlDatabaseId, $tableId, $sqlIndexKey, $sourceProject) { + $index = $this->client->call(Client::METHOD_GET, '/tablesdb/' . $sqlDatabaseId . '/tables/' . $tableId . '/indexes/' . $sqlIndexKey, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $sourceProject['$id'], + 'x-appwrite-key' => $sourceProject['apiKey'], + ]); + + $this->assertEquals(200, $index['headers']['status-code']); + $this->assertEquals('available', $index['body']['status']); + }, 30000, 500); + // Create Row in Table $row = $this->client->call(Client::METHOD_POST, '/tablesdb/' . $sqlDatabaseId . '/tables/' . $tableId . '/rows', [ 'content-type' => 'application/json', @@ -1730,6 +1970,31 @@ trait MigrationsBase $this->assertEquals(201, $collection['headers']['status-code']); $collectionId = $collection['body']['$id']; + $documentsIndexKey = 'email_unique'; + + $documentsIndex = $this->client->call(Client::METHOD_POST, '/documentsdb/' . $docsDatabaseId . '/collections/' . $collectionId . '/indexes', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $sourceProject['$id'], + 'x-appwrite-key' => $sourceProject['apiKey'], + ], [ + 'key' => $documentsIndexKey, + 'type' => Database::INDEX_UNIQUE, + 'attributes' => ['email'], + ]); + + $this->assertEquals(202, $documentsIndex['headers']['status-code']); + + $this->assertEventually(function () use ($docsDatabaseId, $collectionId, $documentsIndexKey, $sourceProject) { + $index = $this->client->call(Client::METHOD_GET, '/documentsdb/' . $docsDatabaseId . '/collections/' . $collectionId . '/indexes/' . $documentsIndexKey, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $sourceProject['$id'], + 'x-appwrite-key' => $sourceProject['apiKey'], + ]); + + $this->assertEquals(200, $index['headers']['status-code']); + $this->assertEquals('available', $index['body']['status']); + }, 30000, 500); + // Create Document in Collection $document = $this->client->call(Client::METHOD_POST, '/documentsdb/' . $docsDatabaseId . '/collections/' . $collectionId . '/documents', [ 'content-type' => 'application/json', @@ -1746,8 +2011,119 @@ trait MigrationsBase $this->assertEquals(201, $document['headers']['status-code']); $documentId = $document['body']['$id']; - // ====== Perform migration including both database kinds with all child resources ====== - $result = $this->performMigrationSync([ + // ====== Create VectorDB (/vectordb) with collection and document ====== + $vector = $this->client->call(Client::METHOD_POST, '/vectordb', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $sourceProject['$id'], + 'x-appwrite-key' => $sourceProject['apiKey'], + ], [ + 'databaseId' => ID::unique(), + 'name' => 'Mixed VectorDB', + ]); + + $this->assertEquals(201, $vector['headers']['status-code']); + $this->assertNotEmpty($vector['body']['$id']); + $vectorDatabaseId = $vector['body']['$id']; + + // Create Collection in VectorDB + $vectorCollection = $this->client->call(Client::METHOD_POST, '/vectordb/' . $vectorDatabaseId . '/collections', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $sourceProject['$id'], + 'x-appwrite-key' => $sourceProject['apiKey'], + ], [ + 'collectionId' => ID::unique(), + 'name' => 'Products', + 'dimensions' => 3, + ]); + + $this->assertEquals(201, $vectorCollection['headers']['status-code']); + $vectorCollectionId = $vectorCollection['body']['$id']; + + // Wait for VectorDB collection attributes to be ready + $this->assertEventually(function () use ($vectorDatabaseId, $vectorCollectionId, $sourceProject) { + $response = $this->client->call(Client::METHOD_GET, '/vectordb/' . $vectorDatabaseId . '/collections/' . $vectorCollectionId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $sourceProject['$id'], + 'x-appwrite-key' => $sourceProject['apiKey'], + ]); + + $this->assertEquals(200, $response['headers']['status-code']); + $this->assertArrayHasKey('attributes', $response['body']); + $this->assertIsArray($response['body']['attributes']); + // Check that default attributes (embeddings and metadata) are present and ready + $attributeKeys = array_column($response['body']['attributes'], 'key'); + $this->assertContains('embeddings', $attributeKeys); + $this->assertContains('metadata', $attributeKeys); + // Check that attributes are available (if status field exists) + foreach ($response['body']['attributes'] as $attribute) { + if (isset($attribute['status']) && $attribute['status'] !== 'available') { + return false; + } + } + return true; + }, 10000, 500); + + $metadataIndexKey = '_key_metadata'; + $vectorIndexes = $this->client->call(Client::METHOD_GET, '/vectordb/' . $vectorDatabaseId . '/collections/' . $vectorCollectionId . '/indexes', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $sourceProject['$id'], + 'x-appwrite-key' => $sourceProject['apiKey'], + ]); + $this->assertEquals(200, $vectorIndexes['headers']['status-code']); + $metadataIndex = null; + foreach ($vectorIndexes['body']['indexes'] ?? [] as $index) { + if (($index['key'] ?? '') === $metadataIndexKey) { + $metadataIndex = $index; + break; + } + } + $this->assertNotNull($metadataIndex, 'Default metadata index should exist on source collection'); + $this->assertEquals(Database::INDEX_OBJECT, $metadataIndex['type']); + + $vectorEmbeddingIndexKey = 'embedding_euclidean'; + $vectorEmbeddingIndex = $this->client->call(Client::METHOD_POST, '/vectordb/' . $vectorDatabaseId . '/collections/' . $vectorCollectionId . '/indexes', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $sourceProject['$id'], + 'x-appwrite-key' => $sourceProject['apiKey'], + ], [ + 'key' => $vectorEmbeddingIndexKey, + 'type' => Database::INDEX_HNSW_EUCLIDEAN, + 'attributes' => ['embeddings'], + ]); + $this->assertEquals(202, $vectorEmbeddingIndex['headers']['status-code']); + + $this->assertEventually(function () use ($vectorDatabaseId, $vectorCollectionId, $vectorEmbeddingIndexKey, $sourceProject) { + $index = $this->client->call(Client::METHOD_GET, '/vectordb/' . $vectorDatabaseId . '/collections/' . $vectorCollectionId . '/indexes/' . $vectorEmbeddingIndexKey, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $sourceProject['$id'], + 'x-appwrite-key' => $sourceProject['apiKey'], + ]); + + $this->assertEquals(200, $index['headers']['status-code']); + $this->assertEquals(Database::INDEX_HNSW_EUCLIDEAN, $index['body']['type']); + if (isset($index['body']['status'])) { + $this->assertEquals('available', $index['body']['status']); + } + }, 30000, 500); + + // Create Document in VectorDB Collection + $vectorDocument = $this->client->call(Client::METHOD_POST, '/vectordb/' . $vectorDatabaseId . '/collections/' . $vectorCollectionId . '/documents', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $sourceProject['$id'], + 'x-appwrite-key' => $sourceProject['apiKey'], + ], [ + 'documentId' => ID::unique(), + 'data' => [ + 'embeddings' => [0.5, 0.3, 0.2], + 'metadata' => ['name' => 'Product Vector'], + ], + ]); + + $this->assertEquals(201, $vectorDocument['headers']['status-code']); + $vectorDocumentId = $vectorDocument['body']['$id']; + + // ====== Perform migration including all three database kinds with all child resources ====== + $migrationConfig = [ 'resources' => [ Resource::TYPE_DATABASE, Resource::TYPE_TABLE, @@ -1756,13 +2132,18 @@ trait MigrationsBase Resource::TYPE_DATABASE_DOCUMENTSDB, Resource::TYPE_COLLECTION, Resource::TYPE_DOCUMENT, + Resource::TYPE_DATABASE_VECTORDB, + Resource::TYPE_ATTRIBUTE, + Resource::TYPE_INDEX, ], 'endpoint' => 'http://localhost/v1', 'projectId' => $sourceProject['$id'], 'apiKey' => $sourceProject['apiKey'], - ]); + ]; - // ====== Assert migration result ====== + // Perform migration sync once and get migration ID + $result = $this->performMigrationSync($migrationConfig); + $migrationId = $result['$id']; $this->assertEquals('completed', $result['status']); $this->assertEquals('Appwrite', $result['source']); $this->assertEquals('Appwrite', $result['destination']); @@ -1774,8 +2155,13 @@ trait MigrationsBase Resource::TYPE_DATABASE_DOCUMENTSDB, Resource::TYPE_COLLECTION, Resource::TYPE_DOCUMENT, + Resource::TYPE_DATABASE_VECTORDB, + Resource::TYPE_ATTRIBUTE, + Resource::TYPE_INDEX, ], $result['resources']); + // Get migration status before asserting SQL Database counters + $result = $this->getMigrationStatus($migrationId); // Assert SQL Database counters $this->assertArrayHasKey(Resource::TYPE_DATABASE, $result['statusCounters']); $this->assertEquals(0, $result['statusCounters'][Resource::TYPE_DATABASE]['error']); @@ -1784,6 +2170,8 @@ trait MigrationsBase $this->assertEquals(0, $result['statusCounters'][Resource::TYPE_DATABASE]['processing']); $this->assertEquals(0, $result['statusCounters'][Resource::TYPE_DATABASE]['warning']); + // Get migration status before asserting Table counters + $result = $this->getMigrationStatus($migrationId); // Assert Table counters $this->assertArrayHasKey(Resource::TYPE_TABLE, $result['statusCounters']); $this->assertEquals(0, $result['statusCounters'][Resource::TYPE_TABLE]['error']); @@ -1792,6 +2180,8 @@ trait MigrationsBase $this->assertEquals(0, $result['statusCounters'][Resource::TYPE_TABLE]['processing']); $this->assertEquals(0, $result['statusCounters'][Resource::TYPE_TABLE]['warning']); + // Get migration status before asserting Column counters + $result = $this->getMigrationStatus($migrationId); // Assert Column counters $this->assertArrayHasKey(Resource::TYPE_COLUMN, $result['statusCounters']); $this->assertEquals(0, $result['statusCounters'][Resource::TYPE_COLUMN]['error']); @@ -1800,6 +2190,8 @@ trait MigrationsBase $this->assertEquals(0, $result['statusCounters'][Resource::TYPE_COLUMN]['processing']); $this->assertEquals(0, $result['statusCounters'][Resource::TYPE_COLUMN]['warning']); + // Get migration status before asserting Row counters + $result = $this->getMigrationStatus($migrationId); // Assert Row counters $this->assertArrayHasKey(Resource::TYPE_ROW, $result['statusCounters']); $this->assertEquals(0, $result['statusCounters'][Resource::TYPE_ROW]['error']); @@ -1808,6 +2200,8 @@ trait MigrationsBase $this->assertEquals(0, $result['statusCounters'][Resource::TYPE_ROW]['processing']); $this->assertEquals(0, $result['statusCounters'][Resource::TYPE_ROW]['warning']); + // Get migration status before asserting DocumentsDB counters + $result = $this->getMigrationStatus($migrationId); // Assert DocumentsDB counters $this->assertArrayHasKey(Resource::TYPE_DATABASE_DOCUMENTSDB, $result['statusCounters']); $this->assertEquals(0, $result['statusCounters'][Resource::TYPE_DATABASE_DOCUMENTSDB]['error']); @@ -1816,24 +2210,75 @@ trait MigrationsBase $this->assertEquals(0, $result['statusCounters'][Resource::TYPE_DATABASE_DOCUMENTSDB]['processing']); $this->assertEquals(0, $result['statusCounters'][Resource::TYPE_DATABASE_DOCUMENTSDB]['warning']); - // Assert Collection counters + // Wait for all collections to be fully processed and status counters to be updated + // Note: Collections are being transferred but status counters may not be updated immediately + // This wait ensures the migration worker has finished processing all collections + $result = null; + $this->assertEventually(function () use ($migrationId, &$result) { + $result = $this->getMigrationStatus($migrationId); + + // Check if collections status counters exist + if (!isset($result['statusCounters'][Resource::TYPE_COLLECTION])) { + return false; + } + + $pendingCount = $result['statusCounters'][Resource::TYPE_COLLECTION]['pending'] ?? 0; + + // Return true only when pending count is 0 + return $pendingCount === 0; + }, 30000, 1000); // 30 second timeout, check every 1 second + + // Assert Collection counters (covers both DocumentsDB and VectorDB collections) $this->assertArrayHasKey(Resource::TYPE_COLLECTION, $result['statusCounters']); $this->assertEquals(0, $result['statusCounters'][Resource::TYPE_COLLECTION]['error']); $this->assertEquals(0, $result['statusCounters'][Resource::TYPE_COLLECTION]['pending']); - $this->assertEquals(1, $result['statusCounters'][Resource::TYPE_COLLECTION]['success']); + $this->assertGreaterThanOrEqual(1, $result['statusCounters'][Resource::TYPE_COLLECTION]['success']); $this->assertEquals(0, $result['statusCounters'][Resource::TYPE_COLLECTION]['processing']); $this->assertEquals(0, $result['statusCounters'][Resource::TYPE_COLLECTION]['warning']); - // Assert Document counters + // Get migration status before asserting Document counters + $result = $this->getMigrationStatus($migrationId); + // Assert Document counters (covers both DocumentsDB and VectorDB documents) $this->assertArrayHasKey(Resource::TYPE_DOCUMENT, $result['statusCounters']); $this->assertEquals(0, $result['statusCounters'][Resource::TYPE_DOCUMENT]['error']); $this->assertEquals(0, $result['statusCounters'][Resource::TYPE_DOCUMENT]['pending']); - $this->assertEquals(1, $result['statusCounters'][Resource::TYPE_DOCUMENT]['success']); + $this->assertGreaterThanOrEqual(1, $result['statusCounters'][Resource::TYPE_DOCUMENT]['success']); $this->assertEquals(0, $result['statusCounters'][Resource::TYPE_DOCUMENT]['processing']); $this->assertEquals(0, $result['statusCounters'][Resource::TYPE_DOCUMENT]['warning']); - // Ensure only expected counters exist (7 total) - $this->assertCount(7, $result['statusCounters']); + // Get migration status before asserting VectorDB counters + $result = $this->getMigrationStatus($migrationId); + // Assert VectorDB counters + $this->assertArrayHasKey(Resource::TYPE_DATABASE_VECTORDB, $result['statusCounters']); + $this->assertEquals(0, $result['statusCounters'][Resource::TYPE_DATABASE_VECTORDB]['error']); + $this->assertEquals(0, $result['statusCounters'][Resource::TYPE_DATABASE_VECTORDB]['pending']); + $this->assertEquals(1, $result['statusCounters'][Resource::TYPE_DATABASE_VECTORDB]['success']); + $this->assertEquals(0, $result['statusCounters'][Resource::TYPE_DATABASE_VECTORDB]['processing']); + $this->assertEquals(0, $result['statusCounters'][Resource::TYPE_DATABASE_VECTORDB]['warning']); + + // Get migration status before asserting Attribute counters + $result = $this->getMigrationStatus($migrationId); + // Assert Attribute counters (for VectorDB) + $this->assertArrayHasKey(Resource::TYPE_ATTRIBUTE, $result['statusCounters']); + $this->assertEquals(0, $result['statusCounters'][Resource::TYPE_ATTRIBUTE]['error']); + $this->assertEquals(0, $result['statusCounters'][Resource::TYPE_ATTRIBUTE]['pending']); + $this->assertGreaterThanOrEqual(1, $result['statusCounters'][Resource::TYPE_ATTRIBUTE]['success']); + $this->assertEquals(0, $result['statusCounters'][Resource::TYPE_ATTRIBUTE]['processing']); + $this->assertEquals(0, $result['statusCounters'][Resource::TYPE_ATTRIBUTE]['warning']); + + // Get migration status before asserting Index counters + $result = $this->getMigrationStatus($migrationId); + $this->assertArrayHasKey(Resource::TYPE_INDEX, $result['statusCounters']); + $this->assertEquals(0, $result['statusCounters'][Resource::TYPE_INDEX]['error']); + $this->assertEquals(0, $result['statusCounters'][Resource::TYPE_INDEX]['pending']); + $this->assertGreaterThanOrEqual(4, $result['statusCounters'][Resource::TYPE_INDEX]['success']); + $this->assertEquals(0, $result['statusCounters'][Resource::TYPE_INDEX]['processing']); + $this->assertEquals(0, $result['statusCounters'][Resource::TYPE_INDEX]['warning']); + + // Get migration status before asserting counter count + $result = $this->getMigrationStatus($migrationId); + // Ensure only expected counters exist (10 total) + $this->assertCount(10, $result['statusCounters']); // ====== Validate on destination: SQL Database resources ====== $response = $this->client->call(Client::METHOD_GET, '/databases/' . $sqlDatabaseId, [ @@ -1880,6 +2325,18 @@ trait MigrationsBase $this->assertEquals($rowId, $response['body']['$id']); $this->assertEquals('Laptop', $response['body']['productName']); + $sqlIndexDestination = $this->client->call(Client::METHOD_GET, '/tablesdb/' . $sqlDatabaseId . '/tables/' . $tableId . '/indexes/' . $sqlIndexKey, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getDestinationProject()['$id'], + 'x-appwrite-key' => $this->getDestinationProject()['apiKey'], + ]); + $this->assertEquals(200, $sqlIndexDestination['headers']['status-code']); + $this->assertEquals($sqlIndexKey, $sqlIndexDestination['body']['key']); + $this->assertEquals(Database::INDEX_UNIQUE, $sqlIndexDestination['body']['type']); + if (isset($sqlIndexDestination['body']['columns'])) { + $this->assertEquals(['productName'], $sqlIndexDestination['body']['columns']); + } + // ====== Validate on destination: DocumentsDB resources ====== $response = $this->client->call(Client::METHOD_GET, '/documentsdb/' . $docsDatabaseId, [ 'content-type' => 'application/json', @@ -1914,7 +2371,72 @@ trait MigrationsBase $this->assertEquals('John Doe', $response['body']['name']); $this->assertEquals('john@example.com', $response['body']['email']); - // ====== Cleanup both destinations ====== + $documentsIndexDestination = $this->client->call(Client::METHOD_GET, '/documentsdb/' . $docsDatabaseId . '/collections/' . $collectionId . '/indexes/' . $documentsIndexKey, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getDestinationProject()['$id'], + 'x-appwrite-key' => $this->getDestinationProject()['apiKey'], + ]); + $this->assertEquals(200, $documentsIndexDestination['headers']['status-code']); + $this->assertEquals($documentsIndexKey, $documentsIndexDestination['body']['key']); + $this->assertEquals(Database::INDEX_UNIQUE, $documentsIndexDestination['body']['type']); + if (isset($documentsIndexDestination['body']['attributes'])) { + $this->assertEquals(['email'], $documentsIndexDestination['body']['attributes']); + } + + // ====== Validate on destination: VectorDB resources ====== + $response = $this->client->call(Client::METHOD_GET, '/vectordb/' . $vectorDatabaseId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getDestinationProject()['$id'], + 'x-appwrite-key' => $this->getDestinationProject()['apiKey'], + ]); + + $this->assertEquals(200, $response['headers']['status-code']); + $this->assertEquals($vectorDatabaseId, $response['body']['$id']); + $this->assertEquals('Mixed VectorDB', $response['body']['name']); + + // Validate VectorDB Collection + $response = $this->client->call(Client::METHOD_GET, '/vectordb/' . $vectorDatabaseId . '/collections/' . $vectorCollectionId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getDestinationProject()['$id'], + 'x-appwrite-key' => $this->getDestinationProject()['apiKey'], + ]); + + $this->assertEquals(200, $response['headers']['status-code']); + $this->assertEquals($vectorCollectionId, $response['body']['$id']); + $this->assertEquals('Products', $response['body']['name']); + // Verify attributes are present (embeddings and metadata are default attributes) + $this->assertArrayHasKey('attributes', $response['body']); + $this->assertIsArray($response['body']['attributes']); + + $vectorIndexesDestination = $this->client->call(Client::METHOD_GET, '/vectordb/' . $vectorDatabaseId . '/collections/' . $vectorCollectionId . '/indexes', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getDestinationProject()['$id'], + 'x-appwrite-key' => $this->getDestinationProject()['apiKey'], + ]); + $this->assertEquals(200, $vectorIndexesDestination['headers']['status-code']); + $indexByKey = []; + foreach ($vectorIndexesDestination['body']['indexes'] ?? [] as $index) { + if (isset($index['key'])) { + $indexByKey[$index['key']] = $index; + } + } + $this->assertArrayHasKey($metadataIndexKey, $indexByKey, 'Metadata index should exist on destination'); + $this->assertEquals(Database::INDEX_OBJECT, $indexByKey[$metadataIndexKey]['type']); + $this->assertArrayHasKey($vectorEmbeddingIndexKey, $indexByKey, 'Embeddings HNSW index should exist on destination'); + $this->assertEquals(Database::INDEX_HNSW_EUCLIDEAN, $indexByKey[$vectorEmbeddingIndexKey]['type']); + + // Validate VectorDB Document + $response = $this->client->call(Client::METHOD_GET, '/vectordb/' . $vectorDatabaseId . '/collections/' . $vectorCollectionId . '/documents/' . $vectorDocumentId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getDestinationProject()['$id'], + 'x-appwrite-key' => $this->getDestinationProject()['apiKey'], + ]); + + $this->assertEquals(200, $response['headers']['status-code']); + $this->assertEquals($vectorDocumentId, $response['body']['$id']); + $this->assertEquals('Product Vector', $response['body']['metadata']['name']); + + // ====== Cleanup all destinations ====== $this->client->call(Client::METHOD_DELETE, '/databases/' . $sqlDatabaseId, [ 'content-type' => 'application/json', 'x-appwrite-project' => $this->getDestinationProject()['$id'], @@ -1927,6 +2449,12 @@ trait MigrationsBase 'x-appwrite-key' => $this->getDestinationProject()['apiKey'], ]); + $this->client->call(Client::METHOD_DELETE, '/vectordb/' . $vectorDatabaseId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getDestinationProject()['$id'], + 'x-appwrite-key' => $this->getDestinationProject()['apiKey'], + ]); + // ====== Cleanup sources ====== $this->client->call(Client::METHOD_DELETE, '/databases/' . $sqlDatabaseId, [ 'content-type' => 'application/json', @@ -1939,5 +2467,11 @@ trait MigrationsBase 'x-appwrite-project' => $sourceProject['$id'], 'x-appwrite-key' => $sourceProject['apiKey'], ]); + + $this->client->call(Client::METHOD_DELETE, '/vectordb/' . $vectorDatabaseId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $sourceProject['$id'], + 'x-appwrite-key' => $sourceProject['apiKey'], + ]); } } From 70a1d5d0f73828396f3d897abf47367a8d0121ab Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Fri, 14 Nov 2025 21:33:17 +0530 Subject: [PATCH 15/27] updated composer lock --- composer.lock | 93 ++++----------------------------------------------- 1 file changed, 7 insertions(+), 86 deletions(-) diff --git a/composer.lock b/composer.lock index 889576ebf8..a0927533ed 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "a2f0965b1d70d23bbfc5a948dafabd1e", + "content-hash": "33efc5db38812c8c9257d0e398948685", "packages": [ { "name": "adhocore/jwt", @@ -162,19 +162,15 @@ { "name": "appwrite/php-runtimes", "version": "0.19.2", - "version": "0.19.2", "source": { "type": "git", "url": "https://github.com/appwrite/runtimes.git", "reference": "e5c142519df5aced37de9c302971c29c079ce3d9" - "reference": "e5c142519df5aced37de9c302971c29c079ce3d9" }, "dist": { "type": "zip", "url": "https://api.github.com/repos/appwrite/runtimes/zipball/e5c142519df5aced37de9c302971c29c079ce3d9", "reference": "e5c142519df5aced37de9c302971c29c079ce3d9", - "url": "https://api.github.com/repos/appwrite/runtimes/zipball/e5c142519df5aced37de9c302971c29c079ce3d9", - "reference": "e5c142519df5aced37de9c302971c29c079ce3d9", "shasum": "" }, "require": { @@ -215,10 +211,8 @@ "support": { "issues": "https://github.com/appwrite/runtimes/issues", "source": "https://github.com/appwrite/runtimes/tree/0.19.2" - "source": "https://github.com/appwrite/runtimes/tree/0.19.2" }, "time": "2025-11-11T13:44:44+00:00" - "time": "2025-11-11T13:44:44+00:00" }, { "name": "beberlei/assert", @@ -761,19 +755,15 @@ { "name": "google/protobuf", "version": "v4.33.1", - "version": "v4.33.1", "source": { "type": "git", "url": "https://github.com/protocolbuffers/protobuf-php.git", "reference": "0cd73ccf0cd26c3e72299cce1ea6144091a57e12" - "reference": "0cd73ccf0cd26c3e72299cce1ea6144091a57e12" }, "dist": { "type": "zip", "url": "https://api.github.com/repos/protocolbuffers/protobuf-php/zipball/0cd73ccf0cd26c3e72299cce1ea6144091a57e12", "reference": "0cd73ccf0cd26c3e72299cce1ea6144091a57e12", - "url": "https://api.github.com/repos/protocolbuffers/protobuf-php/zipball/0cd73ccf0cd26c3e72299cce1ea6144091a57e12", - "reference": "0cd73ccf0cd26c3e72299cce1ea6144091a57e12", "shasum": "" }, "require": { @@ -803,10 +793,8 @@ ], "support": { "source": "https://github.com/protocolbuffers/protobuf-php/tree/v4.33.1" - "source": "https://github.com/protocolbuffers/protobuf-php/tree/v4.33.1" }, "time": "2025-11-12T21:58:05+00:00" - "time": "2025-11-12T21:58:05+00:00" }, { "name": "league/csv", @@ -2681,19 +2669,15 @@ { "name": "symfony/http-client", "version": "v7.3.6", - "version": "v7.3.6", "source": { "type": "git", "url": "https://github.com/symfony/http-client.git", "reference": "3c0a55a2c8e21e30a37022801c11c7ab5a6cb2de" - "reference": "3c0a55a2c8e21e30a37022801c11c7ab5a6cb2de" }, "dist": { "type": "zip", "url": "https://api.github.com/repos/symfony/http-client/zipball/3c0a55a2c8e21e30a37022801c11c7ab5a6cb2de", "reference": "3c0a55a2c8e21e30a37022801c11c7ab5a6cb2de", - "url": "https://api.github.com/repos/symfony/http-client/zipball/3c0a55a2c8e21e30a37022801c11c7ab5a6cb2de", - "reference": "3c0a55a2c8e21e30a37022801c11c7ab5a6cb2de", "shasum": "" }, "require": { @@ -2761,7 +2745,6 @@ ], "support": { "source": "https://github.com/symfony/http-client/tree/v7.3.6" - "source": "https://github.com/symfony/http-client/tree/v7.3.6" }, "funding": [ { @@ -2782,7 +2765,6 @@ } ], "time": "2025-11-05T17:41:46+00:00" - "time": "2025-11-05T17:41:46+00:00" }, { "name": "symfony/http-client-contracts", @@ -3190,19 +3172,15 @@ { "name": "symfony/service-contracts", "version": "v3.6.1", - "version": "v3.6.1", "source": { "type": "git", "url": "https://github.com/symfony/service-contracts.git", "reference": "45112560a3ba2d715666a509a0bc9521d10b6c43" - "reference": "45112560a3ba2d715666a509a0bc9521d10b6c43" }, "dist": { "type": "zip", "url": "https://api.github.com/repos/symfony/service-contracts/zipball/45112560a3ba2d715666a509a0bc9521d10b6c43", "reference": "45112560a3ba2d715666a509a0bc9521d10b6c43", - "url": "https://api.github.com/repos/symfony/service-contracts/zipball/45112560a3ba2d715666a509a0bc9521d10b6c43", - "reference": "45112560a3ba2d715666a509a0bc9521d10b6c43", "shasum": "" }, "require": { @@ -3257,7 +3235,6 @@ ], "support": { "source": "https://github.com/symfony/service-contracts/tree/v3.6.1" - "source": "https://github.com/symfony/service-contracts/tree/v3.6.1" }, "funding": [ { @@ -3272,17 +3249,12 @@ "url": "https://github.com/nicolas-grekas", "type": "github" }, - { - "url": "https://github.com/nicolas-grekas", - "type": "github" - }, { "url": "https://tidelift.com/funding/github/packagist/symfony/symfony", "type": "tidelift" } ], "time": "2025-07-15T11:30:57+00:00" - "time": "2025-07-15T11:30:57+00:00" }, { "name": "tbachert/spi", @@ -3922,19 +3894,15 @@ { "name": "utopia-php/database", "version": "3.4.0", - "version": "3.4.0", "source": { "type": "git", "url": "https://github.com/utopia-php/database.git", "reference": "e10b4faa4f3a3ef30a5f6d76acdb605469924aec" - "reference": "e10b4faa4f3a3ef30a5f6d76acdb605469924aec" }, "dist": { "type": "zip", "url": "https://api.github.com/repos/utopia-php/database/zipball/e10b4faa4f3a3ef30a5f6d76acdb605469924aec", "reference": "e10b4faa4f3a3ef30a5f6d76acdb605469924aec", - "url": "https://api.github.com/repos/utopia-php/database/zipball/e10b4faa4f3a3ef30a5f6d76acdb605469924aec", - "reference": "e10b4faa4f3a3ef30a5f6d76acdb605469924aec", "shasum": "" }, "require": { @@ -3978,27 +3946,21 @@ "support": { "issues": "https://github.com/utopia-php/database/issues", "source": "https://github.com/utopia-php/database/tree/3.4.0" - "source": "https://github.com/utopia-php/database/tree/3.4.0" }, "time": "2025-11-13T06:34:20+00:00" - "time": "2025-11-13T06:34:20+00:00" }, { "name": "utopia-php/detector", "version": "0.2.2", - "version": "0.2.2", "source": { "type": "git", "url": "https://github.com/utopia-php/detector.git", "reference": "9a41be5f21efe2d865de79b08dff94fff85ce5e9" - "reference": "9a41be5f21efe2d865de79b08dff94fff85ce5e9" }, "dist": { "type": "zip", "url": "https://api.github.com/repos/utopia-php/detector/zipball/9a41be5f21efe2d865de79b08dff94fff85ce5e9", "reference": "9a41be5f21efe2d865de79b08dff94fff85ce5e9", - "url": "https://api.github.com/repos/utopia-php/detector/zipball/9a41be5f21efe2d865de79b08dff94fff85ce5e9", - "reference": "9a41be5f21efe2d865de79b08dff94fff85ce5e9", "shasum": "" }, "require": { @@ -4029,10 +3991,8 @@ "support": { "issues": "https://github.com/utopia-php/detector/issues", "source": "https://github.com/utopia-php/detector/tree/0.2.2" - "source": "https://github.com/utopia-php/detector/tree/0.2.2" }, "time": "2025-10-31T12:43:31+00:00" - "time": "2025-10-31T12:43:31+00:00" }, { "name": "utopia-php/dns", @@ -4554,14 +4514,11 @@ "type": "git", "url": "https://github.com/utopia-php/migration.git", "reference": "f5e175fa8da4bee6211dbeb4fe693a3eedb87672" - "reference": "f5e175fa8da4bee6211dbeb4fe693a3eedb87672" }, "dist": { "type": "zip", "url": "https://api.github.com/repos/utopia-php/migration/zipball/f5e175fa8da4bee6211dbeb4fe693a3eedb87672", "reference": "f5e175fa8da4bee6211dbeb4fe693a3eedb87672", - "url": "https://api.github.com/repos/utopia-php/migration/zipball/f5e175fa8da4bee6211dbeb4fe693a3eedb87672", - "reference": "f5e175fa8da4bee6211dbeb4fe693a3eedb87672", "shasum": "" }, "require": { @@ -4571,7 +4528,6 @@ "php": ">=8.1", "utopia-php/console": "0.0.*", "utopia-php/database": "3.*", - "utopia-php/database": "3.*", "utopia-php/dsn": "0.2.*", "utopia-php/storage": "0.18.*" }, @@ -4605,7 +4561,6 @@ "source": "https://github.com/utopia-php/migration/tree/feat-documents-db" }, "time": "2025-11-13T11:11:30+00:00" - "time": "2025-11-13T11:11:30+00:00" }, { "name": "utopia-php/mongo", @@ -5477,16 +5432,16 @@ "packages-dev": [ { "name": "appwrite/sdk-generator", - "version": "1.5.4", + "version": "1.5.5", "source": { "type": "git", "url": "https://github.com/appwrite/sdk-generator.git", - "reference": "958947b6483a79e11c3812f23bb3056199fa4105" + "reference": "1f3686e41a2d10829220b74f54c80a770e9f968a" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/appwrite/sdk-generator/zipball/958947b6483a79e11c3812f23bb3056199fa4105", - "reference": "958947b6483a79e11c3812f23bb3056199fa4105", + "url": "https://api.github.com/repos/appwrite/sdk-generator/zipball/1f3686e41a2d10829220b74f54c80a770e9f968a", + "reference": "1f3686e41a2d10829220b74f54c80a770e9f968a", "shasum": "" }, "require": { @@ -5522,9 +5477,9 @@ "description": "Appwrite PHP library for generating API SDKs for multiple programming languages and platforms", "support": { "issues": "https://github.com/appwrite/sdk-generator/issues", - "source": "https://github.com/appwrite/sdk-generator/tree/1.5.4" + "source": "https://github.com/appwrite/sdk-generator/tree/1.5.5" }, - "time": "2025-11-12T12:43:42+00:00" + "time": "2025-11-14T05:56:33+00:00" }, { "name": "doctrine/annotations", @@ -6178,28 +6133,22 @@ { "name": "phpbench/container", "version": "2.2.3", - "version": "2.2.3", "source": { "type": "git", "url": "https://github.com/phpbench/container.git", "reference": "0c7b2d36c1ea53fe27302fb8873ded7172047196" - "reference": "0c7b2d36c1ea53fe27302fb8873ded7172047196" }, "dist": { "type": "zip", "url": "https://api.github.com/repos/phpbench/container/zipball/0c7b2d36c1ea53fe27302fb8873ded7172047196", "reference": "0c7b2d36c1ea53fe27302fb8873ded7172047196", - "url": "https://api.github.com/repos/phpbench/container/zipball/0c7b2d36c1ea53fe27302fb8873ded7172047196", - "reference": "0c7b2d36c1ea53fe27302fb8873ded7172047196", "shasum": "" }, "require": { "psr/container": "^1.0|^2.0", "symfony/options-resolver": "^4.2 || ^5.0 || ^6.0 || ^7.0 || ^8.0" - "symfony/options-resolver": "^4.2 || ^5.0 || ^6.0 || ^7.0 || ^8.0" }, "require-dev": { - "php-cs-fixer/shim": "^3.89", "php-cs-fixer/shim": "^3.89", "phpstan/phpstan": "^0.12.52", "phpunit/phpunit": "^8" @@ -6229,27 +6178,21 @@ "support": { "issues": "https://github.com/phpbench/container/issues", "source": "https://github.com/phpbench/container/tree/2.2.3" - "source": "https://github.com/phpbench/container/tree/2.2.3" }, "time": "2025-11-06T09:05:13+00:00" - "time": "2025-11-06T09:05:13+00:00" }, { "name": "phpbench/phpbench", "version": "1.4.3", - "version": "1.4.3", "source": { "type": "git", "url": "https://github.com/phpbench/phpbench.git", "reference": "b641dde59d969ea42eed70a39f9b51950bc96878" - "reference": "b641dde59d969ea42eed70a39f9b51950bc96878" }, "dist": { "type": "zip", "url": "https://api.github.com/repos/phpbench/phpbench/zipball/b641dde59d969ea42eed70a39f9b51950bc96878", "reference": "b641dde59d969ea42eed70a39f9b51950bc96878", - "url": "https://api.github.com/repos/phpbench/phpbench/zipball/b641dde59d969ea42eed70a39f9b51950bc96878", - "reference": "b641dde59d969ea42eed70a39f9b51950bc96878", "shasum": "" }, "require": { @@ -6269,11 +6212,6 @@ "symfony/finder": "^6.1 || ^7.0 || ^8.0", "symfony/options-resolver": "^6.1 || ^7.0 || ^8.0", "symfony/process": "^6.1 || ^7.0 || ^8.0", - "symfony/console": "^6.1 || ^7.0 || ^8.0", - "symfony/filesystem": "^6.1 || ^7.0 || ^8.0", - "symfony/finder": "^6.1 || ^7.0 || ^8.0", - "symfony/options-resolver": "^6.1 || ^7.0 || ^8.0", - "symfony/process": "^6.1 || ^7.0 || ^8.0", "webmozart/glob": "^4.6" }, "require-dev": { @@ -6281,7 +6219,6 @@ "ergebnis/composer-normalize": "^2.39", "jangregor/phpstan-prophecy": "^1.0", "php-cs-fixer/shim": "^3.9", - "php-cs-fixer/shim": "^3.9", "phpspec/prophecy": "^1.22", "phpstan/extension-installer": "^1.1", "phpstan/phpstan": "^1.0", @@ -6290,8 +6227,6 @@ "rector/rector": "^1.2", "symfony/error-handler": "^6.1 || ^7.0 || ^8.0", "symfony/var-dumper": "^6.1 || ^7.0 || ^8.0" - "symfony/error-handler": "^6.1 || ^7.0 || ^8.0", - "symfony/var-dumper": "^6.1 || ^7.0 || ^8.0" }, "suggest": { "ext-xdebug": "For Xdebug profiling extension." @@ -6335,7 +6270,6 @@ "support": { "issues": "https://github.com/phpbench/phpbench/issues", "source": "https://github.com/phpbench/phpbench/tree/1.4.3" - "source": "https://github.com/phpbench/phpbench/tree/1.4.3" }, "funding": [ { @@ -6344,7 +6278,6 @@ } ], "time": "2025-11-06T19:07:31+00:00" - "time": "2025-11-06T19:07:31+00:00" }, { "name": "phpstan/phpstan", @@ -7994,19 +7927,15 @@ { "name": "symfony/console", "version": "v7.3.6", - "version": "v7.3.6", "source": { "type": "git", "url": "https://github.com/symfony/console.git", "reference": "c28ad91448f86c5f6d9d2c70f0cf68bf135f252a" - "reference": "c28ad91448f86c5f6d9d2c70f0cf68bf135f252a" }, "dist": { "type": "zip", "url": "https://api.github.com/repos/symfony/console/zipball/c28ad91448f86c5f6d9d2c70f0cf68bf135f252a", "reference": "c28ad91448f86c5f6d9d2c70f0cf68bf135f252a", - "url": "https://api.github.com/repos/symfony/console/zipball/c28ad91448f86c5f6d9d2c70f0cf68bf135f252a", - "reference": "c28ad91448f86c5f6d9d2c70f0cf68bf135f252a", "shasum": "" }, "require": { @@ -8072,7 +8001,6 @@ ], "support": { "source": "https://github.com/symfony/console/tree/v7.3.6" - "source": "https://github.com/symfony/console/tree/v7.3.6" }, "funding": [ { @@ -8093,24 +8021,19 @@ } ], "time": "2025-11-04T01:21:42+00:00" - "time": "2025-11-04T01:21:42+00:00" }, { "name": "symfony/filesystem", "version": "v7.3.6", - "version": "v7.3.6", "source": { "type": "git", "url": "https://github.com/symfony/filesystem.git", "reference": "e9bcfd7837928ab656276fe00464092cc9e1826a" - "reference": "e9bcfd7837928ab656276fe00464092cc9e1826a" }, "dist": { "type": "zip", "url": "https://api.github.com/repos/symfony/filesystem/zipball/e9bcfd7837928ab656276fe00464092cc9e1826a", "reference": "e9bcfd7837928ab656276fe00464092cc9e1826a", - "url": "https://api.github.com/repos/symfony/filesystem/zipball/e9bcfd7837928ab656276fe00464092cc9e1826a", - "reference": "e9bcfd7837928ab656276fe00464092cc9e1826a", "shasum": "" }, "require": { @@ -8148,7 +8071,6 @@ "homepage": "https://symfony.com", "support": { "source": "https://github.com/symfony/filesystem/tree/v7.3.6" - "source": "https://github.com/symfony/filesystem/tree/v7.3.6" }, "funding": [ { @@ -8169,7 +8091,6 @@ } ], "time": "2025-11-05T09:52:27+00:00" - "time": "2025-11-05T09:52:27+00:00" }, { "name": "symfony/finder", From c3ad33739825c08d64e457f681bfa883d4028507 Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Thu, 20 Nov 2025 18:54:41 +0530 Subject: [PATCH 16/27] updated migrations for vectordb importing/exporting in csv endpoints --- app/controllers/api/migrations.php | 3 +- app/init/constants.php | 3 +- composer.lock | 8 +- src/Appwrite/Platform/Workers/Migrations.php | 3 +- .../Services/Migrations/MigrationsBase.php | 250 ++++++++++++++++++ tests/resources/csv/vectordb-documents.csv | 3 + 6 files changed, 263 insertions(+), 7 deletions(-) create mode 100644 tests/resources/csv/vectordb-documents.csv diff --git a/app/controllers/api/migrations.php b/app/controllers/api/migrations.php index be61b1d3e3..5314278e72 100644 --- a/app/controllers/api/migrations.php +++ b/app/controllers/api/migrations.php @@ -47,7 +47,8 @@ function getDatabaseTransferResourceServices(string $databaseType) { return match($databaseType) { DATABASE_LEGACY_TYPE, - DATABASE_TABLESDB_TYPE => Transfer::GROUP_DATABASES_TABLES_DB + DATABASE_TABLESDB_TYPE => Transfer::GROUP_DATABASES_TABLES_DB, + DATABASE_VECTORDB_TYPE => Transfer::GROUP_DATABASES_VECTOR_DB }; } diff --git a/app/init/constants.php b/app/init/constants.php index 4bb4fbf99f..7146e6e118 100644 --- a/app/init/constants.php +++ b/app/init/constants.php @@ -302,6 +302,7 @@ const SCHEDULE_RESOURCE_TYPE_MESSAGE = 'message'; const DATABASE_LEGACY_TYPE = 'legacy'; const DATABASE_TABLESDB_TYPE = 'tablesdb'; const DATABASE_DOCUMENTSDB_TYPE = 'documentsdb'; +const DATABASE_VECTORDB_TYPE = 'vectordb'; // CSV import/export allowed database types -const CSV_ALLOWED_DATABASE_TYPES = [DATABASE_LEGACY_TYPE, DATABASE_LEGACY_TYPE]; +const CSV_ALLOWED_DATABASE_TYPES = [DATABASE_LEGACY_TYPE, DATABASE_LEGACY_TYPE, DATABASE_VECTORDB_TYPE]; diff --git a/composer.lock b/composer.lock index b16f6a1fec..b6bfea6440 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "33efc5db38812c8c9257d0e398948685", + "content-hash": "bdd11652978ebab653ca17a00074343d", "packages": [ { "name": "adhocore/jwt", @@ -4509,12 +4509,12 @@ "source": { "type": "git", "url": "https://github.com/utopia-php/migration.git", - "reference": "105b48dcf2bc8a1a969497813f3d7d06231770e6" + "reference": "bcd7bcb634dea60485c16648e68ed7a255315560" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/migration/zipball/105b48dcf2bc8a1a969497813f3d7d06231770e6", - "reference": "105b48dcf2bc8a1a969497813f3d7d06231770e6", + "url": "https://api.github.com/repos/utopia-php/migration/zipball/bcd7bcb634dea60485c16648e68ed7a255315560", + "reference": "bcd7bcb634dea60485c16648e68ed7a255315560", "shasum": "" }, "require": { diff --git a/src/Appwrite/Platform/Workers/Migrations.php b/src/Appwrite/Platform/Workers/Migrations.php index f84e54ee46..18b0926239 100644 --- a/src/Appwrite/Platform/Workers/Migrations.php +++ b/src/Appwrite/Platform/Workers/Migrations.php @@ -205,7 +205,8 @@ class Migrations extends Action $resourceId, $migrationOptions['path'], $this->deviceForMigrations, - $this->dbForProject + $this->dbForProject, + $getDatabasesDB ), default => throw new \Exception('Invalid source type'), }; diff --git a/tests/e2e/Services/Migrations/MigrationsBase.php b/tests/e2e/Services/Migrations/MigrationsBase.php index b79f5781ad..32004e5b48 100644 --- a/tests/e2e/Services/Migrations/MigrationsBase.php +++ b/tests/e2e/Services/Migrations/MigrationsBase.php @@ -1377,6 +1377,256 @@ trait MigrationsBase ]); } + /** + * Import VectorDB documents from CSV + */ + public function testImportVectordbCSV(): void + { + $databaseId = null; + $collectionId = null; + $bucketId = null; + + try { + $database = $this->client->call(Client::METHOD_POST, '/vectordb', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'databaseId' => ID::unique(), + 'name' => 'Vector CSV Import DB' + ]); + + $this->assertEquals(201, $database['headers']['status-code']); + $databaseId = $database['body']['$id']; + + $collection = $this->client->call(Client::METHOD_POST, '/vectordb/' . $databaseId . '/collections', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'collectionId' => ID::unique(), + 'name' => 'Vector CSV Import Collection', + 'dimensions' => 3, + 'documentSecurity' => true, + ]); + + $this->assertEquals(201, $collection['headers']['status-code']); + $collectionId = $collection['body']['$id']; + + $bucket = $this->client->call(Client::METHOD_POST, '/storage/buckets', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ], [ + 'bucketId' => ID::unique(), + 'name' => 'Vector CSV Bucket', + 'maximumFileSize' => 2000000, + 'allowedFileExtensions' => ['csv'], + ]); + + $this->assertEquals(201, $bucket['headers']['status-code']); + $bucketId = $bucket['body']['$id']; + + $file = $this->client->call(Client::METHOD_POST, '/storage/buckets/' . $bucketId . '/files', [ + 'content-type' => 'multipart/form-data', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ], [ + 'fileId' => ID::unique(), + 'file' => new CURLFile(realpath(__DIR__ . '/../../../resources/csv/vectordb-documents.csv'), 'text/csv', 'vectordb-documents.csv'), + ]); + + $this->assertEquals(201, $file['headers']['status-code']); + $fileId = $file['body']['$id']; + + $migration = $this->performCsvMigration([ + 'fileId' => $fileId, + 'bucketId' => $bucketId, + 'resourceId' => $databaseId . ':' . $collectionId, + ]); + + $this->assertEquals(202, $migration['headers']['status-code']); + + $this->assertEventually(function () use ($migration) { + $migrationId = $migration['body']['$id']; + $status = $this->client->call(Client::METHOD_GET, '/migrations/' . $migrationId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ]); + + $this->assertEquals(200, $status['headers']['status-code']); + $this->assertEquals('finished', $status['body']['stage']); + $this->assertEquals('completed', $status['body']['status']); + $this->assertContains(Resource::TYPE_DOCUMENT, $status['body']['resources']); + $this->assertArrayHasKey(Resource::TYPE_DOCUMENT, $status['body']['statusCounters']); + $this->assertEquals(2, $status['body']['statusCounters'][Resource::TYPE_DOCUMENT]['success']); + + return true; + }, 60_000, 500); + + $documents = $this->client->call(Client::METHOD_GET, '/vectordb/' . $databaseId . '/collections/' . $collectionId . '/documents', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ], [ + 'queries' => [ + Query::limit(10)->toString(), + ], + ]); + + $this->assertEquals(200, $documents['headers']['status-code']); + $this->assertEquals(2, $documents['body']['total']); + + $titles = array_map(fn ($doc) => $doc['metadata']['title'] ?? null, $documents['body']['documents']); + $this->assertContains('Vector Alpha', $titles); + $this->assertContains('Vector Beta', $titles); + } finally { + if ($bucketId) { + $this->client->call(Client::METHOD_DELETE, '/storage/buckets/' . $bucketId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ]); + } + + if ($databaseId) { + $this->client->call(Client::METHOD_DELETE, '/vectordb/' . $databaseId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ]); + } + } + } + + /** + * Export VectorDB documents to CSV + */ + public function testExportVectordbCSV(): void + { + $databaseId = null; + + try { + $database = $this->client->call(Client::METHOD_POST, '/vectordb', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ], [ + 'databaseId' => ID::unique(), + 'name' => 'Vector CSV Export DB', + ]); + + $this->assertEquals(201, $database['headers']['status-code']); + $databaseId = $database['body']['$id']; + + $collection = $this->client->call(Client::METHOD_POST, '/vectordb/' . $databaseId . '/collections', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ], [ + 'collectionId' => ID::unique(), + 'name' => 'Vector CSV Export Collection', + 'dimensions' => 3, + 'documentSecurity' => true, + ]); + + $this->assertEquals(201, $collection['headers']['status-code']); + $collectionId = $collection['body']['$id']; + + $documentsPayload = [ + [ + 'documentId' => ID::unique(), + 'data' => [ + 'embeddings' => [0.11, 0.22, 0.33], + 'metadata' => ['title' => 'Vector Sample One', 'category' => 'alpha'], + ], + ], + [ + 'documentId' => ID::unique(), + 'data' => [ + 'embeddings' => [0.44, 0.55, 0.66], + 'metadata' => ['title' => 'Vector Sample Two', 'category' => 'beta'], + ], + ], + ]; + + foreach ($documentsPayload as $payload) { + $response = $this->client->call(Client::METHOD_POST, '/vectordb/' . $databaseId . '/collections/' . $collectionId . '/documents', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ], $payload); + + $this->assertEquals(201, $response['headers']['status-code']); + } + + $filename = 'vectordb-export-' . ID::unique(); + $migration = $this->client->call(Client::METHOD_POST, '/migrations/csv/exports', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'resourceId' => $databaseId . ':' . $collectionId, + 'filename' => $filename, + 'columns' => [], + 'queries' => [], + 'delimiter' => ',', + 'enclosure' => '"', + 'escape' => '\\', + 'header' => true, + 'notify' => true, + ]); + + $this->assertEquals(202, $migration['headers']['status-code']); + + $migrationId = $migration['body']['$id']; + $this->assertEventually(function () use ($migrationId) { + $response = $this->client->call(Client::METHOD_GET, '/migrations/' . $migrationId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ]); + + $this->assertEquals(200, $response['headers']['status-code']); + $this->assertEquals('finished', $response['body']['stage']); + $this->assertEquals('completed', $response['body']['status']); + + return true; + }, 30_000, 500); + + $lastEmail = $this->getLastEmail(); + $this->assertNotEmpty($lastEmail); + $this->assertEquals('Your CSV export is ready', $lastEmail['subject']); + + \preg_match('/href="([^"]*\/storage\/buckets\/[^"]*\/push[^"]*)"/', $lastEmail['html'], $matches); + $this->assertNotEmpty($matches[1], 'Download URL not found in email'); + $downloadUrl = html_entity_decode($matches[1]); + + $components = \parse_url($downloadUrl); + $this->assertNotEmpty($components); + \parse_str($components['query'] ?? '', $queryParams); + $this->assertArrayHasKey('jwt', $queryParams); + $this->assertArrayHasKey('project', $queryParams); + + $path = \str_replace('/v1', '', $components['path']); + $downloadResponse = $this->client->call(Client::METHOD_GET, $path . '?project=' . $queryParams['project'] . '&jwt=' . $queryParams['jwt']); + $this->assertEquals(200, $downloadResponse['headers']['status-code']); + + $csvData = $downloadResponse['body']; + $this->assertStringContainsString('Vector Sample One', $csvData); + $this->assertStringContainsString('Vector Sample Two', $csvData); + $this->assertStringContainsString('[0.11,0.22,0.33]', $csvData); + } finally { + if ($databaseId) { + $this->client->call(Client::METHOD_DELETE, '/vectordb/' . $databaseId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ]); + } + } + } + /** * DocumentsDB (schemaless) */ diff --git a/tests/resources/csv/vectordb-documents.csv b/tests/resources/csv/vectordb-documents.csv new file mode 100644 index 0000000000..b0b970703e --- /dev/null +++ b/tests/resources/csv/vectordb-documents.csv @@ -0,0 +1,3 @@ +$id,embeddings,metadata +vector-doc-1,"[0.15,0.25,0.35]","{""title"":""Vector Alpha"",""category"":""science""}" +vector-doc-2,"[0.55,0.65,0.75]","{""title"":""Vector Beta"",""category"":""history""}" \ No newline at end of file From 918ebb8eccaa2f4439ae0e4d921b783c7848da66 Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Fri, 21 Nov 2025 18:00:51 +0530 Subject: [PATCH 17/27] Add VectorDB text embedding creation endpoint and update related tests --- app/controllers/api/projects.php | 5 +++- app/controllers/general.php | 1 - app/init/registers.php | 3 --- app/init/resources.php | 2 +- docker-compose.yml | 10 ++++---- .../Documents => Embeddings}/Text/Create.php | 23 ++++++------------- .../Databases/Services/Registry/VectorDB.php | 2 +- src/Appwrite/Utopia/Response.php | 6 +---- .../VectorDB/DatabasesCustomServerTest.php | 12 +++++----- 9 files changed, 24 insertions(+), 40 deletions(-) rename src/Appwrite/Platform/Modules/Databases/Http/VectorDB/{Collections/Documents => Embeddings}/Text/Create.php (76%) diff --git a/app/controllers/api/projects.php b/app/controllers/api/projects.php index c1b6514920..a3b40ce8bd 100644 --- a/app/controllers/api/projects.php +++ b/app/controllers/api/projects.php @@ -96,6 +96,8 @@ function getDatabaseDSN(string $databasetype, $region, ?string $dsn = null): str $databaseKeys = System::getEnv('_APP_DATABASE_VECTORDB_KEYS', ''); $databaseOverride = System::getEnv('_APP_DATABASE_VECTORDB_OVERRIDE'); $dbScheme = System::getEnv('_APP_DB_HOST_VECTORDB', 'postgresql'); + $sharedTables = \explode(',', System::getEnv('_APP_DATABASE_VECTORDB_SHARED_TABLES', '')); + $sharedTablesV1 = \explode(',', System::getEnv('_APP_DATABASE_VECTORDB_SHARED_TABLES_V1', '')); break; default: // legacy/tablesdb @@ -272,7 +274,8 @@ App::post('/v1/projects') 'accessedAt' => DateTime::now(), 'search' => implode(' ', [$projectId, $name]), 'database' => $dsn, - 'documentsDatabase' => getDatabaseDSN('documentsDatabase', $region, $dsn) + 'documentsDatabase' => getDatabaseDSN('documentsDatabase', $region, $dsn), + 'vectorDatabase' => getDatabaseDSN('vectorDatabase', $region, $dsn) ])); } catch (Duplicate) { throw new Exception(Exception::PROJECT_ALREADY_EXISTS); diff --git a/app/controllers/general.php b/app/controllers/general.php index 7f61dc955b..e0435cd499 100644 --- a/app/controllers/general.php +++ b/app/controllers/general.php @@ -1187,7 +1187,6 @@ App::error() $file = $error->getFile(); $line = $error->getLine(); $trace = $error->getTrace(); - $trace = $error->getTrace(); if (php_sapi_name() === 'cli') { Console::error('[Error] Timestamp: ' . date('c', time())); diff --git a/app/init/registers.php b/app/init/registers.php index cb8a3bb9ed..d9dfa5e7ab 100644 --- a/app/init/registers.php +++ b/app/init/registers.php @@ -400,9 +400,6 @@ $register->set('smtp', function () { $register->set('geodb', function () { return new Reader(__DIR__ . '/../assets/dbip/dbip-country-lite-2024-09.mmdb'); }); -$register->set('embeddingAgent', function () { - return System::getEnv('_APP_EMBEDDING_ENDPOINT', 'http://ollama:11434/api/embed'); -}); $register->set('passwordsDictionary', function () { $content = \file_get_contents(__DIR__ . '/../assets/security/10k-common-passwords'); $content = explode("\n", $content); diff --git a/app/init/resources.php b/app/init/resources.php index 18d37c3a37..a7e28ad376 100644 --- a/app/init/resources.php +++ b/app/init/resources.php @@ -1172,6 +1172,6 @@ App::setResource('transactionState', function (Database $dbForProject, callable App::setResource('embeddingAgent', function ($register) { $adapter = new Ollama(); - $adapter->setEndpoint($register->get('embeddingAgent')); + $adapter->setEndpoint(System::getEnv('_APP_EMBEDDING_ENDPOINT', 'http://ollama:11434/api/embed')); return new Agent($adapter); }, ['register']); diff --git a/docker-compose.yml b/docker-compose.yml index ef3ad4f31b..24f555eadc 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1176,8 +1176,7 @@ services: postgresql: build: - context: . - dockerfile: postgres.dockerfile + context: https://github.com/appwrite/docker-postgres.git#feat-dockerfile container_name: appwrite-postgresql <<: *x-logging networks: @@ -1194,8 +1193,7 @@ services: ollama: build: - context: . - dockerfile: ollama.dockerfile + context: https://github.com/appwrite/docker-ollama.git#feat-dockerfile args: MODELS: ${_APP_EMBEDDING_MODELS:-embeddinggemma} # duration to keep model in memory @@ -1206,7 +1204,7 @@ services: restart: unless-stopped # persistent for caching models across restarts and preloading volumes: - - _APP_EMBEDDING_MODELS:/root/.ollama + - appwrite-models:/root/.ollama networks: - appwrite @@ -1320,4 +1318,4 @@ volumes: appwrite-sites: appwrite-builds: appwrite-config: - _APP_EMBEDDING_MODELS: \ No newline at end of file + appwrite-models: \ No newline at end of file diff --git a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Text/Create.php b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Embeddings/Text/Create.php similarity index 76% rename from src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Text/Create.php rename to src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Embeddings/Text/Create.php index aa570ddb2c..559f583cf7 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Text/Create.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Embeddings/Text/Create.php @@ -1,6 +1,6 @@ setHttpMethod(self::HTTP_REQUEST_METHOD_POST) - ->setHttpPath('/v1/vectordb/:databaseId/collections/:collectionId/text') + ->setHttpPath('/v1/vectordb/embeddings/text') ->desc('Create Text Embeddings') ->groups(['api', 'database']) ->label('scope', 'documents.read') ->label('resourceType', RESOURCE_TYPE_DATABASES) ->label('audits.event', 'embedding.create') - ->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}') + ->label('audits.resource', 'vectordb/embeddings/text') ->label('abuse-key', 'ip:{ip},method:{method},url:{url},userId:{userId}') ->label('abuse-limit', APP_LIMIT_WRITE_RATE_DEFAULT * 2) ->label('abuse-time', APP_LIMIT_WRITE_RATE_PERIOD_DEFAULT) @@ -74,8 +73,6 @@ class Create extends CreateDocumentAction ] ) ]) - ->param('databaseId', '', new UID(), 'Database ID.') - ->param('collectionId', '', new UID(), 'Collection ID. You can create a new collection using the Database service [server integration](https://appwrite.io/docs/server/databases#databasesCreateCollection). Make sure to define attributes before creating documents.') ->param('documents', [], fn (array $plan) => new ArrayList(new JSON(), $plan['databasesBatchSize'] ?? APP_LIMIT_DATABASE_BATCH), 'Array of documents data as JSON objects.', true, ['plan']) ->inject('response') ->inject('embeddingAgent') @@ -84,19 +81,13 @@ class Create extends CreateDocumentAction ->callback($this->action(...)); } - public function action(string $databaseId, string $collectionId, array $documents, UtopiaResponse $response, $embeddingAgent, Database $dbForProject, callable $getDatabasesDB): void + public function action(array $documents, UtopiaResponse $response, $embeddingAgent, Database $dbForProject, callable $getDatabasesDB): void { $isAPIKey = Auth::isAppUser(Authorization::getRoles()); $isPrivilegedUser = Auth::isPrivilegedUser(Authorization::getRoles()); - $database = Authorization::skip(fn () => $dbForProject->getDocument('databases', $databaseId)); - if ($database->isEmpty() || (!$database->getAttribute('enabled', false) && !$isAPIKey && !$isPrivilegedUser)) { - throw new Exception(Exception::DATABASE_NOT_FOUND); - } - - $collection = Authorization::skip(fn () => $dbForProject->getDocument('database_' . $database->getSequence(), $collectionId)); - if ($collection->isEmpty() || (!$collection->getAttribute('enabled', false) && !$isAPIKey && !$isPrivilegedUser)) { - throw new Exception($this->getParentNotFoundException()); + if (!$isAPIKey && !$isPrivilegedUser) { + throw new Exception(Exception::GENERAL_BAD_REQUEST); } if (empty($documents)) { @@ -116,7 +107,7 @@ class Create extends CreateDocumentAction } $text = $item['text'] ?? ''; - $model = $item['embeddingModel'] ?? ($item['embeddigModel'] ?? null); + $model = $item['embeddingModel'] ?? ($item['embeddingModel'] ?? null); if (!\is_string($text) || $text === '') { throw new Exception(Exception::GENERAL_ARGUMENT_INVALID, 'Missing or invalid "text" at index ' . $index); diff --git a/src/Appwrite/Platform/Modules/Databases/Services/Registry/VectorDB.php b/src/Appwrite/Platform/Modules/Databases/Services/Registry/VectorDB.php index 22f434d4c4..f6fa731558 100644 --- a/src/Appwrite/Platform/Modules/Databases/Services/Registry/VectorDB.php +++ b/src/Appwrite/Platform/Modules/Databases/Services/Registry/VectorDB.php @@ -10,7 +10,6 @@ use Appwrite\Platform\Modules\Databases\Http\VectorDB\Collections\Documents\Bulk use Appwrite\Platform\Modules\Databases\Http\VectorDB\Collections\Documents\Create as CreateDocument; use Appwrite\Platform\Modules\Databases\Http\VectorDB\Collections\Documents\Delete as DeleteDocument; use Appwrite\Platform\Modules\Databases\Http\VectorDB\Collections\Documents\Get as GetDocument; -use Appwrite\Platform\Modules\Databases\Http\VectorDB\Collections\Documents\Text\Create as CreateTextEmbeddings; use Appwrite\Platform\Modules\Databases\Http\VectorDB\Collections\Documents\Update as UpdateDocument; use Appwrite\Platform\Modules\Databases\Http\VectorDB\Collections\Documents\Upsert as UpsertDocument; use Appwrite\Platform\Modules\Databases\Http\VectorDB\Collections\Documents\XList as ListDocuments; @@ -25,6 +24,7 @@ use Appwrite\Platform\Modules\Databases\Http\VectorDB\Collections\Usage\Get as G use Appwrite\Platform\Modules\Databases\Http\VectorDB\Collections\XList as ListCollections; use Appwrite\Platform\Modules\Databases\Http\VectorDB\Create as CreateVectorDatabase; use Appwrite\Platform\Modules\Databases\Http\VectorDB\Delete as DeleteVectorDatabase; +use Appwrite\Platform\Modules\Databases\Http\VectorDB\Embeddings\Text\Create as CreateTextEmbeddings; use Appwrite\Platform\Modules\Databases\Http\VectorDB\Get as GetVectorDatabase; use Appwrite\Platform\Modules\Databases\Http\VectorDB\Transactions\Create as CreateTransaction; use Appwrite\Platform\Modules\Databases\Http\VectorDB\Transactions\Delete as DeleteTransaction; diff --git a/src/Appwrite/Utopia/Response.php b/src/Appwrite/Utopia/Response.php index 1b04e7ca76..94b2a799a6 100644 --- a/src/Appwrite/Utopia/Response.php +++ b/src/Appwrite/Utopia/Response.php @@ -884,11 +884,7 @@ class Response extends SwooleResponse $this ->setContentType(Response::CONTENT_TYPE_YAML) - ->send((function (array $payload) { - $func = 'yaml_emit'; - $encoding = \defined('YAML_UTF8_ENCODING') ? \constant('YAML_UTF8_ENCODING') : 0; - return \call_user_func($func, $payload, $encoding); - })($data)); + ->send(\yaml_emit($data, YAML_UTF8_ENCODING)); } /** diff --git a/tests/e2e/Services/Databases/VectorDB/DatabasesCustomServerTest.php b/tests/e2e/Services/Databases/VectorDB/DatabasesCustomServerTest.php index 8f9a3fc04f..4203f5fa02 100644 --- a/tests/e2e/Services/Databases/VectorDB/DatabasesCustomServerTest.php +++ b/tests/e2e/Services/Databases/VectorDB/DatabasesCustomServerTest.php @@ -589,7 +589,7 @@ class DatabasesCustomServerTest extends Scope $collectionId = $col['body']['$id']; // Success: two embeddings - $ok = $this->client->call(Client::METHOD_POST, "/vectordb/{$databaseId}/collections/{$collectionId}/text", [ + $ok = $this->client->call(Client::METHOD_POST, "/vectordb/embeddings/text", [ 'content-type' => 'application/json', 'x-appwrite-project' => $this->getProject()['$id'], 'x-appwrite-key' => $this->getProject()['apiKey'] @@ -612,7 +612,7 @@ class DatabasesCustomServerTest extends Scope } // Error: missing documents - $missingDocs = $this->client->call(Client::METHOD_POST, "/vectordb/{$databaseId}/collections/{$collectionId}/text", [ + $missingDocs = $this->client->call(Client::METHOD_POST, "/vectordb/embeddings/text", [ 'content-type' => 'application/json', 'x-appwrite-project' => $this->getProject()['$id'], 'x-appwrite-key' => $this->getProject()['apiKey'] @@ -620,7 +620,7 @@ class DatabasesCustomServerTest extends Scope $this->assertEquals(400, $missingDocs['headers']['status-code']); // Error: invalid item structure (not an object) - $invalidItem = $this->client->call(Client::METHOD_POST, "/vectordb/{$databaseId}/collections/{$collectionId}/text", [ + $invalidItem = $this->client->call(Client::METHOD_POST, "/vectordb/embeddings/text", [ 'content-type' => 'application/json', 'x-appwrite-project' => $this->getProject()['$id'], 'x-appwrite-key' => $this->getProject()['apiKey'] @@ -630,7 +630,7 @@ class DatabasesCustomServerTest extends Scope $this->assertEquals(400, $invalidItem['headers']['status-code']); // Error: missing text - $missingText = $this->client->call(Client::METHOD_POST, "/vectordb/{$databaseId}/collections/{$collectionId}/text", [ + $missingText = $this->client->call(Client::METHOD_POST, "/vectordb/embeddings/text", [ 'content-type' => 'application/json', 'x-appwrite-project' => $this->getProject()['$id'], 'x-appwrite-key' => $this->getProject()['apiKey'] @@ -640,7 +640,7 @@ class DatabasesCustomServerTest extends Scope $this->assertEquals(400, $missingText['headers']['status-code']); // Error: missing embeddingModel - $missingModel = $this->client->call(Client::METHOD_POST, "/vectordb/{$databaseId}/collections/{$collectionId}/text", [ + $missingModel = $this->client->call(Client::METHOD_POST, "/vectordb/embeddings/text", [ 'content-type' => 'application/json', 'x-appwrite-project' => $this->getProject()['$id'], 'x-appwrite-key' => $this->getProject()['apiKey'] @@ -650,7 +650,7 @@ class DatabasesCustomServerTest extends Scope $this->assertEquals(400, $missingModel['headers']['status-code']); // Error: unknown embedding model - $unknownModel = $this->client->call(Client::METHOD_POST, "/vectordb/{$databaseId}/collections/{$collectionId}/text", [ + $unknownModel = $this->client->call(Client::METHOD_POST, "/vectordb/embeddings/text", [ 'content-type' => 'application/json', 'x-appwrite-project' => $this->getProject()['$id'], 'x-appwrite-key' => $this->getProject()['apiKey'] From 6eb17b847f496aa3b894a7b9d8f8bddd6ac94da0 Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Fri, 21 Nov 2025 18:06:32 +0530 Subject: [PATCH 18/27] updated composer --- composer.lock | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/composer.lock b/composer.lock index 5dbb9443eb..32dfedd3be 100644 --- a/composer.lock +++ b/composer.lock @@ -4,8 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "3999dacf9f2fc437123cca148964cb7e", - "content-hash": "bdd11652978ebab653ca17a00074343d", + "content-hash": "b717375900f6871c875e1b61011ff8b5", "packages": [ { "name": "adhocore/jwt", From fdf5ca0067084f7c6d9ea65251f1a7afa933cfee Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Fri, 21 Nov 2025 19:08:35 +0530 Subject: [PATCH 19/27] updated tests condition --- .../Services/Databases/Legacy/DatabasesBase.php | 14 +++++++++++--- .../Services/Databases/TablesDB/DatabasesBase.php | 2 +- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/tests/e2e/Services/Databases/Legacy/DatabasesBase.php b/tests/e2e/Services/Databases/Legacy/DatabasesBase.php index e7ad75d03b..e61f84f183 100644 --- a/tests/e2e/Services/Databases/Legacy/DatabasesBase.php +++ b/tests/e2e/Services/Databases/Legacy/DatabasesBase.php @@ -493,9 +493,17 @@ trait DatabasesBase $this->assertEquals(400, $attribute['headers']['status-code']); - $maxLength = $this->isMongoDB() ? 1024 : 768; - - $this->assertStringContainsString('Index length is longer than the maximum: '.$maxLength, $attribute['body']['message']); + $message = $attribute['body']['message']; + if ($this->MongoDB()) { + $this->assertStringContainsString('Index length is longer than the maximum: 1024', $message); + } else { + // length depends on the shared table + $this->assertTrue( + str_contains($message, 'Index length is longer than the maximum: 767') || + str_contains($message, 'Index length is longer than the maximum: 768'), + "Message does not contain expected max length" + ); + } } public function testUpdateAttributeEnum(): void diff --git a/tests/e2e/Services/Databases/TablesDB/DatabasesBase.php b/tests/e2e/Services/Databases/TablesDB/DatabasesBase.php index 9757b525df..da11582158 100644 --- a/tests/e2e/Services/Databases/TablesDB/DatabasesBase.php +++ b/tests/e2e/Services/Databases/TablesDB/DatabasesBase.php @@ -8663,7 +8663,7 @@ trait DatabasesBase '$id' => 'row1', 'name' => 'Test Row 1', 'pointAttr' => [6.0, 6.0], - 'lineAttr' => [[1.0, 1.0], [2.0, 2.0]], + 'lineAttr' => [[1.0, 1.0], [1.1,1.1] , [2.0, 2.0]], 'polyAttr' => [[[0.0, 0.0], [10.0, 0.0], [10.0, 10.0], [0.0, 10.0], [0.0, 0.0]]] ], [ From d2c9ac079aaf22ea100b272f41c1408921f94c6d Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Tue, 2 Dec 2025 14:20:25 +0530 Subject: [PATCH 20/27] Refactor VectorDB Embeddings API and Update Dimension Handling - Updated the Create.php file to streamline the embedding creation process by removing unnecessary authentication checks and simplifying the parameter structure. - Changed the parameter from 'documents' to 'texts' for clarity and adjusted the embedding model handling. - Modified the response model to include error handling for embedding generation. - Updated the Embedding response model to reflect changes in dimension naming and added error message handling. - Refactored VectorDBCollection model to replace 'dimensions' with 'dimension' for consistency across the codebase. - Adjusted all relevant tests to accommodate the new parameter names and response structures, ensuring comprehensive coverage for the updated functionality. --- app/config/collections/vectordb.php | 2 +- .../Http/Databases/Transactions/Update.php | 39 --------- .../Http/VectorDB/Collections/Create.php | 12 +-- .../Http/VectorDB/Collections/Update.php | 4 +- .../Http/VectorDB/Embeddings/Text/Create.php | 82 ++++++------------- .../Utopia/Response/Model/Embedding.php | 14 +++- .../Response/Model/VectorDBCollection.php | 70 +--------------- .../Databases/VectorDB/DatabasesBase.php | 26 +++--- .../VectorDB/DatabasesConsoleClientTest.php | 4 +- .../VectorDB/DatabasesCustomClientTest.php | 4 +- .../VectorDB/DatabasesCustomServerTest.php | 81 ++++++++---------- .../DatabasesPermissionsGuestTest.php | 6 +- .../DatabasesPermissionsMemberTest.php | 6 +- .../DatabasesPermissionsTeamTest.php | 4 +- .../VectorDB/Transactions/ACIDTest.php | 8 +- .../Transactions/TransactionsBase.php | 36 ++++---- .../Services/Migrations/MigrationsBase.php | 8 +- .../Realtime/RealtimeCustomClientTest.php | 2 +- 18 files changed, 135 insertions(+), 273 deletions(-) diff --git a/app/config/collections/vectordb.php b/app/config/collections/vectordb.php index 6a8a26aabb..817863cffa 100644 --- a/app/config/collections/vectordb.php +++ b/app/config/collections/vectordb.php @@ -41,7 +41,7 @@ return [ 'filters' => [], ], [ - '$id' => ID::custom('dimensions'), + '$id' => ID::custom('dimension'), 'type' => Database::VAR_INTEGER, 'size' => 0, 'required' => true, diff --git a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Transactions/Update.php b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Transactions/Update.php index 6ad0755c9e..d305c35bba 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Transactions/Update.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Transactions/Update.php @@ -298,11 +298,6 @@ class Update extends Action $data = $data->getArrayCopy(); } - // Decode JSON strings that may have been encoded when storing nested arrays/objects - if (\is_array($data)) { - $data = $this->decodeJsonStrings($data); - } - if (!isset($dbCache[$databaseInternalId])) { $databaseDoc = Authorization::skip(fn () => $dbForProject->findOne('databases', [ Query::equal('$sequence', [$databaseInternalId]) @@ -727,8 +722,6 @@ class Update extends Action array &$state ): int { $count = 0; - // Decode JSON strings in bulk create data - $data = $this->decodeJsonStrings($data); $dbForProject->withRequestTimestamp($createdAt, function () use ($dbForProject, $collectionId, $data, &$state, &$count) { $documents = \array_map(function ($doc) { return $doc instanceof Document ? $doc : new Document($doc); @@ -767,11 +760,8 @@ class Update extends Action \DateTime $createdAt, array &$state ): int { - // extracting query first then decoding the nested transaction log data - // otherwise can result error as queries would get transformed to array instead of string $queries = Query::parseQueries($data['queries'] ?? []); $updateData = new Document($data['data']); - $data = $this->decodeJsonStrings($data); $dependentDocs = []; @@ -842,8 +832,6 @@ class Update extends Action \DateTime $createdAt, array &$state ): int { - // Decode JSON strings in bulk upsert data - $data = $this->decodeJsonStrings($data); $documents = \array_map(function ($doc) { return $doc instanceof Document ? $doc : new Document($doc); }, $data); @@ -919,31 +907,4 @@ class Update extends Action return $count; } - - /** - * Recursively decode JSON strings in data array - * This handles cases where nested arrays/objects are stored as JSON strings in the database - * - * @param array $data - * @return array - */ - private function decodeJsonStrings(array $data): array - { - foreach ($data as $key => $value) { - if (\is_string($value)) { - $decoded = \json_decode($value, true); - if (\json_last_error() === JSON_ERROR_NONE && (\is_array($decoded) || \is_object($decoded))) { - $data[$key] = $decoded; - // Recursively decode nested structures - if (\is_array($decoded)) { - $data[$key] = $this->decodeJsonStrings($decoded); - } - } - } elseif (\is_array($value)) { - // Recursively process nested arrays - $data[$key] = $this->decodeJsonStrings($value); - } - } - return $data; - } } diff --git a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Create.php b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Create.php index 76a5a5d173..61adb87dac 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Create.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Create.php @@ -69,7 +69,7 @@ class Create extends CollectionAction ->param('databaseId', '', fn (Database $dbForProject) => new UID($dbForProject->getAdapter()->getMaxUIDLength()), 'Database ID.', false, ['dbForProject']) ->param('collectionId', '', fn (Database $dbForProject) => new CustomId(false, $dbForProject->getAdapter()->getMaxUIDLength()), 'Unique Id. Choose a custom ID or generate a random ID with `ID.unique()`. Valid chars are a-z, A-Z, 0-9, period, hyphen, and underscore. Can\'t start with a special char. Max length is 36 chars.', false, ['dbForProject']) ->param('name', '', new Text(128), 'Collection name. Max length: 128 chars.') - ->param('dimensions', null, new Range(MIN_VECTOR_DIMENSION, MAX_VECTOR_DIMENSION), 'Embedding dimensions.') + ->param('dimension', null, new Range(MIN_VECTOR_DIMENSION, MAX_VECTOR_DIMENSION), 'Embedding dimension.') ->param('permissions', null, new Permissions(APP_LIMIT_ARRAY_PARAMS_SIZE), 'An array of permissions strings. By default, no user is granted with any permissions. [Learn more about permissions](https://appwrite.io/docs/permissions).', true) ->param('documentSecurity', false, new Boolean(true), 'Enables configuring permissions for individual documents. A user needs one of document or collection level permissions to access a document. [Learn more about permissions](https://appwrite.io/docs/permissions).', true) ->param('enabled', true, new Boolean(), 'Is collection enabled? When set to \'disabled\', users cannot access the collection but Server SDKs with and API key can still read and write to the collection. No data is lost when this is toggled.', true) @@ -80,7 +80,7 @@ class Create extends CollectionAction ->callback($this->action(...)); } - public function action(string $databaseId, string $collectionId, string $name, int $dimensions, ?array $permissions, bool $documentSecurity, bool $enabled, UtopiaResponse $response, Database $dbForProject, callable $getDatabasesDB, Event $queueForEvents): void + public function action(string $databaseId, string $collectionId, string $name, int $dimension, ?array $permissions, bool $documentSecurity, bool $enabled, UtopiaResponse $response, Database $dbForProject, callable $getDatabasesDB, Event $queueForEvents): void { $database = Authorization::skip(fn () => $dbForProject->getDocument('databases', $databaseId)); @@ -102,7 +102,7 @@ class Create extends CollectionAction 'documentSecurity' => $documentSecurity, 'enabled' => $enabled, 'name' => $name, - 'dimensions' => $dimensions, + 'dimension' => $dimension, 'search' => \implode(' ', [$collectionId, $name]), ])); @@ -121,7 +121,7 @@ class Create extends CollectionAction $collections = (Config::getParam('collections', [])['vectordb'] ?? [])['collections'] ?? []; foreach ($collections['defaultAttributes'] as $attribute) { if ($attribute['$id'] === 'embeddings') { - $attribute['size'] = $dimensions; + $attribute['size'] = $dimension; } $attributes[] = new Document($attribute); } @@ -142,7 +142,7 @@ class Create extends CollectionAction ); // Create attribute and indexes metadata documents in the attributes and indexes collections // needed for the get and list calls - $attributeDocs = array_map(function ($attributeConfig) use ($database, $collection, $databaseId, $collectionId, $dimensions) { + $attributeDocs = array_map(function ($attributeConfig) use ($database, $collection, $databaseId, $collectionId, $dimension) { $key = \is_string($attributeConfig['$id']) ? $attributeConfig['$id'] : (string) $attributeConfig['$id']; return new Document([ '$id' => ID::custom($database->getSequence() . '_' . $collection->getSequence() . '_' . $key), @@ -153,7 +153,7 @@ class Create extends CollectionAction 'collectionId' => $collectionId, 'type' => $attributeConfig['type'], 'status' => 'available', - 'size' => $dimensions, + 'size' => $dimension, 'required' => $attributeConfig['required'] ?? false, 'signed' => $attributeConfig['signed'] ?? false, 'default' => $attributeConfig['default'] ?? null, diff --git a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Update.php b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Update.php index 06ba3fdc2e..5642b36944 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Update.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Update.php @@ -59,7 +59,7 @@ class Update extends CollectionAction ->param('databaseId', '', new UID(), 'Database ID.') ->param('collectionId', '', new UID(), 'Collection ID.') ->param('name', null, new Text(128), 'Collection name. Max length: 128 chars.') - ->param('dimensions', null, new Range(MIN_VECTOR_DIMENSION, MAX_VECTOR_DIMENSION), 'Embedding dimensions.', true) + ->param('dimension', null, new Range(MIN_VECTOR_DIMENSION, MAX_VECTOR_DIMENSION), 'Embedding dimensions.', true) ->param('permissions', null, new Permissions(APP_LIMIT_ARRAY_PARAMS_SIZE), 'An array of permission strings. By default, the current permissions are inherited. [Learn more about permissions](https://appwrite.io/docs/permissions).', true) ->param('documentSecurity', false, new Boolean(true), 'Enables configuring permissions for individual documents. A user needs one of document or collection level permissions to access a document. [Learn more about permissions](https://appwrite.io/docs/permissions).', true) ->param('enabled', true, new Boolean(), 'Is collection enabled? When set to \'disabled\', users cannot access the collection but Server SDKs with and API key can still read and write to the collection. No data is lost when this is toggled.', true) @@ -94,7 +94,7 @@ class Update extends CollectionAction $collectionId, $collection ->setAttribute('name', $name ?? $collection->getAttribute('name')) - ->setAttribute('dimensions', $dimensions ?? $collection->getAttribute('dimensions')) + ->setAttribute('dimension', $dimensions ?? $collection->getAttribute('dimension')) ->setAttribute('$permissions', $permissions) ->setAttribute('documentSecurity', $documentSecurity) ->setAttribute('enabled', $enabled) diff --git a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Embeddings/Text/Create.php b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Embeddings/Text/Create.php index 559f583cf7..b5d1be8f98 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Embeddings/Text/Create.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Embeddings/Text/Create.php @@ -2,8 +2,6 @@ namespace Appwrite\Platform\Modules\Databases\Http\VectorDB\Embeddings\Text; -use Appwrite\Auth\Auth; -use Appwrite\Extend\Exception; use Appwrite\Platform\Modules\Databases\Http\Databases\Collections\Documents\Action as CreateDocumentAction; use Appwrite\SDK\AuthType; use Appwrite\SDK\ContentType; @@ -11,14 +9,13 @@ use Appwrite\SDK\Method; use Appwrite\SDK\Parameter; use Appwrite\SDK\Response as SDKResponse; use Appwrite\Utopia\Response as UtopiaResponse; -use Utopia\Database\Database; +use Utopia\Agents\Adapters\Ollama; +use Utopia\Agents\Agent; use Utopia\Database\Document; -use Utopia\Database\Validator\Authorization; use Utopia\Swoole\Response as SwooleResponse; use Utopia\Validator\ArrayList; -use Utopia\Validator\JSON; - -// Agent is dynamically provided via container; avoid strict type to pass lints +use Utopia\Validator\Text; +use Utopia\Validator\WhiteList; class Create extends CreateDocumentAction { @@ -58,7 +55,7 @@ class Create extends CreateDocumentAction name: 'createTextEmbeddings', desc: 'Create Text Embedding', description: '/docs/references/vectordb/create-document.md', - auth: [AuthType::SESSION, AuthType::KEY, AuthType::JWT], + auth: [AuthType::KEY, AuthType::JWT], responses: [ new SDKResponse( code: SwooleResponse::STATUS_CODE_OK, @@ -73,73 +70,44 @@ class Create extends CreateDocumentAction ] ) ]) - ->param('documents', [], fn (array $plan) => new ArrayList(new JSON(), $plan['databasesBatchSize'] ?? APP_LIMIT_DATABASE_BATCH), 'Array of documents data as JSON objects.', true, ['plan']) + ->param('embeddingModel', Ollama::MODEL_EMBEDDING_GEMMA, new WhiteList(Ollama::MODELS), 'The embedding model to use for generating vector embeddings.', false) + ->param('texts', [], fn (array $plan) => new ArrayList(new Text(0), $plan['databasesBatchSize'] ?? APP_LIMIT_DATABASE_BATCH), 'Array of text to generate embeddings.', true, ['plan']) ->inject('response') ->inject('embeddingAgent') - ->inject('dbForProject') - ->inject('getDatabasesDB') ->callback($this->action(...)); } - public function action(array $documents, UtopiaResponse $response, $embeddingAgent, Database $dbForProject, callable $getDatabasesDB): void + public function action(string $embeddingModel, array $texts, UtopiaResponse $response, Agent $embeddingAgent): void { - $isAPIKey = Auth::isAppUser(Authorization::getRoles()); - $isPrivilegedUser = Auth::isPrivilegedUser(Authorization::getRoles()); - - if (!$isAPIKey && !$isPrivilegedUser) { - throw new Exception(Exception::GENERAL_BAD_REQUEST); - } - - if (empty($documents)) { - throw new Exception(Exception::DOCUMENT_MISSING_DATA); - } - - // Validate and process each document - $availableModels = []; - $availableModels = $embeddingAgent->getAdapter()->getModels(); - $results = []; + $embeddingAgent->getAdapter()->setModel($embeddingModel); + $dimension = $embeddingAgent->getAdapter()->getEmbeddingDimension(); - // validating all documents first - foreach ($documents as $index => $item) { - if (!\is_array($item)) { - throw new Exception(Exception::DOCUMENT_INVALID_STRUCTURE, 'Invalid item at index ' . $index); - } + foreach ($texts as $text) { - $text = $item['text'] ?? ''; - $model = $item['embeddingModel'] ?? ($item['embeddingModel'] ?? null); - - if (!\is_string($text) || $text === '') { - throw new Exception(Exception::GENERAL_ARGUMENT_INVALID, 'Missing or invalid "text" at index ' . $index); + $embedding = []; + $error = ''; + try { + $embedResult = $embeddingAgent->embed($text); + $embedding = $embedResult['embedding'] ?? []; + } catch (\Exception $e) { + $error = 'Error while generating embedding'; } - if (!\is_string($model) || $model === '') { - throw new Exception(Exception::GENERAL_ARGUMENT_INVALID, 'Missing or invalid "embeddingModel" at index ' . $index); - } - if (!empty($availableModels) && !\in_array($model, $availableModels, true)) { - throw new Exception(Exception::GENERAL_BAD_REQUEST, 'Unknown embedding model: ' . $model); - } - } - foreach ($documents as $index => $item) { - $embeddingAgent->getAdapter()->setModel($model); - - $embedResult = $embeddingAgent->embed($text); - $vector = $embedResult['embedding'] ?? []; - $dimensions = \is_array($vector) ? \count($vector) : 0; $results[] = new Document([ - 'model' => $model, - 'dimensions' => $dimensions, - 'embeddings' => $vector, + 'model' => $embeddingModel, + 'dimension' => $dimension, + 'embedding' => $embedding, + 'error' => $error ]); - } - $list = new Document([ - 'embeddings' => array_map(fn ($d) => $d, $results), + $embeddings = new Document([ + 'embeddings' => $results, 'total' => \count($results), ]); $response ->setStatusCode(SwooleResponse::STATUS_CODE_OK) - ->dynamic($list, $this->getBulkResponseModel()); + ->dynamic($embeddings, $this->getBulkResponseModel()); } } diff --git a/src/Appwrite/Utopia/Response/Model/Embedding.php b/src/Appwrite/Utopia/Response/Model/Embedding.php index 7c881ca1ac..9fce913723 100644 --- a/src/Appwrite/Utopia/Response/Model/Embedding.php +++ b/src/Appwrite/Utopia/Response/Model/Embedding.php @@ -24,16 +24,24 @@ class Embedding extends Model 'description' => 'Embedding model used to generate embeddings.', 'example' => 'embeddinggemma' ]) - ->addRule('dimensions', [ + ->addRule('dimension', [ 'type' => self::TYPE_INTEGER, 'description' => 'Number of dimensions for each embedding vector.', 'example' => 768 ]) - ->addRule('embeddings', [ + ->addRule('embedding', [ 'type' => self::TYPE_FLOAT, 'array' => true, - 'description' => 'Embedding vector values.', + 'default' => [], + 'description' => 'Embedding vector values. If an error occurs, this will be an empty array.', 'example' => [0.01, 0.02, 0.03] + ]) + ->addRule('error', [ + 'type' => self::TYPE_STRING, + 'array' => false, + 'default' => '', + 'description' => 'Error message if embedding generation fails. Empty string if no error.', + 'example' => 'Error message' ]); } } diff --git a/src/Appwrite/Utopia/Response/Model/VectorDBCollection.php b/src/Appwrite/Utopia/Response/Model/VectorDBCollection.php index 017780173b..ccf609132d 100644 --- a/src/Appwrite/Utopia/Response/Model/VectorDBCollection.php +++ b/src/Appwrite/Utopia/Response/Model/VectorDBCollection.php @@ -3,74 +3,19 @@ namespace Appwrite\Utopia\Response\Model; use Appwrite\Utopia\Response; -use Appwrite\Utopia\Response\Model; -class VectorDBCollection extends Model +class VectorDBCollection extends Collection { public function __construct() { + parent::__construct(); $this - ->addRule('$id', [ - 'type' => self::TYPE_STRING, - 'description' => 'Collection ID.', - 'default' => '', - 'example' => '5e5ea5c16897e', - ]) - ->addRule('$createdAt', [ - 'type' => self::TYPE_DATETIME, - 'description' => 'Collection creation date in ISO 8601 format.', - 'default' => '', - 'example' => self::TYPE_DATETIME_EXAMPLE, - ]) - ->addRule('$updatedAt', [ - 'type' => self::TYPE_DATETIME, - 'description' => 'Collection update date in ISO 8601 format.', - 'default' => '', - 'example' => self::TYPE_DATETIME_EXAMPLE, - ]) - ->addRule('$permissions', [ - 'type' => self::TYPE_STRING, - 'description' => 'Collection permissions. [Learn more about permissions](https://appwrite.io/docs/permissions).', - 'default' => '', - 'example' => ['read("any")'], - 'array' => true - ]) - ->addRule('databaseId', [ - 'type' => self::TYPE_STRING, - 'description' => 'Database ID.', - 'default' => '', - 'example' => '5e5ea5c16897e', - ]) - ->addRule('name', [ - 'type' => self::TYPE_STRING, - 'description' => 'Collection name.', - 'default' => '', - 'example' => 'My Collection', - ]) - ->addRule('enabled', [ - 'type' => self::TYPE_BOOLEAN, - 'description' => 'Collection enabled. Can be \'enabled\' or \'disabled\'. When disabled, the collection is inaccessible to users, but remains accessible to Server SDKs using API keys.', - 'default' => true, - 'example' => false, - ]) - ->addRule('documentSecurity', [ - 'type' => self::TYPE_BOOLEAN, - 'description' => 'Whether document-level permissions are enabled. [Learn more about permissions](https://appwrite.io/docs/permissions).', - 'default' => '', - 'example' => true, - ]) - ->addRule('dimensions', [ + ->addRule('dimension', [ 'type' => self::TYPE_INTEGER, - 'description' => 'Embedding dimensions.', + 'description' => 'Embedding dimension.', 'default' => 0, 'example' => 1536, ]) - ->addRule('search', [ - 'type' => self::TYPE_STRING, - 'description' => 'Search text.', - 'default' => '', - 'example' => 'collectionId name', - ]) ->addRule('attributes', [ 'type' => [ Response::MODEL_ATTRIBUTE_OBJECT, @@ -81,13 +26,6 @@ class VectorDBCollection extends Model 'example' => new \stdClass(), 'array' => true, ]) - ->addRule('indexes', [ - 'type' => Response::MODEL_INDEX, - 'description' => 'Collection indexes.', - 'default' => [], - 'example' => new \stdClass(), - 'array' => true - ]) ; } diff --git a/tests/e2e/Services/Databases/VectorDB/DatabasesBase.php b/tests/e2e/Services/Databases/VectorDB/DatabasesBase.php index 443e37a122..5d366af516 100644 --- a/tests/e2e/Services/Databases/VectorDB/DatabasesBase.php +++ b/tests/e2e/Services/Databases/VectorDB/DatabasesBase.php @@ -504,7 +504,7 @@ trait DatabasesBase 'collectionId' => ID::unique(), 'name' => 'Movies', 'documentSecurity' => true, - 'dimensions' => 1536, + 'dimension' => 1536, 'permissions' => [ Permission::create(Role::user($this->getUser()['$id'])), ], @@ -521,7 +521,7 @@ trait DatabasesBase 'collectionId' => ID::unique(), 'name' => 'Actors', 'documentSecurity' => true, - 'dimensions' => 1536, + 'dimension' => 1536, 'permissions' => [ Permission::create(Role::user($this->getUser()['$id'])), ], @@ -570,7 +570,7 @@ trait DatabasesBase ]), [ 'collectionId' => ID::unique(), 'name' => 'Sample Collection', - 'dimensions' => 1536, + 'dimension' => 1536, 'documentSecurity' => true, 'permissions' => [ Permission::create(Role::user($this->getUser()['$id'])), @@ -579,7 +579,7 @@ trait DatabasesBase $this->assertEquals(201, $collection['headers']['status-code']); $this->assertEquals('Sample Collection', $collection['body']['name']); - $this->assertEquals(1536, $collection['body']['dimensions']); + $this->assertEquals(1536, $collection['body']['dimension']); return [ 'databaseId' => $databaseId, @@ -626,7 +626,7 @@ trait DatabasesBase 'collectionId' => ID::unique(), 'name' => $name, 'documentSecurity' => true, - 'dimensions' => $dimensions, + 'dimension' => $dimensions, 'permissions' => [ Permission::create(Role::user($userId)), ], @@ -702,7 +702,7 @@ trait DatabasesBase 'collectionId' => ID::unique(), 'name' => 'ZeroDims', 'documentSecurity' => true, - 'dimensions' => 0, + 'dimension' => 0, 'permissions' => [Permission::create(Role::user($this->getUser()['$id']))], ]); $this->assertGreaterThanOrEqual(400, $col['headers']['status-code']); @@ -717,7 +717,7 @@ trait DatabasesBase 'collectionId' => ID::unique(), 'name' => 'HugeDims', 'documentSecurity' => true, - 'dimensions' => 16001, + 'dimension' => 16001, 'permissions' => [Permission::create(Role::user($this->getUser()['$id']))], ]); $this->assertGreaterThanOrEqual(400, $col2['headers']['status-code']); @@ -745,7 +745,7 @@ trait DatabasesBase 'collectionId' => ID::unique(), 'name' => 'OneDim', 'documentSecurity' => true, - 'dimensions' => 1, + 'dimension' => 1, 'permissions' => [Permission::create(Role::user($this->getUser()['$id']))], ]); $this->assertEquals(201, $col['headers']['status-code']); @@ -802,7 +802,7 @@ trait DatabasesBase 'collectionId' => ID::unique(), 'name' => 'Docs', 'documentSecurity' => true, - 'dimensions' => 3, + 'dimension' => 3, 'permissions' => [Permission::create(Role::user($this->getUser()['$id']))], ]); $this->assertEquals(201, $col['headers']['status-code']); @@ -859,7 +859,7 @@ trait DatabasesBase 'collectionId' => ID::unique(), 'name' => 'Zeros', 'documentSecurity' => true, - 'dimensions' => 3, + 'dimension' => 3, 'permissions' => [Permission::create(Role::user($this->getUser()['$id']))], ]); $this->assertEquals(201, $col['headers']['status-code']); @@ -900,7 +900,7 @@ trait DatabasesBase 'content-type' => 'application/json', 'x-appwrite-project' => $this->getProject()['$id'], 'x-appwrite-key' => $this->getProject()['apiKey'] - ], [ 'collectionId' => ID::unique(), 'name' => 'Docs', 'documentSecurity' => true, 'dimensions' => 3, 'permissions' => [Permission::create(Role::user($this->getUser()['$id']))] ]); + ], [ 'collectionId' => ID::unique(), 'name' => 'Docs', 'documentSecurity' => true, 'dimension' => 3, 'permissions' => [Permission::create(Role::user($this->getUser()['$id']))] ]); $this->assertEquals(201, $col['headers']['status-code']); $collectionId = $col['body']['$id']; @@ -932,7 +932,7 @@ trait DatabasesBase 'content-type' => 'application/json', 'x-appwrite-project' => $this->getProject()['$id'], 'x-appwrite-key' => $this->getProject()['apiKey'] - ], [ 'collectionId' => ID::unique(), 'name' => 'Docs', 'documentSecurity' => true, 'dimensions' => 3, 'permissions' => [Permission::create(Role::user($this->getUser()['$id']))] ]); + ], [ 'collectionId' => ID::unique(), 'name' => 'Docs', 'documentSecurity' => true, 'dimension' => 3, 'permissions' => [Permission::create(Role::user($this->getUser()['$id']))] ]); $this->assertEquals(201, $col['headers']['status-code']); $collectionId = $col['body']['$id']; @@ -959,7 +959,7 @@ trait DatabasesBase 'content-type' => 'application/json', 'x-appwrite-project' => $this->getProject()['$id'], 'x-appwrite-key' => $this->getProject()['apiKey'] - ], [ 'collectionId' => ID::unique(), 'name' => 'Docs', 'documentSecurity' => true, 'dimensions' => 3, 'permissions' => [Permission::create(Role::user($this->getUser()['$id']))] ]); + ], [ 'collectionId' => ID::unique(), 'name' => 'Docs', 'documentSecurity' => true, 'dimension' => 3, 'permissions' => [Permission::create(Role::user($this->getUser()['$id']))] ]); $this->assertEquals(201, $col['headers']['status-code']); $collectionId = $col['body']['$id']; diff --git a/tests/e2e/Services/Databases/VectorDB/DatabasesConsoleClientTest.php b/tests/e2e/Services/Databases/VectorDB/DatabasesConsoleClientTest.php index 7f33777b07..8d711e68a5 100644 --- a/tests/e2e/Services/Databases/VectorDB/DatabasesConsoleClientTest.php +++ b/tests/e2e/Services/Databases/VectorDB/DatabasesConsoleClientTest.php @@ -40,7 +40,7 @@ class DatabasesConsoleClientTest extends Scope ], $this->getHeaders()), [ 'collectionId' => ID::unique(), 'name' => 'Movies', - 'dimensions' => 3, + 'dimension' => 3, 'permissions' => [ Permission::read(Role::any()), Permission::create(Role::any()), @@ -72,7 +72,7 @@ class DatabasesConsoleClientTest extends Scope ], $this->getHeaders()), [ 'collectionId' => ID::unique(), 'name' => 'TvShows', - 'dimensions' => 3, + 'dimension' => 3, 'permissions' => [ Permission::read(Role::any()), Permission::create(Role::any()), diff --git a/tests/e2e/Services/Databases/VectorDB/DatabasesCustomClientTest.php b/tests/e2e/Services/Databases/VectorDB/DatabasesCustomClientTest.php index fafc87e654..cc99806848 100644 --- a/tests/e2e/Services/Databases/VectorDB/DatabasesCustomClientTest.php +++ b/tests/e2e/Services/Databases/VectorDB/DatabasesCustomClientTest.php @@ -42,7 +42,7 @@ class DatabasesCustomClientTest extends Scope ]), [ 'collectionId' => ID::unique(), 'name' => 'Movies', - 'dimensions' => 3, + 'dimension' => 3, 'documentSecurity' => true, 'permissions' => [ Permission::write(Role::user($this->getUser()['$id'])), @@ -129,7 +129,7 @@ class DatabasesCustomClientTest extends Scope ]), [ 'collectionId' => ID::custom('permissionCheck'), 'name' => 'permissionCheck', - 'dimensions' => 3, + 'dimension' => 3, 'permissions' => [], 'documentSecurity' => true, ]); diff --git a/tests/e2e/Services/Databases/VectorDB/DatabasesCustomServerTest.php b/tests/e2e/Services/Databases/VectorDB/DatabasesCustomServerTest.php index 4203f5fa02..7f6aeeccbd 100644 --- a/tests/e2e/Services/Databases/VectorDB/DatabasesCustomServerTest.php +++ b/tests/e2e/Services/Databases/VectorDB/DatabasesCustomServerTest.php @@ -154,7 +154,7 @@ class DatabasesCustomServerTest extends Scope Permission::delete(Role::any()), ], 'documentSecurity' => true, - 'dimensions' => 3, + 'dimension' => 3, ]); $this->assertEquals(201, $col1['headers']['status-code']); // Validate collection response model on create @@ -163,7 +163,7 @@ class DatabasesCustomServerTest extends Scope $this->assertArrayHasKey('$updatedAt', $col1['body']); $this->assertArrayHasKey('enabled', $col1['body']); $this->assertArrayHasKey('documentSecurity', $col1['body']); - $this->assertArrayHasKey('dimensions', $col1['body']); + $this->assertArrayHasKey('dimension', $col1['body']); $col2 = $this->client->call(Client::METHOD_POST, '/vectordb/' . $databaseId . '/collections', [ 'content-type' => 'application/json', @@ -179,7 +179,7 @@ class DatabasesCustomServerTest extends Scope Permission::delete(Role::any()), ], 'documentSecurity' => true, - 'dimensions' => 3, + 'dimension' => 3, ]); $this->assertEquals(201, $col2['headers']['status-code']); $this->assertArrayHasKey('$id', $col2['body']); @@ -198,7 +198,7 @@ class DatabasesCustomServerTest extends Scope $this->assertIsArray($list['body']['collections']); $this->assertArrayHasKey('$id', $list['body']['collections'][0]); $this->assertArrayHasKey('name', $list['body']['collections'][0]); - $this->assertArrayHasKey('dimensions', $list['body']['collections'][0]); + $this->assertArrayHasKey('dimension', $list['body']['collections'][0]); // Get collection $get = $this->client->call(Client::METHOD_GET, '/vectordb/' . $databaseId . '/collections/' . $col1['body']['$id'], [ @@ -209,7 +209,7 @@ class DatabasesCustomServerTest extends Scope $this->assertEquals(200, $get['headers']['status-code']); $this->assertEquals($col1['body']['$id'], $get['body']['$id']); $this->assertEquals('Test 1', $get['body']['name']); - $this->assertEquals(3, $get['body']['dimensions']); + $this->assertEquals(3, $get['body']['dimension']); // Update collection (name only) $upd = $this->client->call(Client::METHOD_PUT, '/vectordb/' . $databaseId . '/collections/' . $col1['body']['$id'], [ @@ -253,11 +253,11 @@ class DatabasesCustomServerTest extends Scope 'x-appwrite-key' => $this->getProject()['apiKey'] ], [ 'name' => 'Test 1 Renamed', - 'dimensions' => 4, + 'dimension' => 4, ]); $this->assertEquals(200, $upd['headers']['status-code']); $this->assertEquals('Test 1 Renamed', $upd['body']['name']); - $this->assertEquals(4, $upd['body']['dimensions']); + $this->assertEquals(4, $upd['body']['dimension']); // Read back to confirm $get = $this->client->call(Client::METHOD_GET, '/vectordb/' . $databaseId . '/collections/' . $collectionId, [ @@ -267,7 +267,7 @@ class DatabasesCustomServerTest extends Scope ]); $this->assertEquals(200, $get['headers']['status-code']); $this->assertEquals('Test 1 Renamed', $get['body']['name']); - $this->assertEquals(4, $get['body']['dimensions']); + $this->assertEquals(4, $get['body']['dimension']); return $data; } @@ -506,7 +506,7 @@ class DatabasesCustomServerTest extends Scope 'collectionId' => ID::unique(), 'name' => 'BulkColCreate', 'documentSecurity' => true, - 'dimensions' => 3, + 'dimension' => 3, 'permissions' => [Permission::read(Role::any())] ]); $this->assertEquals(201, $col['headers']['status-code']); @@ -582,7 +582,7 @@ class DatabasesCustomServerTest extends Scope 'collectionId' => ID::unique(), 'name' => 'EmbedCol', 'documentSecurity' => true, - 'dimensions' => 3, + 'dimension' => 3, 'permissions' => [Permission::read(Role::any())] ]); $this->assertEquals(201, $col['headers']['status-code']); @@ -594,10 +594,11 @@ class DatabasesCustomServerTest extends Scope 'x-appwrite-project' => $this->getProject()['$id'], 'x-appwrite-key' => $this->getProject()['apiKey'] ], [ - 'documents' => [ - ['text' => 'hello world', 'embeddingModel' => 'embeddinggemma'], - ['text' => 'second sentence', 'embeddingModel' => 'embeddinggemma'], - ] + 'embeddingModel' => 'embeddinggemma', + 'texts' => [ + 'hello world', + 'second sentence', + ], ]); $this->assertEquals(200, $ok['headers']['status-code']); $this->assertIsInt($ok['body']['total'] ?? 0); @@ -606,56 +607,42 @@ class DatabasesCustomServerTest extends Scope $this->assertCount(2, $ok['body']['embeddings']); foreach ($ok['body']['embeddings'] as $embed) { $this->assertIsString($embed['model']); - $this->assertIsInt($embed['dimensions']); - $this->assertIsArray($embed['embeddings']); - $this->assertGreaterThan(0, count($embed['embeddings'])); + $this->assertIsInt($embed['dimension']); + $this->assertIsArray($embed['embedding']); + $this->assertGreaterThan(0, count($embed['embedding'])); + $this->assertArrayHasKey('error', $embed); } - // Error: missing documents - $missingDocs = $this->client->call(Client::METHOD_POST, "/vectordb/embeddings/text", [ + // Error: missing texts payload + $missingTexts = $this->client->call(Client::METHOD_POST, "/vectordb/embeddings/text", [ 'content-type' => 'application/json', 'x-appwrite-project' => $this->getProject()['$id'], 'x-appwrite-key' => $this->getProject()['apiKey'] ], []); - $this->assertEquals(400, $missingDocs['headers']['status-code']); + $this->assertEquals(400, $missingTexts['headers']['status-code']); - // Error: invalid item structure (not an object) + // Error: invalid texts item type (must be strings) $invalidItem = $this->client->call(Client::METHOD_POST, "/vectordb/embeddings/text", [ 'content-type' => 'application/json', 'x-appwrite-project' => $this->getProject()['$id'], 'x-appwrite-key' => $this->getProject()['apiKey'] ], [ - 'documents' => [ 'oops' ] + 'embeddingModel' => 'embeddinggemma', + 'texts' => [ + 'valid text', + 123, // invalid, not a string + ], ]); $this->assertEquals(400, $invalidItem['headers']['status-code']); - // Error: missing text - $missingText = $this->client->call(Client::METHOD_POST, "/vectordb/embeddings/text", [ - 'content-type' => 'application/json', - 'x-appwrite-project' => $this->getProject()['$id'], - 'x-appwrite-key' => $this->getProject()['apiKey'] - ], [ - 'documents' => [ ['embeddingModel' => 'embeddinggemma'] ] - ]); - $this->assertEquals(400, $missingText['headers']['status-code']); - - // Error: missing embeddingModel - $missingModel = $this->client->call(Client::METHOD_POST, "/vectordb/embeddings/text", [ - 'content-type' => 'application/json', - 'x-appwrite-project' => $this->getProject()['$id'], - 'x-appwrite-key' => $this->getProject()['apiKey'] - ], [ - 'documents' => [ ['text' => 'no model'] ] - ]); - $this->assertEquals(400, $missingModel['headers']['status-code']); - // Error: unknown embedding model $unknownModel = $this->client->call(Client::METHOD_POST, "/vectordb/embeddings/text", [ 'content-type' => 'application/json', 'x-appwrite-project' => $this->getProject()['$id'], 'x-appwrite-key' => $this->getProject()['apiKey'] ], [ - 'documents' => [ ['text' => 'hello', 'embeddingModel' => 'nonexistent-model'] ] + 'embeddingModel' => 'nonexistent-model', + 'texts' => ['hello'], ]); $this->assertEquals(400, $unknownModel['headers']['status-code']); } @@ -679,7 +666,7 @@ class DatabasesCustomServerTest extends Scope 'collectionId' => ID::unique(), 'name' => 'BulkColUpsert', 'documentSecurity' => true, - 'dimensions' => 3, + 'dimension' => 3, 'permissions' => [Permission::read(Role::any())] ]); $this->assertEquals(201, $col['headers']['status-code']); @@ -772,7 +759,7 @@ class DatabasesCustomServerTest extends Scope 'collectionId' => ID::unique(), 'name' => 'BulkColUpdate', 'documentSecurity' => true, - 'dimensions' => 3, + 'dimension' => 3, 'permissions' => [Permission::read(Role::any())] ]); $this->assertEquals(201, $col['headers']['status-code']); @@ -840,7 +827,7 @@ class DatabasesCustomServerTest extends Scope 'collectionId' => ID::unique(), 'name' => 'BulkColDelete', 'documentSecurity' => true, - 'dimensions' => 3, + 'dimension' => 3, 'permissions' => [Permission::read(Role::any())] ]); $this->assertEquals(201, $col['headers']['status-code']); @@ -905,7 +892,7 @@ class DatabasesCustomServerTest extends Scope 'collectionId' => ID::unique(), 'name' => 'TimestampTestCollection', 'documentSecurity' => true, - 'dimensions' => 1536, + 'dimension' => 1536, 'permissions' => [Permission::read(Role::any())] ]); $this->assertEquals(201, $col['headers']['status-code']); diff --git a/tests/e2e/Services/Databases/VectorDB/Permissions/DatabasesPermissionsGuestTest.php b/tests/e2e/Services/Databases/VectorDB/Permissions/DatabasesPermissionsGuestTest.php index 6851427a09..70f3b0946d 100644 --- a/tests/e2e/Services/Databases/VectorDB/Permissions/DatabasesPermissionsGuestTest.php +++ b/tests/e2e/Services/Databases/VectorDB/Permissions/DatabasesPermissionsGuestTest.php @@ -34,7 +34,7 @@ class DatabasesPermissionsGuestTest extends Scope $publicMovies = $this->client->call(Client::METHOD_POST, '/vectordb/' . $databaseId . '/collections', $this->getServerHeader(), [ 'collectionId' => ID::unique(), 'name' => 'Movies', - 'dimensions' => 3, + 'dimension' => 3, 'permissions' => [ Permission::read(Role::any()), Permission::create(Role::any()), @@ -45,7 +45,7 @@ class DatabasesPermissionsGuestTest extends Scope $privateMovies = $this->client->call(Client::METHOD_POST, '/vectordb/' . $databaseId . '/collections', $this->getServerHeader(), [ 'collectionId' => ID::unique(), 'name' => 'Movies', - 'dimensions' => 3, + 'dimension' => 3, 'permissions' => [], 'documentSecurity' => true, ]); @@ -239,7 +239,7 @@ class DatabasesPermissionsGuestTest extends Scope $movies = $this->client->call(Client::METHOD_POST, '/vectordb/' . $databaseId . '/collections', $this->getServerHeader(), [ 'collectionId' => ID::unique(), 'name' => 'Movies', - 'dimensions' => 3, + 'dimension' => 3, 'permissions' => [ Permission::create(Role::any()), ], diff --git a/tests/e2e/Services/Databases/VectorDB/Permissions/DatabasesPermissionsMemberTest.php b/tests/e2e/Services/Databases/VectorDB/Permissions/DatabasesPermissionsMemberTest.php index d45037999d..f0f34f7f10 100644 --- a/tests/e2e/Services/Databases/VectorDB/Permissions/DatabasesPermissionsMemberTest.php +++ b/tests/e2e/Services/Databases/VectorDB/Permissions/DatabasesPermissionsMemberTest.php @@ -127,7 +127,7 @@ class DatabasesPermissionsMemberTest extends Scope $public = $this->client->call(Client::METHOD_POST, '/vectordb/' . $databaseId . '/collections', $this->getServerHeader(), [ 'collectionId' => ID::unique(), 'name' => 'Movies', - 'dimensions' => 3, + 'dimension' => 3, 'permissions' => [ Permission::read(Role::any()), Permission::create(Role::any()), @@ -142,7 +142,7 @@ class DatabasesPermissionsMemberTest extends Scope $private = $this->client->call(Client::METHOD_POST, '/vectordb/' . $databaseId . '/collections', $this->getServerHeader(), [ 'collectionId' => ID::unique(), 'name' => 'Private Movies', - 'dimensions' => 3, + 'dimension' => 3, 'permissions' => [ Permission::read(Role::users()), Permission::create(Role::users()), @@ -157,7 +157,7 @@ class DatabasesPermissionsMemberTest extends Scope $doconly = $this->client->call(Client::METHOD_POST, '/vectordb/' . $databaseId . '/collections', $this->getServerHeader(), [ 'collectionId' => ID::unique(), 'name' => 'Document Only Movies', - 'dimensions' => 3, + 'dimension' => 3, 'permissions' => [], 'documentSecurity' => true, ]); diff --git a/tests/e2e/Services/Databases/VectorDB/Permissions/DatabasesPermissionsTeamTest.php b/tests/e2e/Services/Databases/VectorDB/Permissions/DatabasesPermissionsTeamTest.php index 4f2a210ef2..9dbace79a2 100644 --- a/tests/e2e/Services/Databases/VectorDB/Permissions/DatabasesPermissionsTeamTest.php +++ b/tests/e2e/Services/Databases/VectorDB/Permissions/DatabasesPermissionsTeamTest.php @@ -47,7 +47,7 @@ class DatabasesPermissionsTeamTest extends Scope $collection1 = $this->client->call(Client::METHOD_POST, '/vectordb/' . $this->databaseId . '/collections', $this->getServerHeader(), [ 'collectionId' => ID::custom('collection1'), 'name' => 'Collection 1', - 'dimensions' => 3, + 'dimension' => 3, 'permissions' => [ Permission::read(Role::team($teams['team1']['$id'])), Permission::create(Role::team($teams['team1']['$id'], 'admin')), @@ -61,7 +61,7 @@ class DatabasesPermissionsTeamTest extends Scope $collection2 = $this->client->call(Client::METHOD_POST, '/vectordb/' . $this->databaseId . '/collections', $this->getServerHeader(), [ 'collectionId' => ID::custom('collection2'), 'name' => 'Collection 2', - 'dimensions' => 3, + 'dimension' => 3, 'permissions' => [ Permission::read(Role::team($teams['team2']['$id'])), Permission::create(Role::team($teams['team2']['$id'], 'owner')), diff --git a/tests/e2e/Services/Databases/VectorDB/Transactions/ACIDTest.php b/tests/e2e/Services/Databases/VectorDB/Transactions/ACIDTest.php index 4de783f2f8..6e0e2dbd28 100644 --- a/tests/e2e/Services/Databases/VectorDB/Transactions/ACIDTest.php +++ b/tests/e2e/Services/Databases/VectorDB/Transactions/ACIDTest.php @@ -48,7 +48,7 @@ class ACIDTest extends Scope ]), [ 'collectionId' => ID::unique(), 'name' => 'AtomicityTest', - 'dimensions' => 3, + 'dimension' => 3, 'documentSecurity' => false, 'permissions' => [ Permission::create(Role::any()), @@ -160,7 +160,7 @@ class ACIDTest extends Scope ]), [ 'collectionId' => ID::unique(), 'name' => 'ConsistencyTest', - 'dimensions' => 3, + 'dimension' => 3, 'documentSecurity' => false, 'permissions' => [ Permission::create(Role::any()), @@ -268,7 +268,7 @@ class ACIDTest extends Scope ]), [ 'collectionId' => ID::unique(), 'name' => 'IsolationTest', - 'dimensions' => 3, + 'dimension' => 3, 'documentSecurity' => false, 'permissions' => [ Permission::create(Role::any()), @@ -416,7 +416,7 @@ class ACIDTest extends Scope ]), [ 'collectionId' => ID::unique(), 'name' => 'DurabilityTest', - 'dimensions' => 3, + 'dimension' => 3, 'documentSecurity' => false, 'permissions' => [ Permission::create(Role::any()), diff --git a/tests/e2e/Services/Databases/VectorDB/Transactions/TransactionsBase.php b/tests/e2e/Services/Databases/VectorDB/Transactions/TransactionsBase.php index 1462921ef1..6fe6c9e3f7 100644 --- a/tests/e2e/Services/Databases/VectorDB/Transactions/TransactionsBase.php +++ b/tests/e2e/Services/Databases/VectorDB/Transactions/TransactionsBase.php @@ -128,7 +128,7 @@ trait TransactionsBase ]), [ 'collectionId' => ID::unique(), 'name' => 'TransactionOperationsTest', - 'dimensions' => 3, + 'dimension' => 3, 'documentSecurity' => false, 'permissions' => [ Permission::create(Role::any()), @@ -267,7 +267,7 @@ trait TransactionsBase ]), [ 'collectionId' => ID::unique(), 'name' => 'TransactionCommitTest', - 'dimensions' => 3, + 'dimension' => 3, 'documentSecurity' => false, 'permissions' => [ Permission::create(Role::any()), @@ -409,7 +409,7 @@ trait TransactionsBase ]), [ 'collectionId' => ID::unique(), 'name' => 'TransactionRollbackTest', - 'dimensions' => 3, + 'dimension' => 3, 'documentSecurity' => false, 'permissions' => [ Permission::create(Role::any()), @@ -489,7 +489,7 @@ trait TransactionsBase ]), [ 'collectionId' => ID::unique(), 'name' => 'TestCollection', - 'dimensions' => 3, + 'dimension' => 3, 'permissions' => [ Permission::read(Role::any()), Permission::create(Role::any()), @@ -575,7 +575,7 @@ trait TransactionsBase ]), [ 'collectionId' => ID::unique(), 'name' => 'TestCollection', - 'dimensions' => 3, + 'dimension' => 3, 'permissions' => [Permission::create(Role::any())], ]); @@ -690,7 +690,7 @@ trait TransactionsBase ]), [ 'collectionId' => ID::unique(), 'name' => 'TestCollection', - 'dimensions' => 3, + 'dimension' => 3, 'permissions' => [ Permission::read(Role::any()), Permission::create(Role::any()), @@ -821,7 +821,7 @@ trait TransactionsBase ]), [ 'collectionId' => ID::unique(), 'name' => 'TestCollection', - 'dimensions' => 3, + 'dimension' => 3, 'permissions' => [ Permission::read(Role::any()), Permission::create(Role::any()), @@ -919,7 +919,7 @@ trait TransactionsBase ]), [ 'collectionId' => ID::unique(), 'name' => 'TestCollection', - 'dimensions' => 3, + 'dimension' => 3, 'permissions' => [ Permission::read(Role::any()), Permission::create(Role::any()), @@ -1079,7 +1079,7 @@ trait TransactionsBase ]), [ 'collectionId' => ID::unique(), 'name' => 'TestCollection', - 'dimensions' => 3, + 'dimension' => 3, 'permissions' => [ Permission::read(Role::any()), Permission::create(Role::any()), @@ -1220,7 +1220,7 @@ trait TransactionsBase ]), [ 'collectionId' => ID::unique(), 'name' => 'TestCollection', - 'dimensions' => 3, + 'dimension' => 3, 'permissions' => [Permission::create(Role::any())], ]); @@ -1331,7 +1331,7 @@ trait TransactionsBase ]), [ 'collectionId' => ID::unique(), 'name' => 'TestCollection', - 'dimensions' => 3, + 'dimension' => 3, 'permissions' => [ Permission::create(Role::any()), Permission::update(Role::any()), @@ -1421,7 +1421,7 @@ trait TransactionsBase ]), [ 'collectionId' => ID::unique(), 'name' => 'TestCollection', - 'dimensions' => 3, + 'dimension' => 3, 'documentSecurity' => false, 'permissions' => [ Permission::create(Role::any()), @@ -1515,7 +1515,7 @@ trait TransactionsBase ]), [ 'collectionId' => ID::unique(), 'name' => 'TestCollection', - 'dimensions' => 3, + 'dimension' => 3, 'permissions' => [ Permission::create(Role::any()), Permission::read(Role::any()), @@ -1624,7 +1624,7 @@ trait TransactionsBase ]), [ 'collectionId' => ID::unique(), 'name' => 'TestCollection', - 'dimensions' => 3, + 'dimension' => 3, 'permissions' => [ Permission::create(Role::any()), Permission::read(Role::any()), @@ -1733,7 +1733,7 @@ trait TransactionsBase ]), [ 'collectionId' => ID::unique(), 'name' => 'TestCollection', - 'dimensions' => 3, + 'dimension' => 3, 'permissions' => [ Permission::create(Role::any()), Permission::read(Role::any()), @@ -1829,7 +1829,7 @@ trait TransactionsBase ]), [ 'collectionId' => ID::unique(), 'name' => 'TestCollection', - 'dimensions' => 3, + 'dimension' => 3, 'permissions' => [ Permission::create(Role::any()), Permission::read(Role::any()), @@ -1959,7 +1959,7 @@ trait TransactionsBase ]), [ 'collectionId' => ID::unique(), 'name' => 'TestCollection', - 'dimensions' => 3, + 'dimension' => 3, 'permissions' => [ Permission::create(Role::any()), Permission::read(Role::any()), @@ -2064,7 +2064,7 @@ trait TransactionsBase ]), [ 'collectionId' => ID::unique(), 'name' => 'TestCollection', - 'dimensions' => 3, + 'dimension' => 3, 'permissions' => [ Permission::create(Role::any()), Permission::read(Role::any()), diff --git a/tests/e2e/Services/Migrations/MigrationsBase.php b/tests/e2e/Services/Migrations/MigrationsBase.php index 32004e5b48..b50ee5e14c 100644 --- a/tests/e2e/Services/Migrations/MigrationsBase.php +++ b/tests/e2e/Services/Migrations/MigrationsBase.php @@ -1406,7 +1406,7 @@ trait MigrationsBase ], [ 'collectionId' => ID::unique(), 'name' => 'Vector CSV Import Collection', - 'dimensions' => 3, + 'dimension' => 3, 'documentSecurity' => true, ]); @@ -1527,7 +1527,7 @@ trait MigrationsBase ], [ 'collectionId' => ID::unique(), 'name' => 'Vector CSV Export Collection', - 'dimensions' => 3, + 'dimension' => 3, 'documentSecurity' => true, ]); @@ -1761,7 +1761,7 @@ trait MigrationsBase ], [ 'collectionId' => ID::unique(), 'name' => 'VDB - Movies', - 'dimensions' => 3, + 'dimension' => 3, ]); $this->assertEquals(201, $collection['headers']['status-code']); @@ -2207,7 +2207,7 @@ trait MigrationsBase ], [ 'collectionId' => ID::unique(), 'name' => 'Products', - 'dimensions' => 3, + 'dimension' => 3, ]); $this->assertEquals(201, $vectorCollection['headers']['status-code']); diff --git a/tests/e2e/Services/Realtime/RealtimeCustomClientTest.php b/tests/e2e/Services/Realtime/RealtimeCustomClientTest.php index 1121d99302..19371a2b6d 100644 --- a/tests/e2e/Services/Realtime/RealtimeCustomClientTest.php +++ b/tests/e2e/Services/Realtime/RealtimeCustomClientTest.php @@ -4347,7 +4347,7 @@ class RealtimeCustomClientTest extends Scope Permission::create(Role::user($this->getUser()['$id'])), ], 'documentSecurity' => true, - 'dimensions' => 3, + 'dimension' => 3, ]); $actorsId = $actors['body']['$id']; From ca4c7b536197bb81a871799b32dbd55277a78ee5 Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Tue, 2 Dec 2025 18:13:49 +0530 Subject: [PATCH 21/27] added stats usage for text embeddings --- app/controllers/api/project.php | 9 +++ app/init/constants.php | 4 ++ composer.lock | 12 ++-- .../Http/VectorDB/Embeddings/Text/Create.php | 43 ++++++++++++-- .../Utopia/Response/Model/UsageProject.php | 21 +++++++ tests/e2e/General/UsageTest.php | 57 +++++++++++++++++++ 6 files changed, 135 insertions(+), 11 deletions(-) diff --git a/app/controllers/api/project.php b/app/controllers/api/project.php index 41200aed35..aefe9d883a 100644 --- a/app/controllers/api/project.php +++ b/app/controllers/api/project.php @@ -71,6 +71,9 @@ App::get('/v1/project/usage') METRIC_DATABASES_OPERATIONS_READS, METRIC_DATABASES_OPERATIONS_WRITES, METRIC_FILES_IMAGES_TRANSFORMED, + METRIC_EMBEDDINGS_TEXT, + METRIC_EMBEDDINGS_TEXT_TOTAL_TOKENS, + METRIC_EMBEDDINGS_TEXT_TOTAL_DURATION, ], 'period' => [ METRIC_NETWORK_REQUESTS, @@ -84,6 +87,9 @@ App::get('/v1/project/usage') METRIC_DATABASES_OPERATIONS_READS, METRIC_DATABASES_OPERATIONS_WRITES, METRIC_FILES_IMAGES_TRANSFORMED, + METRIC_EMBEDDINGS_TEXT, + METRIC_EMBEDDINGS_TEXT_TOTAL_TOKENS, + METRIC_EMBEDDINGS_TEXT_TOTAL_DURATION, ] ]; @@ -379,6 +385,9 @@ App::get('/v1/project/usage') 'authPhoneCountryBreakdown' => $authPhoneCountryBreakdown, 'imageTransformations' => $usage[METRIC_FILES_IMAGES_TRANSFORMED], 'imageTransformationsTotal' => $total[METRIC_FILES_IMAGES_TRANSFORMED], + 'embeddingsText' => $usage[METRIC_EMBEDDINGS_TEXT] ?? [], + 'embeddingsTextTokens' => $usage[METRIC_EMBEDDINGS_TEXT_TOTAL_TOKENS] ?? [], + 'embeddingsTextDuration' => $usage[METRIC_EMBEDDINGS_TEXT_TOTAL_DURATION] ?? [], ]), Response::MODEL_USAGE_PROJECT); }); diff --git a/app/init/constants.php b/app/init/constants.php index 7146e6e118..6c0058f9b0 100644 --- a/app/init/constants.php +++ b/app/init/constants.php @@ -274,6 +274,9 @@ const METRIC_SITES_ID_OUTBOUND = 'sites.{siteInternalId}.outbound'; const METRIC_AVATARS_SCREENSHOTS_GENERATED = 'avatars.screenshotsGenerated'; const METRIC_FUNCTIONS_RUNTIME = 'functions.runtimes.{runtime}'; const METRIC_SITES_FRAMEWORK = 'sites.frameworks.{framework}'; +const METRIC_EMBEDDINGS_TEXT = 'embeddings.text.{embeddingModel}'; +const METRIC_EMBEDDINGS_TEXT_TOTAL_DURATION = 'embeddings.text.{embeddingModel}.totalDuration'; +const METRIC_EMBEDDINGS_TEXT_TOTAL_TOKENS = 'embeddings.text.{embeddingModel}.totalTokens'; // Resource types const RESOURCE_TYPE_PROJECTS = 'projects'; @@ -286,6 +289,7 @@ const RESOURCE_TYPE_TOPICS = 'topics'; const RESOURCE_TYPE_SUBSCRIBERS = 'subscribers'; const RESOURCE_TYPE_MESSAGES = 'messages'; const RESOURCE_TYPE_EXECUTIONS = 'executions'; +const RESOURCE_TYPE_EMBEDDINGS_TEXT = 'embeddingsText'; // Resource types for Tokens const TOKENS_RESOURCE_TYPE_FILES = 'files'; diff --git a/composer.lock b/composer.lock index 32dfedd3be..09d29b9418 100644 --- a/composer.lock +++ b/composer.lock @@ -3500,16 +3500,16 @@ }, { "name": "utopia-php/agents", - "version": "0.5.0", + "version": "0.6.0", "source": { "type": "git", "url": "https://github.com/utopia-php/agents.git", - "reference": "7bbc77d4936ee7e87394a239690c9059b5cdaf1b" + "reference": "ab026e44d332598852b4606808d9cd3bf1de9c96" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/agents/zipball/7bbc77d4936ee7e87394a239690c9059b5cdaf1b", - "reference": "7bbc77d4936ee7e87394a239690c9059b5cdaf1b", + "url": "https://api.github.com/repos/utopia-php/agents/zipball/ab026e44d332598852b4606808d9cd3bf1de9c96", + "reference": "ab026e44d332598852b4606808d9cd3bf1de9c96", "shasum": "" }, "require": { @@ -3548,9 +3548,9 @@ ], "support": { "issues": "https://github.com/utopia-php/agents/issues", - "source": "https://github.com/utopia-php/agents/tree/0.5.0" + "source": "https://github.com/utopia-php/agents/tree/0.6.0" }, - "time": "2025-11-12T06:06:41+00:00" + "time": "2025-12-02T10:51:32+00:00" }, { "name": "utopia-php/analytics", diff --git a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Embeddings/Text/Create.php b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Embeddings/Text/Create.php index b5d1be8f98..704d009fe7 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Embeddings/Text/Create.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Embeddings/Text/Create.php @@ -2,6 +2,7 @@ namespace Appwrite\Platform\Modules\Databases\Http\VectorDB\Embeddings\Text; +use Appwrite\Event\StatsUsage; use Appwrite\Platform\Modules\Databases\Http\Databases\Collections\Documents\Action as CreateDocumentAction; use Appwrite\SDK\AuthType; use Appwrite\SDK\ContentType; @@ -41,8 +42,8 @@ class Create extends CreateDocumentAction ->setHttpPath('/v1/vectordb/embeddings/text') ->desc('Create Text Embeddings') ->groups(['api', 'database']) - ->label('scope', 'documents.read') - ->label('resourceType', RESOURCE_TYPE_DATABASES) + ->label('scope', 'documents.write') + ->label('resourceType', RESOURCE_TYPE_EMBEDDINGS_TEXT) ->label('audits.event', 'embedding.create') ->label('audits.resource', 'vectordb/embeddings/text') ->label('abuse-key', 'ip:{ip},method:{method},url:{url},userId:{userId}') @@ -74,23 +75,27 @@ class Create extends CreateDocumentAction ->param('texts', [], fn (array $plan) => new ArrayList(new Text(0), $plan['databasesBatchSize'] ?? APP_LIMIT_DATABASE_BATCH), 'Array of text to generate embeddings.', true, ['plan']) ->inject('response') ->inject('embeddingAgent') + ->inject('queueForStatsUsage') ->callback($this->action(...)); } - public function action(string $embeddingModel, array $texts, UtopiaResponse $response, Agent $embeddingAgent): void + public function action(string $embeddingModel, array $texts, UtopiaResponse $response, Agent $embeddingAgent, StatsUsage $queueForStatsUsage): void { $results = []; $embeddingAgent->getAdapter()->setModel($embeddingModel); $dimension = $embeddingAgent->getAdapter()->getEmbeddingDimension(); + $totalDuration = 0; + $totalTokens = 0; foreach ($texts as $text) { - $embedding = []; $error = ''; try { $embedResult = $embeddingAgent->embed($text); $embedding = $embedResult['embedding'] ?? []; - } catch (\Exception $e) { + $totalDuration += $embedResult['totalDuration'] ?? 0; + $totalTokens += $embedResult['tokensProcessed'] ?? 0; + } catch (\Exception) { $error = 'Error while generating embedding'; } @@ -109,5 +114,33 @@ class Create extends CreateDocumentAction $response ->setStatusCode(SwooleResponse::STATUS_CODE_OK) ->dynamic($embeddings, $this->getBulkResponseModel()); + + $queueForStatsUsage + ->addMetric( + \str_replace( + '{embeddingModel}', + $embeddingModel, + METRIC_EMBEDDINGS_TEXT + ), + \count($texts) + ) + ->addMetric( + \str_replace( + '{embeddingModel}', + $embeddingModel, + METRIC_EMBEDDINGS_TEXT_TOTAL_TOKENS + ), + $totalTokens + ) + ->addMetric( + \str_replace( + '{embeddingModel}', + $embeddingModel, + METRIC_EMBEDDINGS_TEXT_TOTAL_DURATION + ), + $totalDuration + ) + ->trigger(); + $queueForStatsUsage->reset(); } } diff --git a/src/Appwrite/Utopia/Response/Model/UsageProject.php b/src/Appwrite/Utopia/Response/Model/UsageProject.php index ee644aa845..d9fec1e4a1 100644 --- a/src/Appwrite/Utopia/Response/Model/UsageProject.php +++ b/src/Appwrite/Utopia/Response/Model/UsageProject.php @@ -216,6 +216,27 @@ class UsageProject extends Model 'default' => 0, 'example' => 0, ]) + ->addRule('embeddingsText', [ + 'type' => Response::MODEL_METRIC, + 'description' => 'Aggregated number of text embedding calls per period.', + 'default' => [], + 'example' => [], + 'array' => true + ]) + ->addRule('embeddingsTextTokens', [ + 'type' => Response::MODEL_METRIC, + 'description' => 'Aggregated number of tokens processed by text embeddings per period.', + 'default' => [], + 'example' => [], + 'array' => true + ]) + ->addRule('embeddingsTextDuration', [ + 'type' => Response::MODEL_METRIC, + 'description' => 'Aggregated duration spent generating text embeddings per period.', + 'default' => [], + 'example' => [], + 'array' => true + ]) ; } diff --git a/tests/e2e/General/UsageTest.php b/tests/e2e/General/UsageTest.php index b384d4812b..ff9e0f0ec8 100644 --- a/tests/e2e/General/UsageTest.php +++ b/tests/e2e/General/UsageTest.php @@ -1589,6 +1589,63 @@ class UsageTest extends Scope }); } + public function testEmbeddingsTextUsageDoesNotBreakProjectUsage(): void + { + // Trigger embeddings endpoint a few times so stats usage worker has data to aggregate + for ($i = 0; $i < 3; $i++) { + $response = $this->client->call( + Client::METHOD_POST, + '/vectordb/embeddings/text', + array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ], $this->getHeaders()), + [ + 'embeddingModel' => 'embeddinggemma', + 'texts' => [ + 'usage test text ' . $i, + ], + ] + ); + + $this->assertEquals(200, $response['headers']['status-code']); + $this->assertIsArray($response['body']['embeddings']); + $this->assertGreaterThan(0, $response['body']['total']); + } + + // Ensure project usage endpoint still responds correctly after embeddings calls + $this->assertEventually(function () { + $response = $this->client->call( + Client::METHOD_GET, + '/project/usage', + $this->getConsoleHeaders(), + [ + 'period' => '1h', + 'startDate' => self::getToday(), + 'endDate' => self::getTomorrow(), + ] + ); + + $this->assertEquals(200, $response['headers']['status-code']); + $this->assertArrayHasKey('requests', $response['body']); + $this->assertArrayHasKey('network', $response['body']); + $this->assertArrayHasKey('executionsTotal', $response['body']); + + // New embeddings metrics should be present after calls above + $this->assertArrayHasKey('embeddingsText', $response['body']); + $this->assertArrayHasKey('embeddingsTextTokens', $response['body']); + $this->assertArrayHasKey('embeddingsTextDuration', $response['body']); + $this->assertGreaterThanOrEqual(0, $response['body']['embeddingsText']); + $this->assertGreaterThanOrEqual(0, $response['body']['embeddingsTextTokens']); + $this->assertGreaterThanOrEqual(0, $response['body']['embeddingsTextDuration']); + + $this->validateDates($response['body']['embeddingsText']); + $this->validateDates($response['body']['embeddingsTextTokens']); + $this->validateDates($response['body']['embeddingsTextDuration']); + }); + } + public function tearDown(): void { $this->projectId = ''; From 069f231b3a50a6bb2e484d26434e5bb6299cb210 Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Tue, 2 Dec 2025 18:40:28 +0530 Subject: [PATCH 22/27] updated composer lock --- composer.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/composer.lock b/composer.lock index 2e6376b376..7c082386ce 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "c7d2810ea344ae2dcf2a7e1b66a54c18", + "content-hash": "86c19fdb3d6a9faf68a310d96def667f", "packages": [ { "name": "adhocore/jwt", From 7af361fa164510ecaae6996f9616539ea160d5eb Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Wed, 3 Dec 2025 13:38:24 +0530 Subject: [PATCH 23/27] * added logger in the create text embedding * aadded error text metric --- app/controllers/api/project.php | 3 ++ app/init/constants.php | 1 + .../VectorDB/Collections/Documents/Create.php | 2 +- .../Http/VectorDB/Embeddings/Text/Create.php | 51 +++++++++++++++---- .../Utopia/Response/Model/UsageProject.php | 7 +++ tests/e2e/General/UsageTest.php | 9 ++-- 6 files changed, 60 insertions(+), 13 deletions(-) diff --git a/app/controllers/api/project.php b/app/controllers/api/project.php index aefe9d883a..55d6fb8003 100644 --- a/app/controllers/api/project.php +++ b/app/controllers/api/project.php @@ -74,6 +74,7 @@ App::get('/v1/project/usage') METRIC_EMBEDDINGS_TEXT, METRIC_EMBEDDINGS_TEXT_TOTAL_TOKENS, METRIC_EMBEDDINGS_TEXT_TOTAL_DURATION, + METRIC_EMBEDDINGS_TEXT_TOTAL_ERROR ], 'period' => [ METRIC_NETWORK_REQUESTS, @@ -90,6 +91,7 @@ App::get('/v1/project/usage') METRIC_EMBEDDINGS_TEXT, METRIC_EMBEDDINGS_TEXT_TOTAL_TOKENS, METRIC_EMBEDDINGS_TEXT_TOTAL_DURATION, + METRIC_EMBEDDINGS_TEXT_TOTAL_ERROR ] ]; @@ -388,6 +390,7 @@ App::get('/v1/project/usage') 'embeddingsText' => $usage[METRIC_EMBEDDINGS_TEXT] ?? [], 'embeddingsTextTokens' => $usage[METRIC_EMBEDDINGS_TEXT_TOTAL_TOKENS] ?? [], 'embeddingsTextDuration' => $usage[METRIC_EMBEDDINGS_TEXT_TOTAL_DURATION] ?? [], + 'embeddingsTextErrors' => $usage[METRIC_EMBEDDINGS_TEXT_TOTAL_ERROR] ?? [], ]), Response::MODEL_USAGE_PROJECT); }); diff --git a/app/init/constants.php b/app/init/constants.php index 244d4441a9..e0b72bf1ad 100644 --- a/app/init/constants.php +++ b/app/init/constants.php @@ -338,6 +338,7 @@ const METRIC_AVATARS_SCREENSHOTS_GENERATED = 'avatars.screenshotsGenerated'; const METRIC_FUNCTIONS_RUNTIME = 'functions.runtimes.{runtime}'; const METRIC_SITES_FRAMEWORK = 'sites.frameworks.{framework}'; const METRIC_EMBEDDINGS_TEXT = 'embeddings.text.{embeddingModel}'; +const METRIC_EMBEDDINGS_TEXT_TOTAL_ERROR = 'embeddings.text.{embeddingModel}.totalErrors'; const METRIC_EMBEDDINGS_TEXT_TOTAL_DURATION = 'embeddings.text.{embeddingModel}.totalDuration'; const METRIC_EMBEDDINGS_TEXT_TOTAL_TOKENS = 'embeddings.text.{embeddingModel}.totalTokens'; diff --git a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Create.php b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Create.php index 6c331e40dd..5550e64fab 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Create.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Create.php @@ -95,7 +95,7 @@ class Create extends DocumentCreate ->param('databaseId', '', new UID(), 'Database ID.') ->param('documentId', '', new CustomId(), 'Document ID. Choose a custom ID or generate a random ID with `ID.unique()`. Valid chars are a-z, A-Z, 0-9, period, hyphen, and underscore. Can\'t start with a special char. Max length is 36 chars.', true) ->param('collectionId', '', new UID(), 'Collection ID. You can create a new collection using the Database service [server integration](https://appwrite.io/docs/server/databases#databasesCreateCollection). Make sure to define attributes before creating documents.') - ->param('data', [], new JSON(), 'Document data as JSON object.', true, example: '{"username":"walter.obrien","email":"walter.obrien@example.com","fullName":"Walter O\'Brien","age":30,"isAdmin":false}') + ->param('data', [], new JSON(), 'Document data as JSON object.', true, example: '{"embeddings": [0.12, -0.55, 0.88, 1.02], "metadata": {"key":"value"} }') ->param('permissions', null, new Permissions(APP_LIMIT_ARRAY_PARAMS_SIZE, [Database::PERMISSION_READ, Database::PERMISSION_UPDATE, Database::PERMISSION_DELETE, Database::PERMISSION_WRITE]), 'An array of permissions strings. By default, only the current user is granted all permissions. [Learn more about permissions](https://appwrite.io/docs/permissions).', true) ->param('documents', [], fn (array $plan) => new ArrayList(new JSON(), $plan['databasesBatchSize'] ?? APP_LIMIT_DATABASE_BATCH), 'Array of documents data as JSON objects.', true, ['plan']) ->param('transactionId', null, new UID(), 'Transaction ID for staging the operation.', true) diff --git a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Embeddings/Text/Create.php b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Embeddings/Text/Create.php index 704d009fe7..ba6a4c9a22 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Embeddings/Text/Create.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Embeddings/Text/Create.php @@ -13,7 +13,10 @@ use Appwrite\Utopia\Response as UtopiaResponse; use Utopia\Agents\Adapters\Ollama; use Utopia\Agents\Agent; use Utopia\Database\Document; +use Utopia\Logger\Log; +use Utopia\Logger\Logger; use Utopia\Swoole\Response as SwooleResponse; +use Utopia\System\System; use Utopia\Validator\ArrayList; use Utopia\Validator\Text; use Utopia\Validator\WhiteList; @@ -71,22 +74,26 @@ class Create extends CreateDocumentAction ] ) ]) - ->param('embeddingModel', Ollama::MODEL_EMBEDDING_GEMMA, new WhiteList(Ollama::MODELS), 'The embedding model to use for generating vector embeddings.', false) - ->param('texts', [], fn (array $plan) => new ArrayList(new Text(0), $plan['databasesBatchSize'] ?? APP_LIMIT_DATABASE_BATCH), 'Array of text to generate embeddings.', true, ['plan']) + ->param('texts', [], fn (array $plan) => new ArrayList(new Text(0), $plan['databasesBatchSize'] ?? APP_LIMIT_DATABASE_BATCH), 'Array of text to generate embeddings.', false, ['plan']) + ->param('model', Ollama::MODEL_EMBEDDING_GEMMA, new WhiteList(Ollama::MODELS), 'The embedding model to use for generating vector embeddings.', true) ->inject('response') + ->inject('project') ->inject('embeddingAgent') ->inject('queueForStatsUsage') + ->inject('log') + ->inject('logger') ->callback($this->action(...)); } - public function action(string $embeddingModel, array $texts, UtopiaResponse $response, Agent $embeddingAgent, StatsUsage $queueForStatsUsage): void + public function action(array $texts, string $model, UtopiaResponse $response, Document $project, Agent $embeddingAgent, StatsUsage $queueForStatsUsage, Log $log, ?Logger $logger): void { $results = []; - $embeddingAgent->getAdapter()->setModel($embeddingModel); + $embeddingAgent->getAdapter()->setModel($model); $dimension = $embeddingAgent->getAdapter()->getEmbeddingDimension(); $totalDuration = 0; $totalTokens = 0; + $totalErrors = 0; foreach ($texts as $text) { $embedding = []; $error = ''; @@ -95,12 +102,30 @@ class Create extends CreateDocumentAction $embedding = $embedResult['embedding'] ?? []; $totalDuration += $embedResult['totalDuration'] ?? 0; $totalTokens += $embedResult['tokensProcessed'] ?? 0; - } catch (\Exception) { + } catch (\Exception $e) { $error = 'Error while generating embedding'; + $totalErrors += 1; + if ($logger) { + $log->setNamespace("http"); + $log->setServer(System::getEnv('_APP_LOGGING_SERVICE_IDENTIFIER', \gethostname())); + $log->setVersion(System::getEnv('_APP_VERSION', 'UNKNOWN')); + $log->setType(Log::TYPE_ERROR); + $log->setMessage($e->getMessage()); + + $log->addTag('embeddingModel', $model); + $log->addTag('code', $e->getCode()); + $log->addTag('projectId', $project->getId()); + + $log->addExtra('file', $e->getFile()); + $log->addExtra('line', $e->getLine()); + $log->addExtra('trace', $e->getTraceAsString()); + + $logger->addLog($log); + } } $results[] = new Document([ - 'model' => $embeddingModel, + 'model' => $model, 'dimension' => $dimension, 'embedding' => $embedding, 'error' => $error @@ -119,7 +144,7 @@ class Create extends CreateDocumentAction ->addMetric( \str_replace( '{embeddingModel}', - $embeddingModel, + $model, METRIC_EMBEDDINGS_TEXT ), \count($texts) @@ -127,7 +152,7 @@ class Create extends CreateDocumentAction ->addMetric( \str_replace( '{embeddingModel}', - $embeddingModel, + $model, METRIC_EMBEDDINGS_TEXT_TOTAL_TOKENS ), $totalTokens @@ -135,11 +160,19 @@ class Create extends CreateDocumentAction ->addMetric( \str_replace( '{embeddingModel}', - $embeddingModel, + $model, METRIC_EMBEDDINGS_TEXT_TOTAL_DURATION ), $totalDuration ) + ->addMetric( + \str_replace( + '{embeddingModel}', + $model, + METRIC_EMBEDDINGS_TEXT_TOTAL_ERROR + ), + $totalErrors + ) ->trigger(); $queueForStatsUsage->reset(); } diff --git a/src/Appwrite/Utopia/Response/Model/UsageProject.php b/src/Appwrite/Utopia/Response/Model/UsageProject.php index d9fec1e4a1..fcb073cf61 100644 --- a/src/Appwrite/Utopia/Response/Model/UsageProject.php +++ b/src/Appwrite/Utopia/Response/Model/UsageProject.php @@ -237,6 +237,13 @@ class UsageProject extends Model 'example' => [], 'array' => true ]) + ->addRule('embeddingsTextErrors', [ + 'type' => Response::MODEL_METRIC, + 'description' => 'Aggregated number of errors while generating text embeddings per period.', + 'default' => [], + 'example' => [], + 'array' => true + ]) ; } diff --git a/tests/e2e/General/UsageTest.php b/tests/e2e/General/UsageTest.php index ff9e0f0ec8..6c9024c80f 100644 --- a/tests/e2e/General/UsageTest.php +++ b/tests/e2e/General/UsageTest.php @@ -1634,13 +1634,16 @@ class UsageTest extends Scope // New embeddings metrics should be present after calls above $this->assertArrayHasKey('embeddingsText', $response['body']); + $this->assertArrayHasKey('embeddingsTextErrors', $response['body']); $this->assertArrayHasKey('embeddingsTextTokens', $response['body']); $this->assertArrayHasKey('embeddingsTextDuration', $response['body']); - $this->assertGreaterThanOrEqual(0, $response['body']['embeddingsText']); - $this->assertGreaterThanOrEqual(0, $response['body']['embeddingsTextTokens']); - $this->assertGreaterThanOrEqual(0, $response['body']['embeddingsTextDuration']); + $this->assertGreaterThanOrEqual(0, count($response['body']['embeddingsText'])); + $this->assertGreaterThanOrEqual(0, count($response['body']['embeddingsTextErrors'])); + $this->assertGreaterThanOrEqual(0, count($response['body']['embeddingsTextTokens'])); + $this->assertGreaterThanOrEqual(0, count($response['body']['embeddingsTextDuration'])); $this->validateDates($response['body']['embeddingsText']); + $this->validateDates($response['body']['embeddingsTextErrors']); $this->validateDates($response['body']['embeddingsTextTokens']); $this->validateDates($response['body']['embeddingsTextDuration']); }); From f149ba33d45f98ce8d8fa17239a99b1376900f26 Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Mon, 8 Dec 2025 20:27:30 +0530 Subject: [PATCH 24/27] updated stats for vectordb --- app/controllers/api/project.php | 24 +++- app/controllers/shared/api.php | 1 + app/init/constants.php | 27 ++++- composer.lock | 2 +- .../Databases/Http/Databases/Action.php | 2 +- .../Collections/Documents/Action.php | 2 + .../Databases/Http/Databases/Usage/Get.php | 1 + .../Databases/Http/Databases/Usage/XList.php | 1 + .../Http/VectorDB/Collections/Usage/Get.php | 5 + .../Http/VectorDB/Embeddings/Text/Create.php | 41 ++----- .../Databases/Http/VectorDB/Usage/Get.php | 7 +- .../Databases/Http/VectorDB/Usage/XList.php | 7 +- src/Appwrite/Platform/Workers/Migrations.php | 2 + .../Platform/Workers/StatsResources.php | 18 ++- src/Appwrite/Utopia/Response.php | 6 + .../Utopia/Response/Model/UsageProject.php | 97 ++++++++++++++-- .../Utopia/Response/Model/UsageVectorDB.php | 96 +++++++++++++++ .../Utopia/Response/Model/UsageVectorDBs.php | 109 ++++++++++++++++++ tests/e2e/General/UsageTest.php | 13 +-- .../VectorDB/DatabasesConsoleClientTest.php | 2 +- .../VectorDB/DatabasesCustomServerTest.php | 6 +- 21 files changed, 400 insertions(+), 69 deletions(-) create mode 100644 src/Appwrite/Utopia/Response/Model/UsageVectorDB.php create mode 100644 src/Appwrite/Utopia/Response/Model/UsageVectorDBs.php diff --git a/app/controllers/api/project.php b/app/controllers/api/project.php index 2d17ccbafb..ae1f9041a7 100644 --- a/app/controllers/api/project.php +++ b/app/controllers/api/project.php @@ -76,6 +76,14 @@ App::get('/v1/project/usage') METRIC_DATABASES_OPERATIONS_WRITES, METRIC_DATABASES_OPERATIONS_WRITES_DOCUMENTSDB, METRIC_FILES_IMAGES_TRANSFORMED, + // VectorDB totals + METRIC_DATABASES_VECTORDB, + METRIC_COLLECTIONS_VECTORDB, + METRIC_DOCUMENTS_VECTORDB, + METRIC_DATABASES_STORAGE_VECTORDB, + METRIC_DATABASES_OPERATIONS_READS_VECTORDB, + METRIC_DATABASES_OPERATIONS_WRITES_VECTORDB, + // Embeddings totals METRIC_EMBEDDINGS_TEXT, METRIC_EMBEDDINGS_TEXT_TOTAL_TOKENS, METRIC_EMBEDDINGS_TEXT_TOTAL_DURATION, @@ -96,6 +104,14 @@ App::get('/v1/project/usage') METRIC_DATABASES_OPERATIONS_WRITES, METRIC_DATABASES_OPERATIONS_WRITES_DOCUMENTSDB, METRIC_FILES_IMAGES_TRANSFORMED, + // VectorDB time series + METRIC_DATABASES_VECTORDB, + METRIC_COLLECTIONS_VECTORDB, + METRIC_DOCUMENTS_VECTORDB, + METRIC_DATABASES_STORAGE_VECTORDB, + METRIC_DATABASES_OPERATIONS_READS_VECTORDB, + METRIC_DATABASES_OPERATIONS_WRITES_VECTORDB, + // Embeddings time series METRIC_EMBEDDINGS_TEXT, METRIC_EMBEDDINGS_TEXT_TOTAL_TOKENS, METRIC_EMBEDDINGS_TEXT_TOTAL_DURATION, @@ -403,10 +419,10 @@ App::get('/v1/project/usage') 'authPhoneCountryBreakdown' => $authPhoneCountryBreakdown, 'imageTransformations' => $usage[METRIC_FILES_IMAGES_TRANSFORMED], 'imageTransformationsTotal' => $total[METRIC_FILES_IMAGES_TRANSFORMED], - 'embeddingsText' => $usage[METRIC_EMBEDDINGS_TEXT] ?? [], - 'embeddingsTextTokens' => $usage[METRIC_EMBEDDINGS_TEXT_TOTAL_TOKENS] ?? [], - 'embeddingsTextDuration' => $usage[METRIC_EMBEDDINGS_TEXT_TOTAL_DURATION] ?? [], - 'embeddingsTextErrors' => $usage[METRIC_EMBEDDINGS_TEXT_TOTAL_ERROR] ?? [], + 'embeddingsText' => $total[METRIC_EMBEDDINGS_TEXT] ?? [], + 'embeddingsTextTokens' => $total[METRIC_EMBEDDINGS_TEXT_TOTAL_TOKENS], + 'embeddingsTextDuration' => $total[METRIC_EMBEDDINGS_TEXT_TOTAL_DURATION], + 'embeddingsTextErrors' => $total[METRIC_EMBEDDINGS_TEXT_TOTAL_ERROR], ]), Response::MODEL_USAGE_PROJECT); }); diff --git a/app/controllers/shared/api.php b/app/controllers/shared/api.php index cfb43ce5e7..dea8bd6f42 100644 --- a/app/controllers/shared/api.php +++ b/app/controllers/shared/api.php @@ -529,6 +529,7 @@ App::init() $path = $route->getMatchedPath(); $databaseType = match (true) { str_contains($path, '/documentsdb') => DATABASE_TYPE_DOCUMENTSDB, + str_contains($path, '/vectordb') => DATABASE_TYPE_VECTORDB, default => '', }; diff --git a/app/init/constants.php b/app/init/constants.php index 02576c6478..c8fd7db64f 100644 --- a/app/init/constants.php +++ b/app/init/constants.php @@ -280,6 +280,29 @@ const METRIC_DATABASE_ID_OPERATIONS_READS_DOCUMENTSDB = 'documentsdb.{databaseIn const METRIC_DATABASES_OPERATIONS_WRITES_DOCUMENTSDB = 'documentsdb.databases.operations.writes'; const METRIC_DATABASE_ID_OPERATIONS_WRITES_DOCUMENTSDB = 'documentsdb.{databaseInternalId}.databases.operations.writes'; +// vectordb +const METRIC_DATABASES_VECTORDB = 'vectordb.databases'; +const METRIC_COLLECTIONS_VECTORDB = 'vectordb.collections'; +const METRIC_DATABASES_STORAGE_VECTORDB = 'vectordb.databases.storage'; +const METRIC_DATABASE_ID_COLLECTIONS_VECTORDB = 'vectordb.{databaseInternalId}.collections'; +const METRIC_DATABASE_ID_STORAGE_VECTORDB = 'vectordb.{databaseInternalId}.databases.storage'; +const METRIC_DOCUMENTS_VECTORDB = 'vectordb.documents'; +const METRIC_DATABASE_ID_DOCUMENTS_VECTORDB = 'vectordb.{databaseInternalId}.documents'; +const METRIC_DATABASE_ID_COLLECTION_ID_DOCUMENTS_VECTORDB = 'vectordb.{databaseInternalId}.{collectionInternalId}.documents'; +const METRIC_DATABASE_ID_COLLECTION_ID_STORAGE_VECTORDB = 'vectordb.{databaseInternalId}.{collectionInternalId}.databases.storage'; +const METRIC_DATABASES_OPERATIONS_READS_VECTORDB = 'vectordb.databases.operations.reads'; +const METRIC_DATABASE_ID_OPERATIONS_READS_VECTORDB = 'vectordb.{databaseInternalId}.databases.operations.reads'; +const METRIC_DATABASES_OPERATIONS_WRITES_VECTORDB = 'vectordb.databases.operations.writes'; +const METRIC_DATABASE_ID_OPERATIONS_WRITES_VECTORDB = 'vectordb.{databaseInternalId}.databases.operations.writes'; +const METRIC_EMBEDDINGS_TEXT = 'embeddings.text'; +const METRIC_EMBEDDINGS_MODEL_TEXT = 'embeddings.text.{embeddingModel}'; +const METRIC_EMBEDDINGS_TEXT_TOTAL_ERROR = 'embeddings.text.totalErrors'; +const METRIC_EMBEDDINGS_MODEL_TEXT_TOTAL_ERROR = 'embeddings.text.{embeddingModel}.totalErrors'; +const METRIC_EMBEDDINGS_TEXT_TOTAL_DURATION = 'embeddings.text.totalDuration'; +const METRIC_EMBEDDINGS_MODEL_TEXT_TOTAL_DURATION = 'embeddings.text.{embeddingModel}.totalDuration'; +const METRIC_EMBEDDINGS_TEXT_TOTAL_TOKENS = 'embeddings.text.totalTokens'; +const METRIC_EMBEDDINGS_MODEL_TEXT_TOTAL_TOKENS = 'embeddings.text.{embeddingModel}.totalTokens'; + const METRIC_BUCKETS = 'buckets'; const METRIC_FILES = 'files'; const METRIC_FILES_STORAGE = 'files.storage'; @@ -353,10 +376,6 @@ const METRIC_SITES_ID_OUTBOUND = 'sites.{siteInternalId}.outbound'; const METRIC_AVATARS_SCREENSHOTS_GENERATED = 'avatars.screenshotsGenerated'; const METRIC_FUNCTIONS_RUNTIME = 'functions.runtimes.{runtime}'; const METRIC_SITES_FRAMEWORK = 'sites.frameworks.{framework}'; -const METRIC_EMBEDDINGS_TEXT = 'embeddings.text.{embeddingModel}'; -const METRIC_EMBEDDINGS_TEXT_TOTAL_ERROR = 'embeddings.text.{embeddingModel}.totalErrors'; -const METRIC_EMBEDDINGS_TEXT_TOTAL_DURATION = 'embeddings.text.{embeddingModel}.totalDuration'; -const METRIC_EMBEDDINGS_TEXT_TOTAL_TOKENS = 'embeddings.text.{embeddingModel}.totalTokens'; // Resource types const RESOURCE_TYPE_PROJECTS = 'projects'; diff --git a/composer.lock b/composer.lock index 065713e446..ad58e5406b 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "86c19fdb3d6a9faf68a310d96def667f", + "content-hash": "4a44da365c2e5b931f007f72437144dc", "packages": [ { "name": "adhocore/jwt", diff --git a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Action.php b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Action.php index 0347d4365b..3411e06b27 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Action.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Action.php @@ -26,7 +26,7 @@ class Action extends AppwriteAction $this->context = DATABASE_TYPE_DOCUMENTSDB; } if (\str_contains($path, '/vectordb')) { - $this->context = 'vectordb'; + $this->context = DATABASE_TYPE_VECTORDB; } return parent::setHttpPath($path); } diff --git a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Action.php b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Action.php index b6f95dd27c..83862f8ba8 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Action.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Action.php @@ -28,6 +28,8 @@ abstract class Action extends DatabasesAction $this->context = ROWS; } elseif (str_contains($path, '/documentsdb/')) { $this->databaseType = DATABASE_TYPE_DOCUMENTSDB; + } elseif (str_contains($path, '/vectordb/')) { + $this->databaseType = DATABASE_TYPE_VECTORDB; } $contextId = '$' . $this->getCollectionsEventsContext() . 'Id'; diff --git a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Usage/Get.php b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Usage/Get.php index 45a36a859d..00dc62a9d3 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Usage/Get.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Usage/Get.php @@ -32,6 +32,7 @@ class Get extends Action { $this->databaseType = match (true) { str_contains($path, '/documentsdb') => DATABASE_TYPE_DOCUMENTSDB, + str_contains($path, '/vectordb') => DATABASE_TYPE_VECTORDB, default => DATABASE_TYPE_LEGACY, }; diff --git a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Usage/XList.php b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Usage/XList.php index 5e244bb3d7..70f1792971 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Usage/XList.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Usage/XList.php @@ -30,6 +30,7 @@ class XList extends Action { $this->databaseType = match (true) { str_contains($path, '/documentsdb') => DATABASE_TYPE_DOCUMENTSDB, + str_contains($path, '/vectordb') => DATABASE_TYPE_VECTORDB, default => DATABASE_TYPE_LEGACY, }; diff --git a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Usage/Get.php b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Usage/Get.php index 2e26e9d6b5..00a5eedfb8 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Usage/Get.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Usage/Get.php @@ -24,6 +24,11 @@ class Get extends CollectionUsageGet return UtopiaResponse::MODEL_USAGE_COLLECTION; } + protected function getMetric(): string + { + return METRIC_DATABASE_ID_COLLECTION_ID_DOCUMENTS_VECTORDB; + } + public function __construct() { $this diff --git a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Embeddings/Text/Create.php b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Embeddings/Text/Create.php index ba6a4c9a22..c6c058408a 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Embeddings/Text/Create.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Embeddings/Text/Create.php @@ -141,39 +141,16 @@ class Create extends CreateDocumentAction ->dynamic($embeddings, $this->getBulkResponseModel()); $queueForStatsUsage - ->addMetric( - \str_replace( - '{embeddingModel}', - $model, - METRIC_EMBEDDINGS_TEXT - ), - \count($texts) - ) - ->addMetric( - \str_replace( - '{embeddingModel}', - $model, - METRIC_EMBEDDINGS_TEXT_TOTAL_TOKENS - ), - $totalTokens - ) - ->addMetric( - \str_replace( - '{embeddingModel}', - $model, - METRIC_EMBEDDINGS_TEXT_TOTAL_DURATION - ), - $totalDuration - ) - ->addMetric( - \str_replace( - '{embeddingModel}', - $model, - METRIC_EMBEDDINGS_TEXT_TOTAL_ERROR - ), - $totalErrors - ) + ->addMetric(METRIC_EMBEDDINGS_TEXT, \count($texts)) + ->addMetric(\str_replace('{embeddingModel}', $model, METRIC_EMBEDDINGS_MODEL_TEXT), \count($texts)) + ->addMetric(METRIC_EMBEDDINGS_TEXT_TOTAL_TOKENS, $totalTokens) + ->addMetric(\str_replace('{embeddingModel}', $model, METRIC_EMBEDDINGS_MODEL_TEXT_TOTAL_TOKENS), $totalTokens) + ->addMetric(METRIC_EMBEDDINGS_TEXT_TOTAL_DURATION, $totalDuration) + ->addMetric(\str_replace('{embeddingModel}', $model, METRIC_EMBEDDINGS_MODEL_TEXT_TOTAL_DURATION), $totalDuration) + ->addMetric(METRIC_EMBEDDINGS_TEXT_TOTAL_ERROR, $totalErrors) + ->addMetric(\str_replace('{embeddingModel}', $model, METRIC_EMBEDDINGS_MODEL_TEXT_TOTAL_ERROR), $totalErrors) ->trigger(); + $queueForStatsUsage->reset(); } } diff --git a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Usage/Get.php b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Usage/Get.php index 075837b9cb..5508a9dc91 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Usage/Get.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Usage/Get.php @@ -19,6 +19,11 @@ class Get extends DatabaseUsageGet return 'getVectorDBUsage'; } + public function getResponseModel(): string + { + return UtopiaResponse::MODEL_USAGE_VECTORDB; + } + public function __construct() { $this @@ -38,7 +43,7 @@ class Get extends DatabaseUsageGet responses: [ new SDKResponse( code: SwooleResponse::STATUS_CODE_OK, - model: UtopiaResponse::MODEL_USAGE_DATABASE, + model: UtopiaResponse::MODEL_USAGE_VECTORDB, ) ], contentType: ContentType::JSON, diff --git a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Usage/XList.php b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Usage/XList.php index fe6ed3622b..3201864b50 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Usage/XList.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Usage/XList.php @@ -18,6 +18,11 @@ class XList extends DatabaseUsageXList return 'listVectorDBUsage'; } + public function getResponseModel(): string + { + return UtopiaResponse::MODEL_USAGE_VECTORDBS; + } + public function __construct() { $this @@ -37,7 +42,7 @@ class XList extends DatabaseUsageXList responses: [ new SDKResponse( code: SwooleResponse::STATUS_CODE_OK, - model: UtopiaResponse::MODEL_USAGE_DATABASES, + model: UtopiaResponse::MODEL_USAGE_VECTORDBS, ) ], contentType: ContentType::JSON diff --git a/src/Appwrite/Platform/Workers/Migrations.php b/src/Appwrite/Platform/Workers/Migrations.php index 836cbbc9c8..266eabb4f3 100644 --- a/src/Appwrite/Platform/Workers/Migrations.php +++ b/src/Appwrite/Platform/Workers/Migrations.php @@ -288,6 +288,8 @@ class Migrations extends Action METRIC_DATABASES_OPERATIONS_WRITES, METRIC_DATABASES_OPERATIONS_READS_DOCUMENTSDB, METRIC_DATABASES_OPERATIONS_WRITES_DOCUMENTSDB, + METRIC_DATABASES_OPERATIONS_READS_VECTORDB, + METRIC_DATABASES_OPERATIONS_WRITES_VECTORDB, METRIC_NETWORK_REQUESTS, METRIC_NETWORK_INBOUND, METRIC_NETWORK_OUTBOUND, diff --git a/src/Appwrite/Platform/Workers/StatsResources.php b/src/Appwrite/Platform/Workers/StatsResources.php index b745fac367..dabda146dc 100644 --- a/src/Appwrite/Platform/Workers/StatsResources.php +++ b/src/Appwrite/Platform/Workers/StatsResources.php @@ -125,6 +125,7 @@ class StatsResources extends Action $databases = $dbForProject->count('databases', [Query::equal('type', [DATABASE_TYPE_LEGACY, DATABASE_TYPE_TABLESDB])]); $documentsdb = $dbForProject->count('databases', [Query::equal('type', [DATABASE_TYPE_DOCUMENTSDB])]); + $vectordb = $dbForProject->count('databases', [Query::equal('type', [DATABASE_TYPE_VECTORDB])]); $buckets = $dbForProject->count('buckets'); $users = $dbForProject->count('users'); @@ -160,6 +161,7 @@ class StatsResources extends Action $metrics = [ METRIC_DATABASES => $databases, METRIC_DATABASES_DOCUMENTSDB => $documentsdb, + METRIC_DATABASES_VECTORDB => $vectordb, METRIC_BUCKETS => $buckets, METRIC_USERS => $users, METRIC_FUNCTIONS => $functions, @@ -268,8 +270,13 @@ class StatsResources extends Action $totalDocumentsDocumentsdb = 0; $totalDatabaseStorageDocumentsdb = 0; + // vectordb + $totalCollectionsVectordb = 0; + $totalDocumentsVectordb = 0; + $totalDatabaseStorageVectordb = 0; - $this->foreachDocument($dbForProject, 'databases', [], function ($database) use ($dbForProject, $getDatabasesDB, $region, &$totalCollections, &$totalDocuments, &$totalDatabaseStorage, &$totalCollectionsDocumentsdb, &$totalDocumentsDocumentsdb, &$totalDatabaseStorageDocumentsdb) { + + $this->foreachDocument($dbForProject, 'databases', [], function ($database) use ($dbForProject, $getDatabasesDB, $region, &$totalCollections, &$totalDocuments, &$totalDatabaseStorage, &$totalCollectionsDocumentsdb, &$totalDocumentsDocumentsdb, &$totalDatabaseStorageDocumentsdb, &$totalCollectionsVectordb, &$totalDocumentsVectordb, &$totalDatabaseStorageVectordb) { $dbForDatabases = $getDatabasesDB($database); $collections = $dbForProject->count('database_' . $database->getSequence()); @@ -289,6 +296,11 @@ class StatsResources extends Action $totalDocumentsDocumentsdb += $documents; $totalCollectionsDocumentsdb += $collections; break; + case DATABASE_TYPE_VECTORDB: + $totalDatabaseStorageVectordb += $storage; + $totalDocumentsVectordb += $documents; + $totalCollectionsVectordb += $collections; + break; default: $totalDatabaseStorage += $storage; $totalDocuments += $documents; @@ -303,6 +315,10 @@ class StatsResources extends Action $this->createStatsDocuments($region, METRIC_COLLECTIONS_DOCUMENTSDB, $totalCollectionsDocumentsdb); $this->createStatsDocuments($region, METRIC_DOCUMENTS_DOCUMENTSDB, $totalDocumentsDocumentsdb); $this->createStatsDocuments($region, METRIC_DATABASES_STORAGE_DOCUMENTSDB, $totalDatabaseStorageDocumentsdb); + + $this->createStatsDocuments($region, METRIC_COLLECTIONS_VECTORDB, $totalCollectionsVectordb); + $this->createStatsDocuments($region, METRIC_DOCUMENTS_VECTORDB, $totalDocumentsVectordb); + $this->createStatsDocuments($region, METRIC_DATABASES_STORAGE_VECTORDB, $totalDatabaseStorageVectordb); } protected function countForCollections(Database $dbForProject, Database $dbForDatabases, Document $database, string $region): array { diff --git a/src/Appwrite/Utopia/Response.php b/src/Appwrite/Utopia/Response.php index 80c7060b9a..1843e695cd 100644 --- a/src/Appwrite/Utopia/Response.php +++ b/src/Appwrite/Utopia/Response.php @@ -145,6 +145,8 @@ use Appwrite\Utopia\Response\Model\UsageSites; use Appwrite\Utopia\Response\Model\UsageStorage; use Appwrite\Utopia\Response\Model\UsageTable; use Appwrite\Utopia\Response\Model\UsageUsers; +use Appwrite\Utopia\Response\Model\UsageVectorDB; +use Appwrite\Utopia\Response\Model\UsageVectorDBs; use Appwrite\Utopia\Response\Model\User; use Appwrite\Utopia\Response\Model\Variable; use Appwrite\Utopia\Response\Model\VcsContent; @@ -179,6 +181,8 @@ class Response extends SwooleResponse public const MODEL_USAGE_DATABASE = 'usageDatabase'; public const MODEL_USAGE_DOCUMENTSDBS = 'usageDocumentsDBs'; public const MODEL_USAGE_DOCUMENTSDB = 'usageDocumentsDB'; + public const MODEL_USAGE_VECTORDBS = 'usageVectorDBs'; + public const MODEL_USAGE_VECTORDB = 'usageVectorDB'; public const MODEL_USAGE_TABLE = 'usageTable'; public const MODEL_USAGE_COLLECTION = 'usageCollection'; public const MODEL_USAGE_USERS = 'usageUsers'; @@ -618,6 +622,8 @@ class Response extends SwooleResponse ->setModel(new UsageDatabase()) ->setModel(new UsageDocumentsDBs()) ->setModel(new UsageDocumentsDB()) + ->setModel(new UsageVectorDBs()) + ->setModel(new UsageVectorDB()) ->setModel(new UsageTable()) ->setModel(new UsageCollection()) ->setModel(new UsageUsers()) diff --git a/src/Appwrite/Utopia/Response/Model/UsageProject.php b/src/Appwrite/Utopia/Response/Model/UsageProject.php index 02047265c7..a6ddf4abf0 100644 --- a/src/Appwrite/Utopia/Response/Model/UsageProject.php +++ b/src/Appwrite/Utopia/Response/Model/UsageProject.php @@ -267,33 +267,108 @@ class UsageProject extends Model 'default' => 0, 'example' => 0, ]) - ->addRule('embeddingsText', [ + // VectorDB aggregates + ->addRule('vectordbDatabasesTotal', [ + 'type' => self::TYPE_INTEGER, + 'description' => 'Total aggregated number of VectorDB databases.', + 'default' => 0, + 'example' => 0, + ]) + ->addRule('vectordbCollectionsTotal', [ + 'type' => self::TYPE_INTEGER, + 'description' => 'Total aggregated number of VectorDB collections.', + 'default' => 0, + 'example' => 0, + ]) + ->addRule('vectordbDocumentsTotal', [ + 'type' => self::TYPE_INTEGER, + 'description' => 'Total aggregated number of VectorDB documents.', + 'default' => 0, + 'example' => 0, + ]) + ->addRule('vectordbDatabasesStorageTotal', [ + 'type' => self::TYPE_INTEGER, + 'description' => 'Total aggregated VectorDB storage (bytes).', + 'default' => 0, + 'example' => 0, + ]) + ->addRule('vectordbDatabasesReadsTotal', [ + 'type' => self::TYPE_INTEGER, + 'description' => 'Total aggregated number of VectorDB reads.', + 'default' => 0, + 'example' => 0, + ]) + ->addRule('vectordbDatabasesWritesTotal', [ + 'type' => self::TYPE_INTEGER, + 'description' => 'Total aggregated number of VectorDB writes.', + 'default' => 0, + 'example' => 0, + ]) + ->addRule('vectordbDatabases', [ 'type' => Response::MODEL_METRIC, - 'description' => 'Aggregated number of text embedding calls per period.', + 'description' => 'Aggregated VectorDB databases per period.', 'default' => [], 'example' => [], 'array' => true ]) + ->addRule('vectordbCollections', [ + 'type' => Response::MODEL_METRIC, + 'description' => 'Aggregated VectorDB collections per period.', + 'default' => [], + 'example' => [], + 'array' => true + ]) + ->addRule('vectordbDocuments', [ + 'type' => Response::MODEL_METRIC, + 'description' => 'Aggregated VectorDB documents per period.', + 'default' => [], + 'example' => [], + 'array' => true + ]) + ->addRule('vectordbDatabasesStorage', [ + 'type' => Response::MODEL_METRIC, + 'description' => 'Aggregated VectorDB storage per period.', + 'default' => [], + 'example' => [], + 'array' => true + ]) + ->addRule('vectordbDatabasesReads', [ + 'type' => Response::MODEL_METRIC, + 'description' => 'Aggregated VectorDB reads per period.', + 'default' => [], + 'example' => [], + 'array' => true + ]) + ->addRule('vectordbDatabasesWrites', [ + 'type' => Response::MODEL_METRIC, + 'description' => 'Aggregated VectorDB writes per period.', + 'default' => [], + 'example' => [], + 'array' => true + ]) + ->addRule('embeddingsText', [ + 'type' => Response::MODEL_METRIC, + 'description' => 'Aggregated number of text embedding calls per period.', + 'default' => 0, + 'example' => 0 + ]) ->addRule('embeddingsTextTokens', [ 'type' => Response::MODEL_METRIC, 'description' => 'Aggregated number of tokens processed by text embeddings per period.', - 'default' => [], - 'example' => [], - 'array' => true + 'default' => 0, + 'example' => 0 ]) ->addRule('embeddingsTextDuration', [ 'type' => Response::MODEL_METRIC, 'description' => 'Aggregated duration spent generating text embeddings per period.', - 'default' => [], - 'example' => [], - 'array' => true + 'default' => 0, + 'example' => 0 ]) ->addRule('embeddingsTextErrors', [ 'type' => Response::MODEL_METRIC, 'description' => 'Aggregated number of errors while generating text embeddings per period.', - 'default' => [], - 'example' => [], - 'array' => true + 'default' => 0, + 'example' => 0 ]) ; } diff --git a/src/Appwrite/Utopia/Response/Model/UsageVectorDB.php b/src/Appwrite/Utopia/Response/Model/UsageVectorDB.php new file mode 100644 index 0000000000..a90f593c7e --- /dev/null +++ b/src/Appwrite/Utopia/Response/Model/UsageVectorDB.php @@ -0,0 +1,96 @@ +addRule('range', [ + 'type' => self::TYPE_STRING, + 'description' => 'Time range of the usage stats.', + 'default' => '', + 'example' => '30d', + ]) + ->addRule('collectionsTotal', [ + 'type' => self::TYPE_INTEGER, + 'description' => 'Total aggregated number of collections.', + 'default' => 0, + 'example' => 0, + ]) + ->addRule('documentsTotal', [ + 'type' => self::TYPE_INTEGER, + 'description' => 'Total aggregated number of documents.', + 'default' => 0, + 'example' => 0, + ]) + ->addRule('storageTotal', [ + 'type' => self::TYPE_INTEGER, + 'description' => 'Total aggregated storage used in bytes.', + 'default' => 0, + 'example' => 0, + ]) + ->addRule('databaseReadsTotal', [ + 'type' => self::TYPE_INTEGER, + 'description' => 'Total number of database reads.', + 'default' => 0, + 'example' => 0, + ]) + ->addRule('databaseWritesTotal', [ + 'type' => self::TYPE_INTEGER, + 'description' => 'Total number of database writes.', + 'default' => 0, + 'example' => 0, + ]) + ->addRule('collections', [ + 'type' => Response::MODEL_METRIC, + 'description' => 'Aggregated number of collections per period.', + 'default' => [], + 'example' => [], + 'array' => true + ]) + ->addRule('documents', [ + 'type' => Response::MODEL_METRIC, + 'description' => 'Aggregated number of documents per period.', + 'default' => [], + 'example' => [], + 'array' => true + ]) + ->addRule('storage', [ + 'type' => Response::MODEL_METRIC, + 'description' => 'Aggregated storage used in bytes per period.', + 'default' => [], + 'example' => [], + 'array' => true + ]) + ->addRule('databaseReads', [ + 'type' => Response::MODEL_METRIC, + 'description' => 'An array of aggregated number of database reads.', + 'default' => [], + 'example' => [], + 'array' => true + ]) + ->addRule('databaseWrites', [ + 'type' => Response::MODEL_METRIC, + 'description' => 'An array of aggregated number of database writes.', + 'default' => [], + 'example' => [], + 'array' => true + ]) + ; + } + + public function getName(): string + { + return 'UsageVectorDB'; + } + + public function getType(): string + { + return Response::MODEL_USAGE_VECTORDB; + } +} diff --git a/src/Appwrite/Utopia/Response/Model/UsageVectorDBs.php b/src/Appwrite/Utopia/Response/Model/UsageVectorDBs.php new file mode 100644 index 0000000000..da63120c5c --- /dev/null +++ b/src/Appwrite/Utopia/Response/Model/UsageVectorDBs.php @@ -0,0 +1,109 @@ +addRule('range', [ + 'type' => self::TYPE_STRING, + 'description' => 'Time range of the usage stats.', + 'default' => '', + 'example' => '30d', + ]) + ->addRule('databasesTotal', [ + 'type' => self::TYPE_INTEGER, + 'description' => 'Total aggregated number of VectorDB databases.', + 'default' => 0, + 'example' => 0, + ]) + ->addRule('collectionsTotal', [ + 'type' => self::TYPE_INTEGER, + 'description' => 'Total aggregated number of collections.', + 'default' => 0, + 'example' => 0, + ]) + ->addRule('documentsTotal', [ + 'type' => self::TYPE_INTEGER, + 'description' => 'Total aggregated number of documents.', + 'default' => 0, + 'example' => 0, + ]) + ->addRule('storageTotal', [ + 'type' => self::TYPE_INTEGER, + 'description' => 'Total aggregated storage in bytes.', + 'default' => 0, + 'example' => 0, + ]) + ->addRule('databasesReadsTotal', [ + 'type' => self::TYPE_INTEGER, + 'description' => 'Total number of database reads.', + 'default' => 0, + 'example' => 0, + ]) + ->addRule('databasesWritesTotal', [ + 'type' => self::TYPE_INTEGER, + 'description' => 'Total number of database writes.', + 'default' => 0, + 'example' => 0, + ]) + ->addRule('databases', [ + 'type' => Response::MODEL_METRIC, + 'description' => 'Aggregated number of databases per period.', + 'default' => [], + 'example' => [], + 'array' => true + ]) + ->addRule('collections', [ + 'type' => Response::MODEL_METRIC, + 'description' => 'Aggregated number of collections per period.', + 'default' => [], + 'example' => [], + 'array' => true + ]) + ->addRule('documents', [ + 'type' => Response::MODEL_METRIC, + 'description' => 'Aggregated number of documents per period.', + 'default' => [], + 'example' => [], + 'array' => true + ]) + ->addRule('storage', [ + 'type' => Response::MODEL_METRIC, + 'description' => 'Aggregated storage in bytes per period.', + 'default' => [], + 'example' => [], + 'array' => true + ]) + ->addRule('databasesReads', [ + 'type' => Response::MODEL_METRIC, + 'description' => 'An array of aggregated number of database reads.', + 'default' => [], + 'example' => [], + 'array' => true + ]) + ->addRule('databasesWrites', [ + 'type' => Response::MODEL_METRIC, + 'description' => 'An array of aggregated number of database writes.', + 'default' => [], + 'example' => [], + 'array' => true + ]) + ; + } + + public function getName(): string + { + return 'UsageVectorDBs'; + } + + public function getType(): string + { + return Response::MODEL_USAGE_VECTORDBS; + } +} diff --git a/tests/e2e/General/UsageTest.php b/tests/e2e/General/UsageTest.php index ad5cb1414e..7660861347 100644 --- a/tests/e2e/General/UsageTest.php +++ b/tests/e2e/General/UsageTest.php @@ -1646,15 +1646,10 @@ class UsageTest extends Scope $this->assertArrayHasKey('embeddingsTextErrors', $response['body']); $this->assertArrayHasKey('embeddingsTextTokens', $response['body']); $this->assertArrayHasKey('embeddingsTextDuration', $response['body']); - $this->assertGreaterThanOrEqual(0, count($response['body']['embeddingsText'])); - $this->assertGreaterThanOrEqual(0, count($response['body']['embeddingsTextErrors'])); - $this->assertGreaterThanOrEqual(0, count($response['body']['embeddingsTextTokens'])); - $this->assertGreaterThanOrEqual(0, count($response['body']['embeddingsTextDuration'])); - - $this->validateDates($response['body']['embeddingsText']); - $this->validateDates($response['body']['embeddingsTextErrors']); - $this->validateDates($response['body']['embeddingsTextTokens']); - $this->validateDates($response['body']['embeddingsTextDuration']); + $this->assertGreaterThan(0, $response['body']['embeddingsText']); + $this->assertGreaterThanOrEqual(0, $response['body']['embeddingsTextErrors']); + $this->assertGreaterThan(0, $response['body']['embeddingsTextTokens']); + $this->assertGreaterThan(0, $response['body']['embeddingsTextDuration']); }); } diff --git a/tests/e2e/Services/Databases/VectorDB/DatabasesConsoleClientTest.php b/tests/e2e/Services/Databases/VectorDB/DatabasesConsoleClientTest.php index 8d711e68a5..8ca4822e68 100644 --- a/tests/e2e/Services/Databases/VectorDB/DatabasesConsoleClientTest.php +++ b/tests/e2e/Services/Databases/VectorDB/DatabasesConsoleClientTest.php @@ -226,7 +226,7 @@ class DatabasesConsoleClientTest extends Scope ]); $this->assertEquals(200, $response['headers']['status-code']); - $this->assertEquals(15, count($response['body'])); + $this->assertEquals(11, count($response['body'])); $this->assertEquals('24h', $response['body']['range']); $this->assertIsNumeric($response['body']['documentsTotal']); $this->assertIsNumeric($response['body']['collectionsTotal']); diff --git a/tests/e2e/Services/Databases/VectorDB/DatabasesCustomServerTest.php b/tests/e2e/Services/Databases/VectorDB/DatabasesCustomServerTest.php index 7f6aeeccbd..58cb48440e 100644 --- a/tests/e2e/Services/Databases/VectorDB/DatabasesCustomServerTest.php +++ b/tests/e2e/Services/Databases/VectorDB/DatabasesCustomServerTest.php @@ -594,7 +594,7 @@ class DatabasesCustomServerTest extends Scope 'x-appwrite-project' => $this->getProject()['$id'], 'x-appwrite-key' => $this->getProject()['apiKey'] ], [ - 'embeddingModel' => 'embeddinggemma', + 'model' => 'embeddinggemma', 'texts' => [ 'hello world', 'second sentence', @@ -627,7 +627,7 @@ class DatabasesCustomServerTest extends Scope 'x-appwrite-project' => $this->getProject()['$id'], 'x-appwrite-key' => $this->getProject()['apiKey'] ], [ - 'embeddingModel' => 'embeddinggemma', + 'model' => 'embeddinggemma', 'texts' => [ 'valid text', 123, // invalid, not a string @@ -641,7 +641,7 @@ class DatabasesCustomServerTest extends Scope 'x-appwrite-project' => $this->getProject()['$id'], 'x-appwrite-key' => $this->getProject()['apiKey'] ], [ - 'embeddingModel' => 'nonexistent-model', + 'model' => 'nonexistent-model', 'texts' => ['hello'], ]); $this->assertEquals(400, $unknownModel['headers']['status-code']); From 480757dcfeedddf944fcfc296f4989d30d543e4c Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Tue, 9 Dec 2025 12:07:26 +0530 Subject: [PATCH 25/27] added usage test for the vectordb --- app/controllers/api/project.php | 24 +- src/Appwrite/Platform/Workers/StatsUsage.php | 4 + tests/e2e/General/UsageTest.php | 246 ++++++++++++++++++- 3 files changed, 265 insertions(+), 9 deletions(-) diff --git a/app/controllers/api/project.php b/app/controllers/api/project.php index ae1f9041a7..ba83b02f58 100644 --- a/app/controllers/api/project.php +++ b/app/controllers/api/project.php @@ -403,6 +403,12 @@ App::get('/v1/project/usage') 'databasesWritesTotal' => $total[METRIC_DATABASES_OPERATIONS_WRITES], 'documentsdbDatabasesReadsTotal' => $total[METRIC_DATABASES_OPERATIONS_READS_DOCUMENTSDB], 'documentsdbDatabasesWritesTotal' => $total[METRIC_DATABASES_OPERATIONS_WRITES_DOCUMENTSDB], + 'vectordbDatabasesTotal' => $total[METRIC_DATABASES_VECTORDB] ?? 0, + 'vectordbCollectionsTotal' => $total[METRIC_COLLECTIONS_VECTORDB] ?? 0, + 'vectordbDocumentsTotal' => $total[METRIC_DOCUMENTS_VECTORDB] ?? 0, + 'vectordbDatabasesStorageTotal' => $total[METRIC_DATABASES_STORAGE_VECTORDB] ?? 0, + 'vectordbDatabasesReadsTotal' => $total[METRIC_DATABASES_OPERATIONS_READS_VECTORDB] ?? 0, + 'vectordbDatabasesWritesTotal' => $total[METRIC_DATABASES_OPERATIONS_WRITES_VECTORDB] ?? 0, 'executionsBreakdown' => $executionsBreakdown, 'bucketsBreakdown' => $bucketsBreakdown, 'databasesReads' => $usage[METRIC_DATABASES_OPERATIONS_READS], @@ -410,6 +416,12 @@ App::get('/v1/project/usage') 'documentsdbDatabasesReads' => $usage[METRIC_DATABASES_OPERATIONS_READS_DOCUMENTSDB], 'documentsdbDatabasesWrites' => $usage[METRIC_DATABASES_OPERATIONS_WRITES_DOCUMENTSDB], 'documentsdbDatabasesStorage' => $usage[METRIC_DATABASES_STORAGE_DOCUMENTSDB], + 'vectordbDatabases' => $usage[METRIC_DATABASES_VECTORDB] ?? [], + 'vectordbCollections' => $usage[METRIC_COLLECTIONS_VECTORDB] ?? [], + 'vectordbDocuments' => $usage[METRIC_DOCUMENTS_VECTORDB] ?? [], + 'vectordbDatabasesStorage' => $usage[METRIC_DATABASES_STORAGE_VECTORDB] ?? [], + 'vectordbDatabasesReads' => $usage[METRIC_DATABASES_OPERATIONS_READS_VECTORDB] ?? [], + 'vectordbDatabasesWrites' => $usage[METRIC_DATABASES_OPERATIONS_WRITES_VECTORDB] ?? [], 'databasesStorageBreakdown' => $databasesStorageBreakdown, 'executionsMbSecondsBreakdown' => $executionsMbSecondsBreakdown, 'buildsMbSecondsBreakdown' => $buildsMbSecondsBreakdown, @@ -419,10 +431,14 @@ App::get('/v1/project/usage') 'authPhoneCountryBreakdown' => $authPhoneCountryBreakdown, 'imageTransformations' => $usage[METRIC_FILES_IMAGES_TRANSFORMED], 'imageTransformationsTotal' => $total[METRIC_FILES_IMAGES_TRANSFORMED], - 'embeddingsText' => $total[METRIC_EMBEDDINGS_TEXT] ?? [], - 'embeddingsTextTokens' => $total[METRIC_EMBEDDINGS_TEXT_TOTAL_TOKENS], - 'embeddingsTextDuration' => $total[METRIC_EMBEDDINGS_TEXT_TOTAL_DURATION], - 'embeddingsTextErrors' => $total[METRIC_EMBEDDINGS_TEXT_TOTAL_ERROR], + 'embeddingsText' => $usage[METRIC_EMBEDDINGS_TEXT] ?? [], + 'embeddingsTextTokens' => $usage[METRIC_EMBEDDINGS_TEXT_TOTAL_TOKENS] ?? [], + 'embeddingsTextDuration' => $usage[METRIC_EMBEDDINGS_TEXT_TOTAL_DURATION] ?? [], + 'embeddingsTextErrors' => $usage[METRIC_EMBEDDINGS_TEXT_TOTAL_ERROR] ?? [], + 'embeddingsTextTotal' => $total[METRIC_EMBEDDINGS_TEXT] ?? 0, + 'embeddingsTextTokensTotal' => $total[METRIC_EMBEDDINGS_TEXT_TOTAL_TOKENS] ?? 0, + 'embeddingsTextDurationTotal' => $total[METRIC_EMBEDDINGS_TEXT_TOTAL_DURATION] ?? 0, + 'embeddingsTextErrorsTotal' => $total[METRIC_EMBEDDINGS_TEXT_TOTAL_ERROR] ?? 0, ]), Response::MODEL_USAGE_PROJECT); }); diff --git a/src/Appwrite/Platform/Workers/StatsUsage.php b/src/Appwrite/Platform/Workers/StatsUsage.php index 09860520af..b464d4e1fb 100644 --- a/src/Appwrite/Platform/Workers/StatsUsage.php +++ b/src/Appwrite/Platform/Workers/StatsUsage.php @@ -48,6 +48,7 @@ class StatsUsage extends Action protected array $skipBaseMetrics = [ METRIC_DATABASES => true, METRIC_DATABASES_DOCUMENTSDB => true, + METRIC_DATABASES_VECTORDB => true, METRIC_BUCKETS => true, METRIC_USERS => true, METRIC_FUNCTIONS => true, @@ -67,8 +68,11 @@ class StatsUsage extends Action METRIC_BUILDS => true, METRIC_COLLECTIONS => true, METRIC_DOCUMENTS => true, + METRIC_COLLECTIONS_VECTORDB => true, + METRIC_DOCUMENTS_VECTORDB => true, METRIC_DATABASES_STORAGE => true, METRIC_DATABASES_STORAGE_DOCUMENTSDB => true, + METRIC_DATABASES_STORAGE_VECTORDB => true, ]; /** diff --git a/tests/e2e/General/UsageTest.php b/tests/e2e/General/UsageTest.php index 7660861347..49b554bfb6 100644 --- a/tests/e2e/General/UsageTest.php +++ b/tests/e2e/General/UsageTest.php @@ -1131,6 +1131,228 @@ class UsageTest extends Scope } /** @depends testDocumentsDBStats */ + public function testPrepareVectorDBStats(array $data): array + { + $documentsTotal = 0; + $collectionsTotal = 0; + $vectordbTotal = 0; + $databasesTotal = $data['databasesTotal']; + $requestsTotal = $data['requestsTotal']; + + for ($i = 0; $i < self::CREATE; $i++) { + $name = uniqid() . ' vectordb'; + + $response = $this->client->call( + Client::METHOD_POST, + '/vectordb', + array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'] + ], $this->getHeaders()), + [ + 'databaseId' => 'unique()', + 'name' => $name, + ] + ); + + $this->assertEquals($name, $response['body']['name']); + $this->assertNotEmpty($response['body']['$id']); + + $requestsTotal += 1; + $vectordbTotal += 1; + + $vectordbId = $response['body']['$id']; + + if ($i < (self::CREATE / 2)) { + $response = $this->client->call( + Client::METHOD_DELETE, + '/vectordb/' . $vectordbId, + array_merge([ + 'x-appwrite-project' => $this->getProject()['$id'] + ], $this->getHeaders()), + ); + + $this->assertEmpty($response['body']); + + $vectordbTotal -= 1; + $requestsTotal += 1; + } + } + + for ($i = 0; $i < self::CREATE; $i++) { + $name = uniqid() . ' collection'; + + $response = $this->client->call( + Client::METHOD_POST, + '/vectordb/' . $vectordbId . '/collections', + array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'] + ], $this->getHeaders()), + [ + 'collectionId' => 'unique()', + 'name' => $name, + 'dimension' => 1536, + 'documentSecurity' => false, + 'permissions' => [ + Permission::read(Role::any()), + Permission::create(Role::any()), + Permission::update(Role::any()), + Permission::delete(Role::any()), + ], + ] + ); + + $this->assertEquals($name, $response['body']['name']); + $this->assertNotEmpty($response['body']['$id']); + + $requestsTotal += 1; + $collectionsTotal += 1; + + $collectionId = $response['body']['$id']; + + if ($i < (self::CREATE / 2)) { + $response = $this->client->call( + Client::METHOD_DELETE, + '/vectordb/' . $vectordbId . '/collections/' . $collectionId, + array_merge([ + 'x-appwrite-project' => $this->getProject()['$id'] + ], $this->getHeaders()), + ); + + $this->assertEmpty($response['body']); + + $collectionsTotal -= 1; + $requestsTotal += 1; + } + } + + for ($i = 0; $i < self::CREATE; $i++) { + $response = $this->client->call( + Client::METHOD_POST, + '/vectordb/' . $vectordbId . '/collections/' . $collectionId . '/documents', + array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'] + ], $this->getHeaders()), + [ + 'documentId' => 'unique()', + 'data' => [ + 'embeddings' => array_fill(0, 1536, 0.1), + 'metadata' => [ + 'name' => uniqid() . ' document', + 'value' => $i + ] + ] + ] + ); + + $this->assertNotEmpty($response['body']['$id']); + + $requestsTotal += 1; + $documentsTotal += 1; + + $documentId = $response['body']['$id']; + + if ($i < (self::CREATE / 2)) { + $response = $this->client->call( + Client::METHOD_DELETE, + '/vectordb/' . $vectordbId . '/collections/' . $collectionId . '/documents/' . $documentId, + array_merge([ + 'x-appwrite-project' => $this->getProject()['$id'] + ], $this->getHeaders()), + ); + + $this->assertEmpty($response['body']); + + $documentsTotal -= 1; + $requestsTotal += 1; + } + } + + return array_merge($data, [ + 'vectordbId' => $vectordbId, + 'vectordbCollectionId' => $collectionId, + 'requestsTotal' => $requestsTotal, + 'databasesTotal' => $databasesTotal, + 'vectordbTotal' => $vectordbTotal, + 'vectordbCollectionsTotal' => $collectionsTotal, + 'vectordbDocumentsTotal' => $documentsTotal, + ]); + } + + /** @depends testPrepareVectorDBStats */ + #[Retry(count: 1)] + public function testVectorDBStats(array $data): array + { + $vectordbId = $data['vectordbId']; + $collectionId = $data['vectordbCollectionId']; + $requestsTotal = $data['requestsTotal']; + $databasesTotal = $data['databasesTotal']; + $vectordbTotal = $data['vectordbTotal']; + $collectionsTotal = $data['vectordbCollectionsTotal']; + $documentsTotal = $data['vectordbDocumentsTotal']; + + $this->assertEventually(function () use ($requestsTotal, $vectordbTotal, $documentsTotal) { + $response = $this->client->call( + Client::METHOD_GET, + '/project/usage', + $this->getConsoleHeaders(), + [ + 'period' => '1d', + 'startDate' => self::getToday(), + 'endDate' => self::getTomorrow(), + ] + ); + + $this->assertGreaterThanOrEqual(31, count($response['body'])); + $this->assertCount(1, $response['body']['requests']); + $this->assertCount(1, $response['body']['network']); + $this->assertEquals($requestsTotal, $response['body']['requests'][array_key_last($response['body']['requests'])]['value']); + $this->validateDates($response['body']['requests']); + // vectordbTotal should reflect only VectorDB instances, not relational databases. + $this->assertEquals($vectordbTotal, $response['body']['vectordbDatabasesTotal']); + $this->assertEquals($documentsTotal, $response['body']['vectordbDocumentsTotal']); + }); + + $response = $this->client->call( + Client::METHOD_GET, + '/databases/usage?range=30d', + $this->getConsoleHeaders() + ); + + $this->assertEquals($databasesTotal, $response['body']['databases'][array_key_last($response['body']['databases'])]['value']); + $this->validateDates($response['body']['databases']); + + $this->assertEventually(function () use ($vectordbId, $collectionsTotal, $documentsTotal) { + $response = $this->client->call( + Client::METHOD_GET, + '/vectordb/' . $vectordbId . '/usage?range=30d', + $this->getConsoleHeaders() + ); + + $this->assertEquals($collectionsTotal, $response['body']['collections'][array_key_last($response['body']['collections'])]['value']); + $this->validateDates($response['body']['collections']); + + $this->assertEquals($documentsTotal, $response['body']['documents'][array_key_last($response['body']['documents'])]['value']); + $this->validateDates($response['body']['documents']); + }); + + $this->assertEventually(function () use ($vectordbId, $collectionId, $documentsTotal) { + $response = $this->client->call( + Client::METHOD_GET, + '/vectordb/' . $vectordbId . '/collections/' . $collectionId . '/usage?range=30d', + $this->getConsoleHeaders() + ); + + $this->assertEquals($documentsTotal, $response['body']['documents'][array_key_last($response['body']['documents'])]['value']); + $this->validateDates($response['body']['documents']); + }); + + return $data; + } + + /** @depends testVectorDBStats */ public function testPrepareFunctionsStats(array $data): array { $executionTime = 0; @@ -1611,7 +1833,7 @@ class UsageTest extends Scope 'x-appwrite-key' => $this->getProject()['apiKey'], ], $this->getHeaders()), [ - 'embeddingModel' => 'embeddinggemma', + 'model' => 'embeddinggemma', 'texts' => [ 'usage test text ' . $i, ], @@ -1646,10 +1868,24 @@ class UsageTest extends Scope $this->assertArrayHasKey('embeddingsTextErrors', $response['body']); $this->assertArrayHasKey('embeddingsTextTokens', $response['body']); $this->assertArrayHasKey('embeddingsTextDuration', $response['body']); - $this->assertGreaterThan(0, $response['body']['embeddingsText']); - $this->assertGreaterThanOrEqual(0, $response['body']['embeddingsTextErrors']); - $this->assertGreaterThan(0, $response['body']['embeddingsTextTokens']); - $this->assertGreaterThan(0, $response['body']['embeddingsTextDuration']); + $this->assertArrayHasKey('embeddingsTextTotal', $response['body']); + $this->assertArrayHasKey('embeddingsTextErrorsTotal', $response['body']); + $this->assertArrayHasKey('embeddingsTextTokensTotal', $response['body']); + $this->assertArrayHasKey('embeddingsTextDurationTotal', $response['body']); + + // Time-series arrays should be non-empty + $this->assertNotEmpty($response['body']['embeddingsText']); + $this->assertNotEmpty($response['body']['embeddingsTextTokens']); + $this->assertNotEmpty($response['body']['embeddingsTextDuration']); + $this->validateDates($response['body']['embeddingsText']); + $this->validateDates($response['body']['embeddingsTextTokens']); + $this->validateDates($response['body']['embeddingsTextDuration']); + + // Total scalars should be greater than 0 (or >= 0 for errors) + $this->assertGreaterThan(0, $response['body']['embeddingsTextTotal']); + $this->assertGreaterThanOrEqual(0, $response['body']['embeddingsTextErrorsTotal']); + $this->assertGreaterThan(0, $response['body']['embeddingsTextTokensTotal']); + $this->assertGreaterThan(0, $response['body']['embeddingsTextDurationTotal']); }); } From f3721f712a1e2e5488c05188c0aec5760b4a6998 Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Tue, 9 Dec 2025 15:38:43 +0530 Subject: [PATCH 26/27] fixed missing project usage fields --- .../Utopia/Response/Model/UsageProject.php | 36 +++++++++++++++---- 1 file changed, 30 insertions(+), 6 deletions(-) diff --git a/src/Appwrite/Utopia/Response/Model/UsageProject.php b/src/Appwrite/Utopia/Response/Model/UsageProject.php index a6ddf4abf0..66628a7b13 100644 --- a/src/Appwrite/Utopia/Response/Model/UsageProject.php +++ b/src/Appwrite/Utopia/Response/Model/UsageProject.php @@ -349,24 +349,48 @@ class UsageProject extends Model ->addRule('embeddingsText', [ 'type' => Response::MODEL_METRIC, 'description' => 'Aggregated number of text embedding calls per period.', - 'default' => 0, - 'example' => 0 + 'default' => [], + 'example' => [] ]) ->addRule('embeddingsTextTokens', [ 'type' => Response::MODEL_METRIC, 'description' => 'Aggregated number of tokens processed by text embeddings per period.', - 'default' => 0, - 'example' => 0 + 'default' => [], + 'example' => [] ]) ->addRule('embeddingsTextDuration', [ 'type' => Response::MODEL_METRIC, 'description' => 'Aggregated duration spent generating text embeddings per period.', - 'default' => 0, - 'example' => 0 + 'default' => [], + 'example' => [] ]) ->addRule('embeddingsTextErrors', [ 'type' => Response::MODEL_METRIC, 'description' => 'Aggregated number of errors while generating text embeddings per period.', + 'default' => [], + 'example' => [] + ]) + ->addRule('embeddingsTextTotal', [ + 'type' => Response::MODEL_METRIC, + 'description' => 'Total aggregated number of text embedding calls.', + 'default' => 0, + 'example' => 0 + ]) + ->addRule('embeddingsTextTokensTotal', [ + 'type' => Response::MODEL_METRIC, + 'description' => 'Total aggregated number of tokens processed by text.', + 'default' => 0, + 'example' => 0 + ]) + ->addRule('embeddingsTextDurationTotal', [ + 'type' => Response::MODEL_METRIC, + 'description' => 'Total aggregated duration spent generating text embeddings.', + 'default' => 0, + 'example' => 0 + ]) + ->addRule('embeddingsTextErrorsTotal', [ + 'type' => Response::MODEL_METRIC, + 'description' => 'Total aggregated number of errors while generating text embeddings.', 'default' => 0, 'example' => 0 ]) From 991b2b1c3bdfd89dbc53a7319e13f391a2e2c773 Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Tue, 9 Dec 2025 18:44:26 +0530 Subject: [PATCH 27/27] updated delete worker for vectordb and updated tests --- src/Appwrite/Platform/Workers/Deletes.php | 2 +- .../Projects/ProjectsConsoleClientTest.php | 19 +++++++++++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/src/Appwrite/Platform/Workers/Deletes.php b/src/Appwrite/Platform/Workers/Deletes.php index 8ec487a559..2216b98114 100644 --- a/src/Appwrite/Platform/Workers/Deletes.php +++ b/src/Appwrite/Platform/Workers/Deletes.php @@ -531,7 +531,7 @@ class Deletes extends Action $projectTables = !\in_array($dsn->getHost(), $sharedTables); $sharedTablesV1 = \in_array($dsn->getHost(), $sharedTablesV1); $sharedTablesV2 = !$projectTables && !$sharedTablesV1; - $databaseDSNKeys = ['database','documentsDatabase']; + $databaseDSNKeys = ['database','documentsDatabase','vectorDatabase']; $exectionActionPerDatabase = function (string $databaseDSNKey, $callback) use ($getDatabasesDB, $document) { /** diff --git a/tests/e2e/Services/Projects/ProjectsConsoleClientTest.php b/tests/e2e/Services/Projects/ProjectsConsoleClientTest.php index e71375e517..6896ca8069 100644 --- a/tests/e2e/Services/Projects/ProjectsConsoleClientTest.php +++ b/tests/e2e/Services/Projects/ProjectsConsoleClientTest.php @@ -183,6 +183,25 @@ class ProjectsConsoleClientTest extends Scope ]); $this->assertEquals(201, $documentsCollection['headers']['status-code']); + // Create vectordb database and collection + $vectorDb = $this->client->call(Client::METHOD_POST, '/vectordb', $projectAdminHeaders, [ + 'databaseId' => ID::unique(), + 'name' => 'Vector DB', + ]); + $this->assertEquals(201, $vectorDb['headers']['status-code']); + $vectorDbId = $vectorDb['body']['$id']; + + $vectorCollection = $this->client->call(Client::METHOD_POST, '/vectordb/' . $vectorDbId . '/collections', $projectAdminHeaders, [ + 'collectionId' => ID::unique(), + 'name' => 'Vector Collection', + 'dimension' => 3, + 'documentSecurity' => true, + 'permissions' => [ + Permission::create(Role::any()), + ], + ]); + $this->assertEquals(201, $vectorCollection['headers']['status-code']); + // Delete project $delete = $this->client->call(Client::METHOD_DELETE, '/projects/' . $projectId, array_merge([ 'content-type' => 'application/json',