diff --git a/.env b/.env index 97e41d859c..46b07399a8 100644 --- a/.env +++ b/.env @@ -45,6 +45,11 @@ _APP_DB_ROOT_PASS=rootsecretpassword _APP_DB_ADAPTER_DOCUMENTSDB=mongodb _APP_DB_HOST_DOCUMENTSDB=mongodb _APP_DB_PORT_DOCUMENTSDB=27017 +_APP_DB_ADAPTER_VECTORDB=postgresql +_APP_DB_HOST_VECTORDB=postgresql +_APP_DB_PORT_VECTORDB=5432 +_APP_EMBEDDING_MODELS=embeddinggemma +_APP_EMBEDDING_ENDPOINT='http://ollama:11434/api/embed' _APP_STORAGE_DEVICE=Local _APP_STORAGE_S3_ACCESS_KEY= _APP_STORAGE_S3_SECRET= diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 1d98ee5952..4c26dc0c97 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -172,6 +172,7 @@ jobs: Databases/Legacy, Databases/TablesDB, Databases/DocumentsDB, + Databases/VectorDB, Functions, FunctionsSchedule, GraphQL, diff --git a/app/config/collections.php b/app/config/collections.php index 533dee57a8..887df8d773 100644 --- a/app/config/collections.php +++ b/app/config/collections.php @@ -4,6 +4,7 @@ $common = include __DIR__ . '/collections/common.php'; $projects = include __DIR__ . '/collections/projects.php'; $databases = include __DIR__ . '/collections/databases.php'; +$vectordb = include __DIR__ . '/collections/vectordb.php'; $platform = include __DIR__ . '/collections/platform.php'; $logs = include __DIR__ . '/collections/logs.php'; @@ -26,6 +27,7 @@ unset($common['files']); $collections = [ 'buckets' => $buckets, 'databases' => $databases, + 'vectordb' => $vectordb, 'projects' => array_merge($projects, $common), 'console' => array_merge($platform, $common), 'logs' => $logs, diff --git a/app/config/collections/platform.php b/app/config/collections/platform.php index 09ad724e1f..7db26694c6 100644 --- a/app/config/collections/platform.php +++ b/app/config/collections/platform.php @@ -89,6 +89,17 @@ return [ 'array' => false, 'filters' => [], ], + [ + '$id' => ID::custom('vectorDatabase'), + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => 256, + 'signed' => true, + 'required' => true, + 'default' => null, + 'array' => false, + 'filters' => [], + ], [ '$id' => ID::custom('logo'), 'type' => Database::VAR_STRING, diff --git a/app/config/collections/vectordb.php b/app/config/collections/vectordb.php new file mode 100644 index 0000000000..817863cffa --- /dev/null +++ b/app/config/collections/vectordb.php @@ -0,0 +1,165 @@ + [ + '$collection' => ID::custom('databases'), + '$id' => ID::custom('collections'), + 'name' => 'Collections', + 'attributes' => [ + [ + '$id' => ID::custom('databaseInternalId'), + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => Database::LENGTH_KEY, + 'signed' => true, + 'required' => true, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => ID::custom('databaseId'), + 'type' => Database::VAR_STRING, + 'signed' => true, + 'size' => Database::LENGTH_KEY, + 'format' => '', + 'filters' => [], + 'required' => true, + 'default' => null, + 'array' => false, + ], + [ + '$id' => ID::custom('name'), + 'type' => Database::VAR_STRING, + 'size' => 256, + 'required' => true, + 'signed' => true, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => ID::custom('dimension'), + 'type' => Database::VAR_INTEGER, + 'size' => 0, + 'required' => true, + 'signed' => false, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => ID::custom('enabled'), + 'type' => Database::VAR_BOOLEAN, + 'signed' => true, + 'size' => 0, + 'format' => '', + 'filters' => [], + 'required' => true, + 'default' => null, + 'array' => false, + ], + [ + '$id' => ID::custom('documentSecurity'), + 'type' => Database::VAR_BOOLEAN, + 'signed' => true, + 'size' => 0, + 'format' => '', + 'filters' => [], + 'required' => true, + 'default' => null, + 'array' => false, + ], + [ + '$id' => ID::custom('attributes'), + 'type' => Database::VAR_STRING, + 'size' => 1000000, + 'required' => false, + 'signed' => true, + 'array' => false, + 'filters' => ['subQueryAttributes'], + ], + [ + '$id' => ID::custom('indexes'), + 'type' => Database::VAR_STRING, + 'size' => 1000000, + 'required' => false, + 'signed' => true, + 'array' => false, + 'filters' => ['subQueryIndexes'], + ], + [ + '$id' => ID::custom('search'), + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => 16384, + 'signed' => true, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + ], + 'defaultAttributes' => [ + [ + '$id' => ID::custom('embeddings'), + 'type' => Database::VAR_VECTOR, + 'required' => true, + 'signed' => false, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => ID::custom('metadata'), + 'type' => Database::VAR_OBJECT, + 'default' => [], + 'required' => false, + 'size' => 0, + 'signed' => false, + 'array' => false, + 'filters' => [], + ], + ], + 'indexes' => [ + [ + '$id' => ID::custom('_fulltext_search'), + 'type' => Database::INDEX_FULLTEXT, + 'attributes' => ['search'], + 'lengths' => [], + 'orders' => [], + ], + [ + '$id' => ID::custom('_key_name'), + 'type' => Database::INDEX_KEY, + 'attributes' => ['name'], + 'lengths' => [256], + 'orders' => [Database::ORDER_ASC], + ], + [ + '$id' => ID::custom('_key_enabled'), + 'type' => Database::INDEX_KEY, + 'attributes' => ['enabled'], + 'lengths' => [], + 'orders' => [Database::ORDER_ASC], + ], + [ + '$id' => ID::custom('_key_documentSecurity'), + 'type' => Database::INDEX_KEY, + 'attributes' => ['documentSecurity'], + 'lengths' => [], + 'orders' => [Database::ORDER_ASC], + ], + ], + 'defaultIndexes' => [ + // not creating default indexes on the embeddings as it depends on the type of query users using the most + [ + '$id' => ID::custom('_key_metadata'), + 'type' => Database::INDEX_OBJECT, + 'attributes' => ['metadata'], + 'lengths' => [], + 'orders' => [], + ], + ] + ] +]; diff --git a/app/controllers/api/migrations.php b/app/controllers/api/migrations.php index 404cf02491..14ed30820d 100644 --- a/app/controllers/api/migrations.php +++ b/app/controllers/api/migrations.php @@ -47,7 +47,8 @@ function getDatabaseTransferResourceServices(string $databaseType) { return match($databaseType) { DATABASE_TYPE_LEGACY, - DATABASE_TYPE_TABLESDB => Transfer::GROUP_DATABASES_TABLES_DB + DATABASE_TYPE_TABLESDB => Transfer::GROUP_DATABASES_TABLES_DB, + DATABASE_TYPE_VECTORDB => Transfer::GROUP_DATABASES_VECTOR_DB }; } diff --git a/app/controllers/api/project.php b/app/controllers/api/project.php index 105006e42a..ba83b02f58 100644 --- a/app/controllers/api/project.php +++ b/app/controllers/api/project.php @@ -76,6 +76,18 @@ App::get('/v1/project/usage') METRIC_DATABASES_OPERATIONS_WRITES, METRIC_DATABASES_OPERATIONS_WRITES_DOCUMENTSDB, METRIC_FILES_IMAGES_TRANSFORMED, + // VectorDB totals + METRIC_DATABASES_VECTORDB, + METRIC_COLLECTIONS_VECTORDB, + METRIC_DOCUMENTS_VECTORDB, + METRIC_DATABASES_STORAGE_VECTORDB, + METRIC_DATABASES_OPERATIONS_READS_VECTORDB, + METRIC_DATABASES_OPERATIONS_WRITES_VECTORDB, + // Embeddings totals + METRIC_EMBEDDINGS_TEXT, + METRIC_EMBEDDINGS_TEXT_TOTAL_TOKENS, + METRIC_EMBEDDINGS_TEXT_TOTAL_DURATION, + METRIC_EMBEDDINGS_TEXT_TOTAL_ERROR ], 'period' => [ METRIC_NETWORK_REQUESTS, @@ -92,6 +104,18 @@ App::get('/v1/project/usage') METRIC_DATABASES_OPERATIONS_WRITES, METRIC_DATABASES_OPERATIONS_WRITES_DOCUMENTSDB, METRIC_FILES_IMAGES_TRANSFORMED, + // VectorDB time series + METRIC_DATABASES_VECTORDB, + METRIC_COLLECTIONS_VECTORDB, + METRIC_DOCUMENTS_VECTORDB, + METRIC_DATABASES_STORAGE_VECTORDB, + METRIC_DATABASES_OPERATIONS_READS_VECTORDB, + METRIC_DATABASES_OPERATIONS_WRITES_VECTORDB, + // Embeddings time series + METRIC_EMBEDDINGS_TEXT, + METRIC_EMBEDDINGS_TEXT_TOTAL_TOKENS, + METRIC_EMBEDDINGS_TEXT_TOTAL_DURATION, + METRIC_EMBEDDINGS_TEXT_TOTAL_ERROR ] ]; @@ -379,6 +403,12 @@ App::get('/v1/project/usage') 'databasesWritesTotal' => $total[METRIC_DATABASES_OPERATIONS_WRITES], 'documentsdbDatabasesReadsTotal' => $total[METRIC_DATABASES_OPERATIONS_READS_DOCUMENTSDB], 'documentsdbDatabasesWritesTotal' => $total[METRIC_DATABASES_OPERATIONS_WRITES_DOCUMENTSDB], + 'vectordbDatabasesTotal' => $total[METRIC_DATABASES_VECTORDB] ?? 0, + 'vectordbCollectionsTotal' => $total[METRIC_COLLECTIONS_VECTORDB] ?? 0, + 'vectordbDocumentsTotal' => $total[METRIC_DOCUMENTS_VECTORDB] ?? 0, + 'vectordbDatabasesStorageTotal' => $total[METRIC_DATABASES_STORAGE_VECTORDB] ?? 0, + 'vectordbDatabasesReadsTotal' => $total[METRIC_DATABASES_OPERATIONS_READS_VECTORDB] ?? 0, + 'vectordbDatabasesWritesTotal' => $total[METRIC_DATABASES_OPERATIONS_WRITES_VECTORDB] ?? 0, 'executionsBreakdown' => $executionsBreakdown, 'bucketsBreakdown' => $bucketsBreakdown, 'databasesReads' => $usage[METRIC_DATABASES_OPERATIONS_READS], @@ -386,6 +416,12 @@ App::get('/v1/project/usage') 'documentsdbDatabasesReads' => $usage[METRIC_DATABASES_OPERATIONS_READS_DOCUMENTSDB], 'documentsdbDatabasesWrites' => $usage[METRIC_DATABASES_OPERATIONS_WRITES_DOCUMENTSDB], 'documentsdbDatabasesStorage' => $usage[METRIC_DATABASES_STORAGE_DOCUMENTSDB], + 'vectordbDatabases' => $usage[METRIC_DATABASES_VECTORDB] ?? [], + 'vectordbCollections' => $usage[METRIC_COLLECTIONS_VECTORDB] ?? [], + 'vectordbDocuments' => $usage[METRIC_DOCUMENTS_VECTORDB] ?? [], + 'vectordbDatabasesStorage' => $usage[METRIC_DATABASES_STORAGE_VECTORDB] ?? [], + 'vectordbDatabasesReads' => $usage[METRIC_DATABASES_OPERATIONS_READS_VECTORDB] ?? [], + 'vectordbDatabasesWrites' => $usage[METRIC_DATABASES_OPERATIONS_WRITES_VECTORDB] ?? [], 'databasesStorageBreakdown' => $databasesStorageBreakdown, 'executionsMbSecondsBreakdown' => $executionsMbSecondsBreakdown, 'buildsMbSecondsBreakdown' => $buildsMbSecondsBreakdown, @@ -395,6 +431,14 @@ App::get('/v1/project/usage') 'authPhoneCountryBreakdown' => $authPhoneCountryBreakdown, 'imageTransformations' => $usage[METRIC_FILES_IMAGES_TRANSFORMED], 'imageTransformationsTotal' => $total[METRIC_FILES_IMAGES_TRANSFORMED], + 'embeddingsText' => $usage[METRIC_EMBEDDINGS_TEXT] ?? [], + 'embeddingsTextTokens' => $usage[METRIC_EMBEDDINGS_TEXT_TOTAL_TOKENS] ?? [], + 'embeddingsTextDuration' => $usage[METRIC_EMBEDDINGS_TEXT_TOTAL_DURATION] ?? [], + 'embeddingsTextErrors' => $usage[METRIC_EMBEDDINGS_TEXT_TOTAL_ERROR] ?? [], + 'embeddingsTextTotal' => $total[METRIC_EMBEDDINGS_TEXT] ?? 0, + 'embeddingsTextTokensTotal' => $total[METRIC_EMBEDDINGS_TEXT_TOTAL_TOKENS] ?? 0, + 'embeddingsTextDurationTotal' => $total[METRIC_EMBEDDINGS_TEXT_TOTAL_DURATION] ?? 0, + 'embeddingsTextErrorsTotal' => $total[METRIC_EMBEDDINGS_TEXT_TOTAL_ERROR] ?? 0, ]), Response::MODEL_USAGE_PROJECT); }); diff --git a/app/controllers/api/projects.php b/app/controllers/api/projects.php index bbfbff9945..30fcca8de0 100644 --- a/app/controllers/api/projects.php +++ b/app/controllers/api/projects.php @@ -138,6 +138,14 @@ App::post('/v1/projects') $sharedTables = \explode(',', System::getEnv('_APP_DATABASE_DOCUMENTSDB_SHARED_TABLES', '')); $sharedTablesV1 = \explode(',', System::getEnv('_APP_DATABASE_DOCUMENTSDB_SHARED_TABLES_V1', '')); break; + case 'vectorDatabase': + $databases = Config::getParam('pools-vectordb', []); + $databaseKeys = System::getEnv('_APP_DATABASE_VECTORDB_KEYS', ''); + $databaseOverride = System::getEnv('_APP_DATABASE_VECTORDB_OVERRIDE'); + $dbScheme = System::getEnv('_APP_DB_HOST_VECTORDB', 'postgresql'); + $sharedTables = \explode(',', System::getEnv('_APP_DATABASE_VECTORDB_SHARED_TABLES', '')); + $sharedTablesV1 = \explode(',', System::getEnv('_APP_DATABASE_VECTORDB_SHARED_TABLES_V1', '')); + break; default: // legacy/tablesdb $databases = Config::getParam('pools-database', []); @@ -263,7 +271,8 @@ App::post('/v1/projects') 'accessedAt' => DateTime::now(), 'search' => implode(' ', [$projectId, $name]), 'database' => $dsn, - 'documentsDatabase' => $getDatabaseDSN('documentsDatabase', $region, $dsn) + 'documentsDatabase' => $getDatabaseDSN('documentsDatabase', $region, $dsn), + 'vectorDatabase' => $getDatabaseDSN('vectorDatabase', $region, $dsn) ])); } catch (Duplicate) { throw new Exception(Exception::PROJECT_ALREADY_EXISTS); diff --git a/app/controllers/shared/api.php b/app/controllers/shared/api.php index cfb43ce5e7..dea8bd6f42 100644 --- a/app/controllers/shared/api.php +++ b/app/controllers/shared/api.php @@ -529,6 +529,7 @@ App::init() $path = $route->getMatchedPath(); $databaseType = match (true) { str_contains($path, '/documentsdb') => DATABASE_TYPE_DOCUMENTSDB, + str_contains($path, '/vectordb') => DATABASE_TYPE_VECTORDB, default => '', }; diff --git a/app/init/constants.php b/app/init/constants.php index 8421c3884e..c8fd7db64f 100644 --- a/app/init/constants.php +++ b/app/init/constants.php @@ -280,6 +280,29 @@ const METRIC_DATABASE_ID_OPERATIONS_READS_DOCUMENTSDB = 'documentsdb.{databaseIn const METRIC_DATABASES_OPERATIONS_WRITES_DOCUMENTSDB = 'documentsdb.databases.operations.writes'; const METRIC_DATABASE_ID_OPERATIONS_WRITES_DOCUMENTSDB = 'documentsdb.{databaseInternalId}.databases.operations.writes'; +// vectordb +const METRIC_DATABASES_VECTORDB = 'vectordb.databases'; +const METRIC_COLLECTIONS_VECTORDB = 'vectordb.collections'; +const METRIC_DATABASES_STORAGE_VECTORDB = 'vectordb.databases.storage'; +const METRIC_DATABASE_ID_COLLECTIONS_VECTORDB = 'vectordb.{databaseInternalId}.collections'; +const METRIC_DATABASE_ID_STORAGE_VECTORDB = 'vectordb.{databaseInternalId}.databases.storage'; +const METRIC_DOCUMENTS_VECTORDB = 'vectordb.documents'; +const METRIC_DATABASE_ID_DOCUMENTS_VECTORDB = 'vectordb.{databaseInternalId}.documents'; +const METRIC_DATABASE_ID_COLLECTION_ID_DOCUMENTS_VECTORDB = 'vectordb.{databaseInternalId}.{collectionInternalId}.documents'; +const METRIC_DATABASE_ID_COLLECTION_ID_STORAGE_VECTORDB = 'vectordb.{databaseInternalId}.{collectionInternalId}.databases.storage'; +const METRIC_DATABASES_OPERATIONS_READS_VECTORDB = 'vectordb.databases.operations.reads'; +const METRIC_DATABASE_ID_OPERATIONS_READS_VECTORDB = 'vectordb.{databaseInternalId}.databases.operations.reads'; +const METRIC_DATABASES_OPERATIONS_WRITES_VECTORDB = 'vectordb.databases.operations.writes'; +const METRIC_DATABASE_ID_OPERATIONS_WRITES_VECTORDB = 'vectordb.{databaseInternalId}.databases.operations.writes'; +const METRIC_EMBEDDINGS_TEXT = 'embeddings.text'; +const METRIC_EMBEDDINGS_MODEL_TEXT = 'embeddings.text.{embeddingModel}'; +const METRIC_EMBEDDINGS_TEXT_TOTAL_ERROR = 'embeddings.text.totalErrors'; +const METRIC_EMBEDDINGS_MODEL_TEXT_TOTAL_ERROR = 'embeddings.text.{embeddingModel}.totalErrors'; +const METRIC_EMBEDDINGS_TEXT_TOTAL_DURATION = 'embeddings.text.totalDuration'; +const METRIC_EMBEDDINGS_MODEL_TEXT_TOTAL_DURATION = 'embeddings.text.{embeddingModel}.totalDuration'; +const METRIC_EMBEDDINGS_TEXT_TOTAL_TOKENS = 'embeddings.text.totalTokens'; +const METRIC_EMBEDDINGS_MODEL_TEXT_TOTAL_TOKENS = 'embeddings.text.{embeddingModel}.totalTokens'; + const METRIC_BUCKETS = 'buckets'; const METRIC_FILES = 'files'; const METRIC_FILES_STORAGE = 'files.storage'; @@ -365,6 +388,7 @@ const RESOURCE_TYPE_TOPICS = 'topics'; const RESOURCE_TYPE_SUBSCRIBERS = 'subscribers'; const RESOURCE_TYPE_MESSAGES = 'messages'; const RESOURCE_TYPE_EXECUTIONS = 'executions'; +const RESOURCE_TYPE_EMBEDDINGS_TEXT = 'embeddingsText'; // Resource types for Tokens const TOKENS_RESOURCE_TYPE_FILES = 'files'; @@ -384,6 +408,7 @@ const COOKIE_NAME_PREVIEW = 'a_jwt_console'; const DATABASE_TYPE_LEGACY = 'legacy'; const DATABASE_TYPE_TABLESDB = 'tablesdb'; const DATABASE_TYPE_DOCUMENTSDB = 'documentsdb'; +const DATABASE_TYPE_VECTORDB = 'vectordb'; // CSV import/export allowed database types -const CSV_ALLOWED_DATABASE_TYPES = [DATABASE_TYPE_LEGACY, DATABASE_TYPE_LEGACY]; +const CSV_ALLOWED_DATABASE_TYPES = [DATABASE_TYPE_LEGACY, DATABASE_TYPE_LEGACY, DATABASE_TYPE_VECTORDB]; diff --git a/app/init/registers.php b/app/init/registers.php index 8e25e37466..eadd31527d 100644 --- a/app/init/registers.php +++ b/app/init/registers.php @@ -15,6 +15,7 @@ use Utopia\Config\Config; use Utopia\Database\Adapter\MariaDB; use Utopia\Database\Adapter\Mongo; use Utopia\Database\Adapter\MySQL; +use Utopia\Database\Adapter\Postgres; use Utopia\Database\Adapter\SQL; use Utopia\Database\PDO; use Utopia\Domains\Validator\PublicDomain; @@ -164,6 +165,14 @@ $register->set('pools', function () { 'pass' => System::getEnv('_APP_DB_PASS', ''), 'path' => System::getEnv('_APP_DB_SCHEMA', ''), ]); + $fallbackForVectorDB = 'db_main=' . AppwriteURL::unparse([ + 'scheme' => System::getEnv('_APP_DB_ADAPTER_VECTORDB', 'postgresql'), + 'host' => System::getEnv('_APP_DB_HOST_VECTORDB', 'postgresql'), + 'port' => System::getEnv('_APP_DB_PORT_VECTORDB', '5432'), + 'user' => System::getEnv('_APP_DB_USER', ''), + 'pass' => System::getEnv('_APP_DB_PASS', ''), + 'path' => System::getEnv('_APP_DB_SCHEMA', ''), + ]); $fallbackForRedis = 'redis_main=' . AppwriteURL::unparse([ 'scheme' => 'redis', 'host' => System::getEnv('_APP_REDIS_HOST', 'redis'), @@ -191,6 +200,12 @@ $register->set('pools', function () { 'multiple' => true, 'schemes' => ['mongodb'], ], + 'vectordb' => [ + 'type' => 'database', + 'dsns' => System::getEnv('_APP_CONNECTIONS_DATABASE_VECTORDB', $fallbackForVectorDB), + 'multiple' => true, + 'schemes' => ['postgresql'], + ], 'logs' => [ 'type' => 'database', 'dsns' => System::getEnv('_APP_CONNECTIONS_DB_LOGS', $fallbackForDB), @@ -299,6 +314,17 @@ $register->set('pools', function () { throw new Exception(Exception::GENERAL_SERVER_ERROR, "MongoDB connection failed: " . $e->getMessage()); } }, + 'postgresql' => function () use ($dsnHost, $dsnPort, $dsnUser, $dsnPass, $dsnDatabase) { + return new PDOProxy(function () use ($dsnHost, $dsnPort, $dsnUser, $dsnPass, $dsnDatabase) { + return new PDO("pgsql:host={$dsnHost};port={$dsnPort};dbname={$dsnDatabase}", $dsnUser, $dsnPass, array( + \PDO::ATTR_TIMEOUT => 3, // Seconds + \PDO::ATTR_PERSISTENT => false, + \PDO::ATTR_DEFAULT_FETCH_MODE => \PDO::FETCH_ASSOC, + \PDO::ATTR_EMULATE_PREPARES => true, + \PDO::ATTR_STRINGIFY_FETCHES => true + )); + }); + }, 'redis' => function () use ($dsnHost, $dsnPort, $dsnPass) { $redis = new \Redis(); @$redis->pconnect($dsnHost, (int)$dsnPort); @@ -320,6 +346,7 @@ $register->set('pools', function () { 'mariadb' => new MariaDB($resource()), 'mysql' => new MySQL($resource()), 'mongodb' => new Mongo($resource()), + 'postgresql' => new Postgres($resource()), default => null }; diff --git a/app/init/resources.php b/app/init/resources.php index 92202bd552..994fecea1d 100644 --- a/app/init/resources.php +++ b/app/init/resources.php @@ -27,6 +27,8 @@ use Appwrite\Utopia\Request; use Appwrite\Utopia\Response; use Executor\Executor; use Utopia\Abuse\Adapters\TimeLimit\Redis as TimeLimitRedis; +use Utopia\Agents\Adapters\Ollama; +use Utopia\Agents\Agent; use Utopia\App; use Utopia\Auth\Hashes\Argon2; use Utopia\Auth\Hashes\Sha; @@ -1228,3 +1230,9 @@ App::setResource('httpReferrerSafe', function (Request $request, string $httpRef App::setResource('transactionState', function (Database $dbForProject, callable $getDatabasesDB) { return new TransactionState($dbForProject, $getDatabasesDB); }, ['dbForProject', 'getDatabasesDB']); + +App::setResource('embeddingAgent', function ($register) { + $adapter = new Ollama(); + $adapter->setEndpoint(System::getEnv('_APP_EMBEDDING_ENDPOINT', 'http://ollama:11434/api/embed')); + return new Agent($adapter); +}, ['register']); diff --git a/composer.json b/composer.json index 42b3b1cf43..fe83fd650f 100644 --- a/composer.json +++ b/composer.json @@ -55,6 +55,7 @@ "utopia-php/cli": "0.15.*", "utopia-php/config": "1.*.*", "utopia-php/database": "3.*", + "utopia-php/agents": "0.*", "utopia-php/detector": "0.2.*", "utopia-php/domains": "0.9.*", "utopia-php/emails": "0.6.*", diff --git a/composer.lock b/composer.lock index 00553b7b66..ad58e5406b 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "51d293b5ffae82523da7b15cd0a942dd", + "content-hash": "4a44da365c2e5b931f007f72437144dc", "packages": [ { "name": "adhocore/jwt", @@ -3499,6 +3499,60 @@ }, "time": "2025-10-20T07:18:33+00:00" }, + { + "name": "utopia-php/agents", + "version": "0.6.0", + "source": { + "type": "git", + "url": "https://github.com/utopia-php/agents.git", + "reference": "ab026e44d332598852b4606808d9cd3bf1de9c96" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/utopia-php/agents/zipball/ab026e44d332598852b4606808d9cd3bf1de9c96", + "reference": "ab026e44d332598852b4606808d9cd3bf1de9c96", + "shasum": "" + }, + "require": { + "php": ">=8.3", + "utopia-php/fetch": "0.4.*" + }, + "require-dev": { + "laravel/pint": "1.2.*", + "phpstan/phpstan": "1.9.x-dev", + "phpunit/phpunit": "^9.3", + "vimeo/psalm": "4.0.1" + }, + "type": "library", + "autoload": { + "psr-4": { + "Utopia\\Agents\\": "src/Agents" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Utopia PHP", + "email": "team@appwrite.io" + } + ], + "description": "A simple PHP AI agents library", + "keywords": [ + "agents", + "ai", + "automation", + "machine-learning", + "php" + ], + "support": { + "issues": "https://github.com/utopia-php/agents/issues", + "source": "https://github.com/utopia-php/agents/tree/0.6.0" + }, + "time": "2025-12-02T10:51:32+00:00" + }, { "name": "utopia-php/analytics", "version": "0.10.2", diff --git a/docker-compose.yml b/docker-compose.yml index c1de7997dc..ff797d33b3 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -92,7 +92,9 @@ services: depends_on: - ${_APP_DB_HOST:-mongodb} + - postgresql - redis + - ollama # - clamav entrypoint: - php @@ -137,6 +139,12 @@ services: - _APP_DB_SCHEMA - _APP_DB_USER - _APP_DB_PASS + - _APP_DB_ADAPTER_VECTORDB + - _APP_DB_HOST_VECTORDB + - _APP_DB_PORT_VECTORDB + - _APP_DB_SCHEMA_VECTORDB + - _APP_DB_USER_VECTORDB + - _APP_DB_PASS_VECTORDB - _APP_SMTP_HOST - _APP_SMTP_PORT - _APP_SMTP_SECURE @@ -269,7 +277,9 @@ services: - ./src:/usr/src/code/src depends_on: - ${_APP_DB_HOST:-mongodb} + - postgresql - redis + - ollama environment: - _APP_ENV - _APP_WORKER_PER_CORE @@ -286,6 +296,12 @@ services: - _APP_DB_SCHEMA - _APP_DB_USER - _APP_DB_PASS + - _APP_DB_ADAPTER_VECTORDB + - _APP_DB_HOST_VECTORDB + - _APP_DB_PORT_VECTORDB + - _APP_DB_SCHEMA_VECTORDB + - _APP_DB_USER_VECTORDB + - _APP_DB_PASS_VECTORDB - _APP_USAGE_STATS - _APP_LOGGING_CONFIG - _APP_LOGGING_CONFIG_REALTIME @@ -304,6 +320,8 @@ services: depends_on: - redis - ${_APP_DB_HOST:-mongodb} + - postgresql + - ollama environment: - _APP_ENV - _APP_WORKER_PER_CORE @@ -334,8 +352,10 @@ services: depends_on: - redis - ${_APP_DB_HOST:-mongodb} + - postgresql - request-catcher-sms - request-catcher-webhook + - ollama environment: - _APP_ENV - _APP_WORKER_PER_CORE @@ -365,6 +385,8 @@ services: depends_on: - redis - ${_APP_DB_HOST:-mongodb} + - postgresql + - ollama volumes: - appwrite-uploads:/storage/uploads:rw - appwrite-cache:/storage/cache:rw @@ -432,6 +454,8 @@ services: depends_on: - redis - ${_APP_DB_HOST:-mongodb} + - postgresql + - ollama environment: - _APP_ENV - _APP_WORKER_PER_CORE @@ -446,6 +470,12 @@ services: - _APP_DB_SCHEMA - _APP_DB_USER - _APP_DB_PASS + - _APP_DB_ADAPTER_VECTORDB + - _APP_DB_HOST_VECTORDB + - _APP_DB_PORT_VECTORDB + - _APP_DB_SCHEMA_VECTORDB + - _APP_DB_USER_VECTORDB + - _APP_DB_PASS_VECTORDB - _APP_LOGGING_CONFIG - _APP_WORKERS_NUM - _APP_QUEUE_NAME @@ -468,6 +498,8 @@ services: depends_on: - redis - ${_APP_DB_HOST:-mongodb} + - postgresql + - ollama environment: - _APP_BROWSER_HOST - _APP_ENV @@ -536,6 +568,8 @@ services: depends_on: - redis - ${_APP_DB_HOST:-mongodb} + - postgresql + - ollama volumes: - appwrite-config:/storage/config:rw - appwrite-certificates:/storage/certificates:rw @@ -579,7 +613,9 @@ services: depends_on: - redis - ${_APP_DB_HOST:-mongodb} + - postgresql - openruntimes-executor + - ollama environment: - _APP_ENV - _APP_WORKER_PER_CORE @@ -716,6 +752,8 @@ services: - ./tests:/usr/src/code/tests depends_on: - ${_APP_DB_HOST:-mongodb} + - postgresql + - ollama environment: - _APP_ENV - _APP_WORKER_PER_CORE @@ -799,6 +837,8 @@ services: depends_on: - redis - ${_APP_DB_HOST:-mongodb} + - postgresql + - ollama environment: - _APP_ENV - _APP_WORKER_PER_CORE @@ -831,6 +871,8 @@ services: depends_on: - redis - ${_APP_DB_HOST:-mongodb} + - postgresql + - ollama environment: - _APP_ENV - _APP_WORKER_PER_CORE @@ -863,6 +905,8 @@ services: depends_on: - redis - ${_APP_DB_HOST:-mongodb} + - postgresql + - ollama environment: - _APP_ENV - _APP_WORKER_PER_CORE @@ -894,7 +938,9 @@ services: - ./src:/usr/src/code/src depends_on: - ${_APP_DB_HOST:-mongodb} + - postgresql - redis + - ollama environment: - _APP_ENV - _APP_WORKER_PER_CORE @@ -923,7 +969,9 @@ services: - ./src:/usr/src/code/src depends_on: - ${_APP_DB_HOST:-mongodb} + - postgresql - redis + - ollama environment: - _APP_ENV - _APP_WORKER_PER_CORE @@ -951,7 +999,9 @@ services: - ./src:/usr/src/code/src depends_on: - ${_APP_DB_HOST:-mongodb} + - postgresql - redis + - ollama environment: - _APP_ENV - _APP_WORKER_PER_CORE @@ -1125,6 +1175,40 @@ services: - mongodb + postgresql: + build: + context: https://github.com/appwrite/docker-postgres.git#feat-dockerfile + container_name: appwrite-postgresql + <<: *x-logging + networks: + - appwrite + volumes: + - appwrite-postgresql:/var/lib/postgresql/data:rw + ports: + - "5432:5432" + environment: + - POSTGRES_DB=${_APP_DB_SCHEMA} + - POSTGRES_USER=${_APP_DB_USER} + - POSTGRES_PASSWORD=${_APP_DB_PASS} + command: "postgres" + + ollama: + build: + context: https://github.com/appwrite/docker-ollama.git#feat-dockerfile + args: + MODELS: ${_APP_EMBEDDING_MODELS:-embeddinggemma} + # duration to keep model in memory + OLLAMA_KEEP_ALIVE: 24h + container_name: ollama + ports: + - "11434:11434" + restart: unless-stopped + # persistent for caching models across restarts and preloading + volumes: + - appwrite-models:/root/.ollama + networks: + - appwrite + redis: image: redis:7.2.4-alpine <<: *x-logging @@ -1225,6 +1309,7 @@ volumes: appwrite-mariadb: appwrite-mongodb: appwrite-mongodb-keyfile: + appwrite-postgresql: appwrite-redis: appwrite-cache: appwrite-uploads: @@ -1233,4 +1318,5 @@ volumes: appwrite-functions: appwrite-sites: appwrite-builds: - appwrite-config: \ No newline at end of file + appwrite-config: + appwrite-models: \ No newline at end of file diff --git a/ollama.dockerfile b/ollama.dockerfile new file mode 100644 index 0000000000..fc538ea4d2 --- /dev/null +++ b/ollama.dockerfile @@ -0,0 +1,23 @@ +FROM ollama/ollama:0.12.7 + +# Preload specific models +ARG MODELS +# needed to set in the environment +ARG OLLAMA_KEEP_ALIVE +ENV OLLAMA_KEEP_ALIVE=${OLLAMA_KEEP_ALIVE:-24h} + +# Pre-pull models at build time for Docker layer caching +RUN ollama serve & \ + sleep 5 && \ + for m in $MODELS; do \ + echo "Pulling model $m..."; \ + ollama pull $m || exit 1; \ + done && \ + pkill ollama + +# Expose Ollama default port +EXPOSE 11434 + +# On container start, quickly ensure models exist (no re-download unless missing) +ENTRYPOINT ["/bin/bash", "-c", "(sleep 2; for m in $MODELS; do ollama list | grep -q $m || ollama pull $m; done) & exec ollama $0"] +CMD ["serve"] diff --git a/postgres.dockerfile b/postgres.dockerfile new file mode 100644 index 0000000000..3214121ef7 --- /dev/null +++ b/postgres.dockerfile @@ -0,0 +1,8 @@ +FROM postgres:16 + +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + postgresql-16-postgis-3 \ + postgresql-16-postgis-3-scripts \ + postgresql-16-pgvector \ + && rm -rf /var/lib/apt/lists/* \ No newline at end of file diff --git a/src/Appwrite/Event/Event.php b/src/Appwrite/Event/Event.php index bbe6d1fe9b..9b460541ea 100644 --- a/src/Appwrite/Event/Event.php +++ b/src/Appwrite/Event/Event.php @@ -687,8 +687,10 @@ class Event ]; break; case 'documentsdb': + case 'vectordb': + // sending the type itself(eg: documentsdb, vectordb) $eventMap = [ - 'databases' => 'documentsdb' + 'databases' => $database->getAttribute('type') ]; break; } diff --git a/src/Appwrite/Messaging/Adapter/Realtime.php b/src/Appwrite/Messaging/Adapter/Realtime.php index 95c3c79155..30e5942185 100644 --- a/src/Appwrite/Messaging/Adapter/Realtime.php +++ b/src/Appwrite/Messaging/Adapter/Realtime.php @@ -299,6 +299,7 @@ class Realtime extends MessagingAdapter case 'databases': case 'tablesdb': case 'documentsdb': + case 'vectordb': $resource = $parts[4] ?? ''; if (in_array($resource, ['columns', 'attributes', 'indexes'])) { $channels[] = 'console'; @@ -434,6 +435,7 @@ class Realtime extends MessagingAdapter break; case 'documentsdb': + case 'vectordb': $channels[] = 'documents'; $channels[] = "{$basePrefix}.{$databaseId}.collections.{$resourceId}.documents"; $channels[] = "{$basePrefix}.{$databaseId}.collections.{$resourceId}.documents.{$payloadId}"; diff --git a/src/Appwrite/Platform/Modules/Databases/Constants.php b/src/Appwrite/Platform/Modules/Databases/Constants.php index c8fe6e79bb..6ed373ef77 100644 --- a/src/Appwrite/Platform/Modules/Databases/Constants.php +++ b/src/Appwrite/Platform/Modules/Databases/Constants.php @@ -25,3 +25,7 @@ const COLLECTIONS = 'collection'; const TABLESDB = 'tablesdb'; const DOCUMENTSDB = 'documentsdb'; +const VECTORDB = 'vectordb'; + +const MIN_VECTOR_DIMENSION = 1; +const MAX_VECTOR_DIMENSION = 16000; diff --git a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Action.php b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Action.php index ef6061fd4b..3411e06b27 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Action.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Action.php @@ -25,6 +25,9 @@ class Action extends AppwriteAction if (\str_contains($path, '/documentsdb')) { $this->context = DATABASE_TYPE_DOCUMENTSDB; } + if (\str_contains($path, '/vectordb')) { + $this->context = DATABASE_TYPE_VECTORDB; + } return parent::setHttpPath($path); } diff --git a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Action.php b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Action.php index a148e23845..c1706d69bc 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Action.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Action.php @@ -13,6 +13,8 @@ abstract class Action extends UtopiaAction */ private ?string $context = COLLECTIONS; + private ?string $databaseType = TABLESDB; + /** * Get the response model used in the SDK and HTTP responses. */ @@ -22,6 +24,9 @@ abstract class Action extends UtopiaAction { if (\str_contains($path, '/tablesdb')) { $this->context = TABLES; + $this->databaseType = TABLESDB; + } elseif (\str_contains($path, '/vectordb')) { + $this->databaseType = VECTORDB; } return parent::setHttpPath($path); } @@ -34,6 +39,14 @@ abstract class Action extends UtopiaAction return $this->context; } + /** + * Get the current API database type. + */ + protected function getDatabaseType(): string + { + return $this->databaseType; + } + /** * Get the key used in event parameters (e.g., 'collectionId' or 'tableId'). */ diff --git a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Action.php b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Action.php index b6f95dd27c..83862f8ba8 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Action.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Action.php @@ -28,6 +28,8 @@ abstract class Action extends DatabasesAction $this->context = ROWS; } elseif (str_contains($path, '/documentsdb/')) { $this->databaseType = DATABASE_TYPE_DOCUMENTSDB; + } elseif (str_contains($path, '/vectordb/')) { + $this->databaseType = DATABASE_TYPE_VECTORDB; } $contextId = '$' . $this->getCollectionsEventsContext() . 'Id'; diff --git a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Create.php b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Create.php index d179801eb2..be41037219 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Create.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Create.php @@ -34,10 +34,18 @@ class Create extends Action { return match ($this->getDatabaseType()) { DOCUMENTSDB => $project->getAttribute('documentsDatabase'), + VECTORDB => $project->getAttribute('vectorDatabase'), default => $project->getAttribute('database'), }; } + protected function getDatabaseCollection() + { + return match ($this->getDatabaseType()) { + 'vectordb' => (Config::getParam('collections', [])['vectordb'] ?? [])['collections'] ?? [], + default => (Config::getParam('collections', [])['databases'] ?? [])['collections'] ?? [], + }; + } public function __construct() { $this @@ -102,7 +110,7 @@ class Create extends Action $database = $dbForProject->getDocument('databases', $databaseId); - $collections = (Config::getParam('collections', [])['databases'] ?? [])['collections'] ?? []; + $collections = $this->getDatabaseCollection(); if (empty($collections)) { throw new Exception(Exception::GENERAL_SERVER_ERROR, 'The "collections" collection is not configured.'); } diff --git a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Transactions/Action.php b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Transactions/Action.php index 9a4645446a..033446a00c 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Transactions/Action.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Transactions/Action.php @@ -36,6 +36,7 @@ abstract class Action extends DatabasesAction public function setHttpPath(string $path): DatabasesAction { switch (true) { + // TODO: set the getDatabaseType() from each database group instead of path matching case str_contains($path, '/tablesdb'): $this->context = TABLES; $this->databaseType = TABLESDB; @@ -45,6 +46,10 @@ abstract class Action extends DatabasesAction $this->context = COLLECTIONS; $this->databaseType = DOCUMENTSDB; break; + case str_contains($path, '/vectordb'): + $this->context = COLLECTIONS; + $this->databaseType = VECTORDB; + break; } return parent::setHttpPath($path); } diff --git a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Transactions/Update.php b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Transactions/Update.php index 06acc3a414..87ec52c4e0 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Transactions/Update.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Transactions/Update.php @@ -421,6 +421,7 @@ class Update extends Action { return match ($this->getDatabaseType()) { DOCUMENTSDB => $project->getAttribute('documentsDatabase'), + VECTORDB => $project->getAttribute('vectorDatabase'), default => $project->getAttribute('database'), }; } diff --git a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Usage/Get.php b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Usage/Get.php index 45a36a859d..00dc62a9d3 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Usage/Get.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Usage/Get.php @@ -32,6 +32,7 @@ class Get extends Action { $this->databaseType = match (true) { str_contains($path, '/documentsdb') => DATABASE_TYPE_DOCUMENTSDB, + str_contains($path, '/vectordb') => DATABASE_TYPE_VECTORDB, default => DATABASE_TYPE_LEGACY, }; diff --git a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Usage/XList.php b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Usage/XList.php index 5e244bb3d7..70f1792971 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Usage/XList.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Usage/XList.php @@ -30,6 +30,7 @@ class XList extends Action { $this->databaseType = match (true) { str_contains($path, '/documentsdb') => DATABASE_TYPE_DOCUMENTSDB, + str_contains($path, '/vectordb') => DATABASE_TYPE_VECTORDB, default => DATABASE_TYPE_LEGACY, }; diff --git a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Create.php b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Create.php new file mode 100644 index 0000000000..61adb87dac --- /dev/null +++ b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Create.php @@ -0,0 +1,207 @@ +setHttpMethod(self::HTTP_REQUEST_METHOD_POST) + ->setHttpPath('/v1/vectordb/:databaseId/collections') + ->desc('Create collection') + ->groups(['api', 'database']) + ->label('event', 'databases.[databaseId].collections.[collectionId].create') + ->label('scope', 'collections.write') + ->label('resourceType', RESOURCE_TYPE_DATABASES) + ->label('audits.event', 'collection.create') + ->label('audits.resource', 'database/{request.databaseId}/collection/{response.$id}') + ->label('sdk', new Method( + namespace: 'vectorDB', + group: 'collections', + name: 'createCollection', + description: '/docs/references/vectordb/create-collection.md', + auth: [AuthType::KEY], + responses: [ + new SDKResponse( + code: SwooleResponse::STATUS_CODE_CREATED, + model: $this->getResponseModel(), + ) + ], + contentType: ContentType::JSON + )) + ->param('databaseId', '', fn (Database $dbForProject) => new UID($dbForProject->getAdapter()->getMaxUIDLength()), 'Database ID.', false, ['dbForProject']) + ->param('collectionId', '', fn (Database $dbForProject) => new CustomId(false, $dbForProject->getAdapter()->getMaxUIDLength()), 'Unique Id. Choose a custom ID or generate a random ID with `ID.unique()`. Valid chars are a-z, A-Z, 0-9, period, hyphen, and underscore. Can\'t start with a special char. Max length is 36 chars.', false, ['dbForProject']) + ->param('name', '', new Text(128), 'Collection name. Max length: 128 chars.') + ->param('dimension', null, new Range(MIN_VECTOR_DIMENSION, MAX_VECTOR_DIMENSION), 'Embedding dimension.') + ->param('permissions', null, new Permissions(APP_LIMIT_ARRAY_PARAMS_SIZE), 'An array of permissions strings. By default, no user is granted with any permissions. [Learn more about permissions](https://appwrite.io/docs/permissions).', true) + ->param('documentSecurity', false, new Boolean(true), 'Enables configuring permissions for individual documents. A user needs one of document or collection level permissions to access a document. [Learn more about permissions](https://appwrite.io/docs/permissions).', true) + ->param('enabled', true, new Boolean(), 'Is collection enabled? When set to \'disabled\', users cannot access the collection but Server SDKs with and API key can still read and write to the collection. No data is lost when this is toggled.', true) + ->inject('response') + ->inject('dbForProject') + ->inject('getDatabasesDB') + ->inject('queueForEvents') + ->callback($this->action(...)); + } + + public function action(string $databaseId, string $collectionId, string $name, int $dimension, ?array $permissions, bool $documentSecurity, bool $enabled, UtopiaResponse $response, Database $dbForProject, callable $getDatabasesDB, Event $queueForEvents): void + { + $database = Authorization::skip(fn () => $dbForProject->getDocument('databases', $databaseId)); + + if ($database->isEmpty()) { + throw new Exception(Exception::DATABASE_NOT_FOUND); + } + + $collectionId = $collectionId === 'unique()' ? ID::unique() : $collectionId; + + // Map aggregate permissions into the multiple permissions they represent. + $permissions = Permission::aggregate($permissions) ?? []; + + try { + $collection = $dbForProject->createDocument('database_' . $database->getSequence(), new Document([ + '$id' => $collectionId, + 'databaseInternalId' => $database->getSequence(), + 'databaseId' => $databaseId, + '$permissions' => $permissions, + 'documentSecurity' => $documentSecurity, + 'enabled' => $enabled, + 'name' => $name, + 'dimension' => $dimension, + 'search' => \implode(' ', [$collectionId, $name]), + ])); + + } catch (DuplicateException) { + throw new Exception($this->getDuplicateException()); + } catch (LimitException) { + throw new Exception($this->getLimitException()); + } catch (NotFoundException) { + throw new Exception(Exception::DATABASE_NOT_FOUND); + } + /** @var Database $dbForDatabases */ + $dbForDatabases = $getDatabasesDB($database); + + $attributes = []; + $indexes = []; + $collections = (Config::getParam('collections', [])['vectordb'] ?? [])['collections'] ?? []; + foreach ($collections['defaultAttributes'] as $attribute) { + if ($attribute['$id'] === 'embeddings') { + $attribute['size'] = $dimension; + } + $attributes[] = new Document($attribute); + } + foreach ($collections['defaultIndexes'] as $index) { + $indexes[] = new Document($index); + } + try { + // passing null in creates only creates the metadata collection + if (!$dbForDatabases->exists(null, Database::METADATA)) { + $dbForDatabases->create(); + } + $dbForDatabases->createCollection( + id: 'database_' . $database->getSequence() . '_collection_' . $collection->getSequence(), + permissions: $permissions, + documentSecurity: $documentSecurity, + attributes:$attributes, + indexes:$indexes + ); + // Create attribute and indexes metadata documents in the attributes and indexes collections + // needed for the get and list calls + $attributeDocs = array_map(function ($attributeConfig) use ($database, $collection, $databaseId, $collectionId, $dimension) { + $key = \is_string($attributeConfig['$id']) ? $attributeConfig['$id'] : (string) $attributeConfig['$id']; + return new Document([ + '$id' => ID::custom($database->getSequence() . '_' . $collection->getSequence() . '_' . $key), + 'key' => $key, + 'databaseInternalId' => $database->getSequence(), + 'databaseId' => $databaseId, + 'collectionInternalId' => $collection->getSequence(), + 'collectionId' => $collectionId, + 'type' => $attributeConfig['type'], + 'status' => 'available', + 'size' => $dimension, + 'required' => $attributeConfig['required'] ?? false, + 'signed' => $attributeConfig['signed'] ?? false, + 'default' => $attributeConfig['default'] ?? null, + 'array' => $attributeConfig['array'] ?? false, + 'format' => $attributeConfig['format'] ?? '', + 'formatOptions' => $attributeConfig['formatOptions'] ?? [], + 'filters' => $attributeConfig['filters'] ?? [], + 'options' => $attributeConfig['options'] ?? [], + ]); + }, $collections['defaultAttributes']); + $dbForProject->createDocuments('attributes', $attributeDocs); + + $indexDocs = array_map(function ($indexConfig) use ($database, $collection, $databaseId, $collectionId) { + $key = \is_string($indexConfig['$id']) ? $indexConfig['$id'] : (string) $indexConfig['$id']; + + return new Document([ + '$id' => ID::custom($database->getSequence() . '_' . $collection->getSequence() . '_' . $key), + 'key' => $key, + 'status' => 'available', + 'databaseInternalId' => $database->getSequence(), + 'databaseId' => $databaseId, + 'collectionInternalId' => $collection->getSequence(), + 'collectionId' => $collectionId, + 'type' => $indexConfig['type'], + 'attributes' => $indexConfig['attributes'] ?? [], + 'lengths' => $indexConfig['lengths'] ?? [], + 'orders' => $indexConfig['orders'] ?? [], + ]); + }, $collections['defaultIndexes']); + + if (!empty($indexDocs)) { + $dbForProject->createDocuments('indexes', $indexDocs); + } + } catch (DuplicateException) { + throw new Exception($this->getDuplicateException()); + } catch (IndexException) { + throw new Exception($this->getInvalidIndexException()); + } catch (LimitException) { + throw new Exception($this->getLimitException()); + } + + $queueForEvents + ->setContext('database', $database) + ->setParam('databaseId', $databaseId) + ->setParam($this->getEventsParamKey(), $collection->getId()); + + $response + ->setStatusCode(SwooleResponse::STATUS_CODE_CREATED) + ->dynamic($collection, $this->getResponseModel()); + } +} diff --git a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Delete.php b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Delete.php new file mode 100644 index 0000000000..1d89cb0f19 --- /dev/null +++ b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Delete.php @@ -0,0 +1,61 @@ +setHttpMethod(self::HTTP_REQUEST_METHOD_DELETE) + ->setHttpPath('/v1/vectordb/:databaseId/collections/:collectionId') + ->desc('Delete collection') + ->groups(['api', 'database', 'schema']) + ->label('scope', 'collections.write') + ->label('resourceType', RESOURCE_TYPE_DATABASES) + ->label('event', 'databases.[databaseId].collections.[collectionId].delete') + ->label('audits.event', 'collection.delete') + ->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}') + ->label('sdk', new Method( + namespace: 'vectorDB', + group: 'collections', + name: 'deleteCollection', + description: '/docs/references/vectordb/delete-collection.md', + auth: [AuthType::KEY], + responses: [ + new SDKResponse( + code: SwooleResponse::STATUS_CODE_NOCONTENT, + model: UtopiaResponse::MODEL_NONE, + ) + ], + contentType: ContentType::NONE + )) + ->param('databaseId', '', new UID(), 'Database ID.') + ->param('collectionId', '', new UID(), 'Collection ID.') + ->inject('response') + ->inject('dbForProject') + ->inject('getDatabasesDB') + ->inject('queueForDatabase') + ->inject('queueForEvents') + ->callback($this->action(...)); + } +} diff --git a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Bulk/Delete.php b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Bulk/Delete.php new file mode 100644 index 0000000000..54ff25066c --- /dev/null +++ b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Bulk/Delete.php @@ -0,0 +1,71 @@ +setHttpMethod(self::HTTP_REQUEST_METHOD_DELETE) + ->setHttpPath('/v1/vectordb/:databaseId/collections/:collectionId/documents') + ->desc('Delete documents') + ->groups(['api', 'database']) + ->label('scope', 'documents.write') + ->label('resourceType', RESOURCE_TYPE_DATABASES) + ->label('audits.event', 'documents.delete') + ->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}') + ->label('abuse-key', 'ip:{ip},method:{method},url:{url},userId:{userId}') + ->label('abuse-limit', APP_LIMIT_WRITE_RATE_DEFAULT) + ->label('abuse-time', APP_LIMIT_WRITE_RATE_PERIOD_DEFAULT) + ->label('sdk', new Method( + namespace: 'vectorDB', + group: $this->getSdkGroup(), + name: 'deleteDocuments', + description: '/docs/references/vectordb/delete-documents.md', + auth: [AuthType::ADMIN, AuthType::KEY], + responses: [ + new SDKResponse( + code: SwooleResponse::STATUS_CODE_OK, + model: $this->getResponseModel(), + ) + ], + contentType: ContentType::JSON + )) + ->param('databaseId', '', new UID(), 'Database ID.') + ->param('collectionId', '', new UID(), 'Collection ID. You can create a new collection using the Database service [server integration](https://appwrite.io/docs/server/databases#databasesCreateCollection).') + ->param('queries', [], new ArrayList(new Text(APP_LIMIT_ARRAY_ELEMENT_SIZE), APP_LIMIT_ARRAY_PARAMS_SIZE), 'Array of query strings generated using the Query class provided by the SDK. [Learn more about queries](https://appwrite.io/docs/queries). Maximum of ' . APP_LIMIT_ARRAY_PARAMS_SIZE . ' queries are allowed, each ' . APP_LIMIT_ARRAY_ELEMENT_SIZE . ' characters long.', true) + ->param('transactionId', null, new UID(), 'Transaction ID for staging the operation.', true) + ->inject('response') + ->inject('dbForProject') + ->inject('getDatabasesDB') + ->inject('queueForStatsUsage') + ->inject('queueForEvents') + ->inject('queueForRealtime') + ->inject('queueForFunctions') + ->inject('queueForWebhooks') + ->inject('plan') + ->callback($this->action(...)); + } +} diff --git a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Bulk/Update.php b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Bulk/Update.php new file mode 100644 index 0000000000..fad7adbf34 --- /dev/null +++ b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Bulk/Update.php @@ -0,0 +1,73 @@ +setHttpMethod(self::HTTP_REQUEST_METHOD_PATCH) + ->setHttpPath('/v1/vectordb/:databaseId/collections/:collectionId/documents') + ->desc('Update documents') + ->groups(['api', 'database']) + ->label('scope', 'documents.write') + ->label('resourceType', RESOURCE_TYPE_DATABASES) + ->label('audits.event', 'documents.update') + ->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}') + ->label('abuse-key', 'ip:{ip},method:{method},url:{url},userId:{userId}') + ->label('abuse-limit', APP_LIMIT_WRITE_RATE_DEFAULT * 2) + ->label('abuse-time', APP_LIMIT_WRITE_RATE_PERIOD_DEFAULT) + ->label('sdk', new Method( + namespace: 'vectorDB', + group: $this->getSdkGroup(), + name: 'updateDocuments', + description: '/docs/references/vectordb/update-documents.md', + auth: [AuthType::ADMIN, AuthType::KEY], + responses: [ + new SDKResponse( + code: SwooleResponse::STATUS_CODE_OK, + model: $this->getResponseModel(), + ) + ], + contentType: ContentType::JSON + )) + ->param('databaseId', '', new UID(), 'Database ID.') + ->param('collectionId', '', new UID(), 'Collection ID.') + ->param('data', [], new JSON(), 'Document data as JSON object. Include only attribute and value pairs to be updated.', true) + ->param('queries', [], new ArrayList(new Text(APP_LIMIT_ARRAY_ELEMENT_SIZE), APP_LIMIT_ARRAY_PARAMS_SIZE), 'Array of query strings generated using the Query class provided by the SDK. [Learn more about queries](https://appwrite.io/docs/queries). Maximum of ' . APP_LIMIT_ARRAY_PARAMS_SIZE . ' queries are allowed, each ' . APP_LIMIT_ARRAY_ELEMENT_SIZE . ' characters long.', true) + ->param('transactionId', null, new UID(), 'Transaction ID for staging the operation.', true) + ->inject('response') + ->inject('dbForProject') + ->inject('getDatabasesDB') + ->inject('queueForStatsUsage') + ->inject('queueForEvents') + ->inject('queueForRealtime') + ->inject('queueForFunctions') + ->inject('queueForWebhooks') + ->inject('plan') + ->callback($this->action(...)); + } +} diff --git a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Bulk/Upsert.php b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Bulk/Upsert.php new file mode 100644 index 0000000000..f7d16e4d60 --- /dev/null +++ b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Bulk/Upsert.php @@ -0,0 +1,73 @@ +setHttpMethod(self::HTTP_REQUEST_METHOD_PUT) + ->setHttpPath('/v1/vectordb/:databaseId/collections/:collectionId/documents') + ->desc('Upsert documents') + ->groups(['api', 'database']) + ->label('scope', 'documents.write') + ->label('resourceType', RESOURCE_TYPE_DATABASES) + ->label('audits.event', 'document.create') + ->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}') + ->label('abuse-key', 'ip:{ip},method:{method},url:{url},userId:{userId}') + ->label('abuse-limit', APP_LIMIT_WRITE_RATE_DEFAULT * 2) + ->label('abuse-time', APP_LIMIT_WRITE_RATE_PERIOD_DEFAULT) + ->label('sdk', [ + new Method( + namespace: 'vectorDB', + group: $this->getSdkGroup(), + name: 'upsertDocuments', + description: '/docs/references/vectordb/upsert-documents.md', + auth: [AuthType::ADMIN, AuthType::KEY], + responses: [ + new SDKResponse( + code: SwooleResponse::STATUS_CODE_CREATED, + model: $this->getResponseModel(), + ) + ], + contentType: ContentType::JSON, + ) + ]) + ->param('databaseId', '', new UID(), 'Database ID.') + ->param('collectionId', '', new UID(), 'Collection ID.') + ->param('documents', [], fn (array $plan) => new ArrayList(new JSON(), $plan['databasesBatchSize'] ?? APP_LIMIT_DATABASE_BATCH), 'Array of document data as JSON objects. May contain partial documents.', false, ['plan']) + ->param('transactionId', null, new UID(), 'Transaction ID for staging the operation.', true) + ->inject('response') + ->inject('dbForProject') + ->inject('getDatabasesDB') + ->inject('queueForStatsUsage') + ->inject('queueForEvents') + ->inject('queueForRealtime') + ->inject('queueForFunctions') + ->inject('queueForWebhooks') + ->inject('plan') + ->callback($this->action(...)); + } +} diff --git a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Create.php b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Create.php new file mode 100644 index 0000000000..5550e64fab --- /dev/null +++ b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Create.php @@ -0,0 +1,114 @@ +setHttpMethod(self::HTTP_REQUEST_METHOD_POST) + ->setHttpPath('/v1/vectordb/:databaseId/collections/:collectionId/documents') + ->desc('Create document') + ->groups(['api', 'database']) + ->label('scope', 'documents.write') + ->label('resourceType', RESOURCE_TYPE_DATABASES) + ->label('audits.event', 'document.create') + ->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}') + ->label('abuse-key', 'ip:{ip},method:{method},url:{url},userId:{userId}') + ->label('abuse-limit', APP_LIMIT_WRITE_RATE_DEFAULT * 2) + ->label('abuse-time', APP_LIMIT_WRITE_RATE_PERIOD_DEFAULT) + ->label('sdk', [ + new Method( + namespace: 'vectorDB', + group: $this->getSdkGroup(), + name: 'createDocument', + desc: 'Create document', + description: '/docs/references/vectordb/create-document.md', + auth: [AuthType::SESSION, AuthType::KEY, AuthType::JWT], + responses: [ + new SDKResponse( + code: SwooleResponse::STATUS_CODE_CREATED, + model: $this->getResponseModel(), + ) + ], + contentType: ContentType::JSON, + parameters: [ + new Parameter('databaseId', optional: false), + new Parameter('collectionId', optional: false), + new Parameter('documentId', optional: false), + new Parameter('data', optional: false), + new Parameter('permissions', optional: true), + ] + ), + new Method( + namespace: 'vectorDB', + group: $this->getSdkGroup(), + name: 'createDocuments', + desc: 'Create documents', + description: '/docs/references/vectordb/create-documents.md', + auth: [AuthType::ADMIN, AuthType::KEY], + responses: [ + new SDKResponse( + code: SwooleResponse::STATUS_CODE_CREATED, + model: $this->getBulkResponseModel(), + ) + ], + contentType: ContentType::JSON, + parameters: [ + new Parameter('databaseId', optional: false), + new Parameter('collectionId', optional: false), + new Parameter('documents', optional: false), + ] + ) + ]) + ->param('databaseId', '', new UID(), 'Database ID.') + ->param('documentId', '', new CustomId(), 'Document ID. Choose a custom ID or generate a random ID with `ID.unique()`. Valid chars are a-z, A-Z, 0-9, period, hyphen, and underscore. Can\'t start with a special char. Max length is 36 chars.', true) + ->param('collectionId', '', new UID(), 'Collection ID. You can create a new collection using the Database service [server integration](https://appwrite.io/docs/server/databases#databasesCreateCollection). Make sure to define attributes before creating documents.') + ->param('data', [], new JSON(), 'Document data as JSON object.', true, example: '{"embeddings": [0.12, -0.55, 0.88, 1.02], "metadata": {"key":"value"} }') + ->param('permissions', null, new Permissions(APP_LIMIT_ARRAY_PARAMS_SIZE, [Database::PERMISSION_READ, Database::PERMISSION_UPDATE, Database::PERMISSION_DELETE, Database::PERMISSION_WRITE]), 'An array of permissions strings. By default, only the current user is granted all permissions. [Learn more about permissions](https://appwrite.io/docs/permissions).', true) + ->param('documents', [], fn (array $plan) => new ArrayList(new JSON(), $plan['databasesBatchSize'] ?? APP_LIMIT_DATABASE_BATCH), 'Array of documents data as JSON objects.', true, ['plan']) + ->param('transactionId', null, new UID(), 'Transaction ID for staging the operation.', true) + ->inject('response') + ->inject('dbForProject') + ->inject('getDatabasesDB') + ->inject('user') + ->inject('queueForEvents') + ->inject('queueForStatsUsage') + ->inject('queueForRealtime') + ->inject('queueForFunctions') + ->inject('queueForWebhooks') + ->inject('plan') + ->callback($this->action(...)); + } +} diff --git a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Delete.php b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Delete.php new file mode 100644 index 0000000000..2464f26f91 --- /dev/null +++ b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Delete.php @@ -0,0 +1,75 @@ +setHttpMethod(self::HTTP_REQUEST_METHOD_DELETE) + ->setHttpPath('/v1/vectordb/:databaseId/collections/:collectionId/documents/:documentId') + ->desc('Delete document') + ->groups(['api', 'database']) + ->label('scope', 'documents.write') + ->label('resourceType', RESOURCE_TYPE_DATABASES) + ->label('event', 'databases.[databaseId].collections.[collectionId].documents.[documentId].delete') + ->label('audits.event', 'document.delete') + ->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}/document/{request.documentId}') + ->label('abuse-key', 'ip:{ip},method:{method},url:{url},userId:{userId}') + ->label('abuse-limit', APP_LIMIT_WRITE_RATE_DEFAULT) + ->label('abuse-time', APP_LIMIT_WRITE_RATE_PERIOD_DEFAULT) + ->label('sdk', new Method( + namespace: 'vectorDB', + group: $this->getSdkGroup(), + name: 'deleteDocument', + description: '/docs/references/vectordb/delete-document.md', + auth: [AuthType::SESSION, AuthType::KEY, AuthType::JWT], + responses: [ + new SDKResponse( + code: SwooleResponse::STATUS_CODE_NOCONTENT, + model: UtopiaResponse::MODEL_NONE, + ) + ], + contentType: ContentType::NONE + )) + ->param('databaseId', '', new UID(), 'Database ID.') + ->param('collectionId', '', new UID(), 'Collection ID. You can create a new collection using the Database service [server integration](https://appwrite.io/docs/server/databases#databasesCreateCollection).') + ->param('documentId', '', new UID(), 'Document ID.') + ->param('transactionId', null, new UID(), 'Transaction ID for staging the operation.', true) + ->inject('requestTimestamp') + ->inject('response') + ->inject('dbForProject') + ->inject('getDatabasesDB') + ->inject('queueForEvents') + ->inject('queueForStatsUsage') + ->inject('transactionState') + ->inject('plan') + ->callback($this->action(...)); + } +} diff --git a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Get.php b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Get.php new file mode 100644 index 0000000000..8a1de28148 --- /dev/null +++ b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Get.php @@ -0,0 +1,63 @@ +setHttpMethod(self::HTTP_REQUEST_METHOD_GET) + ->setHttpPath('/v1/vectordb/:databaseId/collections/:collectionId/documents/:documentId') + ->desc('Get document') + ->groups(['api', 'database']) + ->label('scope', 'documents.read') + ->label('resourceType', RESOURCE_TYPE_DATABASES) + ->label('sdk', new Method( + namespace: 'vectorDB', + group: $this->getSdkGroup(), + name: 'getDocument', + description: '/docs/references/vectordb/get-document.md', + auth: [AuthType::SESSION, AuthType::KEY, AuthType::JWT], + responses: [ + new SDKResponse( + code: SwooleResponse::STATUS_CODE_OK, + model: $this->getResponseModel(), + ) + ], + contentType: ContentType::JSON + )) + ->param('databaseId', '', new UID(), 'Database ID.') + ->param('collectionId', '', new UID(), 'Collection ID. You can create a new collection using the Database service [server integration](https://appwrite.io/docs/server/databases#databasesCreateCollection).') + ->param('documentId', '', new UID(), 'Document ID.') + ->param('queries', [], new ArrayList(new Text(APP_LIMIT_ARRAY_ELEMENT_SIZE), APP_LIMIT_ARRAY_PARAMS_SIZE), 'Array of query strings generated using the Query class provided by the SDK. [Learn more about queries](https://appwrite.io/docs/queries). Maximum of ' . APP_LIMIT_ARRAY_PARAMS_SIZE . ' queries are allowed, each ' . APP_LIMIT_ARRAY_ELEMENT_SIZE . ' characters long.', true) + ->param('transactionId', null, new UID(), 'Transaction ID to read uncommitted changes within the transaction.', true) + ->inject('response') + ->inject('dbForProject') + ->inject('getDatabasesDB') + ->inject('queueForStatsUsage') + ->inject('transactionState') + ->callback($this->action(...)); + } +} diff --git a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Logs/XList.php b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Logs/XList.php new file mode 100644 index 0000000000..56ae0d68ab --- /dev/null +++ b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Logs/XList.php @@ -0,0 +1,56 @@ +setHttpMethod(self::HTTP_REQUEST_METHOD_GET) + ->setHttpPath('/v1/vectordb/:databaseId/collections/:collectionId/documents/:documentId/logs') + ->desc('List document logs') + ->groups(['api', 'database']) + ->label('scope', 'documents.read') + ->label('resourceType', RESOURCE_TYPE_DATABASES) + ->label('sdk', new Method( + namespace: 'vectorDB', + group: 'logs', + name: 'listDocumentLogs', + description: '/docs/references/vectordb/get-document-logs.md', + auth: [AuthType::ADMIN], + responses: [ + new SDKResponse( + code: SwooleResponse::STATUS_CODE_OK, + model: $this->getResponseModel(), + ) + ], + contentType: ContentType::JSON, + )) + ->param('databaseId', '', new UID(), 'Database ID.') + ->param('collectionId', '', new UID(), 'Collection ID.') + ->param('documentId', '', new UID(), 'Document ID.') + ->param('queries', [], new Queries([new Limit(), new Offset()]), 'Array of query strings generated using the Query class provided by the SDK. [Learn more about queries](https://appwrite.io/docs/queries). Only supported methods are limit and offset', true) + ->inject('response') + ->inject('dbForProject') + ->inject('locale') + ->inject('geodb') + ->callback($this->action(...)); + } +} diff --git a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Update.php b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Update.php new file mode 100644 index 0000000000..b7b06168ff --- /dev/null +++ b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Update.php @@ -0,0 +1,74 @@ +setHttpMethod(self::HTTP_REQUEST_METHOD_PATCH) + ->setHttpPath('/v1/vectordb/:databaseId/collections/:collectionId/documents/:documentId') + ->desc('Update document') + ->groups(['api', 'database']) + ->label('event', 'databases.[databaseId].collections.[collectionId].documents.[documentId].update') + ->label('scope', 'documents.write') + ->label('resourceType', RESOURCE_TYPE_DATABASES) + ->label('audits.event', 'document.update') + ->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}/document/{response.$id}') + ->label('abuse-key', 'ip:{ip},method:{method},url:{url},userId:{userId}') + ->label('abuse-limit', APP_LIMIT_WRITE_RATE_DEFAULT * 2) + ->label('abuse-time', APP_LIMIT_WRITE_RATE_PERIOD_DEFAULT) + ->label('sdk', new Method( + namespace: 'vectorDB', + group: $this->getSdkGroup(), + name: 'updateDocument', + description: '/docs/references/vectordb/update-document.md', + auth: [AuthType::SESSION, AuthType::KEY, AuthType::JWT], + responses: [ + new SDKResponse( + code: SwooleResponse::STATUS_CODE_OK, + model: $this->getResponseModel(), + ) + ], + contentType: ContentType::JSON + )) + ->param('databaseId', '', new UID(), 'Database ID.') + ->param('collectionId', '', new UID(), 'Collection ID.') + ->param('documentId', '', new UID(), 'Document ID.') + ->param('data', [], new JSON(), 'Document data as JSON object. Include only fields and value pairs to be updated.', true) + ->param('permissions', null, new Permissions(APP_LIMIT_ARRAY_PARAMS_SIZE, [Database::PERMISSION_READ, Database::PERMISSION_UPDATE, Database::PERMISSION_DELETE, Database::PERMISSION_WRITE]), 'An array of permissions strings. By default, the current permissions are inherited. [Learn more about permissions](https://appwrite.io/docs/permissions).', true) + ->param('transactionId', null, new UID(), 'Transaction ID for staging the operation.', true) + ->inject('requestTimestamp') + ->inject('response') + ->inject('dbForProject') + ->inject('getDatabasesDB') + ->inject('queueForEvents') + ->inject('queueForStatsUsage') + ->inject('transactionState') + ->inject('plan') + ->callback($this->action(...)); + } +} diff --git a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Upsert.php b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Upsert.php new file mode 100644 index 0000000000..02c2700e8b --- /dev/null +++ b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/Upsert.php @@ -0,0 +1,78 @@ +setHttpMethod(self::HTTP_REQUEST_METHOD_PUT) + ->setHttpPath('/v1/vectordb/:databaseId/collections/:collectionId/documents/:documentId') + ->desc('Upsert a document') + ->groups(['api', 'database']) + ->label('event', 'databases.[databaseId].collections.[collectionId].documents.[documentId].upsert') + ->label('scope', 'documents.write') + ->label('resourceType', RESOURCE_TYPE_DATABASES) + ->label('audits.event', 'document.upsert') + ->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}/document/{response.$id}') + ->label('abuse-key', 'ip:{ip},method:{method},url:{url},userId:{userId}') + ->label('abuse-limit', APP_LIMIT_WRITE_RATE_DEFAULT * 2) + ->label('abuse-time', APP_LIMIT_WRITE_RATE_PERIOD_DEFAULT) + ->label('sdk', [ + new Method( + namespace: 'vectorDB', + group: $this->getSdkGroup(), + name: 'upsertDocument', + description: '/docs/references/vectordb/upsert-document.md', + auth: [AuthType::SESSION, AuthType::KEY, AuthType::JWT], + responses: [ + new SDKResponse( + code: SwooleResponse::STATUS_CODE_CREATED, + model: $this->getResponseModel(), + ) + ], + contentType: ContentType::JSON + ), + ]) + ->param('databaseId', '', new UID(), 'Database ID.') + ->param('collectionId', '', new UID(), 'Collection ID.') + ->param('documentId', '', fn (Database $dbForProject) => new CustomId(false, $dbForProject->getAdapter()->getMaxUIDLength()), 'Document ID.', false, ['dbForProject']) + ->param('data', [], new JSON(), 'Document data as JSON object. Include all required fields of the document to be created or updated.', true) + ->param('permissions', null, new Permissions(APP_LIMIT_ARRAY_PARAMS_SIZE, [Database::PERMISSION_READ, Database::PERMISSION_UPDATE, Database::PERMISSION_DELETE, Database::PERMISSION_WRITE]), 'An array of permissions strings. By default, the current permissions are inherited. [Learn more about permissions](https://appwrite.io/docs/permissions).', true) + ->param('transactionId', null, new UID(), 'Transaction ID for staging the operation.', true) + ->inject('requestTimestamp') + ->inject('response') + ->inject('user') + ->inject('dbForProject') + ->inject('getDatabasesDB') + ->inject('queueForEvents') + ->inject('queueForStatsUsage') + ->inject('transactionState') + ->inject('plan') + ->callback($this->action(...)); + } +} diff --git a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/XList.php b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/XList.php new file mode 100644 index 0000000000..a55db09a65 --- /dev/null +++ b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Documents/XList.php @@ -0,0 +1,64 @@ +setHttpMethod(self::HTTP_REQUEST_METHOD_GET) + ->setHttpPath('/v1/vectordb/:databaseId/collections/:collectionId/documents') + ->desc('List documents') + ->groups(['api', 'database']) + ->label('scope', 'documents.read') + ->label('resourceType', RESOURCE_TYPE_DATABASES) + ->label('sdk', new Method( + namespace: 'vectorDB', + group: $this->getSdkGroup(), + name: 'listDocuments', + description: '/docs/references/vectordb/list-documents.md', + auth: [AuthType::SESSION, AuthType::KEY, AuthType::JWT], + responses: [ + new SDKResponse( + code: SwooleResponse::STATUS_CODE_OK, + model: $this->getResponseModel(), + ) + ], + contentType: ContentType::JSON + )) + ->param('databaseId', '', new UID(), 'Database ID.') + ->param('collectionId', '', new UID(), 'Collection ID. You can create a new collection using the Database service [server integration](https://appwrite.io/docs/server/databases#databasesCreateCollection).') + ->param('queries', [], new ArrayList(new Text(APP_LIMIT_ARRAY_ELEMENT_SIZE), APP_LIMIT_ARRAY_PARAMS_SIZE), 'Array of query strings generated using the Query class provided by the SDK. [Learn more about queries](https://appwrite.io/docs/queries). Maximum of ' . APP_LIMIT_ARRAY_PARAMS_SIZE . ' queries are allowed, each ' . APP_LIMIT_ARRAY_ELEMENT_SIZE . ' characters long.', true) + ->param('transactionId', null, new UID(), 'Transaction ID to read uncommitted changes within the transaction.', true) + ->param('total', true, new Boolean(true), 'When set to false, the total count returned will be 0 and will not be calculated.', true) + ->inject('response') + ->inject('dbForProject') + ->inject('getDatabasesDB') + ->inject('queueForStatsUsage') + ->inject('transactionState') + ->callback($this->action(...)); + } +} diff --git a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Get.php b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Get.php new file mode 100644 index 0000000000..83bde1bf5f --- /dev/null +++ b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Get.php @@ -0,0 +1,55 @@ +setHttpMethod(self::HTTP_REQUEST_METHOD_GET) + ->setHttpPath('/v1/vectordb/:databaseId/collections/:collectionId') + ->desc('Get collection') + ->groups(['api', 'database']) + ->label('scope', 'collections.read') + ->label('resourceType', RESOURCE_TYPE_DATABASES) + ->label('sdk', new Method( + namespace: 'vectorDB', + group: 'collections', + name: 'getCollection', + description: '/docs/references/vectordb/get-collection.md', + auth: [AuthType::KEY], + responses: [ + new SDKResponse( + code: SwooleResponse::STATUS_CODE_OK, + model: $this->getResponseModel(), + ) + ], + contentType: ContentType::JSON + )) + ->param('databaseId', '', new UID(), 'Database ID.') + ->param('collectionId', '', new UID(), 'Collection ID.') + ->inject('response') + ->inject('dbForProject') + ->callback($this->action(...)); + } +} diff --git a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Indexes/Create.php b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Indexes/Create.php new file mode 100644 index 0000000000..aecab100e8 --- /dev/null +++ b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Indexes/Create.php @@ -0,0 +1,72 @@ +setHttpMethod(self::HTTP_REQUEST_METHOD_POST) + ->setHttpPath('/v1/vectordb/:databaseId/collections/:collectionId/indexes') + ->desc('Create index') + ->groups(['api', 'database']) + ->label('event', 'databases.[databaseId].tables.[tableId].indexes.[indexId].create') + ->label('scope', 'collections.write') + ->label('resourceType', RESOURCE_TYPE_DATABASES) + ->label('audits.event', 'index.create') + ->label('audits.resource', 'database/{request.databaseId}/collection/{request.tableId}') + ->label('sdk', new Method( + namespace: 'vectorDB', + group: $this->getSdkGroup(), + name: 'createIndex', + description: '/docs/references/vectordb/create-index.md', + auth: [AuthType::KEY], + responses: [ + new SDKResponse( + code: SwooleResponse::STATUS_CODE_ACCEPTED, + model: $this->getResponseModel(), + ) + ], + contentType: ContentType::JSON + )) + ->param('databaseId', '', new UID(), 'Database ID.') + ->param('collectionId', '', new UID(), 'Collection ID. You can create a new collection using the Database service [server integration](https://appwrite.io/docs/server/databases#databasesCreateCollection).') + ->param('key', null, new Key(), 'Index Key.') + ->param('type', null, new WhiteList([Database::INDEX_HNSW_EUCLIDEAN,Database::INDEX_HNSW_DOT, Database::INDEX_HNSW_COSINE]), 'Index type.') + ->param('attributes', null, new ArrayList(new Key(true), APP_LIMIT_ARRAY_PARAMS_SIZE), 'Array of attributes to index. Maximum of ' . APP_LIMIT_ARRAY_PARAMS_SIZE . ' attributes are allowed, each 32 characters long.') + ->param('orders', [], new ArrayList(new WhiteList(['ASC', 'DESC'], false, Database::VAR_STRING), APP_LIMIT_ARRAY_PARAMS_SIZE), 'Array of index orders. Maximum of ' . APP_LIMIT_ARRAY_PARAMS_SIZE . ' orders are allowed.', true) + ->param('lengths', [], new ArrayList(new Nullable(new Integer()), APP_LIMIT_ARRAY_PARAMS_SIZE), 'Length of index. Maximum of ' . APP_LIMIT_ARRAY_PARAMS_SIZE, optional: true) + ->inject('response') + ->inject('dbForProject') + ->inject('getDatabasesDB') + ->inject('queueForDatabase') + ->inject('queueForEvents') + ->callback($this->action(...)); + } +} diff --git a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Indexes/Delete.php b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Indexes/Delete.php new file mode 100644 index 0000000000..c6ea3da7fe --- /dev/null +++ b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Indexes/Delete.php @@ -0,0 +1,66 @@ +setHttpMethod(self::HTTP_REQUEST_METHOD_DELETE) + ->setHttpPath('/v1/vectordb/:databaseId/collections/:collectionId/indexes/:key') + ->desc('Delete index') + ->groups(['api', 'database']) + ->label('scope', 'collections.write') + ->label('resourceType', RESOURCE_TYPE_DATABASES) + ->label('event', 'databases.[databaseId].collections.[collectionId].indexes.[indexId].update') + ->label('audits.event', 'index.delete') + ->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}') + ->label('sdk', new Method( + namespace: 'vectorDB', + group: $this->getSdkGroup(), + name: 'deleteIndex', // getName needs to be different from parent action to avoid conflict in path name + description: '/docs/references/vectordb/delete-index.md', + auth: [AuthType::KEY], + responses: [ + new SDKResponse( + code: SwooleResponse::STATUS_CODE_NOCONTENT, + model: UtopiaResponse::MODEL_NONE, + ) + ], + contentType: ContentType::NONE + )) + ->param('databaseId', '', new UID(), 'Database ID.') + ->param('collectionId', '', new UID(), 'Collection ID. You can create a new collection using the Database service [server integration](https://appwrite.io/docs/server/databases#databasesCreateCollection).') + ->param('key', '', new Key(), 'Index Key.') + ->inject('response') + ->inject('dbForProject') + ->inject('queueForDatabase') + ->inject('queueForEvents') + ->callback($this->action(...)); + } +} diff --git a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Indexes/Get.php b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Indexes/Get.php new file mode 100644 index 0000000000..b1dedefe2e --- /dev/null +++ b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Indexes/Get.php @@ -0,0 +1,57 @@ +setHttpMethod(self::HTTP_REQUEST_METHOD_GET) + ->setHttpPath('/v1/vectordb/:databaseId/collections/:collectionId/indexes/:key') + ->desc('Get index') + ->groups(['api', 'database']) + ->label('scope', 'collections.read') + ->label('resourceType', RESOURCE_TYPE_DATABASES) + ->label('sdk', new Method( + namespace: 'vectorDB', + group: $this->getSdkGroup(), + name: 'getIndex', // getName needs to be different from parent action to avoid conflict in path name + description: '/docs/references/vectordb/get-index.md', + auth: [AuthType::KEY], + responses: [ + new SDKResponse( + code: SwooleResponse::STATUS_CODE_OK, + model: $this->getResponseModel(), + ) + ], + contentType: ContentType::JSON + )) + ->param('databaseId', '', new UID(), 'Database ID.') + ->param('collectionId', '', new UID(), 'Collection ID. You can create a new collection using the Database service [server integration](https://appwrite.io/docs/server/databases#databasesCreateCollection).') + ->param('key', null, new Key(), 'Index Key.') + ->inject('response') + ->inject('dbForProject') + ->callback($this->action(...)); + } +} diff --git a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Indexes/XList.php b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Indexes/XList.php new file mode 100644 index 0000000000..c5a1dc48e3 --- /dev/null +++ b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Indexes/XList.php @@ -0,0 +1,59 @@ +setHttpMethod(self::HTTP_REQUEST_METHOD_GET) + ->setHttpPath('/v1/vectordb/:databaseId/collections/:collectionId/indexes') + ->desc('List indexes') + ->groups(['api', 'database']) + ->label('scope', 'collections.read') + ->label('resourceType', RESOURCE_TYPE_DATABASES) + ->label('sdk', new Method( + namespace: 'vectorDB', + group: $this->getSdkGroup(), + name: 'listIndexes', // getName needs to be different from parent action to avoid conflict in path name + description: '/docs/references/vectordb/list-indexes.md', + auth: [AuthType::KEY], + responses: [ + new SDKResponse( + code: SwooleResponse::STATUS_CODE_OK, + model: $this->getResponseModel(), + ) + ], + contentType: ContentType::JSON + )) + ->param('databaseId', '', new UID(), 'Database ID.') + ->param('collectionId', '', new UID(), 'Collection ID. You can create a new collection using the Database service [server integration](https://appwrite.io/docs/server/databases#databasesCreateCollection).') + ->param('queries', [], new Indexes(), 'Array of query strings generated using the Query class provided by the SDK. [Learn more about queries](https://appwrite.io/docs/queries). Maximum of ' . APP_LIMIT_ARRAY_PARAMS_SIZE . ' queries are allowed, each ' . APP_LIMIT_ARRAY_ELEMENT_SIZE . ' characters long. You may filter on the following attributes: ' . implode(', ', Indexes::ALLOWED_ATTRIBUTES), true) + ->param('total', true, new Boolean(true), 'When set to false, the total count returned will be 0 and will not be calculated.', true) + ->inject('response') + ->inject('dbForProject') + ->callback($this->action(...)); + } +} diff --git a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Logs/XList.php b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Logs/XList.php new file mode 100644 index 0000000000..931620edb8 --- /dev/null +++ b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Logs/XList.php @@ -0,0 +1,55 @@ +setHttpMethod(self::HTTP_REQUEST_METHOD_GET) + ->setHttpPath('/v1/vectordb/:databaseId/collections/:collectionId/logs') + ->desc('List collection logs') + ->groups(['api', 'database']) + ->label('scope', 'collections.read') + ->label('resourceType', RESOURCE_TYPE_DATABASES) + ->label('sdk', new Method( + namespace: 'vectorDB', + group: $this->getSdkGroup(), + name: 'listCollectionLogs', + description: '/docs/references/vectordb/get-collection-logs.md', + auth: [AuthType::ADMIN], + responses: [ + new SDKResponse( + code: SwooleResponse::STATUS_CODE_OK, + model: $this->getResponseModel(), + ) + ], + contentType: ContentType::JSON + )) + ->param('databaseId', '', new UID(), 'Database ID.') + ->param('collectionId', '', new UID(), 'Collection ID.') + ->param('queries', [], new Queries([new Limit(), new Offset()]), 'Array of query strings generated using the Query class provided by the SDK. [Learn more about queries](https://appwrite.io/docs/queries). Only supported methods are limit and offset', true) + ->inject('response') + ->inject('dbForProject') + ->inject('locale') + ->inject('geodb') + ->callback($this->action(...)); + } +} diff --git a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Update.php b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Update.php new file mode 100644 index 0000000000..5642b36944 --- /dev/null +++ b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Update.php @@ -0,0 +1,114 @@ +setHttpMethod(self::HTTP_REQUEST_METHOD_PUT) + ->setHttpPath('/v1/vectordb/:databaseId/collections/:collectionId') + ->desc('Update collection') + ->groups(['api', 'database', 'schema']) + ->label('scope', 'collections.write') + ->label('resourceType', RESOURCE_TYPE_DATABASES) + ->label('event', 'databases.[databaseId].collections.[collectionId].update') + ->label('audits.event', 'collection.update') + ->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}') + ->label('sdk', new Method( + namespace: 'vectorDB', + group: 'collections', + name: 'updateCollection', + description: '/docs/references/vectordb/update-collection.md', + auth: [AuthType::KEY], + responses: [ + new SDKResponse( + code: SwooleResponse::STATUS_CODE_OK, + model: UtopiaResponse::MODEL_VECTORDB_COLLECTION, + ) + ], + contentType: ContentType::JSON + )) + ->param('databaseId', '', new UID(), 'Database ID.') + ->param('collectionId', '', new UID(), 'Collection ID.') + ->param('name', null, new Text(128), 'Collection name. Max length: 128 chars.') + ->param('dimension', null, new Range(MIN_VECTOR_DIMENSION, MAX_VECTOR_DIMENSION), 'Embedding dimensions.', true) + ->param('permissions', null, new Permissions(APP_LIMIT_ARRAY_PARAMS_SIZE), 'An array of permission strings. By default, the current permissions are inherited. [Learn more about permissions](https://appwrite.io/docs/permissions).', true) + ->param('documentSecurity', false, new Boolean(true), 'Enables configuring permissions for individual documents. A user needs one of document or collection level permissions to access a document. [Learn more about permissions](https://appwrite.io/docs/permissions).', true) + ->param('enabled', true, new Boolean(), 'Is collection enabled? When set to \'disabled\', users cannot access the collection but Server SDKs with and API key can still read and write to the collection. No data is lost when this is toggled.', true) + ->inject('response') + ->inject('dbForProject') + ->inject('getDatabasesDB') + ->inject('queueForEvents') + ->callback($this->action(...)); + } + + public function action(string $databaseId, string $collectionId, ?string $name, ?int $dimensions, ?array $permissions, bool $documentSecurity, bool $enabled, UtopiaResponse $response, \Utopia\Database\Database $dbForProject, callable $getDatabasesDB, \Appwrite\Event\Event $queueForEvents): void + { + $database = Authorization::skip(fn () => $dbForProject->getDocument('databases', $databaseId)); + if ($database->isEmpty()) { + throw new Exception(Exception::DATABASE_NOT_FOUND); + } + + $collection = $dbForProject->getDocument('database_' . $database->getSequence(), $collectionId); + if ($collection->isEmpty()) { + throw new Exception($this->getNotFoundException()); + } + + $permissions ??= $collection->getPermissions(); + + // Map aggregate permissions into the multiple permissions they represent. + $permissions = Permission::aggregate($permissions); + + $enabled ??= $collection->getAttribute('enabled', true); + + $updated = $dbForProject->updateDocument( + 'database_' . $database->getSequence(), + $collectionId, + $collection + ->setAttribute('name', $name ?? $collection->getAttribute('name')) + ->setAttribute('dimension', $dimensions ?? $collection->getAttribute('dimension')) + ->setAttribute('$permissions', $permissions) + ->setAttribute('documentSecurity', $documentSecurity) + ->setAttribute('enabled', $enabled) + ->setAttribute('search', \implode(' ', [$collectionId, $name ?? $collection->getAttribute('name')])) + ); + + $dbForDatabases = $getDatabasesDB($database); + $dbForDatabases->updateCollection('database_' . $database->getSequence() . '_collection_' . $updated->getSequence(), $permissions, $documentSecurity); + + $queueForEvents + ->setContext('database', $database) + ->setParam('databaseId', $databaseId) + ->setParam($this->getEventsParamKey(), $updated->getId()); + + $response->dynamic($updated, $this->getResponseModel()); + } +} diff --git a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Usage/Get.php b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Usage/Get.php new file mode 100644 index 0000000000..00a5eedfb8 --- /dev/null +++ b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/Usage/Get.php @@ -0,0 +1,63 @@ +setHttpMethod(self::HTTP_REQUEST_METHOD_GET) + ->setHttpPath('/v1/vectordb/:databaseId/collections/:collectionId/usage') + ->desc('Get collection usage stats') + ->groups(['api', 'database', 'usage']) + ->label('scope', 'collections.read') + ->label('resourceType', RESOURCE_TYPE_DATABASES) + ->label('sdk', new Method( + namespace: 'vectorDB', + group: null, + name: 'getCollectionUsage', + description: '/docs/references/vectordb/get-collection-usage.md', + auth: [AuthType::ADMIN], + responses: [ + new SDKResponse( + code: SwooleResponse::STATUS_CODE_OK, + model: $this->getResponseModel(), + ) + ], + contentType: ContentType::JSON, + )) + ->param('databaseId', '', new UID(), 'Database ID.') + ->param('range', '30d', new WhiteList(['24h', '30d', '90d'], true), 'Date range.', true) + ->param('collectionId', '', new UID(), 'Collection ID.') + ->inject('response') + ->inject('dbForProject') + ->inject('getDatabasesDB') + ->callback($this->action(...)); + } +} diff --git a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/XList.php b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/XList.php new file mode 100644 index 0000000000..1095e5ad25 --- /dev/null +++ b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Collections/XList.php @@ -0,0 +1,60 @@ +setHttpMethod(self::HTTP_REQUEST_METHOD_GET) + ->setHttpPath('/v1/vectordb/:databaseId/collections') + ->desc('List collections') + ->groups(['api', 'database']) + ->label('scope', 'collections.read') + ->label('resourceType', RESOURCE_TYPE_DATABASES) + ->label('sdk', new Method( + namespace: 'vectorDB', + group: 'collections', + name: 'listCollections', + description: '/docs/references/vectordb/list-collections.md', + auth: [AuthType::KEY], + responses: [ + new SDKResponse( + code: SwooleResponse::STATUS_CODE_OK, + model: $this->getResponseModel(), + ) + ], + contentType: ContentType::JSON + )) + ->param('databaseId', '', new UID(), 'Database ID.') + ->param('queries', [], new Collections(), 'Array of query strings generated using the Query class provided by the SDK. [Learn more about queries](https://appwrite.io/docs/queries). Maximum of ' . APP_LIMIT_ARRAY_PARAMS_SIZE . ' queries are allowed, each ' . APP_LIMIT_ARRAY_ELEMENT_SIZE . ' characters long. You may filter on the following attributes: ' . implode(', ', Collections::ALLOWED_ATTRIBUTES), true) + ->param('search', '', new Text(256), 'Search term to filter your list results. Max length: 256 chars.', true) + ->param('total', true, new Boolean(true), 'When set to false, the total count returned will be 0 and will not be calculated.', true) + ->inject('response') + ->inject('dbForProject') + ->callback($this->action(...)); + } +} diff --git a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Create.php b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Create.php new file mode 100644 index 0000000000..978f565cec --- /dev/null +++ b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Create.php @@ -0,0 +1,59 @@ +setHttpMethod(self::HTTP_REQUEST_METHOD_POST) + ->setHttpPath('/v1/vectordb') + ->desc('Create database') + ->groups(['api', 'database']) + ->label('event', 'databases.[databaseId].create') + ->label('scope', 'databases.write') + ->label('resourceType', RESOURCE_TYPE_DATABASES) + ->label('audits.event', 'database.create') + ->label('audits.resource', 'database/{response.$id}') + ->label('sdk', new Method( + namespace: 'vectorDB', + group: 'vectordb', + name: 'create', + description: '/docs/references/vectordb/create.md', + auth: [AuthType::KEY], + responses: [ + new SDKResponse( + code: SwooleResponse::STATUS_CODE_CREATED, + model: UtopiaResponse::MODEL_DATABASE, + ) + ], + contentType: ContentType::JSON + )) + ->param('databaseId', '', new CustomId(), 'Unique Id. Choose a custom ID or generate a random ID with `ID.unique()`. Valid chars are a-z, A-Z, 0-9, period, hyphen, and underscore. Can\'t start with a special char. Max length is 36 chars.') + ->param('name', '', new Text(128), 'Database name. Max length: 128 chars.') + ->param('enabled', true, new Boolean(), 'Is the database enabled? When set to \'disabled\', users cannot access the database but Server SDKs with an API key can still read and write to the database. No data is lost when this is toggled.', true) + ->inject('project') + ->inject('response') + ->inject('dbForProject') + ->inject('getDatabasesDB') + ->inject('queueForEvents') + ->callback($this->action(...)); + } +} diff --git a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Delete.php b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Delete.php new file mode 100644 index 0000000000..24b0b6b67b --- /dev/null +++ b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Delete.php @@ -0,0 +1,55 @@ +setHttpMethod(self::HTTP_REQUEST_METHOD_DELETE) + ->setHttpPath('/v1/vectordb/:databaseId') + ->desc('Delete database') + ->groups(['api', 'database', 'schema']) + ->label('scope', 'databases.write') + ->label('resourceType', RESOURCE_TYPE_DATABASES) + ->label('event', 'databases.[databaseId].delete') + ->label('audits.event', 'database.delete') + ->label('audits.resource', 'database/{request.databaseId}') + ->label('sdk', new Method( + namespace: 'vectorDB', + group: 'vectordb', + name: 'delete', + description: '/docs/references/vectordb/delete.md', + auth: [AuthType::KEY], + responses: [ + new SDKResponse( + code: SwooleResponse::STATUS_CODE_NOCONTENT, + model: UtopiaResponse::MODEL_NONE, + ) + ], + contentType: ContentType::NONE + )) + ->param('databaseId', '', new UID(), 'Database ID.') + ->inject('response') + ->inject('dbForProject') + ->inject('queueForDatabase') + ->inject('queueForEvents') + ->inject('queueForStatsUsage') + ->callback($this->action(...)); + } +} diff --git a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Embeddings/Text/Create.php b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Embeddings/Text/Create.php new file mode 100644 index 0000000000..c6c058408a --- /dev/null +++ b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Embeddings/Text/Create.php @@ -0,0 +1,156 @@ +setHttpMethod(self::HTTP_REQUEST_METHOD_POST) + ->setHttpPath('/v1/vectordb/embeddings/text') + ->desc('Create Text Embeddings') + ->groups(['api', 'database']) + ->label('scope', 'documents.write') + ->label('resourceType', RESOURCE_TYPE_EMBEDDINGS_TEXT) + ->label('audits.event', 'embedding.create') + ->label('audits.resource', 'vectordb/embeddings/text') + ->label('abuse-key', 'ip:{ip},method:{method},url:{url},userId:{userId}') + ->label('abuse-limit', APP_LIMIT_WRITE_RATE_DEFAULT * 2) + ->label('abuse-time', APP_LIMIT_WRITE_RATE_PERIOD_DEFAULT) + ->label('sdk', [ + new Method( + namespace: 'vectorDB', + group: $this->getSdkGroup(), + name: 'createTextEmbeddings', + desc: 'Create Text Embedding', + description: '/docs/references/vectordb/create-document.md', + auth: [AuthType::KEY, AuthType::JWT], + responses: [ + new SDKResponse( + code: SwooleResponse::STATUS_CODE_OK, + model: $this->getBulkResponseModel(), + ) + ], + contentType: ContentType::JSON, + parameters: [ + new Parameter('databaseId', optional: false), + new Parameter('collectionId', optional: false), + new Parameter('documents', optional: false), + ] + ) + ]) + ->param('texts', [], fn (array $plan) => new ArrayList(new Text(0), $plan['databasesBatchSize'] ?? APP_LIMIT_DATABASE_BATCH), 'Array of text to generate embeddings.', false, ['plan']) + ->param('model', Ollama::MODEL_EMBEDDING_GEMMA, new WhiteList(Ollama::MODELS), 'The embedding model to use for generating vector embeddings.', true) + ->inject('response') + ->inject('project') + ->inject('embeddingAgent') + ->inject('queueForStatsUsage') + ->inject('log') + ->inject('logger') + ->callback($this->action(...)); + } + + public function action(array $texts, string $model, UtopiaResponse $response, Document $project, Agent $embeddingAgent, StatsUsage $queueForStatsUsage, Log $log, ?Logger $logger): void + { + $results = []; + $embeddingAgent->getAdapter()->setModel($model); + $dimension = $embeddingAgent->getAdapter()->getEmbeddingDimension(); + + $totalDuration = 0; + $totalTokens = 0; + $totalErrors = 0; + foreach ($texts as $text) { + $embedding = []; + $error = ''; + try { + $embedResult = $embeddingAgent->embed($text); + $embedding = $embedResult['embedding'] ?? []; + $totalDuration += $embedResult['totalDuration'] ?? 0; + $totalTokens += $embedResult['tokensProcessed'] ?? 0; + } catch (\Exception $e) { + $error = 'Error while generating embedding'; + $totalErrors += 1; + if ($logger) { + $log->setNamespace("http"); + $log->setServer(System::getEnv('_APP_LOGGING_SERVICE_IDENTIFIER', \gethostname())); + $log->setVersion(System::getEnv('_APP_VERSION', 'UNKNOWN')); + $log->setType(Log::TYPE_ERROR); + $log->setMessage($e->getMessage()); + + $log->addTag('embeddingModel', $model); + $log->addTag('code', $e->getCode()); + $log->addTag('projectId', $project->getId()); + + $log->addExtra('file', $e->getFile()); + $log->addExtra('line', $e->getLine()); + $log->addExtra('trace', $e->getTraceAsString()); + + $logger->addLog($log); + } + } + + $results[] = new Document([ + 'model' => $model, + 'dimension' => $dimension, + 'embedding' => $embedding, + 'error' => $error + ]); + } + $embeddings = new Document([ + 'embeddings' => $results, + 'total' => \count($results), + ]); + + $response + ->setStatusCode(SwooleResponse::STATUS_CODE_OK) + ->dynamic($embeddings, $this->getBulkResponseModel()); + + $queueForStatsUsage + ->addMetric(METRIC_EMBEDDINGS_TEXT, \count($texts)) + ->addMetric(\str_replace('{embeddingModel}', $model, METRIC_EMBEDDINGS_MODEL_TEXT), \count($texts)) + ->addMetric(METRIC_EMBEDDINGS_TEXT_TOTAL_TOKENS, $totalTokens) + ->addMetric(\str_replace('{embeddingModel}', $model, METRIC_EMBEDDINGS_MODEL_TEXT_TOTAL_TOKENS), $totalTokens) + ->addMetric(METRIC_EMBEDDINGS_TEXT_TOTAL_DURATION, $totalDuration) + ->addMetric(\str_replace('{embeddingModel}', $model, METRIC_EMBEDDINGS_MODEL_TEXT_TOTAL_DURATION), $totalDuration) + ->addMetric(METRIC_EMBEDDINGS_TEXT_TOTAL_ERROR, $totalErrors) + ->addMetric(\str_replace('{embeddingModel}', $model, METRIC_EMBEDDINGS_MODEL_TEXT_TOTAL_ERROR), $totalErrors) + ->trigger(); + + $queueForStatsUsage->reset(); + } +} diff --git a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Get.php b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Get.php new file mode 100644 index 0000000000..18c55b7fe8 --- /dev/null +++ b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Get.php @@ -0,0 +1,49 @@ +setHttpMethod(self::HTTP_REQUEST_METHOD_GET) + ->setHttpPath('/v1/vectordb/:databaseId') + ->desc('Get database') + ->groups(['api', 'database']) + ->label('scope', 'databases.read') + ->label('resourceType', RESOURCE_TYPE_DATABASES) + ->label('sdk', new Method( + namespace: 'vectorDB', + group: 'vectordb', + name: 'get', + description: '/docs/references/vectordb/get.md', + auth: [AuthType::KEY], + responses: [ + new SDKResponse( + code: SwooleResponse::STATUS_CODE_OK, + model: UtopiaResponse::MODEL_DATABASE, + ) + ], + contentType: ContentType::JSON + )) + ->param('databaseId', '', new UID(), 'Database ID.') + ->inject('response') + ->inject('dbForProject') + ->callback($this->action(...)); + } +} diff --git a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Logs/XList.php b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Logs/XList.php new file mode 100644 index 0000000000..6232c4892b --- /dev/null +++ b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Logs/XList.php @@ -0,0 +1,57 @@ +setHttpMethod(self::HTTP_REQUEST_METHOD_GET) + ->setHttpPath('/v1/vectordb/:databaseId/logs') + ->desc('List database logs') + ->groups(['api', 'database']) + ->label('scope', 'databases.read') + ->label('resourceType', RESOURCE_TYPE_DATABASES) + ->label('sdk', [ + new Method( + namespace: 'vectorDB', + group: 'logs', + name: 'listDatabaseLogs', + description: '/docs/references/vectordb/get-logs.md', + auth: [AuthType::ADMIN], + responses: [ + new SDKResponse( + code: SwooleResponse::STATUS_CODE_OK, + model: UtopiaResponse::MODEL_LOG_LIST, + ) + ], + contentType: ContentType::JSON + ), + ]) + ->param('databaseId', '', new UID(), 'Database ID.') + ->param('queries', [], new Queries([new Limit(), new Offset()]), 'Array of query strings generated using the Query class provided by the SDK. [Learn more about queries](https://appwrite.io/docs/queries). Only supported methods are limit and offset', true) + ->inject('response') + ->inject('dbForProject') + ->inject('locale') + ->inject('geodb') + ->callback($this->action(...)); + } +} diff --git a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Transactions/Create.php b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Transactions/Create.php new file mode 100644 index 0000000000..5d13c18378 --- /dev/null +++ b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Transactions/Create.php @@ -0,0 +1,55 @@ +setHttpMethod(self::HTTP_REQUEST_METHOD_POST) + ->setHttpPath('/v1/vectordb/transactions') + ->desc('Create transaction') + ->groups(['api', 'database', 'transactions']) + ->label('scope', 'documents.write') + ->label('resourceType', RESOURCE_TYPE_DATABASES) + ->label('sdk', new Method( + namespace: 'vectorDB', + group: 'transactions', + name: 'createTransaction', + description: '/docs/references/vectordb/create-transaction.md', + auth: [AuthType::KEY, AuthType::SESSION, AuthType::JWT], + responses: [ + new SDKResponse( + code: SwooleResponse::STATUS_CODE_CREATED, + model: UtopiaResponse::MODEL_TRANSACTION, + ) + ], + contentType: ContentType::JSON + )) + ->param('ttl', APP_DATABASE_TXN_TTL_DEFAULT, new Range(min: APP_DATABASE_TXN_TTL_MIN, max: APP_DATABASE_TXN_TTL_MAX), 'Seconds before the transaction expires.', true) + ->inject('response') + ->inject('dbForProject') + ->inject('user') + ->callback($this->action(...)); + } +} diff --git a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Transactions/Delete.php b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Transactions/Delete.php new file mode 100644 index 0000000000..b7bdb3db58 --- /dev/null +++ b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Transactions/Delete.php @@ -0,0 +1,55 @@ +setHttpMethod(self::HTTP_REQUEST_METHOD_DELETE) + ->setHttpPath('/v1/vectordb/transactions/:transactionId') + ->desc('Delete transaction') + ->groups(['api', 'database', 'transactions']) + ->label('scope', 'documents.write') + ->label('resourceType', RESOURCE_TYPE_DATABASES) + ->label('sdk', new Method( + namespace: 'vectorDB', + group: 'transactions', + name: 'deleteTransaction', + description: '/docs/references/vectordb/delete-transaction.md', + auth: [AuthType::KEY, AuthType::SESSION, AuthType::JWT], + responses: [ + new SDKResponse( + code: SwooleResponse::STATUS_CODE_NOCONTENT, + model: UtopiaResponse::MODEL_NONE, + ) + ], + contentType: ContentType::NONE + )) + ->param('transactionId', '', new UID(), 'Transaction ID.') + ->inject('response') + ->inject('dbForProject') + ->inject('queueForDeletes') + ->callback($this->action(...)); + } +} diff --git a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Transactions/Get.php b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Transactions/Get.php new file mode 100644 index 0000000000..32a6115f60 --- /dev/null +++ b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Transactions/Get.php @@ -0,0 +1,54 @@ +setHttpMethod(self::HTTP_REQUEST_METHOD_GET) + ->setHttpPath('/v1/vectordb/transactions/:transactionId') + ->desc('Get transaction') + ->groups(['api', 'database', 'transactions']) + ->label('scope', 'documents.read') + ->label('resourceType', RESOURCE_TYPE_DATABASES) + ->label('sdk', new Method( + namespace: 'vectorDB', + group: 'transactions', + name: 'getTransaction', + description: '/docs/references/vectordb/get-transaction.md', + auth: [AuthType::KEY, AuthType::SESSION, AuthType::JWT], + responses: [ + new SDKResponse( + code: SwooleResponse::STATUS_CODE_OK, + model: UtopiaResponse::MODEL_TRANSACTION, + ) + ], + contentType: ContentType::JSON + )) + ->param('transactionId', '', new UID(), 'Transaction ID.') + ->inject('response') + ->inject('dbForProject') + ->callback($this->action(...)); + } +} diff --git a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Transactions/Operations/Create.php b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Transactions/Operations/Create.php new file mode 100644 index 0000000000..eabc4cd825 --- /dev/null +++ b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Transactions/Operations/Create.php @@ -0,0 +1,59 @@ +setHttpMethod(self::HTTP_REQUEST_METHOD_POST) + ->setHttpPath('/v1/vectordb/transactions/:transactionId/operations') + ->desc('Create operations') + ->groups(['api', 'database', 'transactions']) + ->label('scope', 'documents.write') + ->label('resourceType', RESOURCE_TYPE_DATABASES) + ->label('sdk', new Method( + namespace: 'vectorDB', + group: 'transactions', + name: 'createOperations', + description: '/docs/references/vectordb/create-operations.md', + auth: [AuthType::KEY, AuthType::SESSION, AuthType::JWT], + responses: [ + new SDKResponse( + code: SwooleResponse::STATUS_CODE_CREATED, + model: UtopiaResponse::MODEL_TRANSACTION, + ) + ], + contentType: ContentType::JSON + )) + ->param('transactionId', '', new UID(), 'Transaction ID.') + ->param('operations', [], new ArrayList(new Operation(type: 'documentsdb')), 'Array of staged operations.', true) + ->inject('response') + ->inject('dbForProject') + ->inject('transactionState') + ->inject('plan') + ->callback($this->action(...)); + } +} diff --git a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Transactions/Update.php b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Transactions/Update.php new file mode 100644 index 0000000000..61725a65f5 --- /dev/null +++ b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Transactions/Update.php @@ -0,0 +1,67 @@ +setHttpMethod(self::HTTP_REQUEST_METHOD_PATCH) + ->setHttpPath('/v1/vectordb/transactions/:transactionId') + ->desc('Update transaction') + ->groups(['api', 'database', 'transactions']) + ->label('scope', 'documents.write') + ->label('resourceType', RESOURCE_TYPE_DATABASES) + ->label('sdk', new Method( + namespace: 'vectorDB', + group: 'transactions', + name: 'updateTransaction', + description: '/docs/references/vectordb/update-transaction.md', + auth: [AuthType::KEY, AuthType::SESSION, AuthType::JWT], + responses: [ + new SDKResponse( + code: SwooleResponse::STATUS_CODE_OK, + model: UtopiaResponse::MODEL_TRANSACTION, + ) + ], + contentType: ContentType::JSON + )) + ->param('transactionId', '', new UID(), 'Transaction ID.') + ->param('commit', false, new Boolean(), 'Commit transaction?', true) + ->param('rollback', false, new Boolean(), 'Rollback transaction?', true) + ->inject('project') + ->inject('response') + ->inject('dbForProject') + ->inject('getDatabasesDB') + ->inject('user') + ->inject('transactionState') + ->inject('queueForDeletes') + ->inject('queueForEvents') + ->inject('queueForStatsUsage') + ->inject('queueForRealtime') + ->inject('queueForFunctions') + ->inject('queueForWebhooks') + ->callback($this->action(...)); + } +} diff --git a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Transactions/XList.php b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Transactions/XList.php new file mode 100644 index 0000000000..43e4a0b36b --- /dev/null +++ b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Transactions/XList.php @@ -0,0 +1,54 @@ +setHttpMethod(self::HTTP_REQUEST_METHOD_GET) + ->setHttpPath('/v1/vectordb/transactions') + ->desc('List transactions') + ->groups(['api', 'database', 'transactions']) + ->label('scope', 'documents.read') + ->label('resourceType', RESOURCE_TYPE_DATABASES) + ->label('sdk', new Method( + namespace: 'vectorDB', + group: 'transactions', + name: 'listTransactions', + description: '/docs/references/vectordb/list-transactions.md', + auth: [AuthType::KEY, AuthType::SESSION, AuthType::JWT], + responses: [ + new SDKResponse( + code: SwooleResponse::STATUS_CODE_OK, + model: UtopiaResponse::MODEL_TRANSACTION_LIST, + ) + ], + contentType: ContentType::JSON + )) + ->param('queries', [], new Transactions(), 'Array of query strings generated using the Query class provided by the SDK. [Learn more about queries](https://appwrite.io/docs/queries).', true) + ->inject('response') + ->inject('dbForProject') + ->callback($this->action(...)); + } +} diff --git a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Update.php b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Update.php new file mode 100644 index 0000000000..71be301a4d --- /dev/null +++ b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Update.php @@ -0,0 +1,57 @@ +setHttpMethod(self::HTTP_REQUEST_METHOD_PUT) + ->setHttpPath('/v1/vectordb/:databaseId') + ->desc('Update database') + ->groups(['api', 'database', 'schema']) + ->label('scope', 'databases.write') + ->label('resourceType', RESOURCE_TYPE_DATABASES) + ->label('event', 'databases.[databaseId].update') + ->label('audits.event', 'database.update') + ->label('audits.resource', 'database/{response.$id}') + ->label('sdk', new Method( + namespace: 'vectorDB', + group: 'vectordb', + name: 'update', + description: '/docs/references/vectordb/update.md', + auth: [AuthType::KEY], + responses: [ + new SDKResponse( + code: SwooleResponse::STATUS_CODE_OK, + model: UtopiaResponse::MODEL_DATABASE, + ) + ], + contentType: ContentType::JSON + )) + ->param('databaseId', '', new UID(), 'Database ID.') + ->param('name', null, new Text(128), 'Database name. Max length: 128 chars.') + ->param('enabled', true, new Boolean(), 'Is database enabled? When set to \'disabled\', users cannot access the database but Server SDKs with an API key can still read and write to the database. No data is lost when this is toggled.', true) + ->inject('response') + ->inject('dbForProject') + ->inject('queueForEvents') + ->callback($this->action(...)); + } +} diff --git a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Usage/Get.php b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Usage/Get.php new file mode 100644 index 0000000000..5508a9dc91 --- /dev/null +++ b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Usage/Get.php @@ -0,0 +1,58 @@ +setHttpMethod(self::HTTP_REQUEST_METHOD_GET) + ->setHttpPath('/v1/vectordb/:databaseId/usage') + ->desc('Get VectorDB usage stats') + ->groups(['api', 'database', 'usage']) + ->label('scope', 'collections.read') + ->label('resourceType', RESOURCE_TYPE_DATABASES) + ->label('sdk', [ + new Method( + namespace: 'vectorDB', + group: null, + name: 'getUsage', + description: '/docs/references/vectordb/get-database-usage.md', + auth: [AuthType::ADMIN], + responses: [ + new SDKResponse( + code: SwooleResponse::STATUS_CODE_OK, + model: UtopiaResponse::MODEL_USAGE_VECTORDB, + ) + ], + contentType: ContentType::JSON, + ), + ]) + ->param('databaseId', '', new UID(), 'Database ID.') + ->param('range', '30d', new WhiteList(['24h', '30d', '90d'], true), 'Date range.', true) + ->inject('response') + ->inject('dbForProject') + ->callback($this->action(...)); + } +} diff --git a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Usage/XList.php b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Usage/XList.php new file mode 100644 index 0000000000..3201864b50 --- /dev/null +++ b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/Usage/XList.php @@ -0,0 +1,56 @@ +setHttpMethod(self::HTTP_REQUEST_METHOD_GET) + ->setHttpPath('/v1/vectordb/usage') + ->desc('Get VectorDB usage stats') + ->groups(['api', 'database', 'usage']) + ->label('scope', 'collections.read') + ->label('resourceType', RESOURCE_TYPE_DATABASES) + ->label('sdk', [ + new Method( + namespace: 'vectorDB', + group: null, + name: 'listUsage', + description: '/docs/references/vectordb/list-usage.md', + auth: [AuthType::ADMIN], + responses: [ + new SDKResponse( + code: SwooleResponse::STATUS_CODE_OK, + model: UtopiaResponse::MODEL_USAGE_VECTORDBS, + ) + ], + contentType: ContentType::JSON + ), + ]) + ->param('range', '30d', new WhiteList(['24h', '30d', '90d'], true), 'Date range.', true) + ->inject('response') + ->inject('dbForProject') + ->callback($this->action(...)); + } +} diff --git a/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/XList.php b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/XList.php new file mode 100644 index 0000000000..9f902964f4 --- /dev/null +++ b/src/Appwrite/Platform/Modules/Databases/Http/VectorDB/XList.php @@ -0,0 +1,53 @@ +setHttpMethod(self::HTTP_REQUEST_METHOD_GET) + ->setHttpPath('/v1/vectordb') + ->desc('List databases') + ->groups(['api', 'database']) + ->label('scope', 'databases.read') + ->label('resourceType', RESOURCE_TYPE_DATABASES) + ->label('sdk', new Method( + namespace: 'vectorDB', + group: 'vectordb', + name: 'list', + description: '/docs/references/vectordb/list.md', + auth: [AuthType::KEY], + responses: [ + new SDKResponse( + code: SwooleResponse::STATUS_CODE_OK, + model: UtopiaResponse::MODEL_DATABASE_LIST, + ) + ], + contentType: ContentType::JSON + )) + ->param('queries', [], new Databases(), 'Array of query strings generated using the Query class provided by the SDK. [Learn more about queries](https://appwrite.io/docs/queries). Maximum of ' . APP_LIMIT_ARRAY_PARAMS_SIZE . ' queries are allowed, each ' . APP_LIMIT_ARRAY_ELEMENT_SIZE . ' characters long. You may filter on the following columns: ' . implode(', ', Databases::ALLOWED_ATTRIBUTES), true) + ->param('search', '', new Text(256), 'Search term to filter your list results. Max length: 256 chars.', true) + ->param('total', true, new Boolean(true), 'When set to false, the total count returned will be 0 and will not be calculated.', true) + ->inject('response') + ->inject('dbForProject') + ->callback($this->action(...)); + } +} diff --git a/src/Appwrite/Platform/Modules/Databases/Services/Http.php b/src/Appwrite/Platform/Modules/Databases/Services/Http.php index c746249114..682472128e 100644 --- a/src/Appwrite/Platform/Modules/Databases/Services/Http.php +++ b/src/Appwrite/Platform/Modules/Databases/Services/Http.php @@ -6,6 +6,7 @@ use Appwrite\Platform\Modules\Databases\Http\Init\Timeout; use Appwrite\Platform\Modules\Databases\Services\Registry\DocumentsDB as DocumentsDBRegistry; use Appwrite\Platform\Modules\Databases\Services\Registry\Legacy as LegacyRegistry; use Appwrite\Platform\Modules\Databases\Services\Registry\TablesDB as TablesDBDBRegistry; +use Appwrite\Platform\Modules\Databases\Services\Registry\VectorDB as VectorDBRegistry; use Utopia\Platform\Service; class Http extends Service @@ -19,7 +20,8 @@ class Http extends Service foreach ([ LegacyRegistry::class, TablesDBDBRegistry::class, - DocumentsDBRegistry::class + DocumentsDBRegistry::class, + VectorDBRegistry::class ] as $registrar) { new $registrar($this); } diff --git a/src/Appwrite/Platform/Modules/Databases/Services/Registry/VectorDB.php b/src/Appwrite/Platform/Modules/Databases/Services/Registry/VectorDB.php new file mode 100644 index 0000000000..f6fa731558 --- /dev/null +++ b/src/Appwrite/Platform/Modules/Databases/Services/Registry/VectorDB.php @@ -0,0 +1,110 @@ +registerDatabaseActions($service); + $this->registerCollectionActions($service); + $this->registerIndexActions($service); + $this->registerDocumentActions($service); + $this->registerEmbeddingActions($service); + $this->registerTransactionActions($service); + } + + private function registerDatabaseActions(Service $service): void + { + $service->addAction(CreateVectorDatabase::getName(), new CreateVectorDatabase()); + $service->addAction(GetVectorDatabase::getName(), new GetVectorDatabase()); + $service->addAction(UpdateVectorDatabase::getName(), new UpdateVectorDatabase()); + $service->addAction(DeleteVectorDatabase::getName(), new DeleteVectorDatabase()); + $service->addAction(ListVectorDatabases::getName(), new ListVectorDatabases()); + $service->addAction(GetVectorDatabaseUsage::getName(), new GetVectorDatabaseUsage()); + $service->addAction(ListVectorDatabaseUsage::getName(), new ListVectorDatabaseUsage()); + } + + private function registerCollectionActions(Service $service): void + { + $service->addAction(CreateCollection::getName(), new CreateCollection()); + $service->addAction(GetCollection::getName(), new GetCollection()); + $service->addAction(UpdateCollection::getName(), new UpdateCollection()); + $service->addAction(DeleteCollection::getName(), new DeleteCollection()); + $service->addAction(ListCollections::getName(), new ListCollections()); + $service->addAction(ListCollectionLogs::getName(), new ListCollectionLogs()); + $service->addAction(GetCollectionUsage::getName(), new GetCollectionUsage()); + } + + private function registerIndexActions(Service $service): void + { + $service->addAction(CreateIndex::getName(), new CreateIndex()); + $service->addAction(GetIndex::getName(), new GetIndex()); + $service->addAction(DeleteIndex::getName(), new DeleteIndex()); + $service->addAction(ListIndexes::getName(), new ListIndexes()); + } + + private function registerDocumentActions(Service $service): void + { + $service->addAction(CreateDocument::getName(), new CreateDocument()); + $service->addAction(UpdateDocument::getName(), new UpdateDocument()); + $service->addAction(UpsertDocument::getName(), new UpsertDocument()); + $service->addAction(GetDocument::getName(), new GetDocument()); + $service->addAction(ListDocuments::getName(), new ListDocuments()); + $service->addAction(DeleteDocument::getName(), new DeleteDocument()); + $service->addAction(UpdateDocuments::getName(), new UpdateDocuments()); + $service->addAction(UpsertDocuments::getName(), new UpsertDocuments()); + $service->addAction(DeleteDocuments::getName(), new DeleteDocuments()); + } + + private function registerTransactionActions(Service $service): void + { + $service->addAction(CreateTransaction::getName(), new CreateTransaction()); + $service->addAction(GetTransaction::getName(), new GetTransaction()); + $service->addAction(UpdateTransaction::getName(), new UpdateTransaction()); + $service->addAction(DeleteTransaction::getName(), new DeleteTransaction()); + $service->addAction(ListTransactions::getName(), new ListTransactions()); + $service->addAction(CreateOperations::getName(), new CreateOperations()); + } + + private function registerEmbeddingActions(Service $service): void + { + $service->addAction(CreateTextEmbeddings::getName(), new CreateTextEmbeddings()); + } +} diff --git a/src/Appwrite/Platform/Workers/Deletes.php b/src/Appwrite/Platform/Workers/Deletes.php index 8ec487a559..2216b98114 100644 --- a/src/Appwrite/Platform/Workers/Deletes.php +++ b/src/Appwrite/Platform/Workers/Deletes.php @@ -531,7 +531,7 @@ class Deletes extends Action $projectTables = !\in_array($dsn->getHost(), $sharedTables); $sharedTablesV1 = \in_array($dsn->getHost(), $sharedTablesV1); $sharedTablesV2 = !$projectTables && !$sharedTablesV1; - $databaseDSNKeys = ['database','documentsDatabase']; + $databaseDSNKeys = ['database','documentsDatabase','vectorDatabase']; $exectionActionPerDatabase = function (string $databaseDSNKey, $callback) use ($getDatabasesDB, $document) { /** diff --git a/src/Appwrite/Platform/Workers/Migrations.php b/src/Appwrite/Platform/Workers/Migrations.php index d0da532f40..266eabb4f3 100644 --- a/src/Appwrite/Platform/Workers/Migrations.php +++ b/src/Appwrite/Platform/Workers/Migrations.php @@ -288,6 +288,8 @@ class Migrations extends Action METRIC_DATABASES_OPERATIONS_WRITES, METRIC_DATABASES_OPERATIONS_READS_DOCUMENTSDB, METRIC_DATABASES_OPERATIONS_WRITES_DOCUMENTSDB, + METRIC_DATABASES_OPERATIONS_READS_VECTORDB, + METRIC_DATABASES_OPERATIONS_WRITES_VECTORDB, METRIC_NETWORK_REQUESTS, METRIC_NETWORK_INBOUND, METRIC_NETWORK_OUTBOUND, @@ -456,6 +458,7 @@ class Migrations extends Action { return match ($databaseType) { 'documentsdb' => $this->project->getAttribute('documentsDatabase'), + 'vectordb' => $this->project->getAttribute('vectorDatabase'), default => $this->project->getAttribute('database'), }; } diff --git a/src/Appwrite/Platform/Workers/StatsResources.php b/src/Appwrite/Platform/Workers/StatsResources.php index b745fac367..dabda146dc 100644 --- a/src/Appwrite/Platform/Workers/StatsResources.php +++ b/src/Appwrite/Platform/Workers/StatsResources.php @@ -125,6 +125,7 @@ class StatsResources extends Action $databases = $dbForProject->count('databases', [Query::equal('type', [DATABASE_TYPE_LEGACY, DATABASE_TYPE_TABLESDB])]); $documentsdb = $dbForProject->count('databases', [Query::equal('type', [DATABASE_TYPE_DOCUMENTSDB])]); + $vectordb = $dbForProject->count('databases', [Query::equal('type', [DATABASE_TYPE_VECTORDB])]); $buckets = $dbForProject->count('buckets'); $users = $dbForProject->count('users'); @@ -160,6 +161,7 @@ class StatsResources extends Action $metrics = [ METRIC_DATABASES => $databases, METRIC_DATABASES_DOCUMENTSDB => $documentsdb, + METRIC_DATABASES_VECTORDB => $vectordb, METRIC_BUCKETS => $buckets, METRIC_USERS => $users, METRIC_FUNCTIONS => $functions, @@ -268,8 +270,13 @@ class StatsResources extends Action $totalDocumentsDocumentsdb = 0; $totalDatabaseStorageDocumentsdb = 0; + // vectordb + $totalCollectionsVectordb = 0; + $totalDocumentsVectordb = 0; + $totalDatabaseStorageVectordb = 0; - $this->foreachDocument($dbForProject, 'databases', [], function ($database) use ($dbForProject, $getDatabasesDB, $region, &$totalCollections, &$totalDocuments, &$totalDatabaseStorage, &$totalCollectionsDocumentsdb, &$totalDocumentsDocumentsdb, &$totalDatabaseStorageDocumentsdb) { + + $this->foreachDocument($dbForProject, 'databases', [], function ($database) use ($dbForProject, $getDatabasesDB, $region, &$totalCollections, &$totalDocuments, &$totalDatabaseStorage, &$totalCollectionsDocumentsdb, &$totalDocumentsDocumentsdb, &$totalDatabaseStorageDocumentsdb, &$totalCollectionsVectordb, &$totalDocumentsVectordb, &$totalDatabaseStorageVectordb) { $dbForDatabases = $getDatabasesDB($database); $collections = $dbForProject->count('database_' . $database->getSequence()); @@ -289,6 +296,11 @@ class StatsResources extends Action $totalDocumentsDocumentsdb += $documents; $totalCollectionsDocumentsdb += $collections; break; + case DATABASE_TYPE_VECTORDB: + $totalDatabaseStorageVectordb += $storage; + $totalDocumentsVectordb += $documents; + $totalCollectionsVectordb += $collections; + break; default: $totalDatabaseStorage += $storage; $totalDocuments += $documents; @@ -303,6 +315,10 @@ class StatsResources extends Action $this->createStatsDocuments($region, METRIC_COLLECTIONS_DOCUMENTSDB, $totalCollectionsDocumentsdb); $this->createStatsDocuments($region, METRIC_DOCUMENTS_DOCUMENTSDB, $totalDocumentsDocumentsdb); $this->createStatsDocuments($region, METRIC_DATABASES_STORAGE_DOCUMENTSDB, $totalDatabaseStorageDocumentsdb); + + $this->createStatsDocuments($region, METRIC_COLLECTIONS_VECTORDB, $totalCollectionsVectordb); + $this->createStatsDocuments($region, METRIC_DOCUMENTS_VECTORDB, $totalDocumentsVectordb); + $this->createStatsDocuments($region, METRIC_DATABASES_STORAGE_VECTORDB, $totalDatabaseStorageVectordb); } protected function countForCollections(Database $dbForProject, Database $dbForDatabases, Document $database, string $region): array { diff --git a/src/Appwrite/Platform/Workers/StatsUsage.php b/src/Appwrite/Platform/Workers/StatsUsage.php index 092582123a..ab9dd2c194 100644 --- a/src/Appwrite/Platform/Workers/StatsUsage.php +++ b/src/Appwrite/Platform/Workers/StatsUsage.php @@ -48,6 +48,7 @@ class StatsUsage extends Action protected array $skipBaseMetrics = [ METRIC_DATABASES => true, METRIC_DATABASES_DOCUMENTSDB => true, + METRIC_DATABASES_VECTORDB => true, METRIC_BUCKETS => true, METRIC_USERS => true, METRIC_FUNCTIONS => true, @@ -69,8 +70,11 @@ class StatsUsage extends Action METRIC_DOCUMENTS => true, METRIC_COLLECTIONS_DOCUMENTSDB => true, METRIC_DOCUMENTS_DOCUMENTSDB => true, + METRIC_COLLECTIONS_VECTORDB => true, + METRIC_DOCUMENTS_VECTORDB => true, METRIC_DATABASES_STORAGE => true, METRIC_DATABASES_STORAGE_DOCUMENTSDB => true, + METRIC_DATABASES_STORAGE_VECTORDB => true, ]; /** diff --git a/src/Appwrite/Utopia/Response.php b/src/Appwrite/Utopia/Response.php index c0cfc98bf8..1843e695cd 100644 --- a/src/Appwrite/Utopia/Response.php +++ b/src/Appwrite/Utopia/Response.php @@ -25,11 +25,13 @@ use Appwrite\Utopia\Response\Model\AttributeInteger; use Appwrite\Utopia\Response\Model\AttributeIP; use Appwrite\Utopia\Response\Model\AttributeLine; use Appwrite\Utopia\Response\Model\AttributeList; +use Appwrite\Utopia\Response\Model\AttributeObject; use Appwrite\Utopia\Response\Model\AttributePoint; use Appwrite\Utopia\Response\Model\AttributePolygon; use Appwrite\Utopia\Response\Model\AttributeRelationship; use Appwrite\Utopia\Response\Model\AttributeString; use Appwrite\Utopia\Response\Model\AttributeURL; +use Appwrite\Utopia\Response\Model\AttributeVector; use Appwrite\Utopia\Response\Model\AuthProvider; use Appwrite\Utopia\Response\Model\BaseList; use Appwrite\Utopia\Response\Model\Branch; @@ -62,6 +64,7 @@ use Appwrite\Utopia\Response\Model\DetectionRuntime; use Appwrite\Utopia\Response\Model\DetectionVariable; use Appwrite\Utopia\Response\Model\DevKey; use Appwrite\Utopia\Response\Model\Document as ModelDocument; +use Appwrite\Utopia\Response\Model\Embedding; use Appwrite\Utopia\Response\Model\Error; use Appwrite\Utopia\Response\Model\ErrorDev; use Appwrite\Utopia\Response\Model\Execution; @@ -142,9 +145,12 @@ use Appwrite\Utopia\Response\Model\UsageSites; use Appwrite\Utopia\Response\Model\UsageStorage; use Appwrite\Utopia\Response\Model\UsageTable; use Appwrite\Utopia\Response\Model\UsageUsers; +use Appwrite\Utopia\Response\Model\UsageVectorDB; +use Appwrite\Utopia\Response\Model\UsageVectorDBs; use Appwrite\Utopia\Response\Model\User; use Appwrite\Utopia\Response\Model\Variable; use Appwrite\Utopia\Response\Model\VcsContent; +use Appwrite\Utopia\Response\Model\VectorDBCollection; use Appwrite\Utopia\Response\Model\Webhook; use Exception; use JsonException; @@ -175,6 +181,8 @@ class Response extends SwooleResponse public const MODEL_USAGE_DATABASE = 'usageDatabase'; public const MODEL_USAGE_DOCUMENTSDBS = 'usageDocumentsDBs'; public const MODEL_USAGE_DOCUMENTSDB = 'usageDocumentsDB'; + public const MODEL_USAGE_VECTORDBS = 'usageVectorDBs'; + public const MODEL_USAGE_VECTORDB = 'usageVectorDB'; public const MODEL_USAGE_TABLE = 'usageTable'; public const MODEL_USAGE_COLLECTION = 'usageCollection'; public const MODEL_USAGE_USERS = 'usageUsers'; @@ -191,6 +199,10 @@ class Response extends SwooleResponse public const MODEL_DATABASE_LIST = 'databaseList'; public const MODEL_COLLECTION = 'collection'; public const MODEL_COLLECTION_LIST = 'collectionList'; + public const MODEL_VECTORDB_COLLECTION = 'vectordbCollection'; + public const MODEL_VECTORDB_COLLECTION_LIST = 'vectordbCollectionList'; + public const MODEL_EMBEDDING = 'embedding'; + public const MODEL_EMBEDDING_LIST = 'embeddingList'; public const MODEL_TABLE = 'table'; public const MODEL_TABLE_LIST = 'tableList'; public const MODEL_INDEX = 'index'; @@ -218,6 +230,8 @@ class Response extends SwooleResponse public const MODEL_ATTRIBUTE_POINT = 'attributePoint'; public const MODEL_ATTRIBUTE_LINE = 'attributeLine'; public const MODEL_ATTRIBUTE_POLYGON = 'attributePolygon'; + public const MODEL_ATTRIBUTE_OBJECT = 'attributeObject'; + public const MODEL_ATTRIBUTE_VECTOR = 'attributeVector'; // Database Columns public const MODEL_COLUMN = 'column'; @@ -444,6 +458,7 @@ class Response extends SwooleResponse ->setModel(new BaseList('Documents List', self::MODEL_DOCUMENT_LIST, 'documents', self::MODEL_DOCUMENT)) ->setModel(new BaseList('Tables List', self::MODEL_TABLE_LIST, 'tables', self::MODEL_TABLE)) ->setModel(new BaseList('Collections List', self::MODEL_COLLECTION_LIST, 'collections', self::MODEL_COLLECTION)) + ->setModel(new BaseList('VectorDB Collections List', self::MODEL_VECTORDB_COLLECTION_LIST, 'collections', self::MODEL_VECTORDB_COLLECTION)) ->setModel(new BaseList('Databases List', self::MODEL_DATABASE_LIST, 'databases', self::MODEL_DATABASE)) ->setModel(new BaseList('Indexes List', self::MODEL_INDEX_LIST, 'indexes', self::MODEL_INDEX)) ->setModel(new BaseList('Column Indexes List', self::MODEL_COLUMN_INDEX_LIST, 'indexes', self::MODEL_COLUMN_INDEX)) @@ -494,10 +509,13 @@ class Response extends SwooleResponse ->setModel(new BaseList('Migrations Firebase Projects List', self::MODEL_MIGRATION_FIREBASE_PROJECT_LIST, 'projects', self::MODEL_MIGRATION_FIREBASE_PROJECT)) ->setModel(new BaseList('Specifications List', self::MODEL_SPECIFICATION_LIST, 'specifications', self::MODEL_SPECIFICATION)) ->setModel(new BaseList('VCS Content List', self::MODEL_VCS_CONTENT_LIST, 'contents', self::MODEL_VCS_CONTENT)) + ->setModel(new BaseList('Embedding list', self::MODEL_EMBEDDING_LIST, 'embeddings', self::MODEL_EMBEDDING)) // Entities ->setModel(new Database()) + ->setModel(new Embedding()) // Collection API Models ->setModel(new Collection()) + ->setModel(new VectorDBCollection()) ->setModel(new Attribute()) ->setModel(new AttributeList()) ->setModel(new AttributeString()) @@ -513,6 +531,8 @@ class Response extends SwooleResponse ->setModel(new AttributePoint()) ->setModel(new AttributeLine()) ->setModel(new AttributePolygon()) + ->setModel(new AttributeObject()) + ->setModel(new AttributeVector()) // Table API Models ->setModel(new Table()) ->setModel(new Column()) @@ -602,6 +622,8 @@ class Response extends SwooleResponse ->setModel(new UsageDatabase()) ->setModel(new UsageDocumentsDBs()) ->setModel(new UsageDocumentsDB()) + ->setModel(new UsageVectorDBs()) + ->setModel(new UsageVectorDB()) ->setModel(new UsageTable()) ->setModel(new UsageCollection()) ->setModel(new UsageUsers()) diff --git a/src/Appwrite/Utopia/Response/Model/AttributeObject.php b/src/Appwrite/Utopia/Response/Model/AttributeObject.php new file mode 100644 index 0000000000..542f7f744c --- /dev/null +++ b/src/Appwrite/Utopia/Response/Model/AttributeObject.php @@ -0,0 +1,27 @@ + 'object', + ]; + + public function getName(): string + { + return 'AttributeObject'; + } + + public function getType(): string + { + return Response::MODEL_ATTRIBUTE_OBJECT; + } +} diff --git a/src/Appwrite/Utopia/Response/Model/AttributeVector.php b/src/Appwrite/Utopia/Response/Model/AttributeVector.php new file mode 100644 index 0000000000..4b58b979ee --- /dev/null +++ b/src/Appwrite/Utopia/Response/Model/AttributeVector.php @@ -0,0 +1,35 @@ +addRule('size', [ + 'type' => self::TYPE_INTEGER, + 'description' => 'Vector dimensions.', + 'default' => 0, + 'example' => 1536, + ]); + } + + public array $conditions = [ + 'type' => 'vector', + ]; + + public function getName(): string + { + return 'AttributeVector'; + } + + public function getType(): string + { + return Response::MODEL_ATTRIBUTE_VECTOR; + } +} diff --git a/src/Appwrite/Utopia/Response/Model/Embedding.php b/src/Appwrite/Utopia/Response/Model/Embedding.php new file mode 100644 index 0000000000..9fce913723 --- /dev/null +++ b/src/Appwrite/Utopia/Response/Model/Embedding.php @@ -0,0 +1,47 @@ +addRule('model', [ + 'type' => self::TYPE_STRING, + 'description' => 'Embedding model used to generate embeddings.', + 'example' => 'embeddinggemma' + ]) + ->addRule('dimension', [ + 'type' => self::TYPE_INTEGER, + 'description' => 'Number of dimensions for each embedding vector.', + 'example' => 768 + ]) + ->addRule('embedding', [ + 'type' => self::TYPE_FLOAT, + 'array' => true, + 'default' => [], + 'description' => 'Embedding vector values. If an error occurs, this will be an empty array.', + 'example' => [0.01, 0.02, 0.03] + ]) + ->addRule('error', [ + 'type' => self::TYPE_STRING, + 'array' => false, + 'default' => '', + 'description' => 'Error message if embedding generation fails. Empty string if no error.', + 'example' => 'Error message' + ]); + } +} diff --git a/src/Appwrite/Utopia/Response/Model/UsageProject.php b/src/Appwrite/Utopia/Response/Model/UsageProject.php index eeaa30e1c8..66628a7b13 100644 --- a/src/Appwrite/Utopia/Response/Model/UsageProject.php +++ b/src/Appwrite/Utopia/Response/Model/UsageProject.php @@ -267,6 +267,133 @@ class UsageProject extends Model 'default' => 0, 'example' => 0, ]) + // VectorDB aggregates + ->addRule('vectordbDatabasesTotal', [ + 'type' => self::TYPE_INTEGER, + 'description' => 'Total aggregated number of VectorDB databases.', + 'default' => 0, + 'example' => 0, + ]) + ->addRule('vectordbCollectionsTotal', [ + 'type' => self::TYPE_INTEGER, + 'description' => 'Total aggregated number of VectorDB collections.', + 'default' => 0, + 'example' => 0, + ]) + ->addRule('vectordbDocumentsTotal', [ + 'type' => self::TYPE_INTEGER, + 'description' => 'Total aggregated number of VectorDB documents.', + 'default' => 0, + 'example' => 0, + ]) + ->addRule('vectordbDatabasesStorageTotal', [ + 'type' => self::TYPE_INTEGER, + 'description' => 'Total aggregated VectorDB storage (bytes).', + 'default' => 0, + 'example' => 0, + ]) + ->addRule('vectordbDatabasesReadsTotal', [ + 'type' => self::TYPE_INTEGER, + 'description' => 'Total aggregated number of VectorDB reads.', + 'default' => 0, + 'example' => 0, + ]) + ->addRule('vectordbDatabasesWritesTotal', [ + 'type' => self::TYPE_INTEGER, + 'description' => 'Total aggregated number of VectorDB writes.', + 'default' => 0, + 'example' => 0, + ]) + ->addRule('vectordbDatabases', [ + 'type' => Response::MODEL_METRIC, + 'description' => 'Aggregated VectorDB databases per period.', + 'default' => [], + 'example' => [], + 'array' => true + ]) + ->addRule('vectordbCollections', [ + 'type' => Response::MODEL_METRIC, + 'description' => 'Aggregated VectorDB collections per period.', + 'default' => [], + 'example' => [], + 'array' => true + ]) + ->addRule('vectordbDocuments', [ + 'type' => Response::MODEL_METRIC, + 'description' => 'Aggregated VectorDB documents per period.', + 'default' => [], + 'example' => [], + 'array' => true + ]) + ->addRule('vectordbDatabasesStorage', [ + 'type' => Response::MODEL_METRIC, + 'description' => 'Aggregated VectorDB storage per period.', + 'default' => [], + 'example' => [], + 'array' => true + ]) + ->addRule('vectordbDatabasesReads', [ + 'type' => Response::MODEL_METRIC, + 'description' => 'Aggregated VectorDB reads per period.', + 'default' => [], + 'example' => [], + 'array' => true + ]) + ->addRule('vectordbDatabasesWrites', [ + 'type' => Response::MODEL_METRIC, + 'description' => 'Aggregated VectorDB writes per period.', + 'default' => [], + 'example' => [], + 'array' => true + ]) + ->addRule('embeddingsText', [ + 'type' => Response::MODEL_METRIC, + 'description' => 'Aggregated number of text embedding calls per period.', + 'default' => [], + 'example' => [] + ]) + ->addRule('embeddingsTextTokens', [ + 'type' => Response::MODEL_METRIC, + 'description' => 'Aggregated number of tokens processed by text embeddings per period.', + 'default' => [], + 'example' => [] + ]) + ->addRule('embeddingsTextDuration', [ + 'type' => Response::MODEL_METRIC, + 'description' => 'Aggregated duration spent generating text embeddings per period.', + 'default' => [], + 'example' => [] + ]) + ->addRule('embeddingsTextErrors', [ + 'type' => Response::MODEL_METRIC, + 'description' => 'Aggregated number of errors while generating text embeddings per period.', + 'default' => [], + 'example' => [] + ]) + ->addRule('embeddingsTextTotal', [ + 'type' => Response::MODEL_METRIC, + 'description' => 'Total aggregated number of text embedding calls.', + 'default' => 0, + 'example' => 0 + ]) + ->addRule('embeddingsTextTokensTotal', [ + 'type' => Response::MODEL_METRIC, + 'description' => 'Total aggregated number of tokens processed by text.', + 'default' => 0, + 'example' => 0 + ]) + ->addRule('embeddingsTextDurationTotal', [ + 'type' => Response::MODEL_METRIC, + 'description' => 'Total aggregated duration spent generating text embeddings.', + 'default' => 0, + 'example' => 0 + ]) + ->addRule('embeddingsTextErrorsTotal', [ + 'type' => Response::MODEL_METRIC, + 'description' => 'Total aggregated number of errors while generating text embeddings.', + 'default' => 0, + 'example' => 0 + ]) ; } diff --git a/src/Appwrite/Utopia/Response/Model/UsageVectorDB.php b/src/Appwrite/Utopia/Response/Model/UsageVectorDB.php new file mode 100644 index 0000000000..a90f593c7e --- /dev/null +++ b/src/Appwrite/Utopia/Response/Model/UsageVectorDB.php @@ -0,0 +1,96 @@ +addRule('range', [ + 'type' => self::TYPE_STRING, + 'description' => 'Time range of the usage stats.', + 'default' => '', + 'example' => '30d', + ]) + ->addRule('collectionsTotal', [ + 'type' => self::TYPE_INTEGER, + 'description' => 'Total aggregated number of collections.', + 'default' => 0, + 'example' => 0, + ]) + ->addRule('documentsTotal', [ + 'type' => self::TYPE_INTEGER, + 'description' => 'Total aggregated number of documents.', + 'default' => 0, + 'example' => 0, + ]) + ->addRule('storageTotal', [ + 'type' => self::TYPE_INTEGER, + 'description' => 'Total aggregated storage used in bytes.', + 'default' => 0, + 'example' => 0, + ]) + ->addRule('databaseReadsTotal', [ + 'type' => self::TYPE_INTEGER, + 'description' => 'Total number of database reads.', + 'default' => 0, + 'example' => 0, + ]) + ->addRule('databaseWritesTotal', [ + 'type' => self::TYPE_INTEGER, + 'description' => 'Total number of database writes.', + 'default' => 0, + 'example' => 0, + ]) + ->addRule('collections', [ + 'type' => Response::MODEL_METRIC, + 'description' => 'Aggregated number of collections per period.', + 'default' => [], + 'example' => [], + 'array' => true + ]) + ->addRule('documents', [ + 'type' => Response::MODEL_METRIC, + 'description' => 'Aggregated number of documents per period.', + 'default' => [], + 'example' => [], + 'array' => true + ]) + ->addRule('storage', [ + 'type' => Response::MODEL_METRIC, + 'description' => 'Aggregated storage used in bytes per period.', + 'default' => [], + 'example' => [], + 'array' => true + ]) + ->addRule('databaseReads', [ + 'type' => Response::MODEL_METRIC, + 'description' => 'An array of aggregated number of database reads.', + 'default' => [], + 'example' => [], + 'array' => true + ]) + ->addRule('databaseWrites', [ + 'type' => Response::MODEL_METRIC, + 'description' => 'An array of aggregated number of database writes.', + 'default' => [], + 'example' => [], + 'array' => true + ]) + ; + } + + public function getName(): string + { + return 'UsageVectorDB'; + } + + public function getType(): string + { + return Response::MODEL_USAGE_VECTORDB; + } +} diff --git a/src/Appwrite/Utopia/Response/Model/UsageVectorDBs.php b/src/Appwrite/Utopia/Response/Model/UsageVectorDBs.php new file mode 100644 index 0000000000..da63120c5c --- /dev/null +++ b/src/Appwrite/Utopia/Response/Model/UsageVectorDBs.php @@ -0,0 +1,109 @@ +addRule('range', [ + 'type' => self::TYPE_STRING, + 'description' => 'Time range of the usage stats.', + 'default' => '', + 'example' => '30d', + ]) + ->addRule('databasesTotal', [ + 'type' => self::TYPE_INTEGER, + 'description' => 'Total aggregated number of VectorDB databases.', + 'default' => 0, + 'example' => 0, + ]) + ->addRule('collectionsTotal', [ + 'type' => self::TYPE_INTEGER, + 'description' => 'Total aggregated number of collections.', + 'default' => 0, + 'example' => 0, + ]) + ->addRule('documentsTotal', [ + 'type' => self::TYPE_INTEGER, + 'description' => 'Total aggregated number of documents.', + 'default' => 0, + 'example' => 0, + ]) + ->addRule('storageTotal', [ + 'type' => self::TYPE_INTEGER, + 'description' => 'Total aggregated storage in bytes.', + 'default' => 0, + 'example' => 0, + ]) + ->addRule('databasesReadsTotal', [ + 'type' => self::TYPE_INTEGER, + 'description' => 'Total number of database reads.', + 'default' => 0, + 'example' => 0, + ]) + ->addRule('databasesWritesTotal', [ + 'type' => self::TYPE_INTEGER, + 'description' => 'Total number of database writes.', + 'default' => 0, + 'example' => 0, + ]) + ->addRule('databases', [ + 'type' => Response::MODEL_METRIC, + 'description' => 'Aggregated number of databases per period.', + 'default' => [], + 'example' => [], + 'array' => true + ]) + ->addRule('collections', [ + 'type' => Response::MODEL_METRIC, + 'description' => 'Aggregated number of collections per period.', + 'default' => [], + 'example' => [], + 'array' => true + ]) + ->addRule('documents', [ + 'type' => Response::MODEL_METRIC, + 'description' => 'Aggregated number of documents per period.', + 'default' => [], + 'example' => [], + 'array' => true + ]) + ->addRule('storage', [ + 'type' => Response::MODEL_METRIC, + 'description' => 'Aggregated storage in bytes per period.', + 'default' => [], + 'example' => [], + 'array' => true + ]) + ->addRule('databasesReads', [ + 'type' => Response::MODEL_METRIC, + 'description' => 'An array of aggregated number of database reads.', + 'default' => [], + 'example' => [], + 'array' => true + ]) + ->addRule('databasesWrites', [ + 'type' => Response::MODEL_METRIC, + 'description' => 'An array of aggregated number of database writes.', + 'default' => [], + 'example' => [], + 'array' => true + ]) + ; + } + + public function getName(): string + { + return 'UsageVectorDBs'; + } + + public function getType(): string + { + return Response::MODEL_USAGE_VECTORDBS; + } +} diff --git a/src/Appwrite/Utopia/Response/Model/VectorDBCollection.php b/src/Appwrite/Utopia/Response/Model/VectorDBCollection.php new file mode 100644 index 0000000000..ccf609132d --- /dev/null +++ b/src/Appwrite/Utopia/Response/Model/VectorDBCollection.php @@ -0,0 +1,41 @@ +addRule('dimension', [ + 'type' => self::TYPE_INTEGER, + 'description' => 'Embedding dimension.', + 'default' => 0, + 'example' => 1536, + ]) + ->addRule('attributes', [ + 'type' => [ + Response::MODEL_ATTRIBUTE_OBJECT, + Response::MODEL_ATTRIBUTE_VECTOR, + ], + 'description' => 'Collection attributes.', + 'default' => [], + 'example' => new \stdClass(), + 'array' => true, + ]) + ; + } + + public function getName(): string + { + return 'VectorDB Collection'; + } + + public function getType(): string + { + return Response::MODEL_VECTORDB_COLLECTION; + } +} diff --git a/tests/e2e/General/UsageTest.php b/tests/e2e/General/UsageTest.php index 28e5707732..49b554bfb6 100644 --- a/tests/e2e/General/UsageTest.php +++ b/tests/e2e/General/UsageTest.php @@ -1131,6 +1131,228 @@ class UsageTest extends Scope } /** @depends testDocumentsDBStats */ + public function testPrepareVectorDBStats(array $data): array + { + $documentsTotal = 0; + $collectionsTotal = 0; + $vectordbTotal = 0; + $databasesTotal = $data['databasesTotal']; + $requestsTotal = $data['requestsTotal']; + + for ($i = 0; $i < self::CREATE; $i++) { + $name = uniqid() . ' vectordb'; + + $response = $this->client->call( + Client::METHOD_POST, + '/vectordb', + array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'] + ], $this->getHeaders()), + [ + 'databaseId' => 'unique()', + 'name' => $name, + ] + ); + + $this->assertEquals($name, $response['body']['name']); + $this->assertNotEmpty($response['body']['$id']); + + $requestsTotal += 1; + $vectordbTotal += 1; + + $vectordbId = $response['body']['$id']; + + if ($i < (self::CREATE / 2)) { + $response = $this->client->call( + Client::METHOD_DELETE, + '/vectordb/' . $vectordbId, + array_merge([ + 'x-appwrite-project' => $this->getProject()['$id'] + ], $this->getHeaders()), + ); + + $this->assertEmpty($response['body']); + + $vectordbTotal -= 1; + $requestsTotal += 1; + } + } + + for ($i = 0; $i < self::CREATE; $i++) { + $name = uniqid() . ' collection'; + + $response = $this->client->call( + Client::METHOD_POST, + '/vectordb/' . $vectordbId . '/collections', + array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'] + ], $this->getHeaders()), + [ + 'collectionId' => 'unique()', + 'name' => $name, + 'dimension' => 1536, + 'documentSecurity' => false, + 'permissions' => [ + Permission::read(Role::any()), + Permission::create(Role::any()), + Permission::update(Role::any()), + Permission::delete(Role::any()), + ], + ] + ); + + $this->assertEquals($name, $response['body']['name']); + $this->assertNotEmpty($response['body']['$id']); + + $requestsTotal += 1; + $collectionsTotal += 1; + + $collectionId = $response['body']['$id']; + + if ($i < (self::CREATE / 2)) { + $response = $this->client->call( + Client::METHOD_DELETE, + '/vectordb/' . $vectordbId . '/collections/' . $collectionId, + array_merge([ + 'x-appwrite-project' => $this->getProject()['$id'] + ], $this->getHeaders()), + ); + + $this->assertEmpty($response['body']); + + $collectionsTotal -= 1; + $requestsTotal += 1; + } + } + + for ($i = 0; $i < self::CREATE; $i++) { + $response = $this->client->call( + Client::METHOD_POST, + '/vectordb/' . $vectordbId . '/collections/' . $collectionId . '/documents', + array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'] + ], $this->getHeaders()), + [ + 'documentId' => 'unique()', + 'data' => [ + 'embeddings' => array_fill(0, 1536, 0.1), + 'metadata' => [ + 'name' => uniqid() . ' document', + 'value' => $i + ] + ] + ] + ); + + $this->assertNotEmpty($response['body']['$id']); + + $requestsTotal += 1; + $documentsTotal += 1; + + $documentId = $response['body']['$id']; + + if ($i < (self::CREATE / 2)) { + $response = $this->client->call( + Client::METHOD_DELETE, + '/vectordb/' . $vectordbId . '/collections/' . $collectionId . '/documents/' . $documentId, + array_merge([ + 'x-appwrite-project' => $this->getProject()['$id'] + ], $this->getHeaders()), + ); + + $this->assertEmpty($response['body']); + + $documentsTotal -= 1; + $requestsTotal += 1; + } + } + + return array_merge($data, [ + 'vectordbId' => $vectordbId, + 'vectordbCollectionId' => $collectionId, + 'requestsTotal' => $requestsTotal, + 'databasesTotal' => $databasesTotal, + 'vectordbTotal' => $vectordbTotal, + 'vectordbCollectionsTotal' => $collectionsTotal, + 'vectordbDocumentsTotal' => $documentsTotal, + ]); + } + + /** @depends testPrepareVectorDBStats */ + #[Retry(count: 1)] + public function testVectorDBStats(array $data): array + { + $vectordbId = $data['vectordbId']; + $collectionId = $data['vectordbCollectionId']; + $requestsTotal = $data['requestsTotal']; + $databasesTotal = $data['databasesTotal']; + $vectordbTotal = $data['vectordbTotal']; + $collectionsTotal = $data['vectordbCollectionsTotal']; + $documentsTotal = $data['vectordbDocumentsTotal']; + + $this->assertEventually(function () use ($requestsTotal, $vectordbTotal, $documentsTotal) { + $response = $this->client->call( + Client::METHOD_GET, + '/project/usage', + $this->getConsoleHeaders(), + [ + 'period' => '1d', + 'startDate' => self::getToday(), + 'endDate' => self::getTomorrow(), + ] + ); + + $this->assertGreaterThanOrEqual(31, count($response['body'])); + $this->assertCount(1, $response['body']['requests']); + $this->assertCount(1, $response['body']['network']); + $this->assertEquals($requestsTotal, $response['body']['requests'][array_key_last($response['body']['requests'])]['value']); + $this->validateDates($response['body']['requests']); + // vectordbTotal should reflect only VectorDB instances, not relational databases. + $this->assertEquals($vectordbTotal, $response['body']['vectordbDatabasesTotal']); + $this->assertEquals($documentsTotal, $response['body']['vectordbDocumentsTotal']); + }); + + $response = $this->client->call( + Client::METHOD_GET, + '/databases/usage?range=30d', + $this->getConsoleHeaders() + ); + + $this->assertEquals($databasesTotal, $response['body']['databases'][array_key_last($response['body']['databases'])]['value']); + $this->validateDates($response['body']['databases']); + + $this->assertEventually(function () use ($vectordbId, $collectionsTotal, $documentsTotal) { + $response = $this->client->call( + Client::METHOD_GET, + '/vectordb/' . $vectordbId . '/usage?range=30d', + $this->getConsoleHeaders() + ); + + $this->assertEquals($collectionsTotal, $response['body']['collections'][array_key_last($response['body']['collections'])]['value']); + $this->validateDates($response['body']['collections']); + + $this->assertEquals($documentsTotal, $response['body']['documents'][array_key_last($response['body']['documents'])]['value']); + $this->validateDates($response['body']['documents']); + }); + + $this->assertEventually(function () use ($vectordbId, $collectionId, $documentsTotal) { + $response = $this->client->call( + Client::METHOD_GET, + '/vectordb/' . $vectordbId . '/collections/' . $collectionId . '/usage?range=30d', + $this->getConsoleHeaders() + ); + + $this->assertEquals($documentsTotal, $response['body']['documents'][array_key_last($response['body']['documents'])]['value']); + $this->validateDates($response['body']['documents']); + }); + + return $data; + } + + /** @depends testVectorDBStats */ public function testPrepareFunctionsStats(array $data): array { $executionTime = 0; @@ -1598,6 +1820,75 @@ class UsageTest extends Scope }); } + public function testEmbeddingsTextUsageDoesNotBreakProjectUsage(): void + { + // Trigger embeddings endpoint a few times so stats usage worker has data to aggregate + for ($i = 0; $i < 3; $i++) { + $response = $this->client->call( + Client::METHOD_POST, + '/vectordb/embeddings/text', + array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ], $this->getHeaders()), + [ + 'model' => 'embeddinggemma', + 'texts' => [ + 'usage test text ' . $i, + ], + ] + ); + + $this->assertEquals(200, $response['headers']['status-code']); + $this->assertIsArray($response['body']['embeddings']); + $this->assertGreaterThan(0, $response['body']['total']); + } + + // Ensure project usage endpoint still responds correctly after embeddings calls + $this->assertEventually(function () { + $response = $this->client->call( + Client::METHOD_GET, + '/project/usage', + $this->getConsoleHeaders(), + [ + 'period' => '1h', + 'startDate' => self::getToday(), + 'endDate' => self::getTomorrow(), + ] + ); + + $this->assertEquals(200, $response['headers']['status-code']); + $this->assertArrayHasKey('requests', $response['body']); + $this->assertArrayHasKey('network', $response['body']); + $this->assertArrayHasKey('executionsTotal', $response['body']); + + // New embeddings metrics should be present after calls above + $this->assertArrayHasKey('embeddingsText', $response['body']); + $this->assertArrayHasKey('embeddingsTextErrors', $response['body']); + $this->assertArrayHasKey('embeddingsTextTokens', $response['body']); + $this->assertArrayHasKey('embeddingsTextDuration', $response['body']); + $this->assertArrayHasKey('embeddingsTextTotal', $response['body']); + $this->assertArrayHasKey('embeddingsTextErrorsTotal', $response['body']); + $this->assertArrayHasKey('embeddingsTextTokensTotal', $response['body']); + $this->assertArrayHasKey('embeddingsTextDurationTotal', $response['body']); + + // Time-series arrays should be non-empty + $this->assertNotEmpty($response['body']['embeddingsText']); + $this->assertNotEmpty($response['body']['embeddingsTextTokens']); + $this->assertNotEmpty($response['body']['embeddingsTextDuration']); + $this->validateDates($response['body']['embeddingsText']); + $this->validateDates($response['body']['embeddingsTextTokens']); + $this->validateDates($response['body']['embeddingsTextDuration']); + + // Total scalars should be greater than 0 (or >= 0 for errors) + $this->assertGreaterThan(0, $response['body']['embeddingsTextTotal']); + $this->assertGreaterThanOrEqual(0, $response['body']['embeddingsTextErrorsTotal']); + $this->assertGreaterThan(0, $response['body']['embeddingsTextTokensTotal']); + $this->assertGreaterThan(0, $response['body']['embeddingsTextDurationTotal']); + }); + } + public function tearDown(): void { $this->projectId = ''; diff --git a/tests/e2e/Services/Databases/VectorDB/DatabasesBase.php b/tests/e2e/Services/Databases/VectorDB/DatabasesBase.php new file mode 100644 index 0000000000..5d366af516 --- /dev/null +++ b/tests/e2e/Services/Databases/VectorDB/DatabasesBase.php @@ -0,0 +1,1073 @@ +client->call(Client::METHOD_POST, '/vectordb', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'databaseId' => ID::unique(), + 'name' => 'Test Database' + ]); + + $this->assertNotEmpty($database['body']['$id']); + $this->assertEquals(201, $database['headers']['status-code']); + $this->assertEquals('Test Database', $database['body']['name']); + $this->assertEquals('vectordb', $database['body']['type']); + + return ['databaseId' => $database['body']['$id']]; + } + + /** + * @depends testCreateCollectionSample + */ + public function testCreateDocument(array $data): array + { + $databaseId = $data['databaseId']; + $collectionId = $data['collectionId']; + + // Build embedding vector matching collection dimensions (1536) + $vector = array_fill(0, 1536, 0.1); + $vector[0] = 1.0; + + $res = $this->client->call(Client::METHOD_POST, "/vectordb/{$databaseId}/collections/{$collectionId}/documents", [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'documentId' => ID::unique(), + 'data' => [ + 'embeddings' => $vector, + 'metadata' => ['type' => 'sample', 'rank' => 1] + ], + 'permissions' => [ + Permission::read(Role::any()), + Permission::update(Role::any()), + Permission::delete(Role::any()), + ] + ]); + + $this->assertEquals(201, $res['headers']['status-code']); + $this->assertNotEmpty($res['body']['$id']); + $documentId = $res['body']['$id']; + + // createdAt/updatedAt should be present and equal on initial create + $this->assertArrayHasKey('$createdAt', $res['body']); + $this->assertArrayHasKey('$updatedAt', $res['body']); + $this->assertNotEmpty($res['body']['$createdAt']); + $this->assertNotEmpty($res['body']['$updatedAt']); + $this->assertEquals($res['body']['$createdAt'], $res['body']['$updatedAt']); + + // Edge: invalid dimensions (vector too short) → expect 4xx + $badVec = [1.0, 0.0]; + $bad = $this->client->call(Client::METHOD_POST, "/vectordb/{$databaseId}/collections/{$collectionId}/documents", [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'documentId' => ID::unique(), + 'data' => [ + 'embeddings' => $badVec, + 'metadata' => ['type' => 'bad'] + ], + ]); + $this->assertGreaterThanOrEqual(400, $bad['headers']['status-code']); + $this->assertLessThan(500, $bad['headers']['status-code']); + + // Edge: invalid type values (strings) → expect 4xx + $strVec = ['1.0', '0.0', '0.0']; + $bad2 = $this->client->call(Client::METHOD_POST, "/vectordb/{$databaseId}/collections/{$collectionId}/documents", [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'documentId' => ID::unique(), + 'data' => [ + 'embeddings' => $strVec, + 'metadata' => ['type' => 'bad-strings'] + ], + ]); + $this->assertGreaterThanOrEqual(400, $bad2['headers']['status-code']); + $this->assertLessThan(500, $bad2['headers']['status-code']); + + // Create another valid doc to verify list totals later + $res2 = $this->client->call(Client::METHOD_POST, "/vectordb/{$databaseId}/collections/{$collectionId}/documents", [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'documentId' => ID::unique(), + 'data' => [ + 'embeddings' => $vector, + 'metadata' => ['type' => 'sample', 'rank' => 99] + ], + 'permissions' => [Permission::read(Role::any())] + ]); + $this->assertEquals(201, $res2['headers']['status-code']); + $documentId2 = $res2['body']['$id']; + + return [ + 'databaseId' => $databaseId, + 'collectionId' => $collectionId, + 'documentId' => $documentId, + 'documentId2' => $documentId2, + 'createdAt' => $res['body']['$createdAt'], + 'updatedAt' => $res['body']['$updatedAt'], + ]; + } + + /** + * @depends testCreateDocument + */ + public function testGetDocument(array $data): array + { + $databaseId = $data['databaseId']; + $collectionId = $data['collectionId']; + $documentId = $data['documentId']; + + $res = $this->client->call(Client::METHOD_GET, "/vectordb/{$databaseId}/collections/{$collectionId}/documents/{$documentId}", [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]); + + $this->assertEquals(200, $res['headers']['status-code']); + $this->assertEquals($documentId, $res['body']['$id']); + + // Edge: missing document should return 404 + $missing = $this->client->call(Client::METHOD_GET, "/vectordb/{$databaseId}/collections/{$collectionId}/documents/" . ID::unique(), [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]); + $this->assertEquals(404, $missing['headers']['status-code']); + + return $data; + } + + /** + * @depends testCreateDocument + */ + public function testListDocuments(array $data): array + { + $databaseId = $data['databaseId']; + $collectionId = $data['collectionId']; + + $list = $this->client->call(Client::METHOD_GET, "/vectordb/{$databaseId}/collections/{$collectionId}/documents", [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'queries' => [Query::limit(5)->toString()] + ]); + + $this->assertEquals(200, $list['headers']['status-code']); + $this->assertIsInt($list['body']['total']); + $this->assertGreaterThanOrEqual(1, $list['body']['total']); + + // Pagination: limit 1, then offset 1 + $page1 = $this->client->call(Client::METHOD_GET, "/vectordb/{$databaseId}/collections/{$collectionId}/documents", [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'queries' => [ + Query::limit(1)->toString(), + Query::orderAsc('$id')->toString() + ] + ]); + $this->assertEquals(200, $page1['headers']['status-code']); + $this->assertEquals(1, \count($page1['body']['documents'] ?? [])); + + $page2 = $this->client->call(Client::METHOD_GET, "/vectordb/{$databaseId}/collections/{$collectionId}/documents", [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'queries' => [ + Query::limit(1)->toString(), + Query::offset(1)->toString(), + Query::orderAsc('$id')->toString() + ] + ]); + $this->assertEquals(200, $page2['headers']['status-code']); + $this->assertEquals(1, \count($page2['body']['documents'] ?? [])); + + return $data; + } + + /** + * @depends testCreateDocument + */ + public function testUpsertDocument(array $data): array + { + $databaseId = $data['databaseId']; + $collectionId = $data['collectionId']; + $documentId = $data['documentId']; + + $vector = array_fill(0, 1536, 0.0); + // $vector[1] = 1.0; + + $upd = $this->client->call(Client::METHOD_PUT, "/vectordb/{$databaseId}/collections/{$collectionId}/documents/{$documentId}", [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'data' => [ + 'embeddings' => $vector, + 'metadata' => ['type' => 'sample', 'rank' => 2] + ] + ]); + + $this->assertEquals(200, $upd['headers']['status-code']); + + // Verify update took effect + $get = $this->client->call(Client::METHOD_GET, "/vectordb/{$databaseId}/collections/{$collectionId}/documents/{$documentId}", [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]); + $this->assertEquals(200, $get['headers']['status-code']); + $this->assertEquals(2, $get['body']['metadata']['rank']); + // updatedAt should be greater or changed from earlier + $this->assertArrayHasKey('$updatedAt', $get['body']); + + return $data; + } + + /** + * @depends testUpsertDocument + */ + public function testUpdateDocument(array $data): array + { + // Upsert is used for update semantics + $databaseId = $data['databaseId']; + $collectionId = $data['collectionId']; + $documentId = $data['documentId']; + + $vector = array_fill(0, 1536, 0.0); + $vector[2] = 1.0; + + $upd = $this->client->call(Client::METHOD_PUT, "/vectordb/{$databaseId}/collections/{$collectionId}/documents/{$documentId}", [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'data' => [ + 'embeddings' => $vector, + 'metadata' => ['type' => 'sample', 'rank' => 3] + ] + ]); + + $this->assertEquals(200, $upd['headers']['status-code']); + + // Re-update to check idempotence and metadata replacement + $vector2 = array_fill(0, 1536, 0.0); + $vector2[3] = 1.0; + $upd2 = $this->client->call(Client::METHOD_PUT, "/vectordb/{$databaseId}/collections/{$collectionId}/documents/{$documentId}", [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'data' => [ + 'embeddings' => $vector2, + 'metadata' => ['type' => 'sample', 'rank' => 4] + ] + ]); + $this->assertEquals(200, $upd2['headers']['status-code']); + + // Verify updatedAt changed again + $get2 = $this->client->call(Client::METHOD_GET, "/vectordb/{$databaseId}/collections/{$collectionId}/documents/{$documentId}", [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]); + $this->assertEquals(200, $get2['headers']['status-code']); + $this->assertArrayHasKey('$updatedAt', $get2['body']); + + return $data; + } + + /** + * @depends testUpdateDocument + */ + public function testDocumentsVectorQueries(array $data): array + { + $databaseId = $data['databaseId']; + $collectionId = $data['collectionId']; + + // Create two more documents with distinct embeddings + $mk = function (array $vec, string $name) use ($databaseId, $collectionId) { + $this->client->call(Client::METHOD_POST, "/vectordb/{$databaseId}/collections/{$collectionId}/documents", [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'documentId' => ID::unique(), + 'data' => [ + 'embeddings' => $vec, + 'metadata' => ['name' => $name] + ], + 'permissions' => [Permission::read(Role::any())] + ]); + }; + + $vA = array_fill(0, 1536, 0.0); + $vA[0] = 1.0; // close to [1,0,0,...] + $vB = array_fill(0, 1536, 0.0); + $vB[1] = 1.0; // close to [0,1,0,...] + + $mk($vA, 'A'); + $mk($vB, 'B'); + + // Dot product + $dot = $this->client->call(Client::METHOD_GET, "/vectordb/{$databaseId}/collections/{$collectionId}/documents", [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'queries' => [ + Query::vectorDot('embeddings', $vA)->toString(), + Query::limit(2)->toString() + ] + ]); + $this->assertEquals(200, $dot['headers']['status-code']); + $this->assertGreaterThanOrEqual(1, $dot['body']['total']); + + // Cosine + $cos = $this->client->call(Client::METHOD_GET, "/vectordb/{$databaseId}/collections/{$collectionId}/documents", [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'queries' => [ + Query::vectorCosine('embeddings', $vB)->toString(), + Query::limit(2)->toString() + ] + ]); + $this->assertEquals(200, $cos['headers']['status-code']); + $this->assertGreaterThanOrEqual(1, $cos['body']['total']); + + // Euclidean + $eu = $this->client->call(Client::METHOD_GET, "/vectordb/{$databaseId}/collections/{$collectionId}/documents", [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'queries' => [ + Query::vectorEuclidean('embeddings', $vA)->toString(), + Query::limit(2)->toString() + ] + ]); + $this->assertEquals(200, $eu['headers']['status-code']); + $this->assertGreaterThanOrEqual(1, $eu['body']['total']); + + // Combined vector + metadata filters + $combo = $this->client->call(Client::METHOD_GET, "/vectordb/{$databaseId}/collections/{$collectionId}/documents", [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'queries' => [ + Query::vectorCosine('embeddings', $vA)->toString(), + Query::notEqual('metadata', [['name' => 'B']])->toString(), + Query::limit(2)->toString() + ] + ]); + $this->assertEquals(200, $combo['headers']['status-code']); + $this->assertGreaterThanOrEqual(1, $combo['body']['total']); + + // Ordering with $id ascending combined with vector + $ordered = $this->client->call(Client::METHOD_GET, "/vectordb/{$databaseId}/collections/{$collectionId}/documents", [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'queries' => [ + Query::vectorDot('embeddings', $vA)->toString(), + Query::orderAsc('$id')->toString(), + Query::limit(3)->toString() + ] + ]); + $this->assertEquals(200, $ordered['headers']['status-code']); + + return $data; + } + + /** + * @depends testDocumentsVectorQueries + */ + public function testDeleteDocument(array $data): void + { + $databaseId = $data['databaseId']; + $collectionId = $data['collectionId']; + $documentId = $data['documentId']; + + $del = $this->client->call(Client::METHOD_DELETE, "/vectordb/{$databaseId}/collections/{$collectionId}/documents/{$documentId}", [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]); + $this->assertEquals(204, $del['headers']['status-code']); + + // GET after delete should be 404 + $getMissing = $this->client->call(Client::METHOD_GET, "/vectordb/{$databaseId}/collections/{$collectionId}/documents/{$documentId}", [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]); + $this->assertEquals(404, $getMissing['headers']['status-code']); + + // List should still work and reflect at least one less document compared to earlier pages (best-effort) + $list = $this->client->call(Client::METHOD_GET, "/vectordb/{$databaseId}/collections/{$collectionId}/documents", [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'queries' => [Query::limit(5)->toString()] + ]); + $this->assertEquals(200, $list['headers']['status-code']); + } + + /** + * @depends testCreateCollectionSample + */ + public function testDocumentPermissions(array $data): void + { + $databaseId = $data['databaseId']; + $collectionId = $data['collectionId']; + + // Create doc readable only by a specific user + $docId = ID::unique(); + $vector = array_fill(0, 1536, 0.0); + $vector[0] = 1.0; + $create = $this->client->call(Client::METHOD_POST, "/vectordb/{$databaseId}/collections/{$collectionId}/documents", [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'documentId' => $docId, + 'data' => [ + 'embeddings' => $vector, + 'metadata' => ['scope' => 'private'] + ], + 'permissions' => [ + Permission::read(Role::user($this->getUser()['$id'])) + ] + ]); + $this->assertEquals(201, $create['headers']['status-code']); + + $guest = $this->client->call(Client::METHOD_GET, "/vectordb/{$databaseId}/collections/{$collectionId}/documents/{$docId}", [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'] + ]); + $this->assertEquals(404, $guest['headers']['status-code']); + + // GET with key should succeed regardless of document user-level permission + $withKey = $this->client->call(Client::METHOD_GET, "/vectordb/{$databaseId}/collections/{$collectionId}/documents/{$docId}", [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]); + $this->assertEquals(200, $withKey['headers']['status-code']); + } + + /** + * @depends testCreateDatabase + */ + public function testCreateCollection(array $data): array + { + $databaseId = $data['databaseId']; + /** + * Test for SUCCESS + */ + $movies = $this->client->call(Client::METHOD_POST, '/vectordb/' . $databaseId . '/collections', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'collectionId' => ID::unique(), + 'name' => 'Movies', + 'documentSecurity' => true, + 'dimension' => 1536, + 'permissions' => [ + Permission::create(Role::user($this->getUser()['$id'])), + ], + ]); + + $this->assertEquals(201, $movies['headers']['status-code']); + $this->assertEquals($movies['body']['name'], 'Movies'); + + $actors = $this->client->call(Client::METHOD_POST, '/vectordb/' . $databaseId . '/collections', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'collectionId' => ID::unique(), + 'name' => 'Actors', + 'documentSecurity' => true, + 'dimension' => 1536, + 'permissions' => [ + Permission::create(Role::user($this->getUser()['$id'])), + ], + ]); + + $this->assertEquals(201, $actors['headers']['status-code']); + $this->assertEquals($actors['body']['name'], 'Actors'); + + return [ + 'databaseId' => $databaseId, + 'moviesId' => $movies['body']['$id'], + 'actorsId' => $actors['body']['$id'], + ]; + } + + public function testCreateDatabaseSample(): array + { + $database = $this->client->call(Client::METHOD_POST, '/vectordb', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'databaseId' => ID::unique(), + 'name' => 'Sample VectorDB' + ]); + + $this->assertNotEmpty($database['body']['$id']); + $this->assertEquals(201, $database['headers']['status-code']); + $this->assertEquals('Sample VectorDB', $database['body']['name']); + $this->assertEquals('vectordb', $database['body']['type']); + + return ['databaseId' => $database['body']['$id']]; + } + + /** + * @depends testCreateDatabaseSample + */ + public function testCreateCollectionSample(array $data): array + { + $databaseId = $data['databaseId']; + + $collection = $this->client->call(Client::METHOD_POST, '/vectordb/' . $databaseId . '/collections', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'collectionId' => ID::unique(), + 'name' => 'Sample Collection', + 'dimension' => 1536, + 'documentSecurity' => true, + 'permissions' => [ + Permission::create(Role::user($this->getUser()['$id'])), + ], + ]); + + $this->assertEquals(201, $collection['headers']['status-code']); + $this->assertEquals('Sample Collection', $collection['body']['name']); + $this->assertEquals(1536, $collection['body']['dimension']); + + return [ + 'databaseId' => $databaseId, + 'collectionId' => $collection['body']['$id'], + ]; + } + + public function testCreateMultipleDatabasesWithCollections(): array + { + $projectId = $this->getProject()['$id']; + $apiKey = $this->getProject()['apiKey']; + $userId = $this->getUser()['$id']; + + /** + * Helper to create a database + */ + $createDatabase = function (string $name) use ($projectId, $apiKey) { + $db = $this->client->call(Client::METHOD_POST, '/vectordb', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $projectId, + 'x-appwrite-key' => $apiKey + ], [ + 'databaseId' => ID::unique(), + 'name' => $name + ]); + + $this->assertEquals(201, $db['headers']['status-code']); + $this->assertEquals('vectordb', $db['body']['type']); + $this->assertEquals($name, $db['body']['name']); + $this->assertNotEmpty($db['body']['$id']); + + return $db['body']['$id']; + }; + + /** + * Helper to create a collection + */ + $createCollection = function (string $databaseId, string $name, int $dimensions = 1536) use ($projectId, $apiKey, $userId) { + $res = $this->client->call(Client::METHOD_POST, '/vectordb/' . $databaseId . '/collections', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $projectId, + 'x-appwrite-key' => $apiKey + ], [ + 'collectionId' => ID::unique(), + 'name' => $name, + 'documentSecurity' => true, + 'dimension' => $dimensions, + 'permissions' => [ + Permission::create(Role::user($userId)), + ], + ]); + + $this->assertEquals(201, $res['headers']['status-code']); + $this->assertEquals($name, $res['body']['name']); + + return $res['body']['$id']; + }; + + /** + * === Database 1: MediaDB === + */ + $mediaDbId = $createDatabase('MediaDB'); + + $mediaCollections = ['Movies', 'Actors', 'Directors']; + $mediaCollectionIds = []; + + foreach ($mediaCollections as $col) { + $mediaCollectionIds[$col] = $createCollection($mediaDbId, $col); + } + + /** + * === Database 2: ContentDB === + */ + $contentDbId = $createDatabase('ContentDB'); + + $contentCollections = ['Articles', 'Authors']; + $contentCollectionIds = []; + + foreach ($contentCollections as $col) { + $contentCollectionIds[$col] = $createCollection($contentDbId, $col); + } + + // Create a tiny-dimension collection and insert a document to validate vector and object attributes + $tinyCollectionName = 'VectorsTiny'; + $tinyDimensions = 8; + $tinyCollectionId = $createCollection($mediaDbId, $tinyCollectionName, $tinyDimensions); + + return [ + 'databases' => [ + 'MediaDB' => [ + 'id' => $mediaDbId, + 'collections' => $mediaCollectionIds + ['VectorsTiny' => $tinyCollectionId], + ], + 'ContentDB' => [ + 'id' => $contentDbId, + 'collections' => $contentCollectionIds, + ], + ] + ]; + } + + public function testInvalidCollectionDimensions(): void + { + // dimensions = 0 -> expect 4xx + $bad0 = $this->client->call(Client::METHOD_POST, '/vectordb', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'databaseId' => ID::unique(), + 'name' => 'BadDims0' + ]); + $this->assertEquals(201, $bad0['headers']['status-code']); + $dbId = $bad0['body']['$id']; + $col = $this->client->call(Client::METHOD_POST, '/vectordb/' . $dbId . '/collections', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'collectionId' => ID::unique(), + 'name' => 'ZeroDims', + 'documentSecurity' => true, + 'dimension' => 0, + 'permissions' => [Permission::create(Role::user($this->getUser()['$id']))], + ]); + $this->assertGreaterThanOrEqual(400, $col['headers']['status-code']); + $this->assertLessThan(500, $col['headers']['status-code']); + + // dimensions too large -> expect 4xx + $col2 = $this->client->call(Client::METHOD_POST, '/vectordb/' . $dbId . '/collections', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'collectionId' => ID::unique(), + 'name' => 'HugeDims', + 'documentSecurity' => true, + 'dimension' => 16001, + 'permissions' => [Permission::create(Role::user($this->getUser()['$id']))], + ]); + $this->assertGreaterThanOrEqual(400, $col2['headers']['status-code']); + $this->assertLessThan(500, $col2['headers']['status-code']); + } + + public function testSingleDimensionVectorCollection(): void + { + $db = $this->client->call(Client::METHOD_POST, '/vectordb', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'databaseId' => ID::unique(), + 'name' => 'SingleDim' + ]); + $this->assertEquals(201, $db['headers']['status-code']); + $databaseId = $db['body']['$id']; + + $col = $this->client->call(Client::METHOD_POST, '/vectordb/' . $databaseId . '/collections', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'collectionId' => ID::unique(), + 'name' => 'OneDim', + 'documentSecurity' => true, + 'dimension' => 1, + 'permissions' => [Permission::create(Role::user($this->getUser()['$id']))], + ]); + $this->assertEquals(201, $col['headers']['status-code']); + $collectionId = $col['body']['$id']; + + // Create two docs with 1D embeddings + $id1 = ID::unique(); + $this->client->call(Client::METHOD_PUT, "/vectordb/{$databaseId}/collections/{$collectionId}/documents/{$id1}", [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'data' => ['embeddings' => [1.0]] + ]); + $id2 = ID::unique(); + $this->client->call(Client::METHOD_PUT, "/vectordb/{$databaseId}/collections/{$collectionId}/documents/{$id2}", [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'data' => ['embeddings' => [0.5]] + ]); + + // Query with vectorCosine + $res = $this->client->call(Client::METHOD_GET, "/vectordb/{$databaseId}/collections/{$collectionId}/documents", [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'queries' => [Query::vectorCosine('embeddings', [1.0])->toString(), Query::limit(2)->toString()] + ]); + $this->assertEquals(200, $res['headers']['status-code']); + $this->assertGreaterThanOrEqual(1, $res['body']['total']); + } + + public function testVectorInvalidValues(): void + { + $db = $this->client->call(Client::METHOD_POST, '/vectordb', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'databaseId' => ID::unique(), + 'name' => 'InvalidVals' + ]); + $this->assertEquals(201, $db['headers']['status-code']); + $databaseId = $db['body']['$id']; + + $col = $this->client->call(Client::METHOD_POST, '/vectordb/' . $databaseId . '/collections', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'collectionId' => ID::unique(), + 'name' => 'Docs', + 'documentSecurity' => true, + 'dimension' => 3, + 'permissions' => [Permission::create(Role::user($this->getUser()['$id']))], + ]); + $this->assertEquals(201, $col['headers']['status-code']); + $collectionId = $col['body']['$id']; + + $badPayloads = [ + ['embeddings' => [INF, 0.0, 0.0]], + ['embeddings' => [-INF, 0.0, 0.0]], + ['embeddings' => [NAN, 0.0, 0.0]], + ['embeddings' => ['x' => 1.0, 'y' => 0.0, 'z' => 0.0]], + ['embeddings' => [1.0, null, 0.0]], + ['embeddings' => [[1.0], [0.0], [0.0]]], + ['embeddings' => [true, false, true]], + ['embeddings' => [1.0, '2.0', 3.0]], + (function () { + $v = []; + $v[0] = 1.0; + $v[2] = 1.0; + return ['embeddings' => $v]; + })(), + ]; + + foreach ($badPayloads as $payload) { + $resp = $this->client->call(Client::METHOD_PUT, "/vectordb/{$databaseId}/collections/{$collectionId}/documents/" . ID::unique(), [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'data' => $payload + ]); + $this->assertGreaterThanOrEqual(400, $resp['headers']['status-code']); + $this->assertLessThan(500, $resp['headers']['status-code']); + } + } + + public function testVectorAllZerosAndQuery(): void + { + $db = $this->client->call(Client::METHOD_POST, '/vectordb', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'databaseId' => ID::unique(), + 'name' => 'ZerosDB' + ]); + $this->assertEquals(201, $db['headers']['status-code']); + $databaseId = $db['body']['$id']; + + $col = $this->client->call(Client::METHOD_POST, '/vectordb/' . $databaseId . '/collections', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'collectionId' => ID::unique(), + 'name' => 'Zeros', + 'documentSecurity' => true, + 'dimension' => 3, + 'permissions' => [Permission::create(Role::user($this->getUser()['$id']))], + ]); + $this->assertEquals(201, $col['headers']['status-code']); + $collectionId = $col['body']['$id']; + + $this->client->call(Client::METHOD_PUT, "/vectordb/{$databaseId}/collections/{$collectionId}/documents/" . ID::unique(), [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ 'data' => ['embeddings' => [0.0, 0.0, 0.0]] ]); + + $this->client->call(Client::METHOD_PUT, "/vectordb/{$databaseId}/collections/{$collectionId}/documents/" . ID::unique(), [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ 'data' => ['embeddings' => [1.0, 0.0, 0.0]] ]); + + $results = $this->client->call(Client::METHOD_GET, "/vectordb/{$databaseId}/collections/{$collectionId}/documents", [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ 'queries' => [Query::vectorCosine('embeddings', [1.0, 0.0, 0.0])->toString()] ]); + $this->assertEquals(200, $results['headers']['status-code']); + $this->assertGreaterThan(0, $results['body']['total']); + } + + public function testVectorMultipleQueriesRejection(): void + { + // Create a simple DB and collection + $db = $this->client->call(Client::METHOD_POST, '/vectordb', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ 'databaseId' => ID::unique(), 'name' => 'MultiQueryDB' ]); + $this->assertEquals(201, $db['headers']['status-code']); + $databaseId = $db['body']['$id']; + $col = $this->client->call(Client::METHOD_POST, '/vectordb/' . $databaseId . '/collections', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ 'collectionId' => ID::unique(), 'name' => 'Docs', 'documentSecurity' => true, 'dimension' => 3, 'permissions' => [Permission::create(Role::user($this->getUser()['$id']))] ]); + $this->assertEquals(201, $col['headers']['status-code']); + $collectionId = $col['body']['$id']; + + // Two vector queries simultaneously should fail + $fail = $this->client->call(Client::METHOD_GET, "/vectordb/{$databaseId}/collections/{$collectionId}/documents", [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'queries' => [ + Query::vectorCosine('embeddings', [1.0, 0.0, 0.0])->toString(), + Query::vectorEuclidean('embeddings', [1.0, 0.0, 0.0])->toString() + ] + ]); + $this->assertGreaterThanOrEqual(400, $fail['headers']['status-code']); + $this->assertLessThan(500, $fail['headers']['status-code']); + } + + public function testVectorQueryOnNonVectorAttribute(): void + { + $db = $this->client->call(Client::METHOD_POST, '/vectordb', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ 'databaseId' => ID::unique(), 'name' => 'NonVec' ]); + $this->assertEquals(201, $db['headers']['status-code']); + $databaseId = $db['body']['$id']; + $col = $this->client->call(Client::METHOD_POST, '/vectordb/' . $databaseId . '/collections', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ 'collectionId' => ID::unique(), 'name' => 'Docs', 'documentSecurity' => true, 'dimension' => 3, 'permissions' => [Permission::create(Role::user($this->getUser()['$id']))] ]); + $this->assertEquals(201, $col['headers']['status-code']); + $collectionId = $col['body']['$id']; + + // Query on non-vector attribute 'metadata' should fail + $fail = $this->client->call(Client::METHOD_GET, "/vectordb/{$databaseId}/collections/{$collectionId}/documents", [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ 'queries' => [Query::vectorCosine('metadata', [1.0, 0.0, 0.0])->toString()] ]); + $this->assertGreaterThanOrEqual(400, $fail['headers']['status-code']); + $this->assertLessThan(500, $fail['headers']['status-code']); + } + + public function testVectorEmptyQueryCollection(): void + { + $db = $this->client->call(Client::METHOD_POST, '/vectordb', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ 'databaseId' => ID::unique(), 'name' => 'EmptyQ' ]); + $this->assertEquals(201, $db['headers']['status-code']); + $databaseId = $db['body']['$id']; + $col = $this->client->call(Client::METHOD_POST, '/vectordb/' . $databaseId . '/collections', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ 'collectionId' => ID::unique(), 'name' => 'Docs', 'documentSecurity' => true, 'dimension' => 3, 'permissions' => [Permission::create(Role::user($this->getUser()['$id']))] ]); + $this->assertEquals(201, $col['headers']['status-code']); + $collectionId = $col['body']['$id']; + + $res = $this->client->call(Client::METHOD_GET, "/vectordb/{$databaseId}/collections/{$collectionId}/documents", [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ 'queries' => [Query::vectorCosine('embeddings', [1.0, 0.0, 0.0])->toString()] ]); + $this->assertEquals(200, $res['headers']['status-code']); + $this->assertEquals(0, $res['body']['total']); + } + + /** + * @depends testCreateCollection + */ + public function testCreateIndexes(array $data): array + { + $databaseId = $data['databaseId']; + $collectionId = $data['moviesId']; + + // HNSW Euclidean + $idxEuclidean = $this->client->call(Client::METHOD_POST, "/vectordb/{$databaseId}/collections/{$collectionId}/indexes", [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'key' => 'embedding_euclidean', + 'type' => Database::INDEX_HNSW_EUCLIDEAN, + 'attributes' => ['embeddings'] + ]); + $this->assertEquals(202, $idxEuclidean['headers']['status-code']); + + // HNSW Dot (Inner Product) + $idxDot = $this->client->call(Client::METHOD_POST, "/vectordb/{$databaseId}/collections/{$collectionId}/indexes", [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'key' => 'embedding_dot', + 'type' => Database::INDEX_HNSW_DOT, + 'attributes' => ['embeddings'] + ]); + $this->assertEquals(202, $idxDot['headers']['status-code']); + + // HNSW Cosine + $idxCosine = $this->client->call(Client::METHOD_POST, "/vectordb/{$databaseId}/collections/{$collectionId}/indexes", [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'key' => 'embedding_cosine', + 'type' => Database::INDEX_HNSW_COSINE, + 'attributes' => ['embeddings'] + ]); + $this->assertEquals(202, $idxCosine['headers']['status-code']); + + return [ + 'databaseId' => $databaseId, + 'collectionId' => $collectionId, + 'indexes' => ['embedding_euclidean', 'embedding_dot', 'embedding_cosine'] + ]; + } + + /** + * @depends testCreateIndexes + */ + public function testListIndexes(array $data): void + { + $databaseId = $data['databaseId']; + $collectionId = $data['collectionId']; + + $list = $this->client->call(Client::METHOD_GET, "/vectordb/{$databaseId}/collections/{$collectionId}/indexes", [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]); + + $this->assertEquals(200, $list['headers']['status-code']); + $keys = array_map(fn ($i) => $i['key'], $list['body']['indexes'] ?? []); + foreach ($data['indexes'] as $expectedKey) { + $this->assertContains($expectedKey, $keys); + } + } + + /** + * @depends testCreateIndexes + */ + public function testGetIndexByKey(array $data): void + { + $databaseId = $data['databaseId']; + $collectionId = $data['collectionId']; + + $keysToTypes = [ + 'embedding_euclidean' => Database::INDEX_HNSW_EUCLIDEAN, + 'embedding_dot' => Database::INDEX_HNSW_DOT, + 'embedding_cosine' => Database::INDEX_HNSW_COSINE, + ]; + + foreach ($keysToTypes as $key => $type) { + $res = $this->client->call(Client::METHOD_GET, "/vectordb/{$databaseId}/collections/{$collectionId}/indexes/{$key}", [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]); + $this->assertEquals(200, $res['headers']['status-code']); + $this->assertEquals($key, $res['body']['key']); + $this->assertEquals($type, $res['body']['type']); + } + } + +} diff --git a/tests/e2e/Services/Databases/VectorDB/DatabasesConsoleClientTest.php b/tests/e2e/Services/Databases/VectorDB/DatabasesConsoleClientTest.php new file mode 100644 index 0000000000..8ca4822e68 --- /dev/null +++ b/tests/e2e/Services/Databases/VectorDB/DatabasesConsoleClientTest.php @@ -0,0 +1,336 @@ +client->call(Client::METHOD_POST, '/vectordb', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'databaseId' => ID::unique(), + 'name' => 'Vector Console DB', + ]); + $this->assertEquals(201, $database['headers']['status-code']); + $this->assertEquals('Vector Console DB', $database['body']['name']); + $this->assertTrue($database['body']['enabled']); + + $databaseId = $database['body']['$id']; + + /** + * Test for SUCCESS + */ + $movies = $this->client->call(Client::METHOD_POST, '/vectordb/' . $databaseId . '/collections', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'collectionId' => ID::unique(), + 'name' => 'Movies', + 'dimension' => 3, + 'permissions' => [ + Permission::read(Role::any()), + Permission::create(Role::any()), + Permission::update(Role::any()), + Permission::delete(Role::any()), + ], + 'documentSecurity' => true, + ]); + + $this->assertEquals(201, $movies['headers']['status-code']); + $this->assertEquals($movies['body']['name'], 'Movies'); + + /** + * Test when database is disabled but can still create collections + */ + $database = $this->client->call(Client::METHOD_PUT, '/vectordb/' . $databaseId, array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'name' => 'Vector Console DB Updated', + 'enabled' => false, + ]); + + $this->assertFalse($database['body']['enabled']); + + $tvShows = $this->client->call(Client::METHOD_POST, '/vectordb/' . $databaseId . '/collections', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'collectionId' => ID::unique(), + 'name' => 'TvShows', + 'dimension' => 3, + 'permissions' => [ + Permission::read(Role::any()), + Permission::create(Role::any()), + Permission::update(Role::any()), + Permission::delete(Role::any()), + ], + 'documentSecurity' => true, + ]); + + /** + * Test when collection is disabled but can still modify collections + */ + $database = $this->client->call(Client::METHOD_PUT, '/vectordb/' . $databaseId . '/collections/' . $movies['body']['$id'], array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'name' => 'Movies', + 'enabled' => false, + ]); + + $this->assertEquals(201, $tvShows['headers']['status-code']); + $this->assertEquals($tvShows['body']['name'], 'TvShows'); + + return ['moviesId' => $movies['body']['$id'], 'databaseId' => $databaseId, 'tvShowsId' => $tvShows['body']['$id']]; + } + + /** + * @depends testCreateCollection + * @param array $data + * @throws \Exception + */ + public function testListCollection(array $data) + { + /** + * Test when database is disabled but can still call list collections + */ + $databaseId = $data['databaseId']; + + $collections = $this->client->call(Client::METHOD_GET, '/vectordb/' . $databaseId . '/collections', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'] + ], $this->getHeaders())); + + $this->assertEquals(200, $collections['headers']['status-code']); + $this->assertEquals(2, $collections['body']['total']); + } + + /** + * @depends testCreateCollection + * @param array $data + * @throws \Exception + */ + public function testGetCollection(array $data) + { + $databaseId = $data['databaseId']; + $moviesCollectionId = $data['moviesId']; + + /** + * Test when database and collection are disabled but can still call get collection + */ + $collection = $this->client->call(Client::METHOD_GET, '/vectordb/' . $databaseId . '/collections/' . $moviesCollectionId, array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders())); + + $this->assertEquals(200, $collection['headers']['status-code']); + $this->assertEquals('Movies', $collection['body']['name']); + $this->assertEquals($moviesCollectionId, $collection['body']['$id']); + $this->assertFalse($collection['body']['enabled']); + } + + /** + * @depends testCreateCollection + * @param array $data + * @throws \Exception + * @throws \Exception + */ + public function testUpdateCollection(array $data) + { + $databaseId = $data['databaseId']; + $moviesCollectionId = $data['moviesId']; + + /** + * Test When database and collection are disabled but can still call update collection + */ + $collection = $this->client->call(Client::METHOD_PUT, '/vectordb/' . $databaseId . '/collections/' . $moviesCollectionId, array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'name' => 'Movies Updated', + 'enabled' => false + ]); + + $this->assertEquals(200, $collection['headers']['status-code']); + $this->assertEquals('Movies Updated', $collection['body']['name']); + $this->assertEquals($moviesCollectionId, $collection['body']['$id']); + $this->assertFalse($collection['body']['enabled']); + } + + /** + * @depends testCreateCollection + * @param array $data + * @throws \Exception + * @throws \Exception + */ + public function testDeleteCollection(array $data) + { + $databaseId = $data['databaseId']; + $tvShowsId = $data['tvShowsId']; + + /** + * Test when database and collection are disabled but can still call delete collection + */ + $response = $this->client->call(Client::METHOD_DELETE, '/vectordb/' . $databaseId . '/collections/' . $tvShowsId, array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders())); + + $this->assertEquals(204, $response['headers']['status-code']); + $this->assertEquals($response['body'], ""); + } + + /** + * @depends testCreateCollection + */ + public function testGetDatabaseUsage(array $data) + { + $databaseId = $data['databaseId']; + /** + * Test for FAILURE + */ + + $response = $this->client->call(Client::METHOD_GET, '/vectordb/' . $databaseId . '/usage', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'] + ], $this->getHeaders()), [ + 'range' => '32h' + ]); + + $this->assertEquals(400, $response['headers']['status-code']); + + /** + * Test for SUCCESS + */ + + $response = $this->client->call(Client::METHOD_GET, '/vectordb/' . $databaseId . '/usage', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'] + ], $this->getHeaders()), [ + 'range' => '24h' + ]); + + $this->assertEquals(200, $response['headers']['status-code']); + $this->assertEquals(11, count($response['body'])); + $this->assertEquals('24h', $response['body']['range']); + $this->assertIsNumeric($response['body']['documentsTotal']); + $this->assertIsNumeric($response['body']['collectionsTotal']); + $this->assertIsArray($response['body']['collections']); + $this->assertIsArray($response['body']['documents']); + } + + + /** + * @depends testCreateCollection + */ + public function testGetCollectionUsage(array $data) + { + $databaseId = $data['databaseId']; + /** + * Test for FAILURE + */ + + $response = $this->client->call(Client::METHOD_GET, '/vectordb/' . $databaseId . '/collections/' . $data['moviesId'] . '/usage', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'] + ], $this->getHeaders()), [ + 'range' => '32h' + ]); + + $this->assertEquals(400, $response['headers']['status-code']); + + $response = $this->client->call(Client::METHOD_GET, '/vectordb/' . $databaseId . '/collections/randomCollectionId/usage', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'] + ], $this->getHeaders()), [ + 'range' => '24h' + ]); + + $this->assertEquals(404, $response['headers']['status-code']); + + /** + * Test for SUCCESS + */ + $response = $this->client->call(Client::METHOD_GET, '/vectordb/' . $databaseId . '/collections/' . $data['moviesId'] . '/usage', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'] + ], $this->getHeaders()), [ + 'range' => '24h' + ]); + $this->assertEquals(200, $response['headers']['status-code']); + $this->assertEquals(3, count($response['body'])); + $this->assertEquals('24h', $response['body']['range']); + $this->assertIsNumeric($response['body']['documentsTotal']); + $this->assertIsArray($response['body']['documents']); + } + + /** + * @depends testCreateCollection + * @throws \Utopia\Database\Exception\Query + */ + public function testGetCollectionLogs(array $data) + { + $databaseId = $data['databaseId']; + /** + * Test for SUCCESS + */ + $logs = $this->client->call(Client::METHOD_GET, '/vectordb/' . $databaseId . '/collections/' . $data['moviesId'] . '/logs', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders())); + + $this->assertEquals(200, $logs['headers']['status-code']); + $this->assertIsArray($logs['body']['logs']); + $this->assertIsNumeric($logs['body']['total']); + + $logs = $this->client->call(Client::METHOD_GET, '/vectordb/' . $databaseId . '/collections/' . $data['moviesId'] . '/logs', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'queries' => [Query::limit(1)->toString()] + ]); + + $this->assertEquals(200, $logs['headers']['status-code']); + $this->assertIsArray($logs['body']['logs']); + $this->assertLessThanOrEqual(1, count($logs['body']['logs'])); + $this->assertIsNumeric($logs['body']['total']); + + $logs = $this->client->call(Client::METHOD_GET, '/vectordb/' . $databaseId . '/collections/' . $data['moviesId'] . '/logs', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'queries' => [Query::offset(1)->toString()] + ]); + + $this->assertEquals(200, $logs['headers']['status-code']); + $this->assertIsArray($logs['body']['logs']); + $this->assertIsNumeric($logs['body']['total']); + + $logs = $this->client->call(Client::METHOD_GET, '/vectordb/' . $databaseId . '/collections/' . $data['moviesId'] . '/logs', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'queries' => [Query::offset(1)->toString(), Query::limit(1)->toString()] + ]); + + $this->assertEquals(200, $logs['headers']['status-code']); + $this->assertIsArray($logs['body']['logs']); + $this->assertLessThanOrEqual(1, count($logs['body']['logs'])); + $this->assertIsNumeric($logs['body']['total']); + } +} diff --git a/tests/e2e/Services/Databases/VectorDB/DatabasesCustomClientTest.php b/tests/e2e/Services/Databases/VectorDB/DatabasesCustomClientTest.php new file mode 100644 index 0000000000..cc99806848 --- /dev/null +++ b/tests/e2e/Services/Databases/VectorDB/DatabasesCustomClientTest.php @@ -0,0 +1,205 @@ +client->call(Client::METHOD_POST, '/vectordb', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'databaseId' => ID::unique(), + 'name' => 'Test Database' + ]); + + $databaseId = $database['body']['$id']; + + // Collection aliases write to create, update, delete + $movies = $this->client->call(Client::METHOD_POST, '/vectordb/' . $databaseId . '/collections', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'collectionId' => ID::unique(), + 'name' => 'Movies', + 'dimension' => 3, + 'documentSecurity' => true, + 'permissions' => [ + Permission::write(Role::user($this->getUser()['$id'])), + ], + ]); + + $moviesId = $movies['body']['$id']; + + $this->assertContains(Permission::create(Role::user($this->getUser()['$id'])), $movies['body']['$permissions']); + $this->assertContains(Permission::update(Role::user($this->getUser()['$id'])), $movies['body']['$permissions']); + $this->assertContains(Permission::delete(Role::user($this->getUser()['$id'])), $movies['body']['$permissions']); + + // VectorDB uses fixed schema (embeddings, metadata). No attribute creation needed. + + // Document aliases write to update, delete + $document1 = $this->client->call(Client::METHOD_POST, '/vectordb/' . $databaseId . '/collections/' . $moviesId . '/documents', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'documentId' => ID::unique(), + 'data' => [ + 'embeddings' => [1.0, 0.0, 0.0], + 'metadata' => ['k' => 'v'], + ], + 'permissions' => [ + Permission::write(Role::user($this->getUser()['$id'])), + ] + ]); + + $this->assertNotContains(Permission::create(Role::user($this->getUser()['$id'])), $document1['body']['$permissions']); + $this->assertContains(Permission::update(Role::user($this->getUser()['$id'])), $document1['body']['$permissions']); + $this->assertContains(Permission::delete(Role::user($this->getUser()['$id'])), $document1['body']['$permissions']); + + /** + * Test for FAILURE + */ + + // Document does not allow create permission + $document2 = $this->client->call(Client::METHOD_POST, '/vectordb/' . $databaseId . '/collections/' . $moviesId . '/documents', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'documentId' => ID::unique(), + 'data' => [ + 'embeddings' => [0.0, 1.0, 0.0], + 'metadata' => ['k' => 'v'], + ], + 'permissions' => [ + Permission::create(Role::user($this->getUser()['$id'])), + ] + ]); + + $this->assertEquals(400, $document2['headers']['status-code']); + } + + public function testUpdateWithoutPermission(): array + { + // As a part of preparation, we get ID of currently logged-in user + $response = $this->client->call(Client::METHOD_GET, '/account', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders())); + $this->assertEquals(200, $response['headers']['status-code']); + + $userId = $response['body']['$id']; + + $database = $this->client->call(Client::METHOD_POST, '/vectordb', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'databaseId' => ID::custom('permissionCheckDatabase'), + 'name' => 'Test Database', + ]); + $this->assertEquals(201, $database['headers']['status-code']); + $this->assertEquals('Test Database', $database['body']['name']); + + $databaseId = $database['body']['$id']; + // Create collection + $response = $this->client->call(Client::METHOD_POST, '/vectordb/' . $databaseId . '/collections', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'collectionId' => ID::custom('permissionCheck'), + 'name' => 'permissionCheck', + 'dimension' => 3, + 'permissions' => [], + 'documentSecurity' => true, + ]); + $this->assertEquals(201, $response['headers']['status-code']); + + // Creating document by server, give read permission to our user + some other user + $response = $this->client->call(Client::METHOD_POST, '/vectordb/' . $databaseId . '/collections/permissionCheck/documents', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'documentId' => ID::custom('permissionCheckDocument'), + 'data' => [ + 'embeddings' => [1.0, 0.0, 0.0], + 'metadata' => ['name' => 'AppwriteBeginner'], + ], + 'permissions' => [ + Permission::read(Role::user(ID::custom('user2'))), + Permission::read(Role::user($userId)), + Permission::update(Role::user($userId)), + Permission::delete(Role::user($userId)), + ], + ]); + + $this->assertEquals(201, $response['headers']['status-code']); + + // Update document + // This is the point of this test. We should be allowed to do this action, and it should not fail on permission check + $response = $this->client->call(Client::METHOD_PATCH, '/vectordb/' . $databaseId . '/collections/permissionCheck/documents/permissionCheckDocument', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'] + ], $this->getHeaders()), [ + 'data' => [ + 'embeddings' => [0.0, 1.0, 0.0], + 'metadata' => ['name' => 'AppwriteExpert'], + ] + ]); + + $this->assertEquals(200, $response['headers']['status-code']); + + // Get name of the document, should be the new one + $response = $this->client->call(Client::METHOD_GET, '/vectordb/' . $databaseId . '/collections/permissionCheck/documents/permissionCheckDocument', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders())); + $this->assertEquals(200, $response['headers']['status-code']); + $this->assertEquals("AppwriteExpert", $response['body']['metadata']['name']); + + // Cleanup to prevent collision with other tests + // Delete collection + $response = $this->client->call(Client::METHOD_DELETE, '/vectordb/' . $databaseId . '/collections/permissionCheck', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ])); + + $this->assertEquals(204, $response['headers']['status-code']); + + + // Wait for database worker to finish deleting collection + sleep(2); + + // Make sure collection has been deleted + $response = $this->client->call(Client::METHOD_GET, '/vectordb/' . $databaseId . '/collections/permissionCheck', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ])); + $this->assertEquals(404, $response['headers']['status-code']); + + return []; + } +} diff --git a/tests/e2e/Services/Databases/VectorDB/DatabasesCustomServerTest.php b/tests/e2e/Services/Databases/VectorDB/DatabasesCustomServerTest.php new file mode 100644 index 0000000000..58cb48440e --- /dev/null +++ b/tests/e2e/Services/Databases/VectorDB/DatabasesCustomServerTest.php @@ -0,0 +1,975 @@ +client->call(Client::METHOD_POST, '/vectordb', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'databaseId' => ID::custom('first'), + 'name' => 'Test 1', + ]); + $this->assertEquals(201, $db1['headers']['status-code']); + $this->assertEquals('Test 1', $db1['body']['name']); + $this->assertEquals('vectordb', $db1['body']['type']); + // Validate database response model fields on create + $this->assertArrayHasKey('$id', $db1['body']); + $this->assertArrayHasKey('$createdAt', $db1['body']); + $this->assertArrayHasKey('$updatedAt', $db1['body']); + $this->assertArrayHasKey('enabled', $db1['body']); + + $db2 = $this->client->call(Client::METHOD_POST, '/vectordb', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'databaseId' => ID::custom('second'), + 'name' => 'Test 2', + ]); + $this->assertEquals(201, $db2['headers']['status-code']); + $this->assertEquals('Test 2', $db2['body']['name']); + $this->assertEquals('vectordb', $db2['body']['type']); + + $list = $this->client->call(Client::METHOD_GET, '/vectordb', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]); + $this->assertEquals(200, $list['headers']['status-code']); + $this->assertIsInt($list['body']['total']); + $this->assertGreaterThanOrEqual(2, $list['body']['total']); + $this->assertIsArray($list['body']['databases']); + $this->assertArrayHasKey('$id', $list['body']['databases'][0]); + $this->assertArrayHasKey('name', $list['body']['databases'][0]); + $this->assertArrayHasKey('type', $list['body']['databases'][0]); + + return ['databaseId' => $db1['body']['$id']]; + } + + /** + * @depends testListDatabases + */ + public function testGetDatabase(array $data): array + { + $databaseId = $data['databaseId']; + $res = $this->client->call(Client::METHOD_GET, '/vectordb/' . $databaseId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]); + $this->assertEquals(200, $res['headers']['status-code']); + $this->assertEquals($databaseId, $res['body']['$id']); + $this->assertEquals('Test 1', $res['body']['name']); + $this->assertEquals('vectordb', $res['body']['type']); + return ['databaseId' => $databaseId]; + } + + /** + * @depends testListDatabases + */ + public function testUpdateDatabase(array $data): array + { + $databaseId = $data['databaseId']; + $res = $this->client->call(Client::METHOD_PUT, '/vectordb/' . $databaseId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'name' => 'Test 1 Updated', + ]); + $this->assertEquals(200, $res['headers']['status-code']); + $this->assertEquals('Test 1 Updated', $res['body']['name']); + $this->assertEquals('vectordb', $res['body']['type']); + return ['databaseId' => $databaseId]; + } + + /** + * @depends testListDatabases + */ + public function testDeleteDatabase(array $data): void + { + $databaseId = $data['databaseId']; + $del = $this->client->call(Client::METHOD_DELETE, '/vectordb/' . $databaseId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]); + $this->assertEquals(204, $del['headers']['status-code']); + $this->assertEquals("", $del['body']); + + $get = $this->client->call(Client::METHOD_GET, '/vectordb/' . $databaseId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]); + $this->assertEquals(404, $get['headers']['status-code']); + } + + public function testCollectionsCRUD(): array + { + // Create database for collections tests + $database = $this->client->call(Client::METHOD_POST, '/vectordb', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'databaseId' => ID::unique(), + 'name' => 'Collections DB', + ]); + $this->assertEquals(201, $database['headers']['status-code']); + $databaseId = $database['body']['$id']; + + // Create two collections + $col1 = $this->client->call(Client::METHOD_POST, '/vectordb/' . $databaseId . '/collections', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'name' => 'Test 1', + 'collectionId' => ID::custom('first'), + 'permissions' => [ + Permission::read(Role::any()), + Permission::create(Role::any()), + Permission::update(Role::any()), + Permission::delete(Role::any()), + ], + 'documentSecurity' => true, + 'dimension' => 3, + ]); + $this->assertEquals(201, $col1['headers']['status-code']); + // Validate collection response model on create + $this->assertArrayHasKey('$id', $col1['body']); + $this->assertArrayHasKey('$createdAt', $col1['body']); + $this->assertArrayHasKey('$updatedAt', $col1['body']); + $this->assertArrayHasKey('enabled', $col1['body']); + $this->assertArrayHasKey('documentSecurity', $col1['body']); + $this->assertArrayHasKey('dimension', $col1['body']); + + $col2 = $this->client->call(Client::METHOD_POST, '/vectordb/' . $databaseId . '/collections', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'name' => 'Test 2', + 'collectionId' => ID::custom('second'), + 'permissions' => [ + Permission::read(Role::any()), + Permission::create(Role::any()), + Permission::update(Role::any()), + Permission::delete(Role::any()), + ], + 'documentSecurity' => true, + 'dimension' => 3, + ]); + $this->assertEquals(201, $col2['headers']['status-code']); + $this->assertArrayHasKey('$id', $col2['body']); + $this->assertArrayHasKey('$createdAt', $col2['body']); + $this->assertArrayHasKey('$updatedAt', $col2['body']); + + // List collections + $list = $this->client->call(Client::METHOD_GET, '/vectordb/' . $databaseId . '/collections', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]); + $this->assertEquals(200, $list['headers']['status-code']); + $this->assertIsInt($list['body']['total']); + $this->assertGreaterThanOrEqual(2, $list['body']['total']); + $this->assertIsArray($list['body']['collections']); + $this->assertArrayHasKey('$id', $list['body']['collections'][0]); + $this->assertArrayHasKey('name', $list['body']['collections'][0]); + $this->assertArrayHasKey('dimension', $list['body']['collections'][0]); + + // Get collection + $get = $this->client->call(Client::METHOD_GET, '/vectordb/' . $databaseId . '/collections/' . $col1['body']['$id'], [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]); + $this->assertEquals(200, $get['headers']['status-code']); + $this->assertEquals($col1['body']['$id'], $get['body']['$id']); + $this->assertEquals('Test 1', $get['body']['name']); + $this->assertEquals(3, $get['body']['dimension']); + + // Update collection (name only) + $upd = $this->client->call(Client::METHOD_PUT, '/vectordb/' . $databaseId . '/collections/' . $col1['body']['$id'], [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'name' => 'Test 1 Updated', + ]); + $this->assertEquals(200, $upd['headers']['status-code']); + $this->assertEquals('Test 1 Updated', $upd['body']['name']); + $this->assertArrayHasKey('$updatedAt', $upd['body']); + + // Delete collection + $del = $this->client->call(Client::METHOD_DELETE, '/vectordb/' . $databaseId . '/collections/' . $col2['body']['$id'], [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]); + $this->assertEquals(204, $del['headers']['status-code']); + $this->assertEquals("", $del['body']); + + return [ + 'databaseId' => $databaseId, + 'collectionId' => $col1['body']['$id'], + ]; + } + + /** + * @depends testCollectionsCRUD + */ + public function testUpdateCollectionMore(array $data): array + { + $databaseId = $data['databaseId']; + $collectionId = $data['collectionId']; + + // Update collection name and dimensions + $upd = $this->client->call(Client::METHOD_PUT, '/vectordb/' . $databaseId . '/collections/' . $collectionId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'name' => 'Test 1 Renamed', + 'dimension' => 4, + ]); + $this->assertEquals(200, $upd['headers']['status-code']); + $this->assertEquals('Test 1 Renamed', $upd['body']['name']); + $this->assertEquals(4, $upd['body']['dimension']); + + // Read back to confirm + $get = $this->client->call(Client::METHOD_GET, '/vectordb/' . $databaseId . '/collections/' . $collectionId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]); + $this->assertEquals(200, $get['headers']['status-code']); + $this->assertEquals('Test 1 Renamed', $get['body']['name']); + $this->assertEquals(4, $get['body']['dimension']); + + return $data; + } + + /** + * @depends testCollectionsCRUD + */ + public function testUpdateCollectionEnabledFlag(array $data): array + { + $databaseId = $data['databaseId']; + $collectionId = $data['collectionId']; + + // Disable collection + $disable = $this->client->call(Client::METHOD_PUT, '/vectordb/' . $databaseId . '/collections/' . $collectionId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'name' => 'Updated', + 'enabled' => false, + ]); + $this->assertEquals(200, $disable['headers']['status-code']); + $this->assertFalse($disable['body']['enabled']); + + // Re-enable collection + $enable = $this->client->call(Client::METHOD_PUT, '/vectordb/' . $databaseId . '/collections/' . $collectionId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'name' => 'Updated', + 'enabled' => true, + ]); + $this->assertEquals(200, $enable['headers']['status-code']); + $this->assertTrue($enable['body']['enabled']); + + return $data; + } + + public function testUpdateDatabaseNameAndEnabled(): void + { + // Create isolated database for this test to avoid ordering conflicts + $create = $this->client->call(Client::METHOD_POST, '/vectordb', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'databaseId' => ID::unique(), + 'name' => 'Update DB', + ]); + $this->assertEquals(201, $create['headers']['status-code']); + $databaseId = $create['body']['$id']; + + // Update name + $rename = $this->client->call(Client::METHOD_PUT, '/vectordb/' . $databaseId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'name' => 'Test DB Renamed', + ]); + $this->assertEquals(200, $rename['headers']['status-code']); + $this->assertEquals('Test DB Renamed', $rename['body']['name']); + + // Toggle enabled off then on + $disable = $this->client->call(Client::METHOD_PUT, '/vectordb/' . $databaseId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'name' => 'Test DB Renamed', + 'enabled' => false, + ]); + $this->assertEquals(200, $disable['headers']['status-code']); + $this->assertFalse($disable['body']['enabled']); + + $enable = $this->client->call(Client::METHOD_PUT, '/vectordb/' . $databaseId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'name' => 'Test DB Renamed', + 'enabled' => true, + ]); + $this->assertEquals(200, $enable['headers']['status-code']); + $this->assertTrue($enable['body']['enabled']); + + // Cleanup + $del = $this->client->call(Client::METHOD_DELETE, '/vectordb/' . $databaseId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]); + $this->assertEquals(204, $del['headers']['status-code']); + } + + /** + * @depends testCollectionsCRUD + */ + public function testRecreateIndex(array $data): void + { + $databaseId = $data['databaseId']; + $collectionId = $data['collectionId']; + + // Create a new index variant + $create = $this->client->call(Client::METHOD_POST, "/vectordb/{$databaseId}/collections/{$collectionId}/indexes", [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'key' => 'embedding_euclidean_v2', + 'type' => Database::INDEX_HNSW_EUCLIDEAN, + 'attributes' => ['embeddings'] + ]); + $this->assertEquals(202, $create['headers']['status-code']); + + // Ensure it exists + $get = $this->client->call(Client::METHOD_GET, "/vectordb/{$databaseId}/collections/{$collectionId}/indexes/embedding_euclidean_v2", [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]); + $this->assertEquals(200, $get['headers']['status-code']); + $this->assertEquals('embedding_euclidean_v2', $get['body']['key']); + + // Delete it + $del = $this->client->call(Client::METHOD_DELETE, "/vectordb/{$databaseId}/collections/{$collectionId}/indexes/embedding_euclidean_v2", [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]); + $this->assertEquals(204, $del['headers']['status-code']); + } + + /** + * @depends testCollectionsCRUD + */ + public function testIndexesCRUD(array $data): void + { + $databaseId = $data['databaseId']; + $collectionId = $data['collectionId']; + + // Create indexes + $eu = $this->client->call(Client::METHOD_POST, "/vectordb/{$databaseId}/collections/{$collectionId}/indexes", [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'key' => 'embedding_euclidean', + 'type' => Database::INDEX_HNSW_EUCLIDEAN, + 'attributes' => ['embeddings'] + ]); + $this->assertEquals(202, $eu['headers']['status-code']); + + $dot = $this->client->call(Client::METHOD_POST, "/vectordb/{$databaseId}/collections/{$collectionId}/indexes", [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'key' => 'embedding_dot', + 'type' => Database::INDEX_HNSW_DOT, + 'attributes' => ['embeddings'] + ]); + $this->assertEquals(202, $dot['headers']['status-code']); + + $cos = $this->client->call(Client::METHOD_POST, "/vectordb/{$databaseId}/collections/{$collectionId}/indexes", [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'key' => 'embedding_cosine', + 'type' => Database::INDEX_HNSW_COSINE, + 'attributes' => ['embeddings'] + ]); + $this->assertEquals(202, $cos['headers']['status-code']); + + // List indexes + $list = $this->client->call(Client::METHOD_GET, "/vectordb/{$databaseId}/collections/{$collectionId}/indexes", [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]); + $this->assertEquals(200, $list['headers']['status-code']); + $this->assertIsArray($list['body']['indexes']); + $keys = array_map(fn ($i) => $i['key'], $list['body']['indexes']); + $this->assertContains('embedding_euclidean', $keys); + $this->assertContains('embedding_dot', $keys); + $this->assertContains('embedding_cosine', $keys); + + // Get index by key + $get = $this->client->call(Client::METHOD_GET, "/vectordb/{$databaseId}/collections/{$collectionId}/indexes/embedding_euclidean", [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]); + $this->assertEquals(200, $get['headers']['status-code']); + $this->assertEquals('embedding_euclidean', $get['body']['key']); + $this->assertEquals(Database::INDEX_HNSW_EUCLIDEAN, $get['body']['type']); + + // Delete index + $del = $this->client->call(Client::METHOD_DELETE, "/vectordb/{$databaseId}/collections/{$collectionId}/indexes/embedding_dot", [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]); + $this->assertEquals(204, $del['headers']['status-code']); + sleep(4); + // Ensure it's gone + $getMissing = $this->client->call(Client::METHOD_GET, "/vectordb/{$databaseId}/collections/{$collectionId}/indexes/embedding_dot", [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]); + $this->assertEquals(404, $getMissing['headers']['status-code']); + } + + public function testBulkCreate(): array + { + // Setup: create isolated database and collection + $db = $this->client->call(Client::METHOD_POST, '/vectordb', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'databaseId' => ID::unique(), + 'name' => 'BulkDBCreate' + ]); + $this->assertEquals(201, $db['headers']['status-code']); + $databaseId = $db['body']['$id']; + + $col = $this->client->call(Client::METHOD_POST, "/vectordb/{$databaseId}/collections", [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'collectionId' => ID::unique(), + 'name' => 'BulkColCreate', + 'documentSecurity' => true, + 'dimension' => 3, + 'permissions' => [Permission::read(Role::any())] + ]); + $this->assertEquals(201, $col['headers']['status-code']); + $collectionId = $col['body']['$id']; + + $docs = [ + [ + 'embeddings' => [1.0, 0.0, 0.0], + 'metadata' => ['group' => 'bulkA'], + '$permissions' => [Permission::read(Role::any())] + ], + [ + 'embeddings' => [0.0, 1.0, 0.0], + 'metadata' => ['group' => 'bulkB'], + '$permissions' => [Permission::read(Role::any())] + ], + ]; + + $res = $this->client->call(Client::METHOD_POST, "/vectordb/{$databaseId}/collections/{$collectionId}/documents", [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'documents' => $docs + ]); + + $this->assertEquals(201, $res['headers']['status-code']); + $this->assertIsInt($res['body']['total'] ?? 0); + $this->assertGreaterThanOrEqual(2, $res['body']['total']); + $this->assertIsArray($res['body']['documents']); + $this->assertCount(2, $res['body']['documents']); + + $ids = array_map(fn ($d) => $d['$id'], $res['body']['documents']); + $this->assertNotEmpty($ids[0]); + $this->assertNotEmpty($ids[1]); + + // Fetch and validate persisted data via GET + foreach ($ids as $i => $id) { + $get = $this->client->call(Client::METHOD_GET, "/vectordb/{$databaseId}/collections/{$collectionId}/documents/{$id}", [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]); + $this->assertEquals(200, $get['headers']['status-code']); + $this->assertEquals($id, $get['body']['$id']); + $this->assertIsArray($get['body']['embeddings']); + $this->assertCount(3, $get['body']['embeddings']); + $this->assertArrayHasKey('group', $get['body']['metadata']); + } + + return [ 'databaseId' => $databaseId, 'collectionId' => $collectionId, 'bulkIds' => $ids ]; + } + + public function testCreateTextEmbeddingsSuccessAndErrors(): void + { + // Setup new database and collection + $db = $this->client->call(Client::METHOD_POST, '/vectordb', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'databaseId' => ID::unique(), + 'name' => 'EmbedDB', + ]); + $this->assertEquals(201, $db['headers']['status-code']); + $databaseId = $db['body']['$id']; + + $col = $this->client->call(Client::METHOD_POST, "/vectordb/{$databaseId}/collections", [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'collectionId' => ID::unique(), + 'name' => 'EmbedCol', + 'documentSecurity' => true, + 'dimension' => 3, + 'permissions' => [Permission::read(Role::any())] + ]); + $this->assertEquals(201, $col['headers']['status-code']); + $collectionId = $col['body']['$id']; + + // Success: two embeddings + $ok = $this->client->call(Client::METHOD_POST, "/vectordb/embeddings/text", [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'model' => 'embeddinggemma', + 'texts' => [ + 'hello world', + 'second sentence', + ], + ]); + $this->assertEquals(200, $ok['headers']['status-code']); + $this->assertIsInt($ok['body']['total'] ?? 0); + $this->assertEquals(2, $ok['body']['total']); + $this->assertIsArray($ok['body']['embeddings']); + $this->assertCount(2, $ok['body']['embeddings']); + foreach ($ok['body']['embeddings'] as $embed) { + $this->assertIsString($embed['model']); + $this->assertIsInt($embed['dimension']); + $this->assertIsArray($embed['embedding']); + $this->assertGreaterThan(0, count($embed['embedding'])); + $this->assertArrayHasKey('error', $embed); + } + + // Error: missing texts payload + $missingTexts = $this->client->call(Client::METHOD_POST, "/vectordb/embeddings/text", [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], []); + $this->assertEquals(400, $missingTexts['headers']['status-code']); + + // Error: invalid texts item type (must be strings) + $invalidItem = $this->client->call(Client::METHOD_POST, "/vectordb/embeddings/text", [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'model' => 'embeddinggemma', + 'texts' => [ + 'valid text', + 123, // invalid, not a string + ], + ]); + $this->assertEquals(400, $invalidItem['headers']['status-code']); + + // Error: unknown embedding model + $unknownModel = $this->client->call(Client::METHOD_POST, "/vectordb/embeddings/text", [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'model' => 'nonexistent-model', + 'texts' => ['hello'], + ]); + $this->assertEquals(400, $unknownModel['headers']['status-code']); + } + + public function testBulkUpsert(): void + { + // Setup fresh db/collection + $db = $this->client->call(Client::METHOD_POST, '/vectordb', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ 'databaseId' => ID::unique(), 'name' => 'BulkDBUpsert' ]); + $this->assertEquals(201, $db['headers']['status-code']); + $databaseId = $db['body']['$id']; + + $col = $this->client->call(Client::METHOD_POST, "/vectordb/{$databaseId}/collections", [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'collectionId' => ID::unique(), + 'name' => 'BulkColUpsert', + 'documentSecurity' => true, + 'dimension' => 3, + 'permissions' => [Permission::read(Role::any())] + ]); + $this->assertEquals(201, $col['headers']['status-code']); + $collectionId = $col['body']['$id']; + + $docs = [ + [ + 'embeddings' => [0.5, 0.5, 0.0], + 'metadata' => ['group' => 'bulkA', 'updated' => true], + '$permissions' => [Permission::read(Role::any())] + ], + [ + 'embeddings' => [0.2, 0.8, 0.0], + 'metadata' => ['group' => 'bulkB', 'updated' => true], + '$permissions' => [Permission::read(Role::any())] + ], + ]; + + $res = $this->client->call(Client::METHOD_PUT, "/vectordb/{$databaseId}/collections/{$collectionId}/documents", [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'documents' => $docs + ]); + + $this->assertEquals(200, $res['headers']['status-code']); + $this->assertIsArray($res['body']['documents']); + $this->assertCount(2, $res['body']['documents']); + $this->assertTrue($res['body']['documents'][0]['metadata']['updated']); + $this->assertTrue($res['body']['documents'][1]['metadata']['updated']); + + // Fetch and validate updated content + $ids = array_map(fn ($d) => $d['$id'], $res['body']['documents']); + foreach ($ids as $id) { + $get = $this->client->call(Client::METHOD_GET, "/vectordb/{$databaseId}/collections/{$collectionId}/documents/{$id}", [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]); + $this->assertEquals(200, $get['headers']['status-code']); + $this->assertTrue($get['body']['metadata']['updated']); + } + + // Perform another bulk upsert to mutate the same documents + $docs2 = [ + [ 'embeddings' => [0.6, 0.4, 0.0], 'metadata' => ['updatedAgain' => true] ], + [ 'embeddings' => [0.3, 0.7, 0.0], 'metadata' => ['updatedAgain' => true] ], + ]; + $res2 = $this->client->call(Client::METHOD_PUT, "/vectordb/{$databaseId}/collections/{$collectionId}/documents", [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'documents' => $docs2 + ]); + $this->assertEquals(200, $res2['headers']['status-code']); + $this->assertIsArray($res2['body']['documents']); + $this->assertCount(2, $res2['body']['documents']); + + // Fetch again and assert second update persisted + $ids2 = array_map(fn ($d) => $d['$id'], $res2['body']['documents']); + foreach ($ids2 as $id) { + $get2 = $this->client->call(Client::METHOD_GET, "/vectordb/{$databaseId}/collections/{$collectionId}/documents/{$id}", [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]); + $this->assertEquals(200, $get2['headers']['status-code']); + $this->assertTrue($get2['body']['metadata']['updatedAgain']); + } + } + + public function testBulkUpdate(): void + { + // Setup: create db/collection and two docs + $db = $this->client->call(Client::METHOD_POST, '/vectordb', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ 'databaseId' => ID::unique(), 'name' => 'BulkDBUpdate' ]); + $this->assertEquals(201, $db['headers']['status-code']); + $databaseId = $db['body']['$id']; + + $col = $this->client->call(Client::METHOD_POST, "/vectordb/{$databaseId}/collections", [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'collectionId' => ID::unique(), + 'name' => 'BulkColUpdate', + 'documentSecurity' => true, + 'dimension' => 3, + 'permissions' => [Permission::read(Role::any())] + ]); + $this->assertEquals(201, $col['headers']['status-code']); + $collectionId = $col['body']['$id']; + + $seed = $this->client->call(Client::METHOD_PUT, "/vectordb/{$databaseId}/collections/{$collectionId}/documents", [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'documents' => [ + ['embeddings' => [1.0,0.0,0.0], 'metadata' => ['seed' => 1], '$permissions' => [Permission::read(Role::any())]], + ['embeddings' => [0.0,1.0,0.0], 'metadata' => ['seed' => 2], '$permissions' => [Permission::read(Role::any())]] + ] + ]); + $this->assertEquals(200, $seed['headers']['status-code']); + $ids = array_map(fn ($d) => $d['$id'], $seed['body']['documents']); + + $res = $this->client->call(Client::METHOD_PATCH, "/vectordb/{$databaseId}/collections/{$collectionId}/documents", [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'data' => [ 'metadata' => ['bulkUpdated' => true] ], + 'queries' => [ + \Utopia\Database\Query::equal('$id', $ids)->toString() + ] + ]); + + $this->assertEquals(200, $res['headers']['status-code']); + $this->assertIsArray($res['body']['documents']); + $this->assertCount(2, $res['body']['documents']); + foreach ($res['body']['documents'] as $doc) { + $this->assertTrue($doc['metadata']['bulkUpdated']); + } + + // Fetch by IDs and assert update persisted + foreach ($ids as $id) { + $get = $this->client->call(Client::METHOD_GET, "/vectordb/{$databaseId}/collections/{$collectionId}/documents/{$id}", [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]); + $this->assertEquals(200, $get['headers']['status-code']); + $this->assertTrue($get['body']['metadata']['bulkUpdated']); + } + } + + public function testBulkDelete(): void + { + // Setup: create db/collection and two docs + $db = $this->client->call(Client::METHOD_POST, '/vectordb', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ 'databaseId' => ID::unique(), 'name' => 'BulkDBDelete' ]); + $this->assertEquals(201, $db['headers']['status-code']); + $databaseId = $db['body']['$id']; + + $col = $this->client->call(Client::METHOD_POST, "/vectordb/{$databaseId}/collections", [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'collectionId' => ID::unique(), + 'name' => 'BulkColDelete', + 'documentSecurity' => true, + 'dimension' => 3, + 'permissions' => [Permission::read(Role::any())] + ]); + $this->assertEquals(201, $col['headers']['status-code']); + $collectionId = $col['body']['$id']; + + $seed = $this->client->call(Client::METHOD_PUT, "/vectordb/{$databaseId}/collections/{$collectionId}/documents", [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'documents' => [ + ['embeddings' => [1.0,0.0,0.0], 'metadata' => ['seed' => 1], '$permissions' => [Permission::read(Role::any())]], + ['embeddings' => [0.0,1.0,0.0], 'metadata' => ['seed' => 2], '$permissions' => [Permission::read(Role::any())]] + ] + ]); + $this->assertEquals(200, $seed['headers']['status-code']); + $ids = array_map(fn ($d) => $d['$id'], $seed['body']['documents']); + + $res = $this->client->call(Client::METHOD_DELETE, "/vectordb/{$databaseId}/collections/{$collectionId}/documents", [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'queries' => [ + \Utopia\Database\Query::equal('$id', $ids)->toString() + ] + ]); + $this->assertEquals(200, $res['headers']['status-code']); + $this->assertIsInt($res['body']['total'] ?? 0); + $this->assertGreaterThanOrEqual(2, $res['body']['total']); + + // Ensure they are deleted + foreach ($ids as $id) { + $get = $this->client->call(Client::METHOD_GET, "/vectordb/{$databaseId}/collections/{$collectionId}/documents/{$id}", [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]); + $this->assertEquals(404, $get['headers']['status-code']); + } + } + + public function testCustomTimestamps(): void + { + // Setup: create database and collection + $db = $this->client->call(Client::METHOD_POST, '/vectordb', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'databaseId' => ID::unique(), + 'name' => 'TimestampTestDB' + ]); + $this->assertEquals(201, $db['headers']['status-code']); + $databaseId = $db['body']['$id']; + + $col = $this->client->call(Client::METHOD_POST, "/vectordb/{$databaseId}/collections", [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'collectionId' => ID::unique(), + 'name' => 'TimestampTestCollection', + 'documentSecurity' => true, + 'dimension' => 1536, + 'permissions' => [Permission::read(Role::any())] + ]); + $this->assertEquals(201, $col['headers']['status-code']); + $collectionId = $col['body']['$id']; + + // Test: Create document with custom timestamps using PUT (upsert) + $customCreatedAt = '1970-01-01T00:00:00.000+00:00'; + $customUpdatedAt = '1970-01-01T00:00:00.000+00:00'; + $vector = array_fill(0, 1536, 0.0); + $vector[0] = 1.0; + $documentId = ID::unique(); + + $doc = $this->client->call(Client::METHOD_POST, "/vectordb/{$databaseId}/collections/{$collectionId}/documents", [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'documentId' => $documentId, + 'data' => [ + '$createdAt' => $customCreatedAt, + '$updatedAt' => $customUpdatedAt, + 'embeddings' => $vector, + 'metadata' => ['test' => 'custom_timestamps'] + ] + ]); + + $this->assertEquals(201, $doc['headers']['status-code']); + $documentId = $doc['body']['$id']; + $this->assertNotEmpty($documentId); + + // Verify timestamps were set correctly + $this->assertEquals($customCreatedAt, $doc['body']['$createdAt'], 'CreatedAt should match custom timestamp'); + $this->assertEquals($customUpdatedAt, $doc['body']['$updatedAt'], 'UpdatedAt should match custom timestamp'); + + // Fetch document and verify timestamps persist + $fetched = $this->client->call(Client::METHOD_GET, "/vectordb/{$databaseId}/collections/{$collectionId}/documents/{$documentId}", [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]); + + $this->assertEquals(200, $fetched['headers']['status-code']); + $this->assertEquals($customCreatedAt, $fetched['body']['$createdAt'], 'CreatedAt should persist after fetch'); + $this->assertEquals($customUpdatedAt, $fetched['body']['$updatedAt'], 'UpdatedAt should persist after fetch'); + + // Test: Update document with new custom timestamps + $newCustomUpdatedAt = '2000-01-01T12:00:00.000+00:00'; + $vector2 = array_fill(0, 1536, 0.0); + $vector2[1] = 1.0; + + $updated = $this->client->call(Client::METHOD_PUT, "/vectordb/{$databaseId}/collections/{$collectionId}/documents/{$documentId}", [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'data' => [ + '$createdAt' => $customCreatedAt, // Keep original createdAt + '$updatedAt' => $newCustomUpdatedAt, // Update updatedAt + 'embeddings' => $vector2, + 'metadata' => ['test' => 'updated_timestamps'] + ] + ]); + + $this->assertEquals(200, $updated['headers']['status-code']); + $this->assertEquals($customCreatedAt, $updated['body']['$createdAt'], 'CreatedAt should remain unchanged'); + $this->assertEquals($newCustomUpdatedAt, $updated['body']['$updatedAt'], 'UpdatedAt should be updated to new custom timestamp'); + + // Final verification + $final = $this->client->call(Client::METHOD_GET, "/vectordb/{$databaseId}/collections/{$collectionId}/documents/{$documentId}", [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]); + + $this->assertEquals(200, $final['headers']['status-code']); + $this->assertEquals($customCreatedAt, $final['body']['$createdAt'], 'CreatedAt should persist through updates'); + $this->assertEquals($newCustomUpdatedAt, $final['body']['$updatedAt'], 'UpdatedAt should reflect the latest custom timestamp'); + } + +} diff --git a/tests/e2e/Services/Databases/VectorDB/Permissions/DatabasesPermissionsGuestTest.php b/tests/e2e/Services/Databases/VectorDB/Permissions/DatabasesPermissionsGuestTest.php new file mode 100644 index 0000000000..70f3b0946d --- /dev/null +++ b/tests/e2e/Services/Databases/VectorDB/Permissions/DatabasesPermissionsGuestTest.php @@ -0,0 +1,268 @@ +client->call(Client::METHOD_POST, '/vectordb', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'databaseId' => ID::unique(), + 'name' => 'VectorGuestDB', + ]); + $this->assertEquals(201, $database['headers']['status-code']); + $this->assertEquals('VectorGuestDB', $database['body']['name']); + + $databaseId = $database['body']['$id']; + $publicMovies = $this->client->call(Client::METHOD_POST, '/vectordb/' . $databaseId . '/collections', $this->getServerHeader(), [ + 'collectionId' => ID::unique(), + 'name' => 'Movies', + 'dimension' => 3, + 'permissions' => [ + Permission::read(Role::any()), + Permission::create(Role::any()), + Permission::update(Role::any()), + Permission::delete(Role::any()), + ], + ]); + $privateMovies = $this->client->call(Client::METHOD_POST, '/vectordb/' . $databaseId . '/collections', $this->getServerHeader(), [ + 'collectionId' => ID::unique(), + 'name' => 'Movies', + 'dimension' => 3, + 'permissions' => [], + 'documentSecurity' => true, + ]); + + $publicCollection = ['id' => $publicMovies['body']['$id']]; + $privateCollection = ['id' => $privateMovies['body']['$id']]; + + return [ + 'databaseId' => $databaseId, + 'publicCollectionId' => $publicCollection['id'], + 'privateCollectionId' => $privateCollection['id'], + ]; + } + + public function permissionsProvider(): array + { + return [ + [[Permission::read(Role::any())]], + [[Permission::read(Role::users())]], + [[Permission::update(Role::any()), Permission::delete(Role::any())]], + [[Permission::read(Role::any()), Permission::update(Role::any()), Permission::delete(Role::any())]], + [[Permission::read(Role::users()), Permission::update(Role::users()), Permission::delete(Role::users())]], + [[Permission::read(Role::any()), Permission::update(Role::users()), Permission::delete(Role::users())]], + ]; + } + + /** + * @dataProvider permissionsProvider + */ + public function testReadDocuments($permissions) + { + $data = $this->createCollection(); + $publicCollectionId = $data['publicCollectionId']; + $privateCollectionId = $data['privateCollectionId']; + $databaseId = $data['databaseId']; + + $publicResponse = $this->client->call(Client::METHOD_POST, '/vectordb/' . $databaseId . '/collections/' . $publicCollectionId . '/documents', $this->getServerHeader(), [ + 'documentId' => ID::unique(), + 'data' => [ + 'embeddings' => [1.0, 0.0, 0.0], + 'metadata' => ['title' => 'Lorem'], + ], + 'permissions' => $permissions, + ]); + $privateResponse = $this->client->call(Client::METHOD_POST, '/vectordb/' . $databaseId . '/collections/' . $privateCollectionId . '/documents', $this->getServerHeader(), [ + 'documentId' => ID::unique(), + 'data' => [ + 'embeddings' => [0.0, 1.0, 0.0], + 'metadata' => ['title' => 'Lorem'], + ], + 'permissions' => $permissions, + ]); + + $this->assertEquals(201, $publicResponse['headers']['status-code']); + $this->assertEquals(201, $privateResponse['headers']['status-code']); + + $roles = Authorization::getRoles(); + Authorization::cleanRoles(); + + $publicDocuments = $this->client->call(Client::METHOD_GET, '/vectordb/' . $databaseId . '/collections/' . $publicCollectionId . '/documents', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ]); + $privateDocuments = $this->client->call(Client::METHOD_GET, '/vectordb/' . $databaseId . '/collections/' . $privateCollectionId . '/documents', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ]); + + $this->assertEquals(1, $publicDocuments['body']['total']); + $this->assertEquals($permissions, $publicDocuments['body']['documents'][0]['$permissions']); + + if (\in_array(Permission::read(Role::any()), $permissions)) { + $this->assertEquals(1, $privateDocuments['body']['total']); + $this->assertEquals($permissions, $privateDocuments['body']['documents'][0]['$permissions']); + } else { + $this->assertEquals(0, $privateDocuments['body']['total']); + } + + foreach ($roles as $role) { + Authorization::setRole($role); + } + } + + public function testWriteDocument() + { + $data = $this->createCollection(); + $publicCollectionId = $data['publicCollectionId']; + $privateCollectionId = $data['privateCollectionId']; + $databaseId = $data['databaseId']; + + $roles = Authorization::getRoles(); + Authorization::cleanRoles(); + + $publicResponse = $this->client->call(Client::METHOD_POST, '/vectordb/' . $databaseId . '/collections/' . $publicCollectionId . '/documents', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], [ + 'documentId' => ID::unique(), + 'data' => [ + 'embeddings' => [1.0, 0.0, 0.0], + 'metadata' => ['title' => 'Lorem'], + ] + ]); + + $publicDocumentId = $publicResponse['body']['$id']; + $this->assertEquals(201, $publicResponse['headers']['status-code']); + + $privateResponse = $this->client->call(Client::METHOD_POST, '/vectordb/' . $databaseId . '/collections/' . $privateCollectionId . '/documents', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], [ + 'documentId' => ID::unique(), + 'data' => [ + 'embeddings' => [0.0, 1.0, 0.0], + 'metadata' => ['title' => 'Lorem'], + ], + ]); + + $this->assertEquals(401, $privateResponse['headers']['status-code']); + + // Create a document in private collection with API key so we can test that update and delete are also not allowed + $privateResponse = $this->client->call(Client::METHOD_POST, '/vectordb/' . $databaseId . '/collections/' . $privateCollectionId . '/documents', $this->getServerHeader(), [ + 'documentId' => ID::unique(), + 'data' => [ + 'embeddings' => [0.0, 0.0, 1.0], + 'metadata' => ['title' => 'Lorem'], + ], + ]); + + $this->assertEquals(201, $privateResponse['headers']['status-code']); + $privateDocumentId = $privateResponse['body']['$id']; + + $publicDocument = $this->client->call(Client::METHOD_PUT, '/vectordb/' . $databaseId . '/collections/' . $publicCollectionId . '/documents/' . $publicDocumentId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], [ + 'data' => [ + 'embeddings' => [0.5, 0.5, 0.0], + 'metadata' => ['title' => 'Thor: Ragnarok'], + ], + ]); + + $this->assertEquals(200, $publicDocument['headers']['status-code']); + $this->assertEquals('Thor: Ragnarok', $publicDocument['body']['metadata']['title']); + + $privateDocument = $this->client->call(Client::METHOD_PUT, '/vectordb/' . $databaseId . '/collections/' . $privateCollectionId . '/documents/' . $privateDocumentId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], [ + 'data' => [ + 'embeddings' => [0.2, 0.3, 0.5], + 'metadata' => ['title' => 'Thor: Ragnarok'], + ], + ]); + + $this->assertEquals(401, $privateDocument['headers']['status-code']); + + $publicDocument = $this->client->call(Client::METHOD_DELETE, '/vectordb/' . $databaseId . '/collections/' . $publicCollectionId . '/documents/' . $publicDocumentId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ]); + + $this->assertEquals(204, $publicDocument['headers']['status-code']); + + $privateDocument = $this->client->call(Client::METHOD_DELETE, '/vectordb/' . $databaseId . '/collections/' . $privateCollectionId . '/documents/' . $privateDocumentId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ]); + + $this->assertEquals(401, $privateDocument['headers']['status-code']); + + foreach ($roles as $role) { + Authorization::setRole($role); + } + } + + public function testWriteDocumentWithPermissions() + { + $database = $this->client->call(Client::METHOD_POST, '/vectordb', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'databaseId' => ID::unique(), + 'name' => 'VectorGuestPermsWrite', + ]); + $this->assertEquals(201, $database['headers']['status-code']); + $this->assertEquals('VectorGuestPermsWrite', $database['body']['name']); + + $databaseId = $database['body']['$id']; + $movies = $this->client->call(Client::METHOD_POST, '/vectordb/' . $databaseId . '/collections', $this->getServerHeader(), [ + 'collectionId' => ID::unique(), + 'name' => 'Movies', + 'dimension' => 3, + 'permissions' => [ + Permission::create(Role::any()), + ], + 'documentSecurity' => true + ]); + + $moviesId = $movies['body']['$id']; + + $document = $this->client->call(Client::METHOD_POST, '/vectordb/' . $databaseId . '/collections/' . $moviesId . '/documents', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], [ + 'documentId' => ID::unique(), + 'data' => [ + 'embeddings' => [1.0, 0.0, 0.0], + 'metadata' => ['title' => 'Thor: Ragnarok'], + ], + 'permissions' => [ + Permission::read(Role::any()), + ] + ]); + + $this->assertEquals(201, $document['headers']['status-code']); + $this->assertEquals('Thor: Ragnarok', $document['body']['metadata']['title']); + } +} diff --git a/tests/e2e/Services/Databases/VectorDB/Permissions/DatabasesPermissionsMemberTest.php b/tests/e2e/Services/Databases/VectorDB/Permissions/DatabasesPermissionsMemberTest.php new file mode 100644 index 0000000000..f0f34f7f10 --- /dev/null +++ b/tests/e2e/Services/Databases/VectorDB/Permissions/DatabasesPermissionsMemberTest.php @@ -0,0 +1,254 @@ + $this->createUser('user1', 'lorem@ipsum.com'), + 'user2' => $this->createUser('user2', 'dolor@ipsum.com'), + ]; + } + + public function permissionsProvider(): array + { + return [ + [ + 'permissions' => [Permission::read(Role::any())], + 'any' => 1, + 'users' => 1, + 'doconly' => 1, + ], + [ + 'permissions' => [Permission::read(Role::users())], + 'any' => 2, + 'users' => 2, + 'doconly' => 2, + ], + [ + 'permissions' => [Permission::read(Role::user(ID::custom('random')))], + 'any' => 3, + 'users' => 3, + 'doconly' => 2, + ], + [ + 'permissions' => [Permission::read(Role::user(ID::custom('lorem'))), Permission::update(Role::user('lorem')), Permission::delete(Role::user('lorem'))], + 'any' => 4, + 'users' => 4, + 'doconly' => 2, + ], + [ + 'permissions' => [Permission::read(Role::user(ID::custom('dolor'))), Permission::update(Role::user('dolor')), Permission::delete(Role::user('dolor'))], + 'any' => 5, + 'users' => 5, + 'doconly' => 2, + ], + [ + 'permissions' => [Permission::read(Role::user(ID::custom('dolor'))), Permission::read(Role::user('lorem')), Permission::update(Role::user('dolor')), Permission::delete(Role::user('dolor'))], + 'any' => 6, + 'users' => 6, + 'doconly' => 2, + ], + [ + 'permissions' => [Permission::update(Role::any()), Permission::delete(Role::any())], + 'any' => 7, + 'users' => 7, + 'doconly' => 2, + ], + [ + 'permissions' => [Permission::read(Role::any()), Permission::update(Role::any()), Permission::delete(Role::any())], + 'any' => 8, + 'users' => 8, + 'doconly' => 3, + ], + [ + 'permissions' => [Permission::read(Role::any()), Permission::update(Role::users()), Permission::delete(Role::users())], + 'any' => 9, + 'users' => 9, + 'doconly' => 4, + ], + [ + 'permissions' => [Permission::read(Role::user(ID::custom('user1')))], + 'any' => 10, + 'users' => 10, + 'doconly' => 5, + ], + [ + 'permissions' => [Permission::read(Role::user(ID::custom('user1'))), Permission::read(Role::user(ID::custom('user1')))], + 'any' => 11, + 'users' => 11, + 'doconly' => 6, + ], + [ + 'permissions' => [Permission::read(Role::users()), Permission::update(Role::users()), Permission::delete(Role::users())], + 'any' => 12, + 'users' => 12, + 'doconly' => 7, + ], + ]; + } + + /** + * Setup database + * + * Data providers lose object state so explicitly pass [$users, $collections] to each iteration + * + * @return array + * @throws \Exception + */ + public function testSetupDatabase(): array + { + $this->createUsers(); + + $db = $this->client->call(Client::METHOD_POST, '/vectordb', $this->getServerHeader(), [ + 'databaseId' => ID::unique(), + 'name' => 'Test Database', + ]); + $this->assertEquals(201, $db['headers']['status-code']); + + $databaseId = $db['body']['$id']; + + $public = $this->client->call(Client::METHOD_POST, '/vectordb/' . $databaseId . '/collections', $this->getServerHeader(), [ + 'collectionId' => ID::unique(), + 'name' => 'Movies', + 'dimension' => 3, + 'permissions' => [ + Permission::read(Role::any()), + Permission::create(Role::any()), + Permission::update(Role::any()), + Permission::delete(Role::any()), + ], + 'documentSecurity' => true, + ]); + $this->assertEquals(201, $public['headers']['status-code']); + $this->collections = ['public' => $public['body']['$id']]; + + $private = $this->client->call(Client::METHOD_POST, '/vectordb/' . $databaseId . '/collections', $this->getServerHeader(), [ + 'collectionId' => ID::unique(), + 'name' => 'Private Movies', + 'dimension' => 3, + 'permissions' => [ + Permission::read(Role::users()), + Permission::create(Role::users()), + Permission::update(Role::users()), + Permission::delete(Role::users()), + ], + 'documentSecurity' => true, + ]); + $this->assertEquals(201, $private['headers']['status-code']); + $this->collections['private'] = $private['body']['$id']; + + $doconly = $this->client->call(Client::METHOD_POST, '/vectordb/' . $databaseId . '/collections', $this->getServerHeader(), [ + 'collectionId' => ID::unique(), + 'name' => 'Document Only Movies', + 'dimension' => 3, + 'permissions' => [], + 'documentSecurity' => true, + ]); + $this->assertEquals(201, $private['headers']['status-code']); + $this->collections['doconly'] = $doconly['body']['$id']; + + return [ + 'users' => $this->users, + 'collections' => $this->collections, + 'databaseId' => $databaseId + ]; + } + + /** + * Data provider params are passed before test dependencies + * @dataProvider permissionsProvider + * @depends testSetupDatabase + */ + public function testReadDocuments($permissions, $anyCount, $usersCount, $docOnlyCount, $data) + { + $users = $data['users']; + $collections = $data['collections']; + $databaseId = $data['databaseId']; + + $response = $this->client->call(Client::METHOD_POST, '/vectordb/' . $databaseId . '/collections/' . $collections['public'] . '/documents', $this->getServerHeader(), [ + 'documentId' => ID::unique(), + 'data' => [ + 'embeddings' => [1.0, 0.0, 0.0], + 'metadata' => ['title' => 'Lorem'], + ], + 'permissions' => $permissions + ]); + $this->assertEquals(201, $response['headers']['status-code']); + + $response = $this->client->call(Client::METHOD_POST, '/vectordb/' . $databaseId . '/collections/' . $collections['private'] . '/documents', $this->getServerHeader(), [ + 'documentId' => ID::unique(), + 'data' => [ + 'embeddings' => [0.0, 1.0, 0.0], + 'metadata' => ['title' => 'Lorem'], + ], + 'permissions' => $permissions + ]); + $this->assertEquals(201, $response['headers']['status-code']); + + $response = $this->client->call(Client::METHOD_POST, '/vectordb/' . $databaseId . '/collections/' . $collections['doconly'] . '/documents', $this->getServerHeader(), [ + 'documentId' => ID::unique(), + 'data' => [ + 'embeddings' => [0.0, 0.0, 1.0], + 'metadata' => ['title' => 'Lorem'], + ], + 'permissions' => $permissions + ]); + $this->assertEquals(201, $response['headers']['status-code']); + + /** + * Check "any" permission collection + */ + $documents = $this->client->call(Client::METHOD_GET, '/vectordb/' . $databaseId . '/collections/' . $collections['public'] . '/documents', [ + 'origin' => 'http://localhost', + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'cookie' => 'a_session_' . $this->getProject()['$id'] . '=' . $users['user1']['session'], + ]); + + $this->assertEquals(200, $documents['headers']['status-code']); + $this->assertEquals($anyCount, $documents['body']['total']); + + /** + * Check "users" permission collection + */ + $documents = $this->client->call(Client::METHOD_GET, '/vectordb/' . $databaseId . '/collections/' . $collections['private'] . '/documents', [ + 'origin' => 'http://localhost', + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'cookie' => 'a_session_' . $this->getProject()['$id'] . '=' . $users['user1']['session'], + ]); + + $this->assertEquals(200, $documents['headers']['status-code']); + $this->assertEquals($usersCount, $documents['body']['total']); + + /** + * Check "user:user1" document only permission collection + */ + $documents = $this->client->call(Client::METHOD_GET, '/vectordb/' . $databaseId . '/collections/' . $collections['doconly'] . '/documents', [ + 'origin' => 'http://localhost', + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'cookie' => 'a_session_' . $this->getProject()['$id'] . '=' . $users['user1']['session'], + ]); + + $this->assertEquals(200, $documents['headers']['status-code']); + $this->assertEquals($docOnlyCount, $documents['body']['total']); + } +} diff --git a/tests/e2e/Services/Databases/VectorDB/Permissions/DatabasesPermissionsScope.php b/tests/e2e/Services/Databases/VectorDB/Permissions/DatabasesPermissionsScope.php new file mode 100644 index 0000000000..7d1bfb41d6 --- /dev/null +++ b/tests/e2e/Services/Databases/VectorDB/Permissions/DatabasesPermissionsScope.php @@ -0,0 +1,87 @@ +client->call(Client::METHOD_POST, '/account', [ + 'origin' => 'http://localhost', + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-dev-key' => $this->getProject()['devKey'] ?? '', + ], [ + 'userId' => $id, + 'email' => $email, + 'password' => $password + ]); + + $this->assertEquals(201, $user['headers']['status-code']); + + $session = $this->client->call(Client::METHOD_POST, '/account/sessions/email', [ + 'origin' => 'http://localhost', + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], [ + 'email' => $email, + 'password' => $password, + ]); + + $session = $session['cookies']['a_session_' . $this->getProject()['$id']]; + + $user = [ + '$id' => $user['body']['$id'], + 'email' => $user['body']['email'], + 'session' => $session, + ]; + $this->users[$id] = $user; + + return $user; + } + + public function getCreatedUser(string $id): array + { + return $this->users[$id] ?? []; + } + + public function createTeam(string $id, string $name): array + { + $team = $this->client->call(Client::METHOD_POST, '/teams', $this->getServerHeader(), [ + 'teamId' => $id, + 'name' => $name + ]); + $this->teams[$id] = $team['body']; + + return $team['body']; + } + + public function addToTeam(string $user, string $team, array $roles = []): array + { + $membership = $this->client->call(Client::METHOD_POST, '/teams/' . $team . '/memberships', $this->getServerHeader(), [ + 'teamId' => $team, + 'email' => $this->getCreatedUser($user)['email'], + 'roles' => $roles, + 'url' => 'http://localhost:5000/join-us#title' + ]); + + return [ + 'user' => $membership['body']['userId'], + 'membership' => $membership['body']['$id'] + ]; + } + + public function getServerHeader(): array + { + return [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]; + } +} diff --git a/tests/e2e/Services/Databases/VectorDB/Permissions/DatabasesPermissionsTeamTest.php b/tests/e2e/Services/Databases/VectorDB/Permissions/DatabasesPermissionsTeamTest.php new file mode 100644 index 0000000000..9dbace79a2 --- /dev/null +++ b/tests/e2e/Services/Databases/VectorDB/Permissions/DatabasesPermissionsTeamTest.php @@ -0,0 +1,199 @@ + $this->createTeam('team1', 'Team 1'), + 'team2' => $this->createTeam('team2', 'Team 2'), + ]; + } + + public function createUsers(): array + { + return [ + 'user1' => $this->createUser('user1', 'lorem@ipsum.com'), + 'user2' => $this->createUser('user2', 'dolor@ipsum.com'), + 'user3' => $this->createUser('user3', 'sit@ipsum.com'), + ]; + } + + public function createCollections($teams) + { + $db = $this->client->call(Client::METHOD_POST, '/vectordb', $this->getServerHeader(), [ + 'databaseId' => $this->databaseId, + 'name' => 'Test Database', + ]); + $this->assertEquals(201, $db['headers']['status-code']); + + $collection1 = $this->client->call(Client::METHOD_POST, '/vectordb/' . $this->databaseId . '/collections', $this->getServerHeader(), [ + 'collectionId' => ID::custom('collection1'), + 'name' => 'Collection 1', + 'dimension' => 3, + 'permissions' => [ + Permission::read(Role::team($teams['team1']['$id'])), + Permission::create(Role::team($teams['team1']['$id'], 'admin')), + Permission::update(Role::team($teams['team1']['$id'], 'admin')), + Permission::delete(Role::team($teams['team1']['$id'], 'admin')), + ], + ]); + + $this->collections['collection1'] = $collection1['body']['$id']; + + $collection2 = $this->client->call(Client::METHOD_POST, '/vectordb/' . $this->databaseId . '/collections', $this->getServerHeader(), [ + 'collectionId' => ID::custom('collection2'), + 'name' => 'Collection 2', + 'dimension' => 3, + 'permissions' => [ + Permission::read(Role::team($teams['team2']['$id'])), + Permission::create(Role::team($teams['team2']['$id'], 'owner')), + Permission::update(Role::team($teams['team2']['$id'], 'owner')), + Permission::delete(Role::team($teams['team2']['$id'], 'owner')), + ] + ]); + + $this->collections['collection2'] = $collection2['body']['$id']; + + return $this->collections; + } + + /* + * $success = can $user read from $collection + * [$user, $collection, $success] + */ + public function readDocumentsProvider(): array + { + return [ + ['user1', 'collection1', true], + ['user2', 'collection1', false], + ['user3', 'collection1', true], + ['user1', 'collection2', false], + ['user2', 'collection2', true], + ['user3', 'collection2', true], + ]; + } + + /* + * $success = can $user write to $collection + * [$user, $collection, $success] + */ + public function writeDocumentsProvider(): array + { + return [ + ['user1', 'collection1', true], + ['user2', 'collection1', false], + ['user3', 'collection1', false], + ['user1', 'collection2', false], + ['user2', 'collection2', true], + ['user3', 'collection2', false], + ]; + } + + /** + * Setup database + * + * Data providers lose object state + * so explicitly pass $users to each iteration + * @return array $users + */ + public function testSetupDatabase(): array + { + $this->createUsers(); + $this->createTeams(); + + $this->addToTeam('user1', 'team1', ['admin']); + $this->addToTeam('user2', 'team2', ['owner']); + + // user3 in both teams but with no roles + $this->addToTeam('user3', 'team1'); + $this->addToTeam('user3', 'team2'); + + $this->createCollections($this->teams); + + $response = $this->client->call(Client::METHOD_POST, '/vectordb/' . $this->databaseId . '/collections/' . $this->collections['collection1'] . '/documents', $this->getServerHeader(), [ + 'documentId' => ID::unique(), + 'data' => [ + 'embeddings' => [1.0, 0.0, 0.0], + 'metadata' => ['title' => 'Lorem'], + ], + ]); + $this->assertEquals(201, $response['headers']['status-code']); + + $response = $this->client->call(Client::METHOD_POST, '/vectordb/' . $this->databaseId . '/collections/' . $this->collections['collection2'] . '/documents', $this->getServerHeader(), [ + 'documentId' => ID::unique(), + 'data' => [ + 'embeddings' => [0.0, 1.0, 0.0], + 'metadata' => ['title' => 'Ipsum'], + ], + ]); + $this->assertEquals(201, $response['headers']['status-code']); + + return $this->users; + } + + /** + * Data provider params are passed before test dependencies + * @depends testSetupDatabase + * @dataProvider readDocumentsProvider + */ + public function testReadDocuments($user, $collection, $success, $users) + { + $documents = $this->client->call(Client::METHOD_GET, '/vectordb/' . $this->databaseId . '/collections/' . $collection . '/documents', [ + 'origin' => 'http://localhost', + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'cookie' => 'a_session_' . $this->getProject()['$id'] . '=' . $users[$user]['session'], + ]); + + if ($success) { + $this->assertCount(1, $documents['body']['documents']); + } else { + $this->assertEquals(401, $documents['headers']['status-code']); + } + } + + /** + * @depends testSetupDatabase + * @dataProvider writeDocumentsProvider + */ + public function testWriteDocuments($user, $collection, $success, $users) + { + $documents = $this->client->call(Client::METHOD_POST, '/vectordb/' . $this->databaseId . '/collections/' . $collection . '/documents', [ + 'origin' => 'http://localhost', + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'cookie' => 'a_session_' . $this->getProject()['$id'] . '=' . $users[$user]['session'], + ], [ + 'documentId' => ID::unique(), + 'data' => [ + 'embeddings' => [0.2, 0.3, 0.5], + 'metadata' => ['title' => 'Ipsum'], + ], + ]); + + if ($success) { + $this->assertEquals(201, $documents['headers']['status-code']); + } else { + // 401 if user is a part of team, 404 otherwise + $this->assertContains($documents['headers']['status-code'], [401, 404]); + } + } +} diff --git a/tests/e2e/Services/Databases/VectorDB/Transactions/ACIDTest.php b/tests/e2e/Services/Databases/VectorDB/Transactions/ACIDTest.php new file mode 100644 index 0000000000..6e0e2dbd28 --- /dev/null +++ b/tests/e2e/Services/Databases/VectorDB/Transactions/ACIDTest.php @@ -0,0 +1,528 @@ +client->call(Client::METHOD_POST, '/vectordb', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'databaseId' => ID::unique(), + 'name' => 'AtomicityTestDB' + ]); + + $this->assertEquals(201, $database['headers']['status-code']); + $databaseId = $database['body']['$id']; + + // Create collection for the test + $collection = $this->client->call(Client::METHOD_POST, '/vectordb/' . $databaseId . '/collections', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'collectionId' => ID::unique(), + 'name' => 'AtomicityTest', + 'dimension' => 3, + 'documentSecurity' => false, + 'permissions' => [ + Permission::create(Role::any()), + Permission::read(Role::any()), + ], + ]); + + $this->assertEquals(201, $collection['headers']['status-code']); + $collectionId = $collection['body']['$id']; + + // Create a document outside the transaction + $existingDocumentId = 'existing_doc'; + $doc1 = $this->client->call(Client::METHOD_POST, "/vectordb/{$databaseId}/collections/{$collectionId}/documents", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'documentId' => $existingDocumentId, + 'data' => [ + 'embeddings' => $this->generateEmbeddings(3), + 'metadata' => ['email' => 'existing@example.com'], + ], + ]); + + $this->assertEquals(201, $doc1['headers']['status-code']); + + // Create transaction + $transaction = $this->client->call(Client::METHOD_POST, '/vectordb/transactions', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ])); + + $this->assertEquals(201, $transaction['headers']['status-code'], 'Transaction creation should succeed. Response: ' . json_encode($transaction)); + $this->assertArrayHasKey('$id', $transaction['body'], 'Transaction response should have $id. Response body: ' . json_encode($transaction['body'])); + $transactionId = $transaction['body']['$id']; + + // Add operations - second create reuses an existing documentId and should cause the commit to fail + $response = $this->client->call(Client::METHOD_POST, "/vectordb/{$databaseId}/collections/{$collectionId}/documents", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'documents' => [ + [ + '$id' => 'txn_doc_1', + 'embeddings' => $this->generateEmbeddings(3, 0.2), + 'metadata' => ['email' => 'newuser@example.com'], + ], + [ + '$id' => $existingDocumentId, + 'embeddings' => $this->generateEmbeddings(3, 0.3), + 'metadata' => ['email' => 'duplicate@example.com'], + ], + [ + '$id' => 'txn_doc_2', + 'embeddings' => $this->generateEmbeddings(3, 0.4), + 'metadata' => ['email' => 'should-not-exist@example.com'], + ], + ], + 'transactionId' => $transactionId, + ]); + + $this->assertEquals(200, $response['headers']['status-code'], 'Adding documents via normal route should succeed. Response: ' . json_encode($response['body'])); + + // Attempt to commit - should fail due to duplicate document ID + $response = $this->client->call(Client::METHOD_PATCH, "/vectordb/transactions/{$transactionId}", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'commit' => true + ]); + + $this->assertEquals(409, $response['headers']['status-code']); + + // Verify NO new documents were created (atomicity) + $documents = $this->client->call(Client::METHOD_GET, "/vectordb/{$databaseId}/collections/{$collectionId}/documents", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders())); + + $this->assertEquals(1, $documents['body']['total']); + $this->assertEquals('existing@example.com', $documents['body']['documents'][0]['metadata']['email']); + } + + /** + * Test consistency - schema validation and constraints + */ + public function testConsistency(): void + { + // Create database + $database = $this->client->call(Client::METHOD_POST, '/vectordb', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'databaseId' => ID::unique(), + 'name' => 'ConsistencyTestDB' + ]); + + $this->assertEquals(201, $database['headers']['status-code']); + $databaseId = $database['body']['$id']; + + // Create collection + $collection = $this->client->call(Client::METHOD_POST, '/vectordb/' . $databaseId . '/collections', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'collectionId' => ID::unique(), + 'name' => 'ConsistencyTest', + 'dimension' => 3, + 'documentSecurity' => false, + 'permissions' => [ + Permission::create(Role::any()), + Permission::read(Role::any()), + ], + ]); + + $this->assertEquals(201, $collection['headers']['status-code']); + $collectionId = $collection['body']['$id']; + + // Create transaction + $transaction = $this->client->call(Client::METHOD_POST, '/vectordb/transactions', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ])); + + $transactionId = $transaction['body']['$id']; + + // Stage operations with valid and invalid data (embedding length mismatch) + $response = $this->client->call(Client::METHOD_POST, "/vectordb/transactions/{$transactionId}/operations", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'operations' => [ + [ + 'databaseId' => $databaseId, + 'collectionId' => $collectionId, + 'action' => 'create', + 'documentId' => ID::unique(), + 'data' => [ + 'embeddings' => $this->generateEmbeddings(3, 0.2), + 'metadata' => ['name' => 'Valid User'], + ], + ], + [ + 'databaseId' => $databaseId, + 'collectionId' => $collectionId, + 'action' => 'create', + 'documentId' => ID::unique(), + 'data' => [ + 'embeddings' => $this->generateEmbeddings(2, 0.5), // Invalid dimensions + 'metadata' => ['name' => 'Invalid User'], + ], + ], + [ + 'databaseId' => $databaseId, + 'collectionId' => $collectionId, + 'action' => 'create', + 'documentId' => ID::unique(), + 'data' => [ + 'embeddings' => $this->generateEmbeddings(3, 0.6), + 'metadata' => ['name' => 'Should Not Persist'], + ], + ], + ], + ]); + + $this->assertEquals(201, $response['headers']['status-code']); + + // Attempt to commit - should fail due to invalid embeddings + $response = $this->client->call(Client::METHOD_PATCH, "/vectordb/transactions/{$transactionId}", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'commit' => true + ]); + + $this->assertContains($response['headers']['status-code'], [400, 409, 500], 'Transaction commit should fail due to validation. Response: ' . json_encode($response['body'])); + + // Verify no documents were created + $documents = $this->client->call(Client::METHOD_GET, "/vectordb/{$databaseId}/collections/{$collectionId}/documents", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders())); + + $this->assertEquals(0, $documents['body']['total']); + } + + /** + * Test isolation - concurrent transactions on same data + */ + public function testIsolation(): void + { + // Create database + $database = $this->client->call(Client::METHOD_POST, '/vectordb', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'databaseId' => ID::unique(), + 'name' => 'IsolationTestDB' + ]); + + $this->assertEquals(201, $database['headers']['status-code']); + $databaseId = $database['body']['$id']; + + // Create collection + $collection = $this->client->call(Client::METHOD_POST, '/vectordb/' . $databaseId . '/collections', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'collectionId' => ID::unique(), + 'name' => 'IsolationTest', + 'dimension' => 3, + 'documentSecurity' => false, + 'permissions' => [ + Permission::create(Role::any()), + Permission::read(Role::any()), + Permission::update(Role::any()), + ], + ]); + + $collectionId = $collection['body']['$id']; + + // Create initial document with status metadata + $doc = $this->client->call(Client::METHOD_POST, "/vectordb/{$databaseId}/collections/{$collectionId}/documents", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'documentId' => 'shared_doc', + 'data' => [ + 'embeddings' => $this->generateEmbeddings(3), + 'metadata' => ['status' => 'pending'], + ], + ]); + + $this->assertEquals(201, $doc['headers']['status-code']); + + // Create first transaction + $transaction1 = $this->client->call(Client::METHOD_POST, '/vectordb/transactions', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ])); + + $this->assertEquals(201, $transaction1['headers']['status-code'], 'Transaction 1 creation should succeed'); + $this->assertArrayHasKey('$id', $transaction1['body'], 'Transaction 1 response should have $id'); + $transactionId1 = $transaction1['body']['$id']; + + // Transaction 1: update status to approved + $this->client->call(Client::METHOD_POST, "/vectordb/transactions/{$transactionId1}/operations", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'operations' => [ + [ + 'databaseId' => $databaseId, + 'collectionId' => $collectionId, + 'action' => 'update', + 'documentId' => 'shared_doc', + 'data' => [ + 'metadata' => ['status' => 'approved'], + ], + ], + ], + ]); + + // Commit first transaction + $response1 = $this->client->call(Client::METHOD_PATCH, "/vectordb/transactions/{$transactionId1}", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'commit' => true + ]); + $this->assertEquals(200, $response1['headers']['status-code']); + + // Document should reflect the first transaction's update + $document = $this->client->call(Client::METHOD_GET, "/vectordb/{$databaseId}/collections/{$collectionId}/documents/shared_doc", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders())); + $this->assertEquals('approved', $document['body']['metadata']['status']); + + // Create second transaction after first commit + $transaction2 = $this->client->call(Client::METHOD_POST, '/vectordb/transactions', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ])); + + $this->assertEquals(201, $transaction2['headers']['status-code'], 'Transaction 2 creation should succeed'); + $this->assertArrayHasKey('$id', $transaction2['body'], 'Transaction 2 response should have $id'); + $transactionId2 = $transaction2['body']['$id']; + + // Transaction 2: update status to declined + $this->client->call(Client::METHOD_POST, "/vectordb/transactions/{$transactionId2}/operations", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'operations' => [ + [ + 'databaseId' => $databaseId, + 'collectionId' => $collectionId, + 'action' => 'update', + 'documentId' => 'shared_doc', + 'data' => [ + 'metadata' => ['status' => 'declined'], + ], + ], + ], + ]); + + // Commit second transaction and ensure isolation guarantees + $response2 = $this->client->call(Client::METHOD_PATCH, "/vectordb/transactions/{$transactionId2}", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'commit' => true + ]); + + $this->assertEquals(200, $response2['headers']['status-code']); + + // Final document should reflect the second transaction's update + $document = $this->client->call(Client::METHOD_GET, "/vectordb/{$databaseId}/collections/{$collectionId}/documents/shared_doc", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders())); + + $this->assertEquals('declined', $document['body']['metadata']['status']); + } + + /** + * Test durability - committed data persists + */ + public function testDurability(): void + { + // Create database + $database = $this->client->call(Client::METHOD_POST, '/vectordb', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'databaseId' => ID::unique(), + 'name' => 'DurabilityTestDB' + ]); + + $this->assertEquals(201, $database['headers']['status-code']); + $databaseId = $database['body']['$id']; + + // Create collection + $collection = $this->client->call(Client::METHOD_POST, '/vectordb/' . $databaseId . '/collections', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'collectionId' => ID::unique(), + 'name' => 'DurabilityTest', + 'dimension' => 3, + 'documentSecurity' => false, + 'permissions' => [ + Permission::create(Role::any()), + Permission::read(Role::any()), + Permission::update(Role::any()), + Permission::delete(Role::any()), + ], + ]); + + $this->assertEquals(201, $collection['headers']['status-code']); + $collectionId = $collection['body']['$id']; + + // Create transaction with multiple operations + $transaction = $this->client->call(Client::METHOD_POST, '/vectordb/transactions', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ])); + + $this->assertEquals(201, $transaction['headers']['status-code'], 'Transaction creation should succeed'); + $this->assertArrayHasKey('$id', $transaction['body'], 'Transaction response should have $id'); + $transactionId = $transaction['body']['$id']; + + // Create two documents via normal route inside transaction + $this->client->call(Client::METHOD_POST, "/vectordb/{$databaseId}/collections/{$collectionId}/documents", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'documents' => [ + [ + '$id' => 'durable_doc_1', + 'embeddings' => $this->generateEmbeddings(3, 0.3), + 'metadata' => ['data' => 'Important data 1'], + ], + [ + '$id' => 'durable_doc_2', + 'embeddings' => $this->generateEmbeddings(3, 0.5), + 'metadata' => ['data' => 'Important data 2'], + ], + ], + 'transactionId' => $transactionId, + ]); + + // Update first document inside the same transaction + $this->client->call(Client::METHOD_PATCH, "/vectordb/{$databaseId}/collections/{$collectionId}/documents/durable_doc_1", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'data' => [ + 'metadata' => ['data' => 'Updated important data 1'], + ], + 'transactionId' => $transactionId, + ]); + + // Commit transaction + $response = $this->client->call(Client::METHOD_PATCH, "/vectordb/transactions/{$transactionId}", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'commit' => true + ]); + + $this->assertEquals(200, $response['headers']['status-code'], 'Commit should succeed. Response: ' . json_encode($response['body'])); + $this->assertEquals('committed', $response['body']['status']); + + // Verify documents exist and have correct data + $document1 = $this->client->call(Client::METHOD_GET, "/vectordb/{$databaseId}/collections/{$collectionId}/documents/durable_doc_1", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders())); + $this->assertEquals(200, $document1['headers']['status-code']); + $this->assertEquals('Updated important data 1', $document1['body']['metadata']['data']); + + $document2 = $this->client->call(Client::METHOD_GET, "/vectordb/{$databaseId}/collections/{$collectionId}/documents/durable_doc_2", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders())); + $this->assertEquals(200, $document2['headers']['status-code']); + $this->assertEquals('Important data 2', $document2['body']['metadata']['data']); + + // Further update outside transaction to ensure persistence + $update = $this->client->call(Client::METHOD_PATCH, "/vectordb/{$databaseId}/collections/{$collectionId}/documents/durable_doc_1", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'data' => [ + 'metadata' => ['data' => 'Modified outside transaction'], + ], + ]); + $this->assertEquals(200, $update['headers']['status-code']); + + // Verify the update persisted + $document1 = $this->client->call(Client::METHOD_GET, "/vectordb/{$databaseId}/collections/{$collectionId}/documents/durable_doc_1", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders())); + $this->assertEquals('Modified outside transaction', $document1['body']['metadata']['data']); + + // List all documents to verify total count + $documents = $this->client->call(Client::METHOD_GET, "/vectordb/{$databaseId}/collections/{$collectionId}/documents", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders())); + $this->assertEquals(2, $documents['body']['total']); + } +} diff --git a/tests/e2e/Services/Databases/VectorDB/Transactions/TransactionsBase.php b/tests/e2e/Services/Databases/VectorDB/Transactions/TransactionsBase.php new file mode 100644 index 0000000000..6fe6c9e3f7 --- /dev/null +++ b/tests/e2e/Services/Databases/VectorDB/Transactions/TransactionsBase.php @@ -0,0 +1,2371 @@ +client->call(Client::METHOD_POST, '/vectordb', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'databaseId' => ID::unique(), + 'name' => 'TransactionTestDatabase' + ]); + + $this->assertEquals(201, $database['headers']['status-code']); + $databaseId = $database['body']['$id']; + + // Test creating a transaction with default TTL + $response = $this->client->call(Client::METHOD_POST, '/vectordb/transactions', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders())); + + $this->assertEquals(201, $response['headers']['status-code']); + $this->assertArrayHasKey('$id', $response['body']); + $this->assertArrayHasKey('status', $response['body']); + $this->assertArrayHasKey('operations', $response['body']); + $this->assertArrayHasKey('expiresAt', $response['body']); + $this->assertEquals('pending', $response['body']['status']); + $this->assertEquals(0, $response['body']['operations']); + + $transactionId1 = $response['body']['$id']; + + // Test creating a transaction with custom TTL + $response = $this->client->call(Client::METHOD_POST, '/vectordb/transactions', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'ttl' => 900 + ]); + + $this->assertEquals(201, $response['headers']['status-code']); + $this->assertEquals('pending', $response['body']['status']); + + $expiresAt = new \DateTime($response['body']['expiresAt']); + $now = new \DateTime(); + $diff = $expiresAt->getTimestamp() - $now->getTimestamp(); + $this->assertGreaterThan(800, $diff); + $this->assertLessThan(1000, $diff); + + $transactionId2 = $response['body']['$id']; + + // Test invalid TTL values + $response = $this->client->call(Client::METHOD_POST, '/vectordb/transactions', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'ttl' => 30 // Below minimum + ]); + + $this->assertEquals(400, $response['headers']['status-code']); + + $response = $this->client->call(Client::METHOD_POST, '/vectordb/transactions', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'ttl' => 4000 // Above maximum + ]); + + $this->assertEquals(400, $response['headers']['status-code']); + } + + /** + * Test adding operations to a transaction + */ + public function testCreateOperations(): void + { + // Create database first + $database = $this->client->call(Client::METHOD_POST, '/vectordb', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'databaseId' => ID::unique(), + 'name' => 'TransactionOperationsTestDB' + ]); + + $this->assertEquals(201, $database['headers']['status-code']); + $databaseId = $database['body']['$id']; + + // Create transaction + $transaction = $this->client->call(Client::METHOD_POST, '/vectordb/transactions', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders())); + + $this->assertEquals(201, $transaction['headers']['status-code']); + $transactionId = $transaction['body']['$id']; + + // Create a collection for testing + $collection = $this->client->call(Client::METHOD_POST, '/vectordb/' . $databaseId . '/collections', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'collectionId' => ID::unique(), + 'name' => 'TransactionOperationsTest', + 'dimension' => 3, + 'documentSecurity' => false, + 'permissions' => [ + Permission::create(Role::any()), + Permission::read(Role::any()), + Permission::update(Role::any()), + Permission::delete(Role::any()), + ], + ]); + + $this->assertEquals(201, $collection['headers']['status-code']); + $collectionId = $collection['body']['$id']; + + // Add valid operations + $response = $this->client->call(Client::METHOD_POST, "/vectordb/transactions/{$transactionId}/operations", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'operations' => [ + [ + 'databaseId' => $databaseId, + 'collectionId' => $collectionId, + 'action' => 'create', + 'documentId' => 'doc1', + 'data' => [ + 'embeddings' => $this->generateEmbeddings(3), + 'metadata' => ['name' => 'Test Document 1'] + ] + ], + [ + 'databaseId' => $databaseId, + 'collectionId' => $collectionId, + 'action' => 'create', + 'documentId' => 'doc2', + 'data' => [ + 'embeddings' => $this->generateEmbeddings(3, 0.2), + 'metadata' => ['name' => 'Test Document 2'] + ] + ] + ] + ]); + + $this->assertEquals(201, $response['headers']['status-code']); + $this->assertEquals(2, $response['body']['operations']); + + // Test adding more operations + $response = $this->client->call(Client::METHOD_POST, "/vectordb/transactions/{$transactionId}/operations", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'operations' => [ + [ + 'databaseId' => $databaseId, + 'collectionId' => $collectionId, + 'action' => 'update', + 'documentId' => 'doc1', + 'data' => [ + 'metadata' => ['name' => 'Updated Document 1'] + ] + ] + ] + ]); + + $this->assertEquals(201, $response['headers']['status-code']); + $this->assertEquals(3, $response['body']['operations']); + + // Test invalid database ID + $response = $this->client->call(Client::METHOD_POST, "/vectordb/transactions/{$transactionId}/operations", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'operations' => [ + [ + 'databaseId' => 'invalid_database', + 'collectionId' => $collectionId, + 'action' => 'create', + 'documentId' => ID::unique(), + 'data' => [ + 'embeddings' => $this->generateEmbeddings(3), + 'metadata' => ['name' => 'Test'] + ] + ] + ] + ]); + + $this->assertEquals(404, $response['headers']['status-code'], 'Invalid database should return 404. Got: ' . json_encode($response['body'])); + + // Test invalid collection ID + $response = $this->client->call(Client::METHOD_POST, "/vectordb/transactions/{$transactionId}/operations", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'operations' => [ + [ + 'databaseId' => $databaseId, + 'collectionId' => 'invalid_collection', + 'action' => 'create', + 'documentId' => ID::unique(), + 'data' => [ + 'embeddings' => $this->generateEmbeddings(3), + 'metadata' => ['name' => 'Test'] + ] + ] + ] + ]); + + $this->assertEquals(404, $response['headers']['status-code']); + } + + /** + * Test committing a transaction + */ + public function testCommit(): void + { + // Create database first + $database = $this->client->call(Client::METHOD_POST, '/vectordb', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'databaseId' => ID::unique(), + 'name' => 'TransactionCommitTestDB' + ]); + + $this->assertEquals(201, $database['headers']['status-code']); + $databaseId = $database['body']['$id']; + + // Create collection + $collection = $this->client->call(Client::METHOD_POST, '/vectordb/' . $databaseId . '/collections', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'collectionId' => ID::unique(), + 'name' => 'TransactionCommitTest', + 'dimension' => 3, + 'documentSecurity' => false, + 'permissions' => [ + Permission::create(Role::any()), + Permission::read(Role::any()), + Permission::update(Role::any()), + Permission::delete(Role::any()), + ], + ]); + + $this->assertEquals(201, $collection['headers']['status-code']); + $collectionId = $collection['body']['$id']; + + // Create transaction + $transaction = $this->client->call(Client::METHOD_POST, '/vectordb/transactions', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders())); + + $this->assertEquals(201, $transaction['headers']['status-code']); + $transactionId = $transaction['body']['$id']; + + // Add operations + $response = $this->client->call(Client::METHOD_POST, "/vectordb/transactions/{$transactionId}/operations", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'operations' => [ + [ + 'databaseId' => $databaseId, + 'collectionId' => $collectionId, + 'action' => 'create', + 'documentId' => 'doc1', + 'data' => [ + 'embeddings' => $this->generateEmbeddings(3), + 'metadata' => ['name' => 'Test Document 1'] + ] + ], + [ + 'databaseId' => $databaseId, + 'collectionId' => $collectionId, + 'action' => 'create', + 'documentId' => 'doc2', + 'data' => [ + 'embeddings' => $this->generateEmbeddings(3, 0.2), + 'metadata' => ['name' => 'Test Document 2'] + ] + ], + [ + 'databaseId' => $databaseId, + 'collectionId' => $collectionId, + 'action' => 'update', + 'documentId' => 'doc1', + 'data' => [ + 'metadata' => ['name' => 'Updated Document 1'] + ] + ] + ] + ]); + + $this->assertEquals(201, $response['headers']['status-code']); + $this->assertEquals(3, $response['body']['operations']); + + // Commit the transaction + $response = $this->client->call(Client::METHOD_PATCH, "/vectordb/transactions/{$transactionId}", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'commit' => true + ]); + + $this->assertEquals(200, $response['headers']['status-code']); + $this->assertEquals('committed', $response['body']['status']); + + // Verify documents were created + $documents = $this->client->call(Client::METHOD_GET, "/vectordb/{$databaseId}/collections/{$collectionId}/documents", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders())); + + $this->assertEquals(200, $documents['headers']['status-code']); + $this->assertEquals(2, $documents['body']['total']); + + // Verify the update was applied + $doc1Found = false; + foreach ($documents['body']['documents'] as $doc) { + if ($doc['$id'] === 'doc1') { + $this->assertEquals('Updated Document 1', $doc['metadata']['name']); + $doc1Found = true; + } + } + $this->assertTrue($doc1Found, 'Document doc1 should exist with updated name'); + + // Test committing already committed transaction + $response = $this->client->call(Client::METHOD_PATCH, "/vectordb/transactions/{$transactionId}", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'commit' => true + ]); + + $this->assertEquals(400, $response['headers']['status-code']); + } + + /** + * Test rolling back a transaction + */ + public function testRollback(): void + { + // Create database first + $database = $this->client->call(Client::METHOD_POST, '/vectordb', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'databaseId' => ID::unique(), + 'name' => 'TransactionRollbackTestDB' + ]); + + $this->assertEquals(201, $database['headers']['status-code']); + $databaseId = $database['body']['$id']; + + // Create transaction + $transaction = $this->client->call(Client::METHOD_POST, '/vectordb/transactions', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders())); + + $this->assertEquals(201, $transaction['headers']['status-code']); + $transactionId = $transaction['body']['$id']; + + // Create a collection for rollback test + $collection = $this->client->call(Client::METHOD_POST, '/vectordb/' . $databaseId . '/collections', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'collectionId' => ID::unique(), + 'name' => 'TransactionRollbackTest', + 'dimension' => 3, + 'documentSecurity' => false, + 'permissions' => [ + Permission::create(Role::any()), + Permission::read(Role::any()), + Permission::update(Role::any()), + Permission::delete(Role::any()), + ], + ]); + + $collectionId = $collection['body']['$id']; + + // Add operations + $response = $this->client->call(Client::METHOD_POST, "/vectordb/transactions/{$transactionId}/operations", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'operations' => [ + [ + 'databaseId' => $databaseId, + 'collectionId' => $collectionId, + 'action' => 'create', + 'documentId' => 'rollback_doc', + 'data' => [ + 'embeddings' => $this->generateEmbeddings(3), + 'metadata' => ['value' => 'Should not exist'] + ] + ] + ] + ]); + + $this->assertEquals(201, $response['headers']['status-code']); + + // Rollback the transaction + $response = $this->client->call(Client::METHOD_PATCH, "/vectordb/transactions/{$transactionId}", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'rollback' => true + ]); + + $this->assertEquals(200, $response['headers']['status-code']); + $this->assertEquals('failed', $response['body']['status']); + + // Verify no documents were created + $documents = $this->client->call(Client::METHOD_GET, "/vectordb/{$databaseId}/collections/{$collectionId}/documents", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders())); + + $this->assertEquals(200, $documents['headers']['status-code']); + $this->assertEquals(0, $documents['body']['total']); + } + + /** + * Test transaction expiration + */ + public function testTransactionExpiration(): void + { + // Create database and collection + $database = $this->client->call(Client::METHOD_POST, '/vectordb', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'databaseId' => ID::unique(), + 'name' => 'ExpirationTestDB' + ]); + + $databaseId = $database['body']['$id']; + + $collection = $this->client->call(Client::METHOD_POST, "/vectordb/{$databaseId}/collections", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'collectionId' => ID::unique(), + 'name' => 'TestCollection', + 'dimension' => 3, + 'permissions' => [ + Permission::read(Role::any()), + Permission::create(Role::any()), + Permission::update(Role::any()), + Permission::delete(Role::any()), + ], + ]); + + $collectionId = $collection['body']['$id']; + + // Create transaction with minimum TTL (60 seconds) + $transaction = $this->client->call(Client::METHOD_POST, '/vectordb/transactions', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'ttl' => 60 + ]); + + $this->assertEquals(201, $transaction['headers']['status-code']); + $transactionId = $transaction['body']['$id']; + + // Add operation + $response = $this->client->call(Client::METHOD_POST, "/vectordb/transactions/{$transactionId}/operations", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'operations' => [ + [ + 'databaseId' => $databaseId, + 'collectionId' => $collectionId, + 'action' => 'create', + 'documentId' => ID::unique(), + 'data' => [ + 'embeddings' => $this->generateEmbeddings(3), + 'metadata' => ['data' => 'Should expire'] + ] + ] + ] + ]); + + $this->assertEquals(201, $response['headers']['status-code']); + + // Verify transaction was created with correct expiration + $txnDetails = $this->client->call(Client::METHOD_GET, "/vectordb/transactions/{$transactionId}", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ])); + + $this->assertEquals(200, $txnDetails['headers']['status-code']); + $this->assertEquals('pending', $txnDetails['body']['status']); + + // Verify expiration time is approximately 60 seconds from now + $expiresAt = new \DateTime($txnDetails['body']['expiresAt']); + $now = new \DateTime(); + $diff = $expiresAt->getTimestamp() - $now->getTimestamp(); + $this->assertGreaterThan(55, $diff); + $this->assertLessThan(65, $diff); + } + + /** + * Test maximum operations per transaction + */ + public function testTransactionSizeLimit(): void + { + // Create database and collection + $database = $this->client->call(Client::METHOD_POST, '/vectordb', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'databaseId' => ID::unique(), + 'name' => 'SizeLimitTestDB' + ]); + + $databaseId = $database['body']['$id']; + + $collection = $this->client->call(Client::METHOD_POST, "/vectordb/{$databaseId}/collections", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'collectionId' => ID::unique(), + 'name' => 'TestCollection', + 'dimension' => 3, + 'permissions' => [Permission::create(Role::any())], + ]); + + $collectionId = $collection['body']['$id']; + + // Create transaction + $transaction = $this->client->call(Client::METHOD_POST, '/vectordb/transactions', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders())); + + $transactionId = $transaction['body']['$id']; + + // Try to add operations exceeding the limit (assuming limit is 100) + // We'll add 50 operations twice to test incremental limit + $operations = []; + for ($i = 0; $i < 50; $i++) { + $operations[] = [ + 'databaseId' => $databaseId, + 'collectionId' => $collectionId, + 'action' => 'create', + 'documentId' => 'doc_' . $i, + 'data' => [ + 'embeddings' => $this->generateEmbeddings(3, 0.1 + ($i * 0.001)), + 'metadata' => ['value' => 'Test ' . $i] + ] + ]; + } + + // First batch should succeed + $response = $this->client->call(Client::METHOD_POST, "/vectordb/transactions/{$transactionId}/operations", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'operations' => $operations + ]); + + $this->assertEquals(201, $response['headers']['status-code']); + $this->assertEquals(50, $response['body']['operations']); + + // Second batch of 50 more operations + $operations = []; + for ($i = 50; $i < 100; $i++) { + $operations[] = [ + 'databaseId' => $databaseId, + 'collectionId' => $collectionId, + 'documentId' => 'doc_' . $i, + 'action' => 'create', + 'data' => [ + 'embeddings' => $this->generateEmbeddings(3, 0.1 + ($i * 0.001)), + 'metadata' => ['value' => 'Test ' . $i] + ] + ]; + } + + $response = $this->client->call(Client::METHOD_POST, "/vectordb/transactions/{$transactionId}/operations", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'operations' => $operations + ]); + + $this->assertEquals(201, $response['headers']['status-code']); + $this->assertEquals(100, $response['body']['operations']); + + // Try to add one more operation - should fail + $response = $this->client->call(Client::METHOD_POST, "/vectordb/transactions/{$transactionId}/operations", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'operations' => [ + [ + 'databaseId' => $databaseId, + 'collectionId' => $collectionId, + 'action' => 'create', + 'documentId' => 'doc_overflow', + 'data' => [ + 'embeddings' => $this->generateEmbeddings(3), + 'metadata' => ['value' => 'This should fail'] + ] + ] + ] + ]); + + $this->assertEquals(400, $response['headers']['status-code']); + } + + /** + * Test concurrent transactions with conflicting operations + */ + public function testConcurrentTransactionConflicts(): void + { + // Create database and collection + $database = $this->client->call(Client::METHOD_POST, '/vectordb', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'databaseId' => ID::unique(), + 'name' => 'ConflictTestDB' + ]); + + $databaseId = $database['body']['$id']; + + $collection = $this->client->call(Client::METHOD_POST, "/vectordb/{$databaseId}/collections", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'collectionId' => ID::unique(), + 'name' => 'TestCollection', + 'dimension' => 3, + 'permissions' => [ + Permission::read(Role::any()), + Permission::create(Role::any()), + Permission::update(Role::any()), + ], + ]); + + $collectionId = $collection['body']['$id']; + + // Create initial document + $doc = $this->client->call(Client::METHOD_POST, "/vectordb/{$databaseId}/collections/{$collectionId}/documents", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'documentId' => 'shared_doc', + 'data' => [ + 'embeddings' => $this->generateEmbeddings(3), + 'metadata' => ['counter' => 100] + ] + ]); + + $this->assertEquals(201, $doc['headers']['status-code']); + + // Create two transactions + $txn1 = $this->client->call(Client::METHOD_POST, '/vectordb/transactions', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders())); + + $txn2 = $this->client->call(Client::METHOD_POST, '/vectordb/transactions', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders())); + + $transactionId1 = $txn1['body']['$id']; + $transactionId2 = $txn2['body']['$id']; + + // Both transactions try to update the same document + $this->client->call(Client::METHOD_POST, "/vectordb/transactions/{$transactionId1}/operations", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'operations' => [ + [ + 'databaseId' => $databaseId, + 'collectionId' => $collectionId, + 'action' => 'update', + 'documentId' => 'shared_doc', + 'data' => [ + 'metadata' => ['counter' => 200] + ] + ] + ] + ]); + + $this->client->call(Client::METHOD_POST, "/vectordb/transactions/{$transactionId2}/operations", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'operations' => [ + [ + 'databaseId' => $databaseId, + 'collectionId' => $collectionId, + 'action' => 'update', + 'documentId' => 'shared_doc', + 'data' => [ + 'metadata' => ['counter' => 300] + ] + ] + ] + ]); + + // Commit first transaction + $response1 = $this->client->call(Client::METHOD_PATCH, "/vectordb/transactions/{$transactionId1}", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'commit' => true + ]); + + $this->assertEquals(200, $response1['headers']['status-code']); + + // Commit second transaction - should fail with conflict + $response2 = $this->client->call(Client::METHOD_PATCH, "/vectordb/transactions/{$transactionId2}", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'commit' => true + ]); + + $this->assertEquals(409, $response2['headers']['status-code']); // Conflict + + // Verify the document has the value from first transaction + $doc = $this->client->call(Client::METHOD_GET, "/vectordb/{$databaseId}/collections/{$collectionId}/documents/shared_doc", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders())); + + $this->assertEquals(200, $doc['body']['metadata']['counter']); + } + + /** + * Test deleting a document that's being updated in a transaction + */ + public function testDeleteDocumentDuringTransaction(): void + { + // Create database and collection + $database = $this->client->call(Client::METHOD_POST, '/vectordb', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'databaseId' => ID::unique(), + 'name' => 'DeleteConflictDB' + ]); + + $databaseId = $database['body']['$id']; + + $collection = $this->client->call(Client::METHOD_POST, "/vectordb/{$databaseId}/collections", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'collectionId' => ID::unique(), + 'name' => 'TestCollection', + 'dimension' => 3, + 'permissions' => [ + Permission::read(Role::any()), + Permission::create(Role::any()), + Permission::update(Role::any()), + Permission::delete(Role::any()), + ], + ]); + + $collectionId = $collection['body']['$id']; + + // Create document + $doc = $this->client->call(Client::METHOD_POST, "/vectordb/{$databaseId}/collections/{$collectionId}/documents", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'documentId' => 'target_doc', + 'data' => [ + 'embeddings' => $this->generateEmbeddings(3), + 'metadata' => ['data' => 'Original'] + ] + ]); + + $this->assertEquals(201, $doc['headers']['status-code']); + + // Create transaction + $transaction = $this->client->call(Client::METHOD_POST, '/vectordb/transactions', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders())); + + $transactionId = $transaction['body']['$id']; + + // Add update operation to transaction + $this->client->call(Client::METHOD_POST, "/vectordb/transactions/{$transactionId}/operations", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'operations' => [ + [ + 'databaseId' => $databaseId, + 'collectionId' => $collectionId, + 'action' => 'update', + 'documentId' => 'target_doc', + 'data' => [ + 'metadata' => ['data' => 'Updated in transaction'] + ] + ] + ] + ]); + + // Delete the document outside of transaction + $response = $this->client->call(Client::METHOD_DELETE, "/vectordb/{$databaseId}/collections/{$collectionId}/documents/target_doc", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ])); + + $this->assertEquals(204, $response['headers']['status-code']); + + // Try to commit transaction - should fail because document no longer exists + $response = $this->client->call(Client::METHOD_PATCH, "/vectordb/transactions/{$transactionId}", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'commit' => true + ]); + + $this->assertEquals(404, $response['headers']['status-code']); // Conflict + } + + /** + * Test bulk operations in transactions + */ + public function testBulkOperations(): void + { + // Create database and collection + $database = $this->client->call(Client::METHOD_POST, '/vectordb', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'databaseId' => ID::unique(), + 'name' => 'BulkOpsDB' + ]); + + $databaseId = $database['body']['$id']; + + $collection = $this->client->call(Client::METHOD_POST, "/vectordb/{$databaseId}/collections", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'collectionId' => ID::unique(), + 'name' => 'TestCollection', + 'dimension' => 3, + 'permissions' => [ + Permission::read(Role::any()), + Permission::create(Role::any()), + Permission::update(Role::any()), + Permission::delete(Role::any()), + ], + ]); + + $collectionId = $collection['body']['$id']; + + // Create some initial documents + for ($i = 1; $i <= 5; $i++) { + $this->client->call(Client::METHOD_POST, "/vectordb/{$databaseId}/collections/{$collectionId}/documents", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'documentId' => 'existing_' . $i, + 'data' => [ + 'embeddings' => $this->generateEmbeddings(3, 0.1 + ($i * 0.01)), + 'metadata' => [ + 'name' => 'Existing ' . $i, + 'category' => 'old' + ] + ] + ]); + } + + // Create transaction + $transaction = $this->client->call(Client::METHOD_POST, '/vectordb/transactions', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders())); + + $transactionId = $transaction['body']['$id']; + + // Add bulk operations + $response = $this->client->call(Client::METHOD_POST, "/vectordb/transactions/{$transactionId}/operations", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'operations' => [ + // Bulk create + [ + 'databaseId' => $databaseId, + 'collectionId' => $collectionId, + 'action' => 'bulkCreate', + 'data' => [ + [ + '$id' => 'bulk_1', + 'embeddings' => $this->generateEmbeddings(3, 0.2), + 'metadata' => ['name' => 'Bulk 1', 'category' => 'new'] + ], + [ + '$id' => 'bulk_2', + 'embeddings' => $this->generateEmbeddings(3, 0.3), + 'metadata' => ['name' => 'Bulk 2', 'category' => 'new'] + ], + [ + '$id' => 'bulk_3', + 'embeddings' => $this->generateEmbeddings(3, 0.4), + 'metadata' => ['name' => 'Bulk 3', 'category' => 'new'] + ], + ] + ], + // Bulk update + [ + 'databaseId' => $databaseId, + 'collectionId' => $collectionId, + 'action' => 'bulkUpdate', + 'data' => [ + 'queries' => [Query::equal('metadata', [['category' => 'old']])->toString()], + 'data' => ['metadata' => ['category' => 'updated']] + ] + ], + // Bulk delete + [ + 'databaseId' => $databaseId, + 'collectionId' => $collectionId, + 'action' => 'bulkDelete', + 'data' => [ + 'queries' => [Query::equal('$id', ['existing_5'])->toString()] + ] + ] + ] + ]); + + $this->assertEquals(201, $response['headers']['status-code']); + + // Commit transaction + $response = $this->client->call(Client::METHOD_PATCH, "/vectordb/transactions/{$transactionId}", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'commit' => true + ]); + + $this->assertEquals(200, $response['headers']['status-code']); + + // Verify results + $documents = $this->client->call(Client::METHOD_GET, "/vectordb/{$databaseId}/collections/{$collectionId}/documents", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders())); + + // Should have 7 documents (5 existing - 1 deleted + 3 new) + $this->assertEquals(7, $documents['body']['total']); + + // Check categories were updated + $oldCategoryCount = 0; + $updatedCategoryCount = 0; + $newCategoryCount = 0; + + foreach ($documents['body']['documents'] as $doc) { + $category = $doc['metadata']['category'] ?? null; + switch ($category) { + case 'old': + $oldCategoryCount++; + break; + case 'updated': + $updatedCategoryCount++; + break; + case 'new': + $newCategoryCount++; + break; + } + } + + $this->assertEquals(0, $oldCategoryCount); + $this->assertEquals(4, $updatedCategoryCount); // 4 existing docs updated + $this->assertEquals(3, $newCategoryCount); // 3 new docs + } + + /** + * Test transaction with mixed success and failure operations + */ + public function testPartialFailureRollback(): void + { + // Create database and collection + $database = $this->client->call(Client::METHOD_POST, '/vectordb', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'databaseId' => ID::unique(), + 'name' => 'PartialFailureDB' + ]); + + $databaseId = $database['body']['$id']; + + $collection = $this->client->call(Client::METHOD_POST, "/vectordb/{$databaseId}/collections", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'collectionId' => ID::unique(), + 'name' => 'TestCollection', + 'dimension' => 3, + 'permissions' => [ + Permission::read(Role::any()), + Permission::create(Role::any()), + ], + ]); + + $collectionId = $collection['body']['$id']; + + // Create HNSW index on embeddings + $this->client->call(Client::METHOD_POST, "/vectordb/{$databaseId}/collections/{$collectionId}/indexes", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'key' => 'embeddings_index', + 'type' => Database::INDEX_HNSW_EUCLIDEAN, + 'attributes' => ['embeddings'], + ]); + + sleep(2); + + // Create an existing document + $duplicateId = ID::unique(); + $this->client->call(Client::METHOD_POST, "/vectordb/{$databaseId}/collections/{$collectionId}/documents", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'documentId' => $duplicateId, + 'data' => [ + 'embeddings' => $this->generateEmbeddings(3), + 'metadata' => ['email' => 'existing@example.com'] + ] + ]); + + // Create transaction + $transaction = $this->client->call(Client::METHOD_POST, '/vectordb/transactions', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders())); + + $transactionId = $transaction['body']['$id']; + + // Add operations - mix of valid and invalid (duplicate id) + $response = $this->client->call(Client::METHOD_POST, "/vectordb/transactions/{$transactionId}/operations", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'operations' => [ + [ + 'databaseId' => $databaseId, + 'collectionId' => $collectionId, + 'action' => 'create', + 'documentId' => ID::unique(), + 'data' => [ + 'embeddings' => $this->generateEmbeddings(3, 0.2), + 'metadata' => ['email' => 'valid1@example.com'] + ] + ], + [ + 'databaseId' => $databaseId, + 'collectionId' => $collectionId, + 'action' => 'create', + 'documentId' => ID::unique(), + 'data' => [ + 'embeddings' => $this->generateEmbeddings(3, 0.3), + 'metadata' => ['email' => 'valid2@example.com'] + ] + ], + [ + 'databaseId' => $databaseId, + 'collectionId' => $collectionId, + 'action' => 'create', + 'documentId' => $duplicateId, + 'data' => [ + 'embeddings' => $this->generateEmbeddings(3, 0.4), + 'metadata' => ['email' => 'existing@example.com'] + ] + ], + [ + 'databaseId' => $databaseId, + 'collectionId' => $collectionId, + 'action' => 'create', + 'documentId' => ID::unique(), + 'data' => [ + 'embeddings' => $this->generateEmbeddings(3, 0.5), + 'metadata' => ['email' => 'valid3@example.com'] + ] + ], + ] + ]); + + $this->assertEquals(201, $response['headers']['status-code']); + + // Try to commit - should fail and rollback all operations + $response = $this->client->call(Client::METHOD_PATCH, "/vectordb/transactions/{$transactionId}", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'commit' => true + ]); + + $this->assertEquals(409, $response['headers']['status-code']); // Conflict due to duplicate + + // Verify NO new documents were created (atomicity) + $documents = $this->client->call(Client::METHOD_GET, "/vectordb/{$databaseId}/collections/{$collectionId}/documents", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders())); + + $this->assertEquals(1, $documents['body']['total']); // Only the original document + $this->assertEquals('existing@example.com', $documents['body']['documents'][0]['metadata']['email']); + } + + /** + * Test double commit/rollback attempts + */ + public function testDoubleCommitRollback(): void + { + // Create database and collection + $database = $this->client->call(Client::METHOD_POST, '/vectordb', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'databaseId' => ID::unique(), + 'name' => 'DoubleCommitDB' + ]); + + $databaseId = $database['body']['$id']; + + $collection = $this->client->call(Client::METHOD_POST, "/vectordb/{$databaseId}/collections", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'collectionId' => ID::unique(), + 'name' => 'TestCollection', + 'dimension' => 3, + 'permissions' => [Permission::create(Role::any())], + ]); + + $collectionId = $collection['body']['$id']; + + // Test double commit + $transaction = $this->client->call(Client::METHOD_POST, '/vectordb/transactions', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders())); + + $transactionId = $transaction['body']['$id']; + + // Add operation + $this->client->call(Client::METHOD_POST, "/vectordb/transactions/{$transactionId}/operations", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'operations' => [ + [ + 'databaseId' => $databaseId, + 'collectionId' => $collectionId, + 'action' => 'create', + 'documentId' => ID::unique(), + 'data' => [ + 'embeddings' => $this->generateEmbeddings(3), + 'metadata' => ['data' => 'Test'] + ] + ] + ] + ]); + + // First commit + $response = $this->client->call(Client::METHOD_PATCH, "/vectordb/transactions/{$transactionId}", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'commit' => true + ]); + + $this->assertEquals(200, $response['headers']['status-code']); + + // Second commit attempt - should fail + $response = $this->client->call(Client::METHOD_PATCH, "/vectordb/transactions/{$transactionId}", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'commit' => true + ]); + + $this->assertEquals(400, $response['headers']['status-code']); // Bad request - already committed + + // Test double rollback + $transaction2 = $this->client->call(Client::METHOD_POST, '/vectordb/transactions', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders())); + + $transactionId2 = $transaction2['body']['$id']; + + // First rollback + $response = $this->client->call(Client::METHOD_PATCH, "/vectordb/transactions/{$transactionId2}", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'rollback' => true + ]); + + $this->assertEquals(200, $response['headers']['status-code']); + + // Second rollback attempt - should fail + $response = $this->client->call(Client::METHOD_PATCH, "/vectordb/transactions/{$transactionId2}", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'rollback' => true + ]); + + $this->assertEquals(400, $response['headers']['status-code']); // Bad request - already rolled back + } + + /** + * Test operations on non-existent documents + */ + public function testOperationsOnNonExistentDocuments(): void + { + // Create database and collection + $database = $this->client->call(Client::METHOD_POST, '/vectordb', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'databaseId' => ID::unique(), + 'name' => 'NonExistentDocDB' + ]); + + $databaseId = $database['body']['$id']; + + $collection = $this->client->call(Client::METHOD_POST, "/vectordb/{$databaseId}/collections", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'collectionId' => ID::unique(), + 'name' => 'TestCollection', + 'dimension' => 3, + 'permissions' => [ + Permission::create(Role::any()), + Permission::update(Role::any()), + Permission::delete(Role::any()), + ], + ]); + + $collectionId = $collection['body']['$id']; + + // Create transaction + $transaction = $this->client->call(Client::METHOD_POST, '/vectordb/transactions', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders())); + + $transactionId = $transaction['body']['$id']; + + // Try to update non-existent document - should fail at staging time with early validation + $response = $this->client->call(Client::METHOD_POST, "/vectordb/transactions/{$transactionId}/operations", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'operations' => [ + [ + 'databaseId' => $databaseId, + 'collectionId' => $collectionId, + 'action' => 'update', + 'documentId' => 'non_existent_doc', + 'data' => [ + 'metadata' => ['data' => 'Should fail'] + ] + ] + ] + ]); + + $this->assertEquals(404, $response['headers']['status-code']); // Document not found at staging time + + // Test delete non-existent document - should also fail at staging time with early validation + $transaction2 = $this->client->call(Client::METHOD_POST, '/vectordb/transactions', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders())); + + $transactionId2 = $transaction2['body']['$id']; + + $response = $this->client->call(Client::METHOD_POST, "/vectordb/transactions/{$transactionId2}/operations", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'operations' => [ + [ + 'databaseId' => $databaseId, + 'collectionId' => $collectionId, + 'action' => 'delete', + 'documentId' => 'non_existent_doc', + 'data' => [] + ] + ] + ]); + + $this->assertEquals(404, $response['headers']['status-code']); // Document not found at staging time + } + + /** + * Test createDocument with transactionId via normal route + */ + public function testCreateDocument(): void + { + // Create database and collection + $database = $this->client->call(Client::METHOD_POST, '/vectordb', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'databaseId' => ID::unique(), + 'name' => 'WriteRoutesTestDB' + ]); + + $databaseId = $database['body']['$id']; + + $collection = $this->client->call(Client::METHOD_POST, "/vectordb/{$databaseId}/collections", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'collectionId' => ID::unique(), + 'name' => 'TestCollection', + 'dimension' => 3, + 'documentSecurity' => false, + 'permissions' => [ + Permission::create(Role::any()), + Permission::read(Role::any()), + Permission::update(Role::any()), + Permission::delete(Role::any()), + ], + ]); + + $collectionId = $collection['body']['$id']; + + // Create transaction + $transaction = $this->client->call(Client::METHOD_POST, '/vectordb/transactions', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders())); + + $this->assertEquals(201, $transaction['headers']['status-code']); + $transactionId = $transaction['body']['$id']; + + // Create document via normal route with transactionId + $response = $this->client->call(Client::METHOD_POST, "/vectordb/{$databaseId}/collections/{$collectionId}/documents", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'documentId' => 'doc_from_route', + 'data' => [ + 'embeddings' => $this->generateEmbeddings(3), + 'metadata' => [ + 'name' => 'Created via normal route', + 'counter' => 100, + 'category' => 'test' + ] + ], + 'transactionId' => $transactionId + ]); + + $this->assertEquals(201, $response['headers']['status-code']); + + // Document should not exist outside transaction yet + $response = $this->client->call(Client::METHOD_GET, "/vectordb/{$databaseId}/collections/{$collectionId}/documents/doc_from_route", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders())); + + $this->assertEquals(404, $response['headers']['status-code']); + + // Commit transaction + $response = $this->client->call(Client::METHOD_PATCH, "/vectordb/transactions/{$transactionId}", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'commit' => true + ]); + + $this->assertEquals(200, $response['headers']['status-code']); + + // Document should now exist + $response = $this->client->call(Client::METHOD_GET, "/vectordb/{$databaseId}/collections/{$collectionId}/documents/doc_from_route", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders())); + + $this->assertEquals(200, $response['headers']['status-code']); + $this->assertEquals('Created via normal route', $response['body']['metadata']['name']); + } + + /** + * Test updateDocument with transactionId via normal route + */ + public function testUpdateDocument(): void + { + // Create database and collection + $database = $this->client->call(Client::METHOD_POST, '/vectordb', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'databaseId' => ID::unique(), + 'name' => 'UpdateRouteTestDB' + ]); + + $databaseId = $database['body']['$id']; + + $collection = $this->client->call(Client::METHOD_POST, "/vectordb/{$databaseId}/collections", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'collectionId' => ID::unique(), + 'name' => 'TestCollection', + 'dimension' => 3, + 'permissions' => [ + Permission::create(Role::any()), + Permission::read(Role::any()), + Permission::update(Role::any()), + ], + ]); + + $collectionId = $collection['body']['$id']; + + // Create document outside transaction + $doc = $this->client->call(Client::METHOD_POST, "/vectordb/{$databaseId}/collections/{$collectionId}/documents", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'documentId' => 'doc_to_update', + 'data' => [ + 'embeddings' => $this->generateEmbeddings(3), + 'metadata' => [ + 'name' => 'Original name', + 'counter' => 50, + 'category' => 'original' + ] + ] + ]); + + $this->assertEquals(201, $doc['headers']['status-code']); + + // Create transaction + $transaction = $this->client->call(Client::METHOD_POST, '/vectordb/transactions', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders())); + + $transactionId = $transaction['body']['$id']; + + // Update document via normal route with transactionId + $response = $this->client->call(Client::METHOD_PATCH, "/vectordb/{$databaseId}/collections/{$collectionId}/documents/doc_to_update", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'data' => [ + 'metadata' => [ + 'name' => 'Updated via normal route', + 'counter' => 150, + 'category' => 'updated' + ] + ], + 'transactionId' => $transactionId + ]); + + $this->assertEquals(200, $response['headers']['status-code']); + + // Document should still have original values outside transaction + $response = $this->client->call(Client::METHOD_GET, "/vectordb/{$databaseId}/collections/{$collectionId}/documents/doc_to_update", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders())); + + $this->assertEquals('Original name', $response['body']['metadata']['name']); + $this->assertEquals(50, $response['body']['metadata']['counter']); + + // Commit transaction + $response = $this->client->call(Client::METHOD_PATCH, "/vectordb/transactions/{$transactionId}", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'commit' => true + ]); + + $this->assertEquals(200, $response['headers']['status-code']); + + // Document should now have updated values + $response = $this->client->call(Client::METHOD_GET, "/vectordb/{$databaseId}/collections/{$collectionId}/documents/doc_to_update", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders())); + + $this->assertEquals('Updated via normal route', $response['body']['metadata']['name']); + $this->assertEquals(150, $response['body']['metadata']['counter']); + } + + /** + * Test upsertDocument with transactionId via normal route + */ + public function testUpsertDocument(): void + { + // Create database and collection + $database = $this->client->call(Client::METHOD_POST, '/vectordb', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'databaseId' => ID::unique(), + 'name' => 'UpsertRouteTestDB' + ]); + + $databaseId = $database['body']['$id']; + + $collection = $this->client->call(Client::METHOD_POST, "/vectordb/{$databaseId}/collections", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'collectionId' => ID::unique(), + 'name' => 'TestCollection', + 'dimension' => 3, + 'permissions' => [ + Permission::create(Role::any()), + Permission::read(Role::any()), + Permission::update(Role::any()), + ], + ]); + + $collectionId = $collection['body']['$id']; + + // Create transaction + $transaction = $this->client->call(Client::METHOD_POST, '/vectordb/transactions', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders())); + + $transactionId = $transaction['body']['$id']; + + // Upsert document (create) via normal route with transactionId + $response = $this->client->call(Client::METHOD_PUT, "/vectordb/{$databaseId}/collections/{$collectionId}/documents/doc_upsert", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'documentId' => 'doc_upsert', + 'data' => [ + 'embeddings' => $this->generateEmbeddings(3), + 'metadata' => [ + 'name' => 'Created by upsert', + 'counter' => 25 + ] + ], + 'transactionId' => $transactionId + ]); + + $this->assertEquals(201, $response['headers']['status-code']); + + // Document should not exist outside transaction yet + $response = $this->client->call(Client::METHOD_GET, "/vectordb/{$databaseId}/collections/{$collectionId}/documents/doc_upsert", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders())); + + $this->assertEquals(404, $response['headers']['status-code']); + + // Upsert same document (update) in same transaction + $response = $this->client->call(Client::METHOD_PUT, "/vectordb/{$databaseId}/collections/{$collectionId}/documents/doc_upsert", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'documentId' => 'doc_upsert', + 'data' => [ + 'metadata' => [ + 'name' => 'Updated by upsert', + 'counter' => 75 + ] + ], + 'transactionId' => $transactionId + ]); + + $this->assertEquals(201, $response['headers']['status-code']); // Upsert in transaction returns 201 + + // Commit transaction + $response = $this->client->call(Client::METHOD_PATCH, "/vectordb/transactions/{$transactionId}", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'commit' => true + ]); + + $this->assertEquals(200, $response['headers']['status-code']); + + // Document should now exist with updated values + $response = $this->client->call(Client::METHOD_GET, "/vectordb/{$databaseId}/collections/{$collectionId}/documents/doc_upsert", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders())); + + $this->assertEquals(200, $response['headers']['status-code']); + $this->assertEquals('Updated by upsert', $response['body']['metadata']['name']); + $this->assertEquals(75, $response['body']['metadata']['counter']); + } + + /** + * Test deleteDocument with transactionId via normal route + */ + public function testDeleteDocument(): void + { + // Create database and collection + $database = $this->client->call(Client::METHOD_POST, '/vectordb', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'databaseId' => ID::unique(), + 'name' => 'DeleteRouteTestDB' + ]); + + $databaseId = $database['body']['$id']; + + $collection = $this->client->call(Client::METHOD_POST, "/vectordb/{$databaseId}/collections", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'collectionId' => ID::unique(), + 'name' => 'TestCollection', + 'dimension' => 3, + 'permissions' => [ + Permission::create(Role::any()), + Permission::read(Role::any()), + Permission::delete(Role::any()), + ], + ]); + + $collectionId = $collection['body']['$id']; + + // Create document outside transaction + $doc = $this->client->call(Client::METHOD_POST, "/vectordb/{$databaseId}/collections/{$collectionId}/documents", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'documentId' => 'doc_to_delete', + 'data' => [ + 'embeddings' => $this->generateEmbeddings(3), + 'metadata' => ['name' => 'Will be deleted'] + ] + ]); + + $this->assertEquals(201, $doc['headers']['status-code']); + + // Create transaction + $transaction = $this->client->call(Client::METHOD_POST, '/vectordb/transactions', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders())); + + $transactionId = $transaction['body']['$id']; + + // Delete document via normal route with transactionId + $response = $this->client->call(Client::METHOD_DELETE, "/vectordb/{$databaseId}/collections/{$collectionId}/documents/doc_to_delete", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'transactionId' => $transactionId + ]); + + $this->assertEquals(204, $response['headers']['status-code']); + + // Document should still exist outside transaction + $response = $this->client->call(Client::METHOD_GET, "/vectordb/{$databaseId}/collections/{$collectionId}/documents/doc_to_delete", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders())); + + $this->assertEquals(200, $response['headers']['status-code']); + + // Commit transaction + $response = $this->client->call(Client::METHOD_PATCH, "/vectordb/transactions/{$transactionId}", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'commit' => true + ]); + + $this->assertEquals(200, $response['headers']['status-code']); + + // Document should no longer exist + $response = $this->client->call(Client::METHOD_GET, "/vectordb/{$databaseId}/collections/{$collectionId}/documents/doc_to_delete", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders())); + + $this->assertEquals(404, $response['headers']['status-code']); + } + + /** + * Test bulkCreate with transactionId via normal route + */ + public function testBulkCreate(): void + { + // Create database and collection + $database = $this->client->call(Client::METHOD_POST, '/vectordb', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'databaseId' => ID::unique(), + 'name' => 'BulkCreateTestDB' + ]); + + $databaseId = $database['body']['$id']; + + $collection = $this->client->call(Client::METHOD_POST, "/vectordb/{$databaseId}/collections", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'collectionId' => ID::unique(), + 'name' => 'TestCollection', + 'dimension' => 3, + 'permissions' => [ + Permission::create(Role::any()), + Permission::read(Role::any()), + ], + ]); + + $collectionId = $collection['body']['$id']; + + // Create transaction + $transaction = $this->client->call(Client::METHOD_POST, '/vectordb/transactions', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders())); + + $transactionId = $transaction['body']['$id']; + + // Bulk create via normal route with transactionId + $response = $this->client->call(Client::METHOD_POST, "/vectordb/{$databaseId}/collections/{$collectionId}/documents", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'documents' => [ + [ + '$id' => 'bulk_create_1', + 'embeddings' => $this->generateEmbeddings(3), + 'metadata' => [ + 'name' => 'Bulk created 1', + 'category' => 'bulk_created' + ] + ], + [ + '$id' => 'bulk_create_2', + 'embeddings' => $this->generateEmbeddings(3, 0.2), + 'metadata' => [ + 'name' => 'Bulk created 2', + 'category' => 'bulk_created' + ] + ], + [ + '$id' => 'bulk_create_3', + 'embeddings' => $this->generateEmbeddings(3, 0.3), + 'metadata' => [ + 'name' => 'Bulk created 3', + 'category' => 'bulk_created' + ] + ] + ], + 'transactionId' => $transactionId + ]); + + $this->assertEquals(200, $response['headers']['status-code']); // Bulk operations return 200 + + // Documents should not exist outside transaction yet + $response = $this->client->call(Client::METHOD_GET, "/vectordb/{$databaseId}/collections/{$collectionId}/documents", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'queries' => [Query::equal('metadata', [['metadata' => ['category' => 'bulk_created']]])->toString()] + ]); + + $this->assertEquals(0, $response['body']['total']); + + // Individual document check + $response = $this->client->call(Client::METHOD_GET, "/vectordb/{$databaseId}/collections/{$collectionId}/documents/bulk_create_1", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders())); + + $this->assertEquals(404, $response['headers']['status-code']); + + // Commit transaction + $response = $this->client->call(Client::METHOD_PATCH, "/vectordb/transactions/{$transactionId}", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'commit' => true + ]); + + $this->assertEquals(200, $response['headers']['status-code']); + + // Documents should now exist + $response = $this->client->call(Client::METHOD_GET, "/vectordb/{$databaseId}/collections/{$collectionId}/documents", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'queries' => [Query::equal('metadata', ['metadata' => ['category' => 'bulk_created']])->toString()] + ]); + + $this->assertEquals(3, $response['body']['total']); + + // Verify individual documents + for ($i = 1; $i <= 3; $i++) { + $response = $this->client->call(Client::METHOD_GET, "/vectordb/{$databaseId}/collections/{$collectionId}/documents/bulk_create_{$i}", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders())); + + $this->assertEquals(200, $response['headers']['status-code']); + $this->assertEquals("Bulk created {$i}", $response['body']['metadata']['name']); + $this->assertEquals('bulk_created', $response['body']['metadata']['category']); + } + } + + /** + * Test bulkUpdate with transactionId via normal route + */ + public function testBulkUpdate(): void + { + // Create database and collection + $database = $this->client->call(Client::METHOD_POST, '/vectordb', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'databaseId' => ID::unique(), + 'name' => 'BulkUpdateTestDB' + ]); + + $databaseId = $database['body']['$id']; + + $collection = $this->client->call(Client::METHOD_POST, "/vectordb/{$databaseId}/collections", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'collectionId' => ID::unique(), + 'name' => 'TestCollection', + 'dimension' => 3, + 'permissions' => [ + Permission::create(Role::any()), + Permission::read(Role::any()), + Permission::update(Role::any()), + ], + ]); + + $collectionId = $collection['body']['$id']; + + // Create documents for bulk testing + for ($i = 1; $i <= 3; $i++) { + $this->client->call(Client::METHOD_POST, "/vectordb/{$databaseId}/collections/{$collectionId}/documents", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'documentId' => 'bulk_update_' . $i, + 'data' => [ + 'embeddings' => $this->generateEmbeddings(3, 0.1 * $i), + 'metadata' => [ + 'name' => 'Bulk doc ' . $i, + 'category' => 'bulk_test' + ] + ] + ]); + } + + // Create transaction + $transaction = $this->client->call(Client::METHOD_POST, '/vectordb/transactions', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders())); + + $transactionId = $transaction['body']['$id']; + + // Bulk update via normal route with transactionId + $response = $this->client->call(Client::METHOD_PATCH, "/vectordb/{$databaseId}/collections/{$collectionId}/documents", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'queries' => [Query::equal('metadata', ['metadata' => ['category' => 'bulk_test']])->toString()], + 'data' => ['metadata' => ['category' => 'bulk_updated']], + 'transactionId' => $transactionId + ]); + + $this->assertEquals(200, $response['headers']['status-code']); + + // Documents should still have original category outside transaction + $response = $this->client->call(Client::METHOD_GET, "/vectordb/{$databaseId}/collections/{$collectionId}/documents", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'queries' => [Query::equal('metadata', ['metadata' => ['category' => 'bulk_test']])->toString()] + ]); + + $this->assertEquals(3, $response['body']['total']); + + // Commit transaction + $response = $this->client->call(Client::METHOD_PATCH, "/vectordb/transactions/{$transactionId}", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'commit' => true + ]); + + $this->assertEquals(200, $response['headers']['status-code']); + + // Documents should now have updated category + $response = $this->client->call(Client::METHOD_GET, "/vectordb/{$databaseId}/collections/{$collectionId}/documents", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'queries' => [Query::equal('metadata', ['metadata' => ['category' => 'bulk_updated']])->toString()] + ]); + + $this->assertEquals(3, $response['body']['total']); + } + + /** + * Test bulkUpsert with transactionId via normal route + */ + public function testBulkUpsert(): void + { + // Create database and collection + $database = $this->client->call(Client::METHOD_POST, '/vectordb', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'databaseId' => ID::unique(), + 'name' => 'BulkUpsertTestDB' + ]); + + $databaseId = $database['body']['$id']; + + $collection = $this->client->call(Client::METHOD_POST, "/vectordb/{$databaseId}/collections", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'collectionId' => ID::unique(), + 'name' => 'TestCollection', + 'dimension' => 3, + 'permissions' => [ + Permission::create(Role::any()), + Permission::read(Role::any()), + Permission::update(Role::any()), + ], + ]); + + $collectionId = $collection['body']['$id']; + + // Create transaction + $transaction = $this->client->call(Client::METHOD_POST, '/vectordb/transactions', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders())); + + $this->assertEquals(201, $transaction['headers']['status-code']); + $transactionId = $transaction['body']['$id']; + + // Test 1: Invalid action type + $response = $this->client->call(Client::METHOD_POST, "/vectordb/transactions/{$transactionId}/operations", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'operations' => [ + [ + 'action' => 'invalidAction', + 'databaseId' => $databaseId, + 'collectionId' => $collectionId, + 'documentId' => ID::unique(), + 'data' => ['name' => 'Test'] + ] + ] + ]); + + $this->assertEquals(400, $response['headers']['status-code']); + + // Test 2: Missing required action field + $response = $this->client->call(Client::METHOD_POST, "/vectordb/transactions/{$transactionId}/operations", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'operations' => [ + [ + 'databaseId' => $databaseId, + 'collectionId' => $collectionId, + 'documentId' => ID::unique(), + 'data' => ['name' => 'Test'] + ] + ] + ]); + + $this->assertEquals(400, $response['headers']['status-code']); + + // Test 3: Missing required databaseId field + $response = $this->client->call(Client::METHOD_POST, "/vectordb/transactions/{$transactionId}/operations", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'operations' => [ + [ + 'action' => 'create', + 'collectionId' => $collectionId, + 'documentId' => ID::unique(), + 'data' => ['name' => 'Test'] + ] + ] + ]); + + $this->assertEquals(400, $response['headers']['status-code']); + + // Test 4: Missing documentId for create operation + $response = $this->client->call(Client::METHOD_POST, "/vectordb/transactions/{$transactionId}/operations", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'operations' => [ + [ + 'action' => 'create', + 'databaseId' => $databaseId, + 'collectionId' => $collectionId, + 'data' => ['name' => 'Test'] + ] + ] + ]); + + $this->assertEquals(400, $response['headers']['status-code']); + + // Test 5: Missing data for create operation + $response = $this->client->call(Client::METHOD_POST, "/vectordb/transactions/{$transactionId}/operations", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'operations' => [ + [ + 'action' => 'create', + 'databaseId' => $databaseId, + 'collectionId' => $collectionId, + 'documentId' => ID::unique() + ] + ] + ]); + + $this->assertEquals(400, $response['headers']['status-code']); + + // Test 6: BulkCreate with non-array data + $response = $this->client->call(Client::METHOD_POST, "/vectordb/transactions/{$transactionId}/operations", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'operations' => [ + [ + 'action' => 'bulkCreate', + 'databaseId' => $databaseId, + 'collectionId' => $collectionId, + 'data' => 'not an array' + ] + ] + ]); + + $this->assertEquals(400, $response['headers']['status-code']); + + // Test 7: BulkUpdate with missing queries + $response = $this->client->call(Client::METHOD_POST, "/vectordb/transactions/{$transactionId}/operations", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'operations' => [ + [ + 'action' => 'bulkUpdate', + 'databaseId' => $databaseId, + 'collectionId' => $collectionId, + 'data' => [ + 'data' => ['name' => 'Updated'] + ] + ] + ] + ]); + + $this->assertEquals(400, $response['headers']['status-code']); + + // Test 8: Empty operations array + $response = $this->client->call(Client::METHOD_POST, "/vectordb/transactions/{$transactionId}/operations", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'operations' => [] + ]); + + $this->assertEquals(400, $response['headers']['status-code']); + + // Test 9: Operations not an array + $response = $this->client->call(Client::METHOD_POST, "/vectordb/transactions/{$transactionId}/operations", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'operations' => 'not an array' + ]); + + $this->assertEquals(400, $response['headers']['status-code']); + } + + /** + * Test validation for committing/rolling back transactions + */ + public function testCommitRollbackValidation(): void + { + // Create transaction + $transaction = $this->client->call(Client::METHOD_POST, '/vectordb/transactions', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders())); + + $this->assertEquals(201, $transaction['headers']['status-code']); + $transactionId = $transaction['body']['$id']; + + // Test 1: Missing both commit and rollback + $response = $this->client->call(Client::METHOD_PATCH, "/vectordb/transactions/{$transactionId}", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), []); + + $this->assertEquals(400, $response['headers']['status-code']); + + // Test 2: Both commit and rollback set to true + $response = $this->client->call(Client::METHOD_PATCH, "/vectordb/transactions/{$transactionId}", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'commit' => true, + 'rollback' => true + ]); + + $this->assertEquals(400, $response['headers']['status-code']); + + // Test 3: Invalid transaction ID + $response = $this->client->call(Client::METHOD_PATCH, "/vectordb/transactions/invalid_id", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'commit' => true + ]); + + $this->assertEquals(404, $response['headers']['status-code']); + + // Commit the transaction + $response = $this->client->call(Client::METHOD_PATCH, "/vectordb/transactions/{$transactionId}", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'commit' => true + ]); + + $this->assertEquals(200, $response['headers']['status-code']); + + // Test 4: Attempt to commit already committed transaction + $response = $this->client->call(Client::METHOD_PATCH, "/vectordb/transactions/{$transactionId}", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'commit' => true + ]); + + $this->assertEquals(400, $response['headers']['status-code']); + } + + /** + * Test validation for non-existent resources + */ + public function testNonExistentResources(): void + { + // Create database and transaction + $database = $this->client->call(Client::METHOD_POST, '/vectordb', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'databaseId' => ID::unique(), + 'name' => 'ResourceTestDatabase' + ]); + + $this->assertEquals(201, $database['headers']['status-code']); + $databaseId = $database['body']['$id']; + + $transaction = $this->client->call(Client::METHOD_POST, '/vectordb/transactions', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders())); + + $this->assertEquals(201, $transaction['headers']['status-code']); + $transactionId = $transaction['body']['$id']; + + // Test 1: Non-existent database + $response = $this->client->call(Client::METHOD_POST, "/vectordb/transactions/{$transactionId}/operations", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'operations' => [ + [ + 'action' => 'create', + 'databaseId' => 'nonExistentDatabase', + 'collectionId' => 'someCollection', + 'documentId' => ID::unique(), + 'data' => ['name' => 'Test'] + ] + ] + ]); + + $this->assertEquals(404, $response['headers']['status-code']); + + // Test 2: Non-existent collection + $response = $this->client->call(Client::METHOD_POST, "/vectordb/transactions/{$transactionId}/operations", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'operations' => [ + [ + 'action' => 'create', + 'databaseId' => $databaseId, + 'collectionId' => 'nonExistentCollection', + 'documentId' => ID::unique(), + 'data' => ['name' => 'Test'] + ] + ] + ]); + + $this->assertEquals(404, $response['headers']['status-code']); + } +} diff --git a/tests/e2e/Services/Databases/VectorDB/Transactions/TransactionsConsoleClientTest.php b/tests/e2e/Services/Databases/VectorDB/Transactions/TransactionsConsoleClientTest.php new file mode 100644 index 0000000000..02f7c57da8 --- /dev/null +++ b/tests/e2e/Services/Databases/VectorDB/Transactions/TransactionsConsoleClientTest.php @@ -0,0 +1,14 @@ +client->call(Client::METHOD_GET, '/migrations/' . $migrationId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getDestinationProject()['$id'], + 'x-appwrite-key' => $this->getDestinationProject()['apiKey'], + ]); + + $this->assertEquals(200, $response['headers']['status-code']); + $this->assertNotEmpty($response['body']); + + return $response['body']; + } + /** * Appwrite E2E Migration Tests */ @@ -1356,6 +1377,256 @@ trait MigrationsBase ]); } + /** + * Import VectorDB documents from CSV + */ + public function testImportVectordbCSV(): void + { + $databaseId = null; + $collectionId = null; + $bucketId = null; + + try { + $database = $this->client->call(Client::METHOD_POST, '/vectordb', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'databaseId' => ID::unique(), + 'name' => 'Vector CSV Import DB' + ]); + + $this->assertEquals(201, $database['headers']['status-code']); + $databaseId = $database['body']['$id']; + + $collection = $this->client->call(Client::METHOD_POST, '/vectordb/' . $databaseId . '/collections', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'collectionId' => ID::unique(), + 'name' => 'Vector CSV Import Collection', + 'dimension' => 3, + 'documentSecurity' => true, + ]); + + $this->assertEquals(201, $collection['headers']['status-code']); + $collectionId = $collection['body']['$id']; + + $bucket = $this->client->call(Client::METHOD_POST, '/storage/buckets', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ], [ + 'bucketId' => ID::unique(), + 'name' => 'Vector CSV Bucket', + 'maximumFileSize' => 2000000, + 'allowedFileExtensions' => ['csv'], + ]); + + $this->assertEquals(201, $bucket['headers']['status-code']); + $bucketId = $bucket['body']['$id']; + + $file = $this->client->call(Client::METHOD_POST, '/storage/buckets/' . $bucketId . '/files', [ + 'content-type' => 'multipart/form-data', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ], [ + 'fileId' => ID::unique(), + 'file' => new CURLFile(realpath(__DIR__ . '/../../../resources/csv/vectordb-documents.csv'), 'text/csv', 'vectordb-documents.csv'), + ]); + + $this->assertEquals(201, $file['headers']['status-code']); + $fileId = $file['body']['$id']; + + $migration = $this->performCsvMigration([ + 'fileId' => $fileId, + 'bucketId' => $bucketId, + 'resourceId' => $databaseId . ':' . $collectionId, + ]); + + $this->assertEquals(202, $migration['headers']['status-code']); + + $this->assertEventually(function () use ($migration) { + $migrationId = $migration['body']['$id']; + $status = $this->client->call(Client::METHOD_GET, '/migrations/' . $migrationId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ]); + + $this->assertEquals(200, $status['headers']['status-code']); + $this->assertEquals('finished', $status['body']['stage']); + $this->assertEquals('completed', $status['body']['status']); + $this->assertContains(Resource::TYPE_DOCUMENT, $status['body']['resources']); + $this->assertArrayHasKey(Resource::TYPE_DOCUMENT, $status['body']['statusCounters']); + $this->assertEquals(2, $status['body']['statusCounters'][Resource::TYPE_DOCUMENT]['success']); + + return true; + }, 60_000, 500); + + $documents = $this->client->call(Client::METHOD_GET, '/vectordb/' . $databaseId . '/collections/' . $collectionId . '/documents', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ], [ + 'queries' => [ + Query::limit(10)->toString(), + ], + ]); + + $this->assertEquals(200, $documents['headers']['status-code']); + $this->assertEquals(2, $documents['body']['total']); + + $titles = array_map(fn ($doc) => $doc['metadata']['title'] ?? null, $documents['body']['documents']); + $this->assertContains('Vector Alpha', $titles); + $this->assertContains('Vector Beta', $titles); + } finally { + if ($bucketId) { + $this->client->call(Client::METHOD_DELETE, '/storage/buckets/' . $bucketId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ]); + } + + if ($databaseId) { + $this->client->call(Client::METHOD_DELETE, '/vectordb/' . $databaseId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ]); + } + } + } + + /** + * Export VectorDB documents to CSV + */ + public function testExportVectordbCSV(): void + { + $databaseId = null; + + try { + $database = $this->client->call(Client::METHOD_POST, '/vectordb', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ], [ + 'databaseId' => ID::unique(), + 'name' => 'Vector CSV Export DB', + ]); + + $this->assertEquals(201, $database['headers']['status-code']); + $databaseId = $database['body']['$id']; + + $collection = $this->client->call(Client::METHOD_POST, '/vectordb/' . $databaseId . '/collections', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ], [ + 'collectionId' => ID::unique(), + 'name' => 'Vector CSV Export Collection', + 'dimension' => 3, + 'documentSecurity' => true, + ]); + + $this->assertEquals(201, $collection['headers']['status-code']); + $collectionId = $collection['body']['$id']; + + $documentsPayload = [ + [ + 'documentId' => ID::unique(), + 'data' => [ + 'embeddings' => [0.11, 0.22, 0.33], + 'metadata' => ['title' => 'Vector Sample One', 'category' => 'alpha'], + ], + ], + [ + 'documentId' => ID::unique(), + 'data' => [ + 'embeddings' => [0.44, 0.55, 0.66], + 'metadata' => ['title' => 'Vector Sample Two', 'category' => 'beta'], + ], + ], + ]; + + foreach ($documentsPayload as $payload) { + $response = $this->client->call(Client::METHOD_POST, '/vectordb/' . $databaseId . '/collections/' . $collectionId . '/documents', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ], $payload); + + $this->assertEquals(201, $response['headers']['status-code']); + } + + $filename = 'vectordb-export-' . ID::unique(); + $migration = $this->client->call(Client::METHOD_POST, '/migrations/csv/exports', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'resourceId' => $databaseId . ':' . $collectionId, + 'filename' => $filename, + 'columns' => [], + 'queries' => [], + 'delimiter' => ',', + 'enclosure' => '"', + 'escape' => '\\', + 'header' => true, + 'notify' => true, + ]); + + $this->assertEquals(202, $migration['headers']['status-code']); + + $migrationId = $migration['body']['$id']; + $this->assertEventually(function () use ($migrationId) { + $response = $this->client->call(Client::METHOD_GET, '/migrations/' . $migrationId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ]); + + $this->assertEquals(200, $response['headers']['status-code']); + $this->assertEquals('finished', $response['body']['stage']); + $this->assertEquals('completed', $response['body']['status']); + + return true; + }, 30_000, 500); + + $lastEmail = $this->getLastEmail(); + $this->assertNotEmpty($lastEmail); + $this->assertEquals('Your CSV export is ready', $lastEmail['subject']); + + \preg_match('/href="([^"]*\/storage\/buckets\/[^"]*\/push[^"]*)"/', $lastEmail['html'], $matches); + $this->assertNotEmpty($matches[1], 'Download URL not found in email'); + $downloadUrl = html_entity_decode($matches[1]); + + $components = \parse_url($downloadUrl); + $this->assertNotEmpty($components); + \parse_str($components['query'] ?? '', $queryParams); + $this->assertArrayHasKey('jwt', $queryParams); + $this->assertArrayHasKey('project', $queryParams); + + $path = \str_replace('/v1', '', $components['path']); + $downloadResponse = $this->client->call(Client::METHOD_GET, $path . '?project=' . $queryParams['project'] . '&jwt=' . $queryParams['jwt']); + $this->assertEquals(200, $downloadResponse['headers']['status-code']); + + $csvData = $downloadResponse['body']; + $this->assertStringContainsString('Vector Sample One', $csvData); + $this->assertStringContainsString('Vector Sample Two', $csvData); + $this->assertStringContainsString('[0.11,0.22,0.33]', $csvData); + } finally { + if ($databaseId) { + $this->client->call(Client::METHOD_DELETE, '/vectordb/' . $databaseId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ]); + } + } + } + /** * DocumentsDB (schemaless) */ @@ -1418,6 +1689,200 @@ trait MigrationsBase ]; } + /** + * VectorDB (embeddings collections) + */ + public function testAppwriteMigrationVectorDBDatabase(): array + { + $response = $this->client->call(Client::METHOD_POST, '/vectordb', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ], [ + 'databaseId' => ID::unique(), + 'name' => 'VDB - Migration DB' + ]); + + $this->assertEquals(201, $response['headers']['status-code']); + $this->assertNotEmpty($response['body']); + $this->assertNotEmpty($response['body']['$id']); + + $databaseId = $response['body']['$id']; + + $result = $this->performMigrationSync([ + 'resources' => [ + Resource::TYPE_DATABASE_VECTORDB, + ], + 'endpoint' => 'http://localhost/v1', + 'projectId' => $this->getProject()['$id'], + 'apiKey' => $this->getProject()['apiKey'], + ]); + + $this->assertEquals('completed', $result['status']); + $this->assertEquals([Resource::TYPE_DATABASE_VECTORDB], $result['resources']); + $this->assertArrayHasKey(Resource::TYPE_DATABASE_VECTORDB, $result['statusCounters']); + $this->assertEquals(0, $result['statusCounters'][Resource::TYPE_DATABASE_VECTORDB]['error'] ?? 0); + + $response = $this->client->call(Client::METHOD_GET, '/vectordb/' . $databaseId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getDestinationProject()['$id'], + 'x-appwrite-key' => $this->getDestinationProject()['apiKey'], + ]); + + $this->assertEquals(200, $response['headers']['status-code']); + $this->assertNotEmpty($response['body']); + $this->assertNotEmpty($response['body']['$id']); + $this->assertEquals($databaseId, $response['body']['$id']); + $this->assertEquals('VDB - Migration DB', $response['body']['name']); + + // Cleanup on destination + $this->client->call(Client::METHOD_DELETE, '/vectordb/' . $databaseId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getDestinationProject()['$id'], + 'x-appwrite-key' => $this->getDestinationProject()['apiKey'], + ]); + + return [ + 'databaseId' => $databaseId, + ]; + } + + /** + * @depends testAppwriteMigrationVectorDBDatabase + */ + public function testAppwriteMigrationVectorDBCollection(array $data): array + { + $databaseId = $data['databaseId']; + + $collection = $this->client->call(Client::METHOD_POST, '/vectordb/' . $databaseId . '/collections', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ], [ + 'collectionId' => ID::unique(), + 'name' => 'VDB - Movies', + 'dimension' => 3, + ]); + + $this->assertEquals(201, $collection['headers']['status-code']); + + $collectionId = $collection['body']['$id']; + + $result = $this->performMigrationSync([ + 'resources' => [ + Resource::TYPE_DATABASE_VECTORDB, + Resource::TYPE_COLLECTION, + Resource::TYPE_ATTRIBUTE, + ], + 'endpoint' => 'http://localhost/v1', + 'projectId' => $this->getProject()['$id'], + 'apiKey' => $this->getProject()['apiKey'], + ]); + $this->assertEquals('completed', $result['status']); + + $response = $this->client->call(Client::METHOD_GET, '/vectordb/' . $databaseId . '/collections/' . $collectionId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getDestinationProject()['$id'], + 'x-appwrite-key' => $this->getDestinationProject()['apiKey'], + ]); + + $this->assertEquals(200, $response['headers']['status-code']); + $this->assertNotEmpty($response['body']); + $this->assertEquals($collectionId, $response['body']['$id']); + $this->assertEquals('VDB - Movies', $response['body']['name']); + // Verify attributes are present (embeddings and metadata are default attributes) + $this->assertArrayHasKey('attributes', $response['body']); + $this->assertIsArray($response['body']['attributes']); + + // Cleanup + $this->client->call(Client::METHOD_DELETE, '/vectordb/' . $databaseId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getDestinationProject()['$id'], + 'x-appwrite-key' => $this->getDestinationProject()['apiKey'], + ]); + + return [ + 'databaseId' => $databaseId, + 'collectionId' => $collectionId, + ]; + } + + /** + * @depends testAppwriteMigrationVectorDBCollection + */ + public function testAppwriteMigrationVectorDBDocument(array $data): void + { + $databaseId = $data['databaseId']; + $collectionId = $data['collectionId']; + + $document = $this->client->call(Client::METHOD_POST, '/vectordb/' . $databaseId . '/collections/' . $collectionId . '/documents', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ], [ + 'documentId' => ID::unique(), + 'data' => [ + 'embeddings' => [1.0, 0.0, 0.0], + 'metadata' => ['title' => 'Migration Test Movie'], + ] + ]); + + $this->assertEquals(201, $document['headers']['status-code']); + $documentId = $document['body']['$id']; + + // Ensure attributes are exported before documents + $result = $this->performMigrationSync([ + 'resources' => [ + Resource::TYPE_DATABASE_VECTORDB, + Resource::TYPE_COLLECTION, + Resource::TYPE_ATTRIBUTE, + Resource::TYPE_DOCUMENT, + ], + 'endpoint' => 'http://localhost/v1', + 'projectId' => $this->getProject()['$id'], + 'apiKey' => $this->getProject()['apiKey'], + ]); + + $this->assertEquals('completed', $result['status']); + // Verify that TYPE_ATTRIBUTE appears in the resources array for VectorDB + $this->assertContains(Resource::TYPE_ATTRIBUTE, $result['resources'], 'TYPE_ATTRIBUTE should be in resources array for VectorDB'); + + // Verify attributes exist on destination before checking document + $collectionResponse = $this->client->call(Client::METHOD_GET, '/vectordb/' . $databaseId . '/collections/' . $collectionId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getDestinationProject()['$id'], + 'x-appwrite-key' => $this->getDestinationProject()['apiKey'], + ]); + + $this->assertEquals(200, $collectionResponse['headers']['status-code']); + $this->assertArrayHasKey('attributes', $collectionResponse['body']); + $this->assertIsArray($collectionResponse['body']['attributes']); + + $response = $this->client->call(Client::METHOD_GET, '/vectordb/' . $databaseId . '/collections/' . $collectionId . '/documents/' . $documentId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getDestinationProject()['$id'], + 'x-appwrite-key' => $this->getDestinationProject()['apiKey'], + ]); + + $this->assertEquals(200, $response['headers']['status-code']); + $this->assertNotEmpty($response['body']); + $this->assertEquals($documentId, $response['body']['$id']); + $this->assertEquals('Migration Test Movie', $response['body']['metadata']['title']); + + // Cleanup + $this->client->call(Client::METHOD_DELETE, '/vectordb/' . $databaseId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getDestinationProject()['$id'], + 'x-appwrite-key' => $this->getDestinationProject()['apiKey'], + ]); + + $this->client->call(Client::METHOD_DELETE, '/vectordb/' . $databaseId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ]); + } + /** * @depends testAppwriteMigrationDocumentsDBDatabase */ @@ -1612,6 +2077,31 @@ trait MigrationsBase $this->assertEquals('available', $response['body']['status']); }, 5000, 500); + $sqlIndexKey = 'product_unique'; + + $sqlIndex = $this->client->call(Client::METHOD_POST, '/tablesdb/' . $sqlDatabaseId . '/tables/' . $tableId . '/indexes', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $sourceProject['$id'], + 'x-appwrite-key' => $sourceProject['apiKey'], + ], [ + 'key' => $sqlIndexKey, + 'type' => Database::INDEX_UNIQUE, + 'columns' => ['productName'], + ]); + + $this->assertEquals(202, $sqlIndex['headers']['status-code']); + + $this->assertEventually(function () use ($sqlDatabaseId, $tableId, $sqlIndexKey, $sourceProject) { + $index = $this->client->call(Client::METHOD_GET, '/tablesdb/' . $sqlDatabaseId . '/tables/' . $tableId . '/indexes/' . $sqlIndexKey, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $sourceProject['$id'], + 'x-appwrite-key' => $sourceProject['apiKey'], + ]); + + $this->assertEquals(200, $index['headers']['status-code']); + $this->assertEquals('available', $index['body']['status']); + }, 30000, 500); + // Create Row in Table $row = $this->client->call(Client::METHOD_POST, '/tablesdb/' . $sqlDatabaseId . '/tables/' . $tableId . '/rows', [ 'content-type' => 'application/json', @@ -1654,6 +2144,31 @@ trait MigrationsBase $this->assertEquals(201, $collection['headers']['status-code']); $collectionId = $collection['body']['$id']; + $documentsIndexKey = 'email_unique'; + + $documentsIndex = $this->client->call(Client::METHOD_POST, '/documentsdb/' . $docsDatabaseId . '/collections/' . $collectionId . '/indexes', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $sourceProject['$id'], + 'x-appwrite-key' => $sourceProject['apiKey'], + ], [ + 'key' => $documentsIndexKey, + 'type' => Database::INDEX_UNIQUE, + 'attributes' => ['email'], + ]); + + $this->assertEquals(202, $documentsIndex['headers']['status-code']); + + $this->assertEventually(function () use ($docsDatabaseId, $collectionId, $documentsIndexKey, $sourceProject) { + $index = $this->client->call(Client::METHOD_GET, '/documentsdb/' . $docsDatabaseId . '/collections/' . $collectionId . '/indexes/' . $documentsIndexKey, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $sourceProject['$id'], + 'x-appwrite-key' => $sourceProject['apiKey'], + ]); + + $this->assertEquals(200, $index['headers']['status-code']); + $this->assertEquals('available', $index['body']['status']); + }, 30000, 500); + // Create Document in Collection $document = $this->client->call(Client::METHOD_POST, '/documentsdb/' . $docsDatabaseId . '/collections/' . $collectionId . '/documents', [ 'content-type' => 'application/json', @@ -1670,8 +2185,119 @@ trait MigrationsBase $this->assertEquals(201, $document['headers']['status-code']); $documentId = $document['body']['$id']; - // ====== Perform migration including both database kinds with all child resources ====== - $result = $this->performMigrationSync([ + // ====== Create VectorDB (/vectordb) with collection and document ====== + $vector = $this->client->call(Client::METHOD_POST, '/vectordb', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $sourceProject['$id'], + 'x-appwrite-key' => $sourceProject['apiKey'], + ], [ + 'databaseId' => ID::unique(), + 'name' => 'Mixed VectorDB', + ]); + + $this->assertEquals(201, $vector['headers']['status-code']); + $this->assertNotEmpty($vector['body']['$id']); + $vectorDatabaseId = $vector['body']['$id']; + + // Create Collection in VectorDB + $vectorCollection = $this->client->call(Client::METHOD_POST, '/vectordb/' . $vectorDatabaseId . '/collections', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $sourceProject['$id'], + 'x-appwrite-key' => $sourceProject['apiKey'], + ], [ + 'collectionId' => ID::unique(), + 'name' => 'Products', + 'dimension' => 3, + ]); + + $this->assertEquals(201, $vectorCollection['headers']['status-code']); + $vectorCollectionId = $vectorCollection['body']['$id']; + + // Wait for VectorDB collection attributes to be ready + $this->assertEventually(function () use ($vectorDatabaseId, $vectorCollectionId, $sourceProject) { + $response = $this->client->call(Client::METHOD_GET, '/vectordb/' . $vectorDatabaseId . '/collections/' . $vectorCollectionId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $sourceProject['$id'], + 'x-appwrite-key' => $sourceProject['apiKey'], + ]); + + $this->assertEquals(200, $response['headers']['status-code']); + $this->assertArrayHasKey('attributes', $response['body']); + $this->assertIsArray($response['body']['attributes']); + // Check that default attributes (embeddings and metadata) are present and ready + $attributeKeys = array_column($response['body']['attributes'], 'key'); + $this->assertContains('embeddings', $attributeKeys); + $this->assertContains('metadata', $attributeKeys); + // Check that attributes are available (if status field exists) + foreach ($response['body']['attributes'] as $attribute) { + if (isset($attribute['status']) && $attribute['status'] !== 'available') { + return false; + } + } + return true; + }, 10000, 500); + + $metadataIndexKey = '_key_metadata'; + $vectorIndexes = $this->client->call(Client::METHOD_GET, '/vectordb/' . $vectorDatabaseId . '/collections/' . $vectorCollectionId . '/indexes', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $sourceProject['$id'], + 'x-appwrite-key' => $sourceProject['apiKey'], + ]); + $this->assertEquals(200, $vectorIndexes['headers']['status-code']); + $metadataIndex = null; + foreach ($vectorIndexes['body']['indexes'] ?? [] as $index) { + if (($index['key'] ?? '') === $metadataIndexKey) { + $metadataIndex = $index; + break; + } + } + $this->assertNotNull($metadataIndex, 'Default metadata index should exist on source collection'); + $this->assertEquals(Database::INDEX_OBJECT, $metadataIndex['type']); + + $vectorEmbeddingIndexKey = 'embedding_euclidean'; + $vectorEmbeddingIndex = $this->client->call(Client::METHOD_POST, '/vectordb/' . $vectorDatabaseId . '/collections/' . $vectorCollectionId . '/indexes', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $sourceProject['$id'], + 'x-appwrite-key' => $sourceProject['apiKey'], + ], [ + 'key' => $vectorEmbeddingIndexKey, + 'type' => Database::INDEX_HNSW_EUCLIDEAN, + 'attributes' => ['embeddings'], + ]); + $this->assertEquals(202, $vectorEmbeddingIndex['headers']['status-code']); + + $this->assertEventually(function () use ($vectorDatabaseId, $vectorCollectionId, $vectorEmbeddingIndexKey, $sourceProject) { + $index = $this->client->call(Client::METHOD_GET, '/vectordb/' . $vectorDatabaseId . '/collections/' . $vectorCollectionId . '/indexes/' . $vectorEmbeddingIndexKey, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $sourceProject['$id'], + 'x-appwrite-key' => $sourceProject['apiKey'], + ]); + + $this->assertEquals(200, $index['headers']['status-code']); + $this->assertEquals(Database::INDEX_HNSW_EUCLIDEAN, $index['body']['type']); + if (isset($index['body']['status'])) { + $this->assertEquals('available', $index['body']['status']); + } + }, 30000, 500); + + // Create Document in VectorDB Collection + $vectorDocument = $this->client->call(Client::METHOD_POST, '/vectordb/' . $vectorDatabaseId . '/collections/' . $vectorCollectionId . '/documents', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $sourceProject['$id'], + 'x-appwrite-key' => $sourceProject['apiKey'], + ], [ + 'documentId' => ID::unique(), + 'data' => [ + 'embeddings' => [0.5, 0.3, 0.2], + 'metadata' => ['name' => 'Product Vector'], + ], + ]); + + $this->assertEquals(201, $vectorDocument['headers']['status-code']); + $vectorDocumentId = $vectorDocument['body']['$id']; + + // ====== Perform migration including all three database kinds with all child resources ====== + $migrationConfig = [ 'resources' => [ Resource::TYPE_DATABASE, Resource::TYPE_TABLE, @@ -1680,13 +2306,18 @@ trait MigrationsBase Resource::TYPE_DATABASE_DOCUMENTSDB, Resource::TYPE_COLLECTION, Resource::TYPE_DOCUMENT, + Resource::TYPE_DATABASE_VECTORDB, + Resource::TYPE_ATTRIBUTE, + Resource::TYPE_INDEX, ], 'endpoint' => 'http://localhost/v1', 'projectId' => $sourceProject['$id'], 'apiKey' => $sourceProject['apiKey'], - ]); + ]; - // ====== Assert migration result ====== + // Perform migration sync once and get migration ID + $result = $this->performMigrationSync($migrationConfig); + $migrationId = $result['$id']; $this->assertEquals('completed', $result['status']); $this->assertEquals('Appwrite', $result['source']); $this->assertEquals('Appwrite', $result['destination']); @@ -1698,8 +2329,13 @@ trait MigrationsBase Resource::TYPE_DATABASE_DOCUMENTSDB, Resource::TYPE_COLLECTION, Resource::TYPE_DOCUMENT, + Resource::TYPE_DATABASE_VECTORDB, + Resource::TYPE_ATTRIBUTE, + Resource::TYPE_INDEX, ], $result['resources']); + // Get migration status before asserting SQL Database counters + $result = $this->getMigrationStatus($migrationId); // Assert SQL Database counters $this->assertArrayHasKey(Resource::TYPE_DATABASE, $result['statusCounters']); $this->assertEquals(0, $result['statusCounters'][Resource::TYPE_DATABASE]['error']); @@ -1708,6 +2344,8 @@ trait MigrationsBase $this->assertEquals(0, $result['statusCounters'][Resource::TYPE_DATABASE]['processing']); $this->assertEquals(0, $result['statusCounters'][Resource::TYPE_DATABASE]['warning']); + // Get migration status before asserting Table counters + $result = $this->getMigrationStatus($migrationId); // Assert Table counters $this->assertArrayHasKey(Resource::TYPE_TABLE, $result['statusCounters']); $this->assertEquals(0, $result['statusCounters'][Resource::TYPE_TABLE]['error']); @@ -1716,6 +2354,8 @@ trait MigrationsBase $this->assertEquals(0, $result['statusCounters'][Resource::TYPE_TABLE]['processing']); $this->assertEquals(0, $result['statusCounters'][Resource::TYPE_TABLE]['warning']); + // Get migration status before asserting Column counters + $result = $this->getMigrationStatus($migrationId); // Assert Column counters $this->assertArrayHasKey(Resource::TYPE_COLUMN, $result['statusCounters']); $this->assertEquals(0, $result['statusCounters'][Resource::TYPE_COLUMN]['error']); @@ -1724,6 +2364,8 @@ trait MigrationsBase $this->assertEquals(0, $result['statusCounters'][Resource::TYPE_COLUMN]['processing']); $this->assertEquals(0, $result['statusCounters'][Resource::TYPE_COLUMN]['warning']); + // Get migration status before asserting Row counters + $result = $this->getMigrationStatus($migrationId); // Assert Row counters $this->assertArrayHasKey(Resource::TYPE_ROW, $result['statusCounters']); $this->assertEquals(0, $result['statusCounters'][Resource::TYPE_ROW]['error']); @@ -1732,6 +2374,8 @@ trait MigrationsBase $this->assertEquals(0, $result['statusCounters'][Resource::TYPE_ROW]['processing']); $this->assertEquals(0, $result['statusCounters'][Resource::TYPE_ROW]['warning']); + // Get migration status before asserting DocumentsDB counters + $result = $this->getMigrationStatus($migrationId); // Assert DocumentsDB counters $this->assertArrayHasKey(Resource::TYPE_DATABASE_DOCUMENTSDB, $result['statusCounters']); $this->assertEquals(0, $result['statusCounters'][Resource::TYPE_DATABASE_DOCUMENTSDB]['error']); @@ -1740,24 +2384,75 @@ trait MigrationsBase $this->assertEquals(0, $result['statusCounters'][Resource::TYPE_DATABASE_DOCUMENTSDB]['processing']); $this->assertEquals(0, $result['statusCounters'][Resource::TYPE_DATABASE_DOCUMENTSDB]['warning']); - // Assert Collection counters + // Wait for all collections to be fully processed and status counters to be updated + // Note: Collections are being transferred but status counters may not be updated immediately + // This wait ensures the migration worker has finished processing all collections + $result = null; + $this->assertEventually(function () use ($migrationId, &$result) { + $result = $this->getMigrationStatus($migrationId); + + // Check if collections status counters exist + if (!isset($result['statusCounters'][Resource::TYPE_COLLECTION])) { + return false; + } + + $pendingCount = $result['statusCounters'][Resource::TYPE_COLLECTION]['pending'] ?? 0; + + // Return true only when pending count is 0 + return $pendingCount === 0; + }, 30000, 1000); // 30 second timeout, check every 1 second + + // Assert Collection counters (covers both DocumentsDB and VectorDB collections) $this->assertArrayHasKey(Resource::TYPE_COLLECTION, $result['statusCounters']); $this->assertEquals(0, $result['statusCounters'][Resource::TYPE_COLLECTION]['error']); $this->assertEquals(0, $result['statusCounters'][Resource::TYPE_COLLECTION]['pending']); - $this->assertEquals(1, $result['statusCounters'][Resource::TYPE_COLLECTION]['success']); + $this->assertGreaterThanOrEqual(1, $result['statusCounters'][Resource::TYPE_COLLECTION]['success']); $this->assertEquals(0, $result['statusCounters'][Resource::TYPE_COLLECTION]['processing']); $this->assertEquals(0, $result['statusCounters'][Resource::TYPE_COLLECTION]['warning']); - // Assert Document counters + // Get migration status before asserting Document counters + $result = $this->getMigrationStatus($migrationId); + // Assert Document counters (covers both DocumentsDB and VectorDB documents) $this->assertArrayHasKey(Resource::TYPE_DOCUMENT, $result['statusCounters']); $this->assertEquals(0, $result['statusCounters'][Resource::TYPE_DOCUMENT]['error']); $this->assertEquals(0, $result['statusCounters'][Resource::TYPE_DOCUMENT]['pending']); - $this->assertEquals(1, $result['statusCounters'][Resource::TYPE_DOCUMENT]['success']); + $this->assertGreaterThanOrEqual(1, $result['statusCounters'][Resource::TYPE_DOCUMENT]['success']); $this->assertEquals(0, $result['statusCounters'][Resource::TYPE_DOCUMENT]['processing']); $this->assertEquals(0, $result['statusCounters'][Resource::TYPE_DOCUMENT]['warning']); - // Ensure only expected counters exist (7 total) - $this->assertCount(7, $result['statusCounters']); + // Get migration status before asserting VectorDB counters + $result = $this->getMigrationStatus($migrationId); + // Assert VectorDB counters + $this->assertArrayHasKey(Resource::TYPE_DATABASE_VECTORDB, $result['statusCounters']); + $this->assertEquals(0, $result['statusCounters'][Resource::TYPE_DATABASE_VECTORDB]['error']); + $this->assertEquals(0, $result['statusCounters'][Resource::TYPE_DATABASE_VECTORDB]['pending']); + $this->assertEquals(1, $result['statusCounters'][Resource::TYPE_DATABASE_VECTORDB]['success']); + $this->assertEquals(0, $result['statusCounters'][Resource::TYPE_DATABASE_VECTORDB]['processing']); + $this->assertEquals(0, $result['statusCounters'][Resource::TYPE_DATABASE_VECTORDB]['warning']); + + // Get migration status before asserting Attribute counters + $result = $this->getMigrationStatus($migrationId); + // Assert Attribute counters (for VectorDB) + $this->assertArrayHasKey(Resource::TYPE_ATTRIBUTE, $result['statusCounters']); + $this->assertEquals(0, $result['statusCounters'][Resource::TYPE_ATTRIBUTE]['error']); + $this->assertEquals(0, $result['statusCounters'][Resource::TYPE_ATTRIBUTE]['pending']); + $this->assertGreaterThanOrEqual(1, $result['statusCounters'][Resource::TYPE_ATTRIBUTE]['success']); + $this->assertEquals(0, $result['statusCounters'][Resource::TYPE_ATTRIBUTE]['processing']); + $this->assertEquals(0, $result['statusCounters'][Resource::TYPE_ATTRIBUTE]['warning']); + + // Get migration status before asserting Index counters + $result = $this->getMigrationStatus($migrationId); + $this->assertArrayHasKey(Resource::TYPE_INDEX, $result['statusCounters']); + $this->assertEquals(0, $result['statusCounters'][Resource::TYPE_INDEX]['error']); + $this->assertEquals(0, $result['statusCounters'][Resource::TYPE_INDEX]['pending']); + $this->assertGreaterThanOrEqual(4, $result['statusCounters'][Resource::TYPE_INDEX]['success']); + $this->assertEquals(0, $result['statusCounters'][Resource::TYPE_INDEX]['processing']); + $this->assertEquals(0, $result['statusCounters'][Resource::TYPE_INDEX]['warning']); + + // Get migration status before asserting counter count + $result = $this->getMigrationStatus($migrationId); + // Ensure only expected counters exist (10 total) + $this->assertCount(10, $result['statusCounters']); // ====== Validate on destination: SQL Database resources ====== $response = $this->client->call(Client::METHOD_GET, '/databases/' . $sqlDatabaseId, [ @@ -1804,6 +2499,18 @@ trait MigrationsBase $this->assertEquals($rowId, $response['body']['$id']); $this->assertEquals('Laptop', $response['body']['productName']); + $sqlIndexDestination = $this->client->call(Client::METHOD_GET, '/tablesdb/' . $sqlDatabaseId . '/tables/' . $tableId . '/indexes/' . $sqlIndexKey, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getDestinationProject()['$id'], + 'x-appwrite-key' => $this->getDestinationProject()['apiKey'], + ]); + $this->assertEquals(200, $sqlIndexDestination['headers']['status-code']); + $this->assertEquals($sqlIndexKey, $sqlIndexDestination['body']['key']); + $this->assertEquals(Database::INDEX_UNIQUE, $sqlIndexDestination['body']['type']); + if (isset($sqlIndexDestination['body']['columns'])) { + $this->assertEquals(['productName'], $sqlIndexDestination['body']['columns']); + } + // ====== Validate on destination: DocumentsDB resources ====== $response = $this->client->call(Client::METHOD_GET, '/documentsdb/' . $docsDatabaseId, [ 'content-type' => 'application/json', @@ -1838,7 +2545,72 @@ trait MigrationsBase $this->assertEquals('John Doe', $response['body']['name']); $this->assertEquals('john@example.com', $response['body']['email']); - // ====== Cleanup both destinations ====== + $documentsIndexDestination = $this->client->call(Client::METHOD_GET, '/documentsdb/' . $docsDatabaseId . '/collections/' . $collectionId . '/indexes/' . $documentsIndexKey, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getDestinationProject()['$id'], + 'x-appwrite-key' => $this->getDestinationProject()['apiKey'], + ]); + $this->assertEquals(200, $documentsIndexDestination['headers']['status-code']); + $this->assertEquals($documentsIndexKey, $documentsIndexDestination['body']['key']); + $this->assertEquals(Database::INDEX_UNIQUE, $documentsIndexDestination['body']['type']); + if (isset($documentsIndexDestination['body']['attributes'])) { + $this->assertEquals(['email'], $documentsIndexDestination['body']['attributes']); + } + + // ====== Validate on destination: VectorDB resources ====== + $response = $this->client->call(Client::METHOD_GET, '/vectordb/' . $vectorDatabaseId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getDestinationProject()['$id'], + 'x-appwrite-key' => $this->getDestinationProject()['apiKey'], + ]); + + $this->assertEquals(200, $response['headers']['status-code']); + $this->assertEquals($vectorDatabaseId, $response['body']['$id']); + $this->assertEquals('Mixed VectorDB', $response['body']['name']); + + // Validate VectorDB Collection + $response = $this->client->call(Client::METHOD_GET, '/vectordb/' . $vectorDatabaseId . '/collections/' . $vectorCollectionId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getDestinationProject()['$id'], + 'x-appwrite-key' => $this->getDestinationProject()['apiKey'], + ]); + + $this->assertEquals(200, $response['headers']['status-code']); + $this->assertEquals($vectorCollectionId, $response['body']['$id']); + $this->assertEquals('Products', $response['body']['name']); + // Verify attributes are present (embeddings and metadata are default attributes) + $this->assertArrayHasKey('attributes', $response['body']); + $this->assertIsArray($response['body']['attributes']); + + $vectorIndexesDestination = $this->client->call(Client::METHOD_GET, '/vectordb/' . $vectorDatabaseId . '/collections/' . $vectorCollectionId . '/indexes', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getDestinationProject()['$id'], + 'x-appwrite-key' => $this->getDestinationProject()['apiKey'], + ]); + $this->assertEquals(200, $vectorIndexesDestination['headers']['status-code']); + $indexByKey = []; + foreach ($vectorIndexesDestination['body']['indexes'] ?? [] as $index) { + if (isset($index['key'])) { + $indexByKey[$index['key']] = $index; + } + } + $this->assertArrayHasKey($metadataIndexKey, $indexByKey, 'Metadata index should exist on destination'); + $this->assertEquals(Database::INDEX_OBJECT, $indexByKey[$metadataIndexKey]['type']); + $this->assertArrayHasKey($vectorEmbeddingIndexKey, $indexByKey, 'Embeddings HNSW index should exist on destination'); + $this->assertEquals(Database::INDEX_HNSW_EUCLIDEAN, $indexByKey[$vectorEmbeddingIndexKey]['type']); + + // Validate VectorDB Document + $response = $this->client->call(Client::METHOD_GET, '/vectordb/' . $vectorDatabaseId . '/collections/' . $vectorCollectionId . '/documents/' . $vectorDocumentId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getDestinationProject()['$id'], + 'x-appwrite-key' => $this->getDestinationProject()['apiKey'], + ]); + + $this->assertEquals(200, $response['headers']['status-code']); + $this->assertEquals($vectorDocumentId, $response['body']['$id']); + $this->assertEquals('Product Vector', $response['body']['metadata']['name']); + + // ====== Cleanup all destinations ====== $this->client->call(Client::METHOD_DELETE, '/databases/' . $sqlDatabaseId, [ 'content-type' => 'application/json', 'x-appwrite-project' => $this->getDestinationProject()['$id'], @@ -1851,6 +2623,12 @@ trait MigrationsBase 'x-appwrite-key' => $this->getDestinationProject()['apiKey'], ]); + $this->client->call(Client::METHOD_DELETE, '/vectordb/' . $vectorDatabaseId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getDestinationProject()['$id'], + 'x-appwrite-key' => $this->getDestinationProject()['apiKey'], + ]); + // ====== Cleanup sources ====== $this->client->call(Client::METHOD_DELETE, '/databases/' . $sqlDatabaseId, [ 'content-type' => 'application/json', @@ -1863,5 +2641,11 @@ trait MigrationsBase 'x-appwrite-project' => $sourceProject['$id'], 'x-appwrite-key' => $sourceProject['apiKey'], ]); + + $this->client->call(Client::METHOD_DELETE, '/vectordb/' . $vectorDatabaseId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $sourceProject['$id'], + 'x-appwrite-key' => $sourceProject['apiKey'], + ]); } } diff --git a/tests/e2e/Services/Projects/ProjectsConsoleClientTest.php b/tests/e2e/Services/Projects/ProjectsConsoleClientTest.php index e71375e517..6896ca8069 100644 --- a/tests/e2e/Services/Projects/ProjectsConsoleClientTest.php +++ b/tests/e2e/Services/Projects/ProjectsConsoleClientTest.php @@ -183,6 +183,25 @@ class ProjectsConsoleClientTest extends Scope ]); $this->assertEquals(201, $documentsCollection['headers']['status-code']); + // Create vectordb database and collection + $vectorDb = $this->client->call(Client::METHOD_POST, '/vectordb', $projectAdminHeaders, [ + 'databaseId' => ID::unique(), + 'name' => 'Vector DB', + ]); + $this->assertEquals(201, $vectorDb['headers']['status-code']); + $vectorDbId = $vectorDb['body']['$id']; + + $vectorCollection = $this->client->call(Client::METHOD_POST, '/vectordb/' . $vectorDbId . '/collections', $projectAdminHeaders, [ + 'collectionId' => ID::unique(), + 'name' => 'Vector Collection', + 'dimension' => 3, + 'documentSecurity' => true, + 'permissions' => [ + Permission::create(Role::any()), + ], + ]); + $this->assertEquals(201, $vectorCollection['headers']['status-code']); + // Delete project $delete = $this->client->call(Client::METHOD_DELETE, '/projects/' . $projectId, array_merge([ 'content-type' => 'application/json', diff --git a/tests/e2e/Services/Realtime/RealtimeCustomClientTest.php b/tests/e2e/Services/Realtime/RealtimeCustomClientTest.php index 50430122f8..19371a2b6d 100644 --- a/tests/e2e/Services/Realtime/RealtimeCustomClientTest.php +++ b/tests/e2e/Services/Realtime/RealtimeCustomClientTest.php @@ -4299,4 +4299,192 @@ class RealtimeCustomClientTest extends Scope $client->close(); } + + public function testChannelVectorDB() + { + $user = $this->getUser(); + $session = $user['session'] ?? ''; + $projectId = $this->getProject()['$id']; + + $client = $this->getWebsocket(['documents', 'collections'], [ + 'origin' => 'http://localhost', + 'cookie' => 'a_session_' . $projectId . '=' . $session + ]); + + $response = json_decode($client->receive(), true); + + $this->assertArrayHasKey('type', $response); + $this->assertArrayHasKey('data', $response); + $this->assertEquals('connected', $response['type']); + $this->assertNotEmpty($response['data']); + $this->assertCount(2, $response['data']['channels']); + $this->assertContains('documents', $response['data']['channels']); + $this->assertContains('collections', $response['data']['channels']); + $this->assertNotEmpty($response['data']['user']); + $this->assertEquals($user['$id'], $response['data']['user']['$id']); + + // Create VectorDB database + $database = $this->client->call(Client::METHOD_POST, '/vectordb', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'databaseId' => ID::unique(), + 'name' => 'Actors VDB', + ]); + + $databaseId = $database['body']['$id']; + + // Create collection in VectorDB + $actors = $this->client->call(Client::METHOD_POST, '/vectordb/' . $databaseId . '/collections', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'collectionId' => ID::unique(), + 'name' => 'Actors', + 'permissions' => [ + Permission::create(Role::user($this->getUser()['$id'])), + ], + 'documentSecurity' => true, + 'dimension' => 3, + ]); + + $actorsId = $actors['body']['$id']; + + // Create document in VectorDB + $document = $this->client->call(Client::METHOD_POST, '/vectordb/' . $databaseId . '/collections/' . $actorsId . '/documents', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'documentId' => ID::unique(), + 'data' => [ + 'embeddings' => [1.0, 0.0, 0.0], + 'metadata' => ['name' => 'Chris Evans'] + ], + 'permissions' => [ + Permission::read(Role::any()), + Permission::update(Role::any()), + Permission::delete(Role::any()), + ], + ]); + + $response = json_decode($client->receive(), true); + + $documentId = $document['body']['$id']; + $this->assertArrayHasKey('type', $response); + $this->assertArrayHasKey('data', $response); + $this->assertEquals('event', $response['type']); + $this->assertNotEmpty($response['data']); + $this->assertArrayHasKey('timestamp', $response['data']); + // vectordb channels should include 3 items like documentsdb + $this->assertCount(3, $response['data']['channels']); + $this->assertContains('documents', $response['data']['channels']); + $this->assertContains('vectordb.' . $databaseId . '.collections.' . $actorsId . '.documents.' . $documentId, $response['data']['channels']); + $this->assertContains('vectordb.' . $databaseId . '.collections.' . $actorsId . '.documents', $response['data']['channels']); + $this->assertNotEmpty($response['data']['payload']); + $this->assertIsArray($response['data']['payload']['embeddings']); + $this->assertCount(3, $response['data']['payload']['embeddings']); + $this->assertEquals('Chris Evans', $response['data']['payload']['metadata']['name']); + + // Update document + $this->client->call(Client::METHOD_PATCH, '/vectordb/' . $databaseId . '/collections/' . $actorsId . '/documents/' . $documentId, array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'data' => [ + 'embeddings' => [0.0, 1.0, 0.0], + 'metadata' => ['name' => 'Chris Evans 2'] + ], + 'permissions' => [ + Permission::read(Role::any()), + Permission::update(Role::any()), + Permission::delete(Role::any()), + ], + ]); + + $response = json_decode($client->receive(), true); + $this->assertArrayHasKey('type', $response); + $this->assertArrayHasKey('data', $response); + $this->assertEquals('event', $response['type']); + $this->assertNotEmpty($response['data']); + $this->assertArrayHasKey('timestamp', $response['data']); + $this->assertCount(3, $response['data']['channels']); + $this->assertContains('vectordb.' . $databaseId . '.collections.' . $actorsId . '.documents.' . $documentId, $response['data']['channels']); + $this->assertContains('vectordb.' . $databaseId . '.collections.' . $actorsId . '.documents', $response['data']['channels']); + $this->assertNotEmpty($response['data']['payload']); + $this->assertIsArray($response['data']['payload']['embeddings']); + $this->assertEquals('Chris Evans 2', $response['data']['payload']['metadata']['name']); + + // Delete document + $this->client->call(Client::METHOD_DELETE, '/vectordb/' . $databaseId . '/collections/' . $actorsId . '/documents/' . $documentId, array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders())); + + $response = json_decode($client->receive(), true); + $this->assertArrayHasKey('type', $response); + $this->assertArrayHasKey('data', $response); + $this->assertEquals('event', $response['type']); + $this->assertNotEmpty($response['data']); + $this->assertArrayHasKey('timestamp', $response['data']); + $this->assertCount(3, $response['data']['channels']); + $this->assertContains('vectordb.' . $databaseId . '.collections.' . $actorsId . '.documents.' . $documentId, $response['data']['channels']); + $this->assertContains('vectordb.' . $databaseId . '.collections.' . $actorsId . '.documents', $response['data']['channels']); + + // Bulk create two documents + $this->client->call(Client::METHOD_POST, "/vectordb/{$databaseId}/collections/{$actorsId}/documents", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'documents' => [ + [ + 'embeddings' => [1.0, 0.0, 0.0], + 'metadata' => ['name' => 'Robert Downey Jr.'], + '$permissions' => [ + Permission::read(Role::any()), + Permission::update(Role::any()), + Permission::delete(Role::any()), + ], + ], + [ + 'embeddings' => [0.0, 1.0, 0.0], + 'metadata' => ['name' => 'Scarlett Johansson'], + '$permissions' => [ + Permission::read(Role::any()), + Permission::update(Role::any()), + Permission::delete(Role::any()), + ], + ] + ], + ]); + + // Receive first bulk document event + $response = json_decode($client->receive(), true); + $this->assertArrayHasKey('type', $response); + $this->assertArrayHasKey('data', $response); + $this->assertEquals('event', $response['type']); + $this->assertNotEmpty($response['data']); + $this->assertArrayHasKey('timestamp', $response['data']); + $this->assertCount(3, $response['data']['channels']); + $this->assertContains('vectordb.' . $databaseId . '.collections.' . $actorsId . '.documents.' . $response['data']['payload']['$id'] . '.create', $response['data']['events']); + $this->assertContains('vectordb.*.collections.*.documents.*.create', $response['data']['events']); + $this->assertContains('vectordb.' . $databaseId . '.collections.*.documents.*.create', $response['data']['events']); + $this->assertContains('vectordb.*.collections.' . $actorsId . '.documents.*.create', $response['data']['events']); + $this->assertNotEmpty($response['data']['payload']); + $this->assertIsArray($response['data']['payload']); + + // Receive second bulk document event + $response = json_decode($client->receive(), true); + $this->assertArrayHasKey('type', $response); + $this->assertArrayHasKey('data', $response); + $this->assertEquals('event', $response['type']); + $this->assertNotEmpty($response['data']); + $this->assertArrayHasKey('timestamp', $response['data']); + $this->assertCount(3, $response['data']['channels']); + $this->assertContains('vectordb.' . $databaseId . '.collections.' . $actorsId . '.documents.' . $response['data']['payload']['$id'] . '.create', $response['data']['events']); + + $client->close(); + } } diff --git a/tests/resources/csv/vectordb-documents.csv b/tests/resources/csv/vectordb-documents.csv new file mode 100644 index 0000000000..b0b970703e --- /dev/null +++ b/tests/resources/csv/vectordb-documents.csv @@ -0,0 +1,3 @@ +$id,embeddings,metadata +vector-doc-1,"[0.15,0.25,0.35]","{""title"":""Vector Alpha"",""category"":""science""}" +vector-doc-2,"[0.55,0.65,0.75]","{""title"":""Vector Beta"",""category"":""history""}" \ No newline at end of file