diff --git a/.env b/.env index 0df9cb42f4..9abfa756e1 100644 --- a/.env +++ b/.env @@ -39,7 +39,7 @@ _APP_REDIS_HOST=redis _APP_REDIS_PORT=6379 _APP_REDIS_PASS= _APP_REDIS_USER= -COMPOSE_PROFILES=mongodb +COMPOSE_PROFILES=mariadb,mongodb,postgresql _APP_DB_ADAPTER=mongodb _APP_DB_HOST=mongodb _APP_DB_PORT=27017 @@ -47,6 +47,15 @@ _APP_DB_SCHEMA=appwrite _APP_DB_USER=user _APP_DB_PASS=password _APP_DB_ROOT_PASS=rootsecretpassword +_APP_DB_ADAPTER_DOCUMENTSDB=mongodb +_APP_DB_HOST_DOCUMENTSDB=mongodb +_APP_DB_PORT_DOCUMENTSDB=27017 +_APP_DB_ADAPTER_VECTORSDB=postgresql +_APP_DB_HOST_VECTORSDB=postgresql +_APP_DB_PORT_VECTORSDB=5432 +_APP_EMBEDDING_MODELS=embeddinggemma +_APP_EMBEDDING_ENDPOINT='http://ollama:11434/api/embed' +_APP_EMBEDDING_TIMEOUT=30000 _APP_STORAGE_DEVICE=Local _APP_STORAGE_S3_ACCESS_KEY= _APP_STORAGE_S3_SECRET= diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8f4df90398..aa6dbe2bc3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -7,7 +7,6 @@ concurrency: env: COMPOSE_FILE: docker-compose.yml IMAGE: appwrite-dev - CACHE_KEY: appwrite-dev-${{ github.event.pull_request.head.sha }} on: pull_request: @@ -248,11 +247,12 @@ jobs: TESTING=true VERSION=dev - - name: Cache Docker Image - uses: actions/cache@v5 + - name: Upload Docker Image + uses: actions/upload-artifact@v7 with: - key: ${{ env.CACHE_KEY }} + name: ${{ env.IMAGE }} path: /tmp/${{ env.IMAGE }}.tar + retention-days: 1 unit: name: Tests / Unit @@ -265,12 +265,11 @@ jobs: - name: checkout uses: actions/checkout@v6 - - name: Load Cache - uses: actions/cache@v5 + - name: Download Docker Image + uses: actions/download-artifact@v7 with: - key: ${{ env.CACHE_KEY }} - path: /tmp/${{ env.IMAGE }}.tar - fail-on-cache-miss: true + name: ${{ env.IMAGE }} + path: /tmp - name: Login to Docker Hub uses: docker/login-action@v4 @@ -313,12 +312,11 @@ jobs: - name: checkout uses: actions/checkout@v6 - - name: Load Cache - uses: actions/cache@v5 + - name: Download Docker Image + uses: actions/download-artifact@v7 with: - key: ${{ env.CACHE_KEY }} - path: /tmp/${{ env.IMAGE }}.tar - fail-on-cache-miss: true + name: ${{ env.IMAGE }} + path: /tmp - name: Login to Docker Hub uses: docker/login-action@v4 @@ -363,7 +361,7 @@ jobs: e2e_service: name: Tests / E2E / ${{ matrix.database }} (${{ matrix.mode }}) / ${{ matrix.service }} - runs-on: ubuntu-latest + runs-on: ${{ matrix.runner || 'ubuntu-latest' }} needs: [build, matrix] permissions: contents: read @@ -399,16 +397,28 @@ jobs: Migrations, Project ] + include: + - service: Databases + runner: blacksmith-4vcpu-ubuntu-2404 + - service: Sites + runner: blacksmith-4vcpu-ubuntu-2404 + - service: Functions + runner: blacksmith-4vcpu-ubuntu-2404 + - service: Avatars + runner: blacksmith-4vcpu-ubuntu-2404 + - service: Realtime + runner: blacksmith-4vcpu-ubuntu-2404 + - service: TablesDB + runner: blacksmith-4vcpu-ubuntu-2404 steps: - name: Checkout repository uses: actions/checkout@v6 - - name: Load Cache - uses: actions/cache@v5 + - name: Download Docker Image + uses: actions/download-artifact@v7 with: - key: ${{ env.CACHE_KEY }} - path: /tmp/${{ env.IMAGE }}.tar - fail-on-cache-miss: true + name: ${{ env.IMAGE }} + path: /tmp - name: Set database environment run: | @@ -497,12 +507,11 @@ jobs: - name: Checkout repository uses: actions/checkout@v6 - - name: Load Cache - uses: actions/cache@v5 + - name: Download Docker Image + uses: actions/download-artifact@v7 with: - key: ${{ env.CACHE_KEY }} - path: /tmp/${{ env.IMAGE }}.tar - fail-on-cache-miss: true + name: ${{ env.IMAGE }} + path: /tmp - name: Login to Docker Hub uses: docker/login-action@v4 @@ -556,12 +565,11 @@ jobs: - name: Checkout repository uses: actions/checkout@v6 - - name: Load Cache - uses: actions/cache@v5 + - name: Download Docker Image + uses: actions/download-artifact@v7 with: - key: ${{ env.CACHE_KEY }} - path: /tmp/${{ env.IMAGE }}.tar - fail-on-cache-miss: true + name: ${{ env.IMAGE }} + path: /tmp - name: Login to Docker Hub uses: docker/login-action@v4 @@ -617,12 +625,11 @@ jobs: - name: Checkout repository uses: actions/checkout@v6 - - name: Load Cache - uses: actions/cache@v5 + - name: Download Docker Image + uses: actions/download-artifact@v7 with: - key: ${{ env.CACHE_KEY }} - path: /tmp/${{ env.IMAGE }}.tar - fail-on-cache-miss: true + name: ${{ env.IMAGE }} + path: /tmp - name: Login to Docker Hub uses: docker/login-action@v4 diff --git a/app/config/collections.php b/app/config/collections.php index a74e079dce..3af20ff2ac 100644 --- a/app/config/collections.php +++ b/app/config/collections.php @@ -4,6 +4,7 @@ $common = include __DIR__ . '/collections/common.php'; $projects = include __DIR__ . '/collections/projects.php'; $databases = include __DIR__ . '/collections/databases.php'; +$vectorsdb = include __DIR__ . '/collections/vectorsdb.php'; $platform = include __DIR__ . '/collections/platform.php'; $logs = include __DIR__ . '/collections/logs.php'; @@ -26,6 +27,7 @@ unset($common['files']); $collections = [ 'buckets' => $buckets, 'databases' => $databases, + 'vectorsdb' => $vectorsdb, 'projects' => array_merge_recursive($projects, $common), 'console' => array_merge_recursive($platform, $common), 'logs' => $logs, diff --git a/app/config/collections/projects.php b/app/config/collections/projects.php index 55dceb9b40..b41e8f0fd5 100644 --- a/app/config/collections/projects.php +++ b/app/config/collections/projects.php @@ -61,6 +61,15 @@ return [ 'array' => false, 'filters' => [], ], + [ + '$id' => ID::custom('database'), + 'type' => Database::VAR_STRING, + 'size' => 128, + 'required' => false, + 'signed' => true, + 'array' => false, + 'filters' => [], + ] ], 'indexes' => [ [ diff --git a/app/config/collections/vectorsdb.php b/app/config/collections/vectorsdb.php new file mode 100644 index 0000000000..817863cffa --- /dev/null +++ b/app/config/collections/vectorsdb.php @@ -0,0 +1,165 @@ + [ + '$collection' => ID::custom('databases'), + '$id' => ID::custom('collections'), + 'name' => 'Collections', + 'attributes' => [ + [ + '$id' => ID::custom('databaseInternalId'), + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => Database::LENGTH_KEY, + 'signed' => true, + 'required' => true, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => ID::custom('databaseId'), + 'type' => Database::VAR_STRING, + 'signed' => true, + 'size' => Database::LENGTH_KEY, + 'format' => '', + 'filters' => [], + 'required' => true, + 'default' => null, + 'array' => false, + ], + [ + '$id' => ID::custom('name'), + 'type' => Database::VAR_STRING, + 'size' => 256, + 'required' => true, + 'signed' => true, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => ID::custom('dimension'), + 'type' => Database::VAR_INTEGER, + 'size' => 0, + 'required' => true, + 'signed' => false, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => ID::custom('enabled'), + 'type' => Database::VAR_BOOLEAN, + 'signed' => true, + 'size' => 0, + 'format' => '', + 'filters' => [], + 'required' => true, + 'default' => null, + 'array' => false, + ], + [ + '$id' => ID::custom('documentSecurity'), + 'type' => Database::VAR_BOOLEAN, + 'signed' => true, + 'size' => 0, + 'format' => '', + 'filters' => [], + 'required' => true, + 'default' => null, + 'array' => false, + ], + [ + '$id' => ID::custom('attributes'), + 'type' => Database::VAR_STRING, + 'size' => 1000000, + 'required' => false, + 'signed' => true, + 'array' => false, + 'filters' => ['subQueryAttributes'], + ], + [ + '$id' => ID::custom('indexes'), + 'type' => Database::VAR_STRING, + 'size' => 1000000, + 'required' => false, + 'signed' => true, + 'array' => false, + 'filters' => ['subQueryIndexes'], + ], + [ + '$id' => ID::custom('search'), + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => 16384, + 'signed' => true, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + ], + 'defaultAttributes' => [ + [ + '$id' => ID::custom('embeddings'), + 'type' => Database::VAR_VECTOR, + 'required' => true, + 'signed' => false, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => ID::custom('metadata'), + 'type' => Database::VAR_OBJECT, + 'default' => [], + 'required' => false, + 'size' => 0, + 'signed' => false, + 'array' => false, + 'filters' => [], + ], + ], + 'indexes' => [ + [ + '$id' => ID::custom('_fulltext_search'), + 'type' => Database::INDEX_FULLTEXT, + 'attributes' => ['search'], + 'lengths' => [], + 'orders' => [], + ], + [ + '$id' => ID::custom('_key_name'), + 'type' => Database::INDEX_KEY, + 'attributes' => ['name'], + 'lengths' => [256], + 'orders' => [Database::ORDER_ASC], + ], + [ + '$id' => ID::custom('_key_enabled'), + 'type' => Database::INDEX_KEY, + 'attributes' => ['enabled'], + 'lengths' => [], + 'orders' => [Database::ORDER_ASC], + ], + [ + '$id' => ID::custom('_key_documentSecurity'), + 'type' => Database::INDEX_KEY, + 'attributes' => ['documentSecurity'], + 'lengths' => [], + 'orders' => [Database::ORDER_ASC], + ], + ], + 'defaultIndexes' => [ + // not creating default indexes on the embeddings as it depends on the type of query users using the most + [ + '$id' => ID::custom('_key_metadata'), + 'type' => Database::INDEX_OBJECT, + 'attributes' => ['metadata'], + 'lengths' => [], + 'orders' => [], + ], + ] + ] +]; diff --git a/app/config/errors.php b/app/config/errors.php index 278dbb3458..ec2593d207 100644 --- a/app/config/errors.php +++ b/app/config/errors.php @@ -1206,6 +1206,11 @@ return [ 'description' => 'Migration is already in progress. You can check the status of the migration in your Appwrite Console\'s "Settings" > "Migrations".', 'code' => 409, ], + Exception::MIGRATION_DATABASE_TYPE_UNSUPPORTED => [ + 'name' => Exception::MIGRATION_DATABASE_TYPE_UNSUPPORTED, + 'description' => 'The specified database type is not supported for CSV import or export operations.', + 'code' => 400, + ], /** Realtime */ Exception::REALTIME_MESSAGE_FORMAT_INVALID => [ diff --git a/app/controllers/api/migrations.php b/app/controllers/api/migrations.php index bfb73189b5..5a87293b49 100644 --- a/app/controllers/api/migrations.php +++ b/app/controllers/api/migrations.php @@ -43,6 +43,16 @@ use Utopia\Validator\WhiteList; include_once __DIR__ . '/../shared/api.php'; +function getDatabaseTransferResourceServices(string $databaseType) +{ + return match($databaseType) { + DATABASE_TYPE_LEGACY, + DATABASE_TYPE_TABLESDB => Transfer::GROUP_DATABASES_TABLES_DB, + DATABASE_TYPE_VECTORSDB => Transfer::GROUP_DATABASES_VECTOR_DB, + DATABASE_TYPE_DOCUMENTSDB => Transfer::GROUP_DATABASES_DOCUMENTS_DB + }; +} + Http::post('/v1/migrations/appwrite') ->groups(['api', 'migrations']) ->desc('Create Appwrite migration') @@ -427,8 +437,16 @@ Http::post('/v1/migrations/csv/imports') throw new \Exception('Unable to copy file'); } + // getting databasetype + $resources = explode(':', $resourceId); + $databaseId = $resources[0]; + $database = $authorization->skip(fn () => $dbForProject->getDocument('databases', $databaseId)); + $databaseType = $database->getAttribute('type'); + if (!in_array($databaseType, CSV_ALLOWED_DATABASE_TYPES)) { + throw new Exception(Exception::MIGRATION_DATABASE_TYPE_UNSUPPORTED, 'Database type not supported for csv'); + } $fileSize = $deviceForMigrations->getFileSize($newPath); - $resources = Transfer::extractServices([Transfer::GROUP_DATABASES]); + $resources = Transfer::extractServices([getDatabaseTransferResourceServices($databaseType)]); $migration = $dbForProject->createDocument('migrations', new Document([ '$id' => $migrationId, @@ -557,13 +575,23 @@ Http::post('/v1/migrations/csv/exports') throw new Exception(Exception::GENERAL_QUERY_INVALID, $validator->getDescription()); } + // getting databasetype + $resources = explode(':', $resourceId); + $databaseId = $resources[0]; + $database = $authorization->skip(fn () => $dbForProject->getDocument('databases', $databaseId)); + $databaseType = $database->getAttribute('type'); + if (!in_array($databaseType, CSV_ALLOWED_DATABASE_TYPES)) { + throw new Exception(Exception::MIGRATION_DATABASE_TYPE_UNSUPPORTED, 'Database type not supported for csv'); + } + $resources = Transfer::extractServices([getDatabaseTransferResourceServices($databaseType)]); + $migration = $dbForProject->createDocument('migrations', new Document([ '$id' => ID::unique(), 'status' => 'pending', 'stage' => 'init', 'source' => Appwrite::getName(), 'destination' => CSV::getName(), - 'resources' => Transfer::extractServices([Transfer::GROUP_DATABASES]), + 'resources' => $resources, 'resourceId' => $resourceId, 'resourceType' => Resource::TYPE_DATABASE, 'statusCounters' => '{}', @@ -713,12 +741,11 @@ Http::get('/v1/migrations/appwrite/report') ->param('projectID', '', new Text(512), "Source's Project ID") ->param('key', '', new Text(512), "Source's API Key") ->inject('response') - ->inject('dbForProject') - ->inject('project') - ->inject('user') - ->action(function (array $resources, string $endpoint, string $projectID, string $key, Response $response) { + ->inject('getDatabasesDB') + ->action(function (array $resources, string $endpoint, string $projectID, string $key, Response $response, callable $getDatabasesDB) { + try { - $appwrite = new Appwrite($projectID, $endpoint, $key); + $appwrite = new Appwrite($projectID, $endpoint, $key, $getDatabasesDB); $report = $appwrite->report($resources); } catch (\Throwable $e) { throw new Exception( diff --git a/app/controllers/api/project.php b/app/controllers/api/project.php index 8668a21e24..054a7c8f0d 100644 --- a/app/controllers/api/project.php +++ b/app/controllers/api/project.php @@ -52,16 +52,33 @@ Http::get('/v1/project/usage') METRIC_EXECUTIONS_MB_SECONDS, METRIC_BUILDS_MB_SECONDS, METRIC_DOCUMENTS, + METRIC_DOCUMENTS_DOCUMENTSDB, METRIC_DATABASES, + METRIC_DATABASES_DOCUMENTSDB, METRIC_USERS, METRIC_BUCKETS, METRIC_FILES_STORAGE, METRIC_DATABASES_STORAGE, + METRIC_DATABASES_STORAGE_DOCUMENTSDB, METRIC_DEPLOYMENTS_STORAGE, METRIC_BUILDS_STORAGE, METRIC_DATABASES_OPERATIONS_READS, + METRIC_DATABASES_OPERATIONS_READS_DOCUMENTSDB, METRIC_DATABASES_OPERATIONS_WRITES, + METRIC_DATABASES_OPERATIONS_WRITES_DOCUMENTSDB, METRIC_FILES_IMAGES_TRANSFORMED, + // VectorsDB totals + METRIC_DATABASES_VECTORSDB, + METRIC_COLLECTIONS_VECTORSDB, + METRIC_DOCUMENTS_VECTORSDB, + METRIC_DATABASES_STORAGE_VECTORSDB, + METRIC_DATABASES_OPERATIONS_READS_VECTORSDB, + METRIC_DATABASES_OPERATIONS_WRITES_VECTORSDB, + // Embeddings totals + METRIC_EMBEDDINGS_TEXT, + METRIC_EMBEDDINGS_TEXT_TOTAL_TOKENS, + METRIC_EMBEDDINGS_TEXT_TOTAL_DURATION, + METRIC_EMBEDDINGS_TEXT_TOTAL_ERROR ], 'period' => [ METRIC_NETWORK_REQUESTS, @@ -70,11 +87,26 @@ Http::get('/v1/project/usage') METRIC_USERS, METRIC_EXECUTIONS, METRIC_DATABASES_STORAGE, + METRIC_DATABASES_STORAGE_DOCUMENTSDB, METRIC_EXECUTIONS_MB_SECONDS, METRIC_BUILDS_MB_SECONDS, METRIC_DATABASES_OPERATIONS_READS, + METRIC_DATABASES_OPERATIONS_READS_DOCUMENTSDB, METRIC_DATABASES_OPERATIONS_WRITES, + METRIC_DATABASES_OPERATIONS_WRITES_DOCUMENTSDB, METRIC_FILES_IMAGES_TRANSFORMED, + // VectorsDB time series + METRIC_DATABASES_VECTORSDB, + METRIC_COLLECTIONS_VECTORSDB, + METRIC_DOCUMENTS_VECTORSDB, + METRIC_DATABASES_STORAGE_VECTORSDB, + METRIC_DATABASES_OPERATIONS_READS_VECTORSDB, + METRIC_DATABASES_OPERATIONS_WRITES_VECTORSDB, + // Embeddings time series + METRIC_EMBEDDINGS_TEXT, + METRIC_EMBEDDINGS_TEXT_TOTAL_TOKENS, + METRIC_EMBEDDINGS_TEXT_TOTAL_DURATION, + METRIC_EMBEDDINGS_TEXT_TOTAL_ERROR ] ]; @@ -347,8 +379,11 @@ Http::get('/v1/project/usage') 'buildsMbSecondsTotal' => $total[METRIC_BUILDS_MB_SECONDS], 'documentsTotal' => $total[METRIC_DOCUMENTS], 'rowsTotal' => $total[METRIC_DOCUMENTS], + 'documentsdbDocumentsTotal' => $total[METRIC_DOCUMENTS_DOCUMENTSDB], 'databasesTotal' => $total[METRIC_DATABASES], + 'documentsdbTotal' => $total[METRIC_DATABASES_DOCUMENTSDB], 'databasesStorageTotal' => $total[METRIC_DATABASES_STORAGE], + 'documentsdbDatabasesStorageTotal' => $total[METRIC_DATABASES_STORAGE_DOCUMENTSDB], 'usersTotal' => $total[METRIC_USERS], 'bucketsTotal' => $total[METRIC_BUCKETS], 'filesStorageTotal' => $total[METRIC_FILES_STORAGE], @@ -357,10 +392,27 @@ Http::get('/v1/project/usage') 'deploymentsStorageTotal' => $total[METRIC_DEPLOYMENTS_STORAGE], 'databasesReadsTotal' => $total[METRIC_DATABASES_OPERATIONS_READS], 'databasesWritesTotal' => $total[METRIC_DATABASES_OPERATIONS_WRITES], + 'documentsdbDatabasesReadsTotal' => $total[METRIC_DATABASES_OPERATIONS_READS_DOCUMENTSDB], + 'documentsdbDatabasesWritesTotal' => $total[METRIC_DATABASES_OPERATIONS_WRITES_DOCUMENTSDB], + 'vectorsdbDatabasesTotal' => $total[METRIC_DATABASES_VECTORSDB] ?? 0, + 'vectorsdbCollectionsTotal' => $total[METRIC_COLLECTIONS_VECTORSDB] ?? 0, + 'vectorsdbDocumentsTotal' => $total[METRIC_DOCUMENTS_VECTORSDB] ?? 0, + 'vectorsdbDatabasesStorageTotal' => $total[METRIC_DATABASES_STORAGE_VECTORSDB] ?? 0, + 'vectorsdbDatabasesReadsTotal' => $total[METRIC_DATABASES_OPERATIONS_READS_VECTORSDB] ?? 0, + 'vectorsdbDatabasesWritesTotal' => $total[METRIC_DATABASES_OPERATIONS_WRITES_VECTORSDB] ?? 0, 'executionsBreakdown' => $executionsBreakdown, 'bucketsBreakdown' => $bucketsBreakdown, 'databasesReads' => $usage[METRIC_DATABASES_OPERATIONS_READS], 'databasesWrites' => $usage[METRIC_DATABASES_OPERATIONS_WRITES], + 'documentsdbDatabasesReads' => $usage[METRIC_DATABASES_OPERATIONS_READS_DOCUMENTSDB], + 'documentsdbDatabasesWrites' => $usage[METRIC_DATABASES_OPERATIONS_WRITES_DOCUMENTSDB], + 'documentsdbDatabasesStorage' => $usage[METRIC_DATABASES_STORAGE_DOCUMENTSDB], + 'vectorsdbDatabases' => $usage[METRIC_DATABASES_VECTORSDB] ?? [], + 'vectorsdbCollections' => $usage[METRIC_COLLECTIONS_VECTORSDB] ?? [], + 'vectorsdbDocuments' => $usage[METRIC_DOCUMENTS_VECTORSDB] ?? [], + 'vectorsdbDatabasesStorage' => $usage[METRIC_DATABASES_STORAGE_VECTORSDB] ?? [], + 'vectorsdbDatabasesReads' => $usage[METRIC_DATABASES_OPERATIONS_READS_VECTORSDB] ?? [], + 'vectorsdbDatabasesWrites' => $usage[METRIC_DATABASES_OPERATIONS_WRITES_VECTORSDB] ?? [], 'databasesStorageBreakdown' => $databasesStorageBreakdown, 'executionsMbSecondsBreakdown' => $executionsMbSecondsBreakdown, 'buildsMbSecondsBreakdown' => $buildsMbSecondsBreakdown, @@ -370,5 +422,13 @@ Http::get('/v1/project/usage') 'authPhoneCountryBreakdown' => $authPhoneCountryBreakdown, 'imageTransformations' => $usage[METRIC_FILES_IMAGES_TRANSFORMED], 'imageTransformationsTotal' => $total[METRIC_FILES_IMAGES_TRANSFORMED], + 'embeddingsText' => $usage[METRIC_EMBEDDINGS_TEXT] ?? [], + 'embeddingsTextTokens' => $usage[METRIC_EMBEDDINGS_TEXT_TOTAL_TOKENS] ?? [], + 'embeddingsTextDuration' => $usage[METRIC_EMBEDDINGS_TEXT_TOTAL_DURATION] ?? [], + 'embeddingsTextErrors' => $usage[METRIC_EMBEDDINGS_TEXT_TOTAL_ERROR] ?? [], + 'embeddingsTextTotal' => $total[METRIC_EMBEDDINGS_TEXT] ?? 0, + 'embeddingsTextTokensTotal' => $total[METRIC_EMBEDDINGS_TEXT_TOTAL_TOKENS] ?? 0, + 'embeddingsTextDurationTotal' => $total[METRIC_EMBEDDINGS_TEXT_TOTAL_DURATION] ?? 0, + 'embeddingsTextErrorsTotal' => $total[METRIC_EMBEDDINGS_TEXT_TOTAL_ERROR] ?? 0, ]), Response::MODEL_USAGE_PROJECT); }); diff --git a/app/controllers/api/projects.php b/app/controllers/api/projects.php index 24a1b28cdd..cef64ffdeb 100644 --- a/app/controllers/api/projects.php +++ b/app/controllers/api/projects.php @@ -4,7 +4,6 @@ use Ahc\Jwt\JWT; use Appwrite\Auth\Validator\MockNumber; use Appwrite\Event\Delete; use Appwrite\Event\Mail; -use Appwrite\Event\Validator\Event; use Appwrite\Extend\Exception; use Appwrite\Network\Platform; use Appwrite\Network\Validator\Email; @@ -30,7 +29,6 @@ use Utopia\Database\Query; use Utopia\Database\Validator\Datetime as DatetimeValidator; use Utopia\Database\Validator\Query\Cursor; use Utopia\Database\Validator\UID; -use Utopia\Domains\Validator\PublicDomain; use Utopia\Http\Http; use Utopia\Locale\Locale; use Utopia\System\System; @@ -38,11 +36,9 @@ use Utopia\Validator\ArrayList; use Utopia\Validator\Boolean; use Utopia\Validator\Hostname; use Utopia\Validator\Integer; -use Utopia\Validator\Multiple; use Utopia\Validator\Nullable; use Utopia\Validator\Range; use Utopia\Validator\Text; -use Utopia\Validator\URL; use Utopia\Validator\WhiteList; Http::init() @@ -773,312 +769,6 @@ Http::delete('/v1/projects/:projectId') $response->noContent(); }); -// Webhooks - -Http::post('/v1/projects/:projectId/webhooks') - ->desc('Create webhook') - ->groups(['api', 'projects']) - ->label('scope', 'projects.write') - ->label('sdk', new Method( - namespace: 'projects', - group: 'webhooks', - name: 'createWebhook', - description: '/docs/references/projects/create-webhook.md', - auth: [AuthType::ADMIN], - responses: [ - new SDKResponse( - code: Response::STATUS_CODE_CREATED, - model: Response::MODEL_WEBHOOK, - ) - ] - )) - ->param('projectId', '', fn (Database $dbForPlatform) => new UID($dbForPlatform->getAdapter()->getMaxUIDLength()), 'Project unique ID.', false, ['dbForPlatform']) - ->param('name', null, new Text(128), 'Webhook name. Max length: 128 chars.') - ->param('enabled', true, new Boolean(true), 'Enable or disable a webhook.', true) - ->param('events', null, new ArrayList(new Event(), APP_LIMIT_ARRAY_PARAMS_SIZE), 'Events list. Maximum of ' . APP_LIMIT_ARRAY_PARAMS_SIZE . ' events are allowed.') - ->param('url', '', fn ($request) => new Multiple([new URL(['http', 'https']), new PublicDomain()], Multiple::TYPE_STRING), 'Webhook URL.', false, ['request']) - ->param('security', false, new Boolean(true), 'Certificate verification, false for disabled or true for enabled.') - ->param('httpUser', '', new Text(256), 'Webhook HTTP user. Max length: 256 chars.', true) - ->param('httpPass', '', new Text(256), 'Webhook HTTP password. Max length: 256 chars.', true) - ->inject('response') - ->inject('dbForPlatform') - ->action(function (string $projectId, string $name, bool $enabled, array $events, string $url, bool $security, string $httpUser, string $httpPass, Response $response, Database $dbForPlatform) { - - $project = $dbForPlatform->getDocument('projects', $projectId); - - if ($project->isEmpty()) { - throw new Exception(Exception::PROJECT_NOT_FOUND); - } - - $security = (bool) filter_var($security, FILTER_VALIDATE_BOOLEAN); - - $webhook = new Document([ - '$id' => ID::unique(), - '$permissions' => [ - Permission::read(Role::any()), - Permission::update(Role::any()), - Permission::delete(Role::any()), - ], - 'projectInternalId' => $project->getSequence(), - 'projectId' => $project->getId(), - 'name' => $name, - 'events' => $events, - 'url' => $url, - 'security' => $security, - 'httpUser' => $httpUser, - 'httpPass' => $httpPass, - 'signatureKey' => \bin2hex(\random_bytes(64)), - 'enabled' => $enabled, - ]); - - $webhook = $dbForPlatform->createDocument('webhooks', $webhook); - - $dbForPlatform->purgeCachedDocument('projects', $project->getId()); - - $response - ->setStatusCode(Response::STATUS_CODE_CREATED) - ->dynamic($webhook, Response::MODEL_WEBHOOK); - }); - -Http::get('/v1/projects/:projectId/webhooks') - ->desc('List webhooks') - ->groups(['api', 'projects']) - ->label('scope', 'projects.read') - ->label('sdk', new Method( - namespace: 'projects', - group: 'webhooks', - name: 'listWebhooks', - description: '/docs/references/projects/list-webhooks.md', - auth: [AuthType::ADMIN], - responses: [ - new SDKResponse( - code: Response::STATUS_CODE_OK, - model: Response::MODEL_WEBHOOK_LIST, - ) - ] - )) - ->param('projectId', '', fn (Database $dbForPlatform) => new UID($dbForPlatform->getAdapter()->getMaxUIDLength()), 'Project unique ID.', false, ['dbForPlatform']) - ->param('total', true, new Boolean(true), 'When set to false, the total count returned will be 0 and will not be calculated.', true) - ->inject('response') - ->inject('dbForPlatform') - ->action(function (string $projectId, bool $includeTotal, Response $response, Database $dbForPlatform) { - - $project = $dbForPlatform->getDocument('projects', $projectId); - - if ($project->isEmpty()) { - throw new Exception(Exception::PROJECT_NOT_FOUND); - } - - $webhooks = $dbForPlatform->find('webhooks', [ - Query::equal('projectInternalId', [$project->getSequence()]), - Query::limit(5000), - ]); - - $response->dynamic(new Document([ - 'webhooks' => $webhooks, - 'total' => $includeTotal ? count($webhooks) : 0, - ]), Response::MODEL_WEBHOOK_LIST); - }); - -Http::get('/v1/projects/:projectId/webhooks/:webhookId') - ->desc('Get webhook') - ->groups(['api', 'projects']) - ->label('scope', 'projects.read') - ->label('sdk', new Method( - namespace: 'projects', - group: 'webhooks', - name: 'getWebhook', - description: '/docs/references/projects/get-webhook.md', - auth: [AuthType::ADMIN], - responses: [ - new SDKResponse( - code: Response::STATUS_CODE_OK, - model: Response::MODEL_WEBHOOK, - ) - ] - )) - ->param('projectId', '', fn (Database $dbForPlatform) => new UID($dbForPlatform->getAdapter()->getMaxUIDLength()), 'Project unique ID.', false, ['dbForPlatform']) - ->param('webhookId', '', fn (Database $dbForPlatform) => new UID($dbForPlatform->getAdapter()->getMaxUIDLength()), 'Webhook unique ID.', false, ['dbForPlatform']) - ->inject('response') - ->inject('dbForPlatform') - ->action(function (string $projectId, string $webhookId, Response $response, Database $dbForPlatform) { - - $project = $dbForPlatform->getDocument('projects', $projectId); - - if ($project->isEmpty()) { - throw new Exception(Exception::PROJECT_NOT_FOUND); - } - - $webhook = $dbForPlatform->findOne('webhooks', [ - Query::equal('$id', [$webhookId]), - Query::equal('projectInternalId', [$project->getSequence()]), - ]); - - if ($webhook->isEmpty()) { - throw new Exception(Exception::WEBHOOK_NOT_FOUND); - } - - $response->dynamic($webhook, Response::MODEL_WEBHOOK); - }); - -Http::put('/v1/projects/:projectId/webhooks/:webhookId') - ->desc('Update webhook') - ->groups(['api', 'projects']) - ->label('scope', 'projects.write') - ->label('sdk', new Method( - namespace: 'projects', - group: 'webhooks', - name: 'updateWebhook', - description: '/docs/references/projects/update-webhook.md', - auth: [AuthType::ADMIN], - responses: [ - new SDKResponse( - code: Response::STATUS_CODE_OK, - model: Response::MODEL_WEBHOOK, - ) - ] - )) - ->param('projectId', '', fn (Database $dbForPlatform) => new UID($dbForPlatform->getAdapter()->getMaxUIDLength()), 'Project unique ID.', false, ['dbForPlatform']) - ->param('webhookId', '', fn (Database $dbForPlatform) => new UID($dbForPlatform->getAdapter()->getMaxUIDLength()), 'Webhook unique ID.', false, ['dbForPlatform']) - ->param('name', null, new Text(128), 'Webhook name. Max length: 128 chars.') - ->param('enabled', true, new Boolean(true), 'Enable or disable a webhook.', true) - ->param('events', null, new ArrayList(new Event(), APP_LIMIT_ARRAY_PARAMS_SIZE), 'Events list. Maximum of ' . APP_LIMIT_ARRAY_PARAMS_SIZE . ' events are allowed.') - ->param('url', '', fn ($request) => new Multiple([new URL(['http', 'https']), new PublicDomain()], Multiple::TYPE_STRING), 'Webhook URL.', false, ['request']) - ->param('security', false, new Boolean(true), 'Certificate verification, false for disabled or true for enabled.') - ->param('httpUser', '', new Text(256), 'Webhook HTTP user. Max length: 256 chars.', true) - ->param('httpPass', '', new Text(256), 'Webhook HTTP password. Max length: 256 chars.', true) - ->inject('response') - ->inject('dbForPlatform') - ->action(function (string $projectId, string $webhookId, string $name, bool $enabled, array $events, string $url, bool $security, string $httpUser, string $httpPass, Response $response, Database $dbForPlatform) { - - $project = $dbForPlatform->getDocument('projects', $projectId); - - if ($project->isEmpty()) { - throw new Exception(Exception::PROJECT_NOT_FOUND); - } - - $security = ($security === '1' || $security === 'true' || $security === 1 || $security === true); - - $webhook = $dbForPlatform->findOne('webhooks', [ - Query::equal('$id', [$webhookId]), - Query::equal('projectInternalId', [$project->getSequence()]), - ]); - - if ($webhook->isEmpty()) { - throw new Exception(Exception::WEBHOOK_NOT_FOUND); - } - - $webhook - ->setAttribute('name', $name) - ->setAttribute('events', $events) - ->setAttribute('url', $url) - ->setAttribute('security', $security) - ->setAttribute('httpUser', $httpUser) - ->setAttribute('httpPass', $httpPass) - ->setAttribute('enabled', $enabled); - - if ($enabled) { - $webhook->setAttribute('attempts', 0); - } - - $dbForPlatform->updateDocument('webhooks', $webhook->getId(), $webhook); - $dbForPlatform->purgeCachedDocument('projects', $project->getId()); - - $response->dynamic($webhook, Response::MODEL_WEBHOOK); - }); - -Http::patch('/v1/projects/:projectId/webhooks/:webhookId/signature') - ->desc('Update webhook signature key') - ->groups(['api', 'projects']) - ->label('scope', 'projects.write') - ->label('sdk', new Method( - namespace: 'projects', - group: 'webhooks', - name: 'updateWebhookSignature', - description: '/docs/references/projects/update-webhook-signature.md', - auth: [AuthType::ADMIN], - responses: [ - new SDKResponse( - code: Response::STATUS_CODE_OK, - model: Response::MODEL_WEBHOOK, - ) - ] - )) - ->param('projectId', '', fn (Database $dbForPlatform) => new UID($dbForPlatform->getAdapter()->getMaxUIDLength()), 'Project unique ID.', false, ['dbForPlatform']) - ->param('webhookId', '', fn (Database $dbForPlatform) => new UID($dbForPlatform->getAdapter()->getMaxUIDLength()), 'Webhook unique ID.', false, ['dbForPlatform']) - ->inject('response') - ->inject('dbForPlatform') - ->action(function (string $projectId, string $webhookId, Response $response, Database $dbForPlatform) { - - $project = $dbForPlatform->getDocument('projects', $projectId); - - if ($project->isEmpty()) { - throw new Exception(Exception::PROJECT_NOT_FOUND); - } - - $webhook = $dbForPlatform->findOne('webhooks', [ - Query::equal('$id', [$webhookId]), - Query::equal('projectInternalId', [$project->getSequence()]), - ]); - - if ($webhook->isEmpty()) { - throw new Exception(Exception::WEBHOOK_NOT_FOUND); - } - - $webhook->setAttribute('signatureKey', \bin2hex(\random_bytes(64))); - - $dbForPlatform->updateDocument('webhooks', $webhook->getId(), $webhook); - $dbForPlatform->purgeCachedDocument('projects', $project->getId()); - - $response->dynamic($webhook, Response::MODEL_WEBHOOK); - }); - -Http::delete('/v1/projects/:projectId/webhooks/:webhookId') - ->desc('Delete webhook') - ->groups(['api', 'projects']) - ->label('scope', 'projects.write') - ->label('sdk', new Method( - namespace: 'projects', - group: 'webhooks', - name: 'deleteWebhook', - description: '/docs/references/projects/delete-webhook.md', - auth: [AuthType::ADMIN], - responses: [ - new SDKResponse( - code: Response::STATUS_CODE_NOCONTENT, - model: Response::MODEL_NONE, - ) - ], - contentType: ContentType::NONE - )) - ->param('projectId', '', fn (Database $dbForPlatform) => new UID($dbForPlatform->getAdapter()->getMaxUIDLength()), 'Project unique ID.', false, ['dbForPlatform']) - ->param('webhookId', '', fn (Database $dbForPlatform) => new UID($dbForPlatform->getAdapter()->getMaxUIDLength()), 'Webhook unique ID.', false, ['dbForPlatform']) - ->inject('response') - ->inject('dbForPlatform') - ->action(function (string $projectId, string $webhookId, Response $response, Database $dbForPlatform) { - - $project = $dbForPlatform->getDocument('projects', $projectId); - - if ($project->isEmpty()) { - throw new Exception(Exception::PROJECT_NOT_FOUND); - } - - $webhook = $dbForPlatform->findOne('webhooks', [ - Query::equal('$id', [$webhookId]), - Query::equal('projectInternalId', [$project->getSequence()]), - ]); - - if ($webhook->isEmpty()) { - throw new Exception(Exception::WEBHOOK_NOT_FOUND); - } - - $dbForPlatform->deleteDocument('webhooks', $webhook->getId()); - - $dbForPlatform->purgeCachedDocument('projects', $project->getId()); - - $response->noContent(); - }); - // Keys Http::post('/v1/projects/:projectId/keys') diff --git a/app/controllers/general.php b/app/controllers/general.php index 1a099c4bde..341ac59f7a 100644 --- a/app/controllers/general.php +++ b/app/controllers/general.php @@ -888,7 +888,7 @@ Http::init() $dbForProject = $getProjectDB($project); $request->addFilter(new RequestV20($dbForProject, $route->getPathValues($request))); } - if (version_compare($requestFormat, '1.9.0', '<')) { + if (version_compare($requestFormat, '1.8.2', '<')) { $request->addFilter(new RequestV21()); } } diff --git a/app/controllers/shared/api.php b/app/controllers/shared/api.php index acc9c2dc9a..fc8b9b589b 100644 --- a/app/controllers/shared/api.php +++ b/app/controllers/shared/api.php @@ -470,6 +470,12 @@ Http::init() ->action(function (Http $utopia, Request $request, Response $response, Document $project, Document $user, Event $queueForEvents, Messaging $queueForMessaging, Audit $queueForAudits, Delete $queueForDeletes, EventDatabase $queueForDatabase, Build $queueForBuilds, Context $usage, Func $queueForFunctions, Mail $queueForMails, Database $dbForProject, callable $timelimit, Document $resourceToken, string $mode, ?Key $apiKey, array $plan, Document $devKey, Telemetry $telemetry, array $platform, Authorization $authorization) { $route = $utopia->getRoute(); + $path = $route->getMatchedPath(); + $databaseType = match (true) { + str_contains($path, '/documentsdb') => DATABASE_TYPE_DOCUMENTSDB, + str_contains($path, '/vectorsdb') => DATABASE_TYPE_VECTORSDB, + default => '', + }; if ( array_key_exists('rest', $project->getAttribute('apis', [])) diff --git a/app/http.php b/app/http.php index 1302940856..517a1aa544 100644 --- a/app/http.php +++ b/app/http.php @@ -196,6 +196,8 @@ include __DIR__ . '/controllers/general.php'; function createDatabase(Http $app, string $resourceKey, string $dbName, array $collections, mixed $pools, ?callable $extraSetup = null): void { + $max = 15; + $sleep = 2; $max = 15; $sleep = 2; $attempts = 0; @@ -409,13 +411,29 @@ $http->on(Constant::EVENT_START, function (Server $http) use ($payloadSize, $tot }); $projectCollections = $collections['projects']; + $sharedTables = \explode(',', System::getEnv('_APP_DATABASE_SHARED_TABLES', '')); $sharedTablesV1 = \explode(',', System::getEnv('_APP_DATABASE_SHARED_TABLES_V1', '')); $sharedTablesV2 = \array_diff($sharedTables, $sharedTablesV1); + $documentsSharedTables = \explode(',', System::getEnv('_APP_DATABASE_DOCUMENTSDB_SHARED_TABLES', '')); + $documentsSharedTablesV1 = \explode(',', System::getEnv('_APP_DATABASE_DOCUMENTSDB_SHARED_TABLES_V1', '')); + $documentsSharedTablesV2 = \array_diff($documentsSharedTables, $documentsSharedTablesV1); + + $vectorSharedTables = \explode(',', System::getEnv('_APP_DATABASE_VECTORSDB_SHARED_TABLES', '')); + $vectorSharedTablesV1 = \explode(',', System::getEnv('_APP_DATABASE_VECTORSDB_SHARED_TABLES_V1', '')); + $vectorSharedTablesV2 = \array_diff($vectorSharedTables, $vectorSharedTablesV1); + $cache = $app->getResource('cache'); - foreach ($sharedTablesV2 as $hostname) { + // All shared tables V2 pools that need project metadata collections + $sharedTablesV2All = \array_values(\array_unique(\array_filter([ + ...$sharedTablesV2, + ...$documentsSharedTablesV2, + ...$vectorSharedTablesV2, + ]))); + + foreach ($sharedTablesV2All as $hostname) { Span::init('database.setup'); Span::add('database.hostname', $hostname); diff --git a/app/init/constants.php b/app/init/constants.php index 7a484c7f4e..fe61316955 100644 --- a/app/init/constants.php +++ b/app/init/constants.php @@ -47,7 +47,7 @@ const APP_RESOURCE_TOKEN_ACCESS = 24 * 60 * 60; // 24 hours const APP_FILE_ACCESS = 24 * 60 * 60; // 24 hours const APP_CACHE_UPDATE = 24 * 60 * 60; // 24 hours const APP_CACHE_BUSTER = 4321; -const APP_VERSION_STABLE = '1.8.1'; +const APP_VERSION_STABLE = '1.8.2'; const APP_DATABASE_ATTRIBUTE_EMAIL = 'email'; const APP_DATABASE_ATTRIBUTE_ENUM = 'enum'; const APP_DATABASE_ATTRIBUTE_IP = 'ip'; @@ -288,6 +288,45 @@ const METRIC_DATABASES_OPERATIONS_READS = 'databases.operations.reads'; const METRIC_DATABASE_ID_OPERATIONS_READS = '{databaseInternalId}.databases.operations.reads'; const METRIC_DATABASES_OPERATIONS_WRITES = 'databases.operations.writes'; const METRIC_DATABASE_ID_OPERATIONS_WRITES = '{databaseInternalId}.databases.operations.writes'; + +// documentsdb +const METRIC_DATABASES_DOCUMENTSDB = 'documentsdb.databases'; +const METRIC_COLLECTIONS_DOCUMENTSDB = 'documentsdb.collections'; +const METRIC_DATABASES_STORAGE_DOCUMENTSDB = 'documentsdb.databases.storage'; +const METRIC_DATABASE_ID_COLLECTIONS_DOCUMENTSDB = 'documentsdb.{databaseInternalId}.collections'; +const METRIC_DATABASE_ID_STORAGE_DOCUMENTSDB = 'documentsdb.{databaseInternalId}.databases.storage'; +const METRIC_DOCUMENTS_DOCUMENTSDB = 'documentsdb.documents'; +const METRIC_DATABASE_ID_DOCUMENTS_DOCUMENTSDB = 'documentsdb.{databaseInternalId}.documents'; +const METRIC_DATABASE_ID_COLLECTION_ID_DOCUMENTS_DOCUMENTSDB = 'documentsdb.{databaseInternalId}.{collectionInternalId}.documents'; +const METRIC_DATABASE_ID_COLLECTION_ID_STORAGE_DOCUMENTSDB = 'documentsdb.{databaseInternalId}.{collectionInternalId}.databases.storage'; +const METRIC_DATABASES_OPERATIONS_READS_DOCUMENTSDB = 'documentsdb.databases.operations.reads'; +const METRIC_DATABASE_ID_OPERATIONS_READS_DOCUMENTSDB = 'documentsdb.{databaseInternalId}.databases.operations.reads'; +const METRIC_DATABASES_OPERATIONS_WRITES_DOCUMENTSDB = 'documentsdb.databases.operations.writes'; +const METRIC_DATABASE_ID_OPERATIONS_WRITES_DOCUMENTSDB = 'documentsdb.{databaseInternalId}.databases.operations.writes'; + +// vectorsdb +const METRIC_DATABASES_VECTORSDB = 'vectorsdb.databases'; +const METRIC_COLLECTIONS_VECTORSDB = 'vectorsdb.collections'; +const METRIC_DATABASES_STORAGE_VECTORSDB = 'vectorsdb.databases.storage'; +const METRIC_DATABASE_ID_COLLECTIONS_VECTORSDB = 'vectorsdb.{databaseInternalId}.collections'; +const METRIC_DATABASE_ID_STORAGE_VECTORSDB = 'vectorsdb.{databaseInternalId}.databases.storage'; +const METRIC_DOCUMENTS_VECTORSDB = 'vectorsdb.documents'; +const METRIC_DATABASE_ID_DOCUMENTS_VECTORSDB = 'vectorsdb.{databaseInternalId}.documents'; +const METRIC_DATABASE_ID_COLLECTION_ID_DOCUMENTS_VECTORSDB = 'vectorsdb.{databaseInternalId}.{collectionInternalId}.documents'; +const METRIC_DATABASE_ID_COLLECTION_ID_STORAGE_VECTORSDB = 'vectorsdb.{databaseInternalId}.{collectionInternalId}.databases.storage'; +const METRIC_DATABASES_OPERATIONS_READS_VECTORSDB = 'vectorsdb.databases.operations.reads'; +const METRIC_DATABASE_ID_OPERATIONS_READS_VECTORSDB = 'vectorsdb.{databaseInternalId}.databases.operations.reads'; +const METRIC_DATABASES_OPERATIONS_WRITES_VECTORSDB = 'vectorsdb.databases.operations.writes'; +const METRIC_DATABASE_ID_OPERATIONS_WRITES_VECTORSDB = 'vectorsdb.{databaseInternalId}.databases.operations.writes'; +const METRIC_EMBEDDINGS_TEXT = 'embeddings.text'; +const METRIC_EMBEDDINGS_MODEL_TEXT = 'embeddings.text.{embeddingModel}'; +const METRIC_EMBEDDINGS_TEXT_TOTAL_ERROR = 'embeddings.text.totalErrors'; +const METRIC_EMBEDDINGS_MODEL_TEXT_TOTAL_ERROR = 'embeddings.text.{embeddingModel}.totalErrors'; +const METRIC_EMBEDDINGS_TEXT_TOTAL_DURATION = 'embeddings.text.totalDuration'; +const METRIC_EMBEDDINGS_MODEL_TEXT_TOTAL_DURATION = 'embeddings.text.{embeddingModel}.totalDuration'; +const METRIC_EMBEDDINGS_TEXT_TOTAL_TOKENS = 'embeddings.text.totalTokens'; +const METRIC_EMBEDDINGS_MODEL_TEXT_TOTAL_TOKENS = 'embeddings.text.{embeddingModel}.totalTokens'; + const METRIC_BUCKETS = 'buckets'; const METRIC_FILES = 'files'; const METRIC_FILES_STORAGE = 'files.storage'; @@ -380,6 +419,7 @@ const RESOURCE_TYPE_SUBSCRIBERS = 'subscribers'; const RESOURCE_TYPE_MESSAGES = 'messages'; const RESOURCE_TYPE_EXECUTIONS = 'executions'; const RESOURCE_TYPE_VCS = 'vcs'; +const RESOURCE_TYPE_EMBEDDINGS_TEXT = 'embeddingsText'; // Resource types for Tokens const TOKENS_RESOURCE_TYPE_FILES = 'files'; @@ -401,3 +441,16 @@ const CACHE_RECONNECT_RETRY_DELAY = 1000; // Project status const PROJECT_STATUS_ACTIVE = 'active'; + +// Database types +const DATABASE_TYPE_LEGACY = 'legacy'; +const DATABASE_TYPE_TABLESDB = 'tablesdb'; +const DATABASE_TYPE_DOCUMENTSDB = 'documentsdb'; +const DATABASE_TYPE_VECTORSDB = 'vectorsdb'; + +// CSV import/export allowed database types +const CSV_ALLOWED_DATABASE_TYPES = [ + DATABASE_TYPE_LEGACY, + DATABASE_TYPE_TABLESDB, + DATABASE_TYPE_VECTORSDB +]; diff --git a/app/init/models.php b/app/init/models.php index d432852660..bf6d67dd95 100644 --- a/app/init/models.php +++ b/app/init/models.php @@ -22,6 +22,7 @@ use Appwrite\Utopia\Response\Model\AttributeLine; use Appwrite\Utopia\Response\Model\AttributeList; use Appwrite\Utopia\Response\Model\AttributeLongtext; use Appwrite\Utopia\Response\Model\AttributeMediumtext; +use Appwrite\Utopia\Response\Model\AttributeObject; use Appwrite\Utopia\Response\Model\AttributePoint; use Appwrite\Utopia\Response\Model\AttributePolygon; use Appwrite\Utopia\Response\Model\AttributeRelationship; @@ -29,6 +30,7 @@ use Appwrite\Utopia\Response\Model\AttributeString; use Appwrite\Utopia\Response\Model\AttributeText; use Appwrite\Utopia\Response\Model\AttributeURL; use Appwrite\Utopia\Response\Model\AttributeVarchar; +use Appwrite\Utopia\Response\Model\AttributeVector; use Appwrite\Utopia\Response\Model\AuthProvider; use Appwrite\Utopia\Response\Model\BaseList; use Appwrite\Utopia\Response\Model\Branch; @@ -65,6 +67,7 @@ use Appwrite\Utopia\Response\Model\DetectionRuntime; use Appwrite\Utopia\Response\Model\DetectionVariable; use Appwrite\Utopia\Response\Model\DevKey; use Appwrite\Utopia\Response\Model\Document as ModelDocument; +use Appwrite\Utopia\Response\Model\Embedding; use Appwrite\Utopia\Response\Model\Error; use Appwrite\Utopia\Response\Model\ErrorDev; use Appwrite\Utopia\Response\Model\Execution; @@ -136,6 +139,8 @@ use Appwrite\Utopia\Response\Model\UsageBuckets; use Appwrite\Utopia\Response\Model\UsageCollection; use Appwrite\Utopia\Response\Model\UsageDatabase; use Appwrite\Utopia\Response\Model\UsageDatabases; +use Appwrite\Utopia\Response\Model\UsageDocumentsDB; +use Appwrite\Utopia\Response\Model\UsageDocumentsDBs; use Appwrite\Utopia\Response\Model\UsageFunction; use Appwrite\Utopia\Response\Model\UsageFunctions; use Appwrite\Utopia\Response\Model\UsageProject; @@ -144,9 +149,12 @@ use Appwrite\Utopia\Response\Model\UsageSites; use Appwrite\Utopia\Response\Model\UsageStorage; use Appwrite\Utopia\Response\Model\UsageTable; use Appwrite\Utopia\Response\Model\UsageUsers; +use Appwrite\Utopia\Response\Model\UsageVectorsDB; +use Appwrite\Utopia\Response\Model\UsageVectorsDBs; use Appwrite\Utopia\Response\Model\User; use Appwrite\Utopia\Response\Model\Variable; use Appwrite\Utopia\Response\Model\VcsContent; +use Appwrite\Utopia\Response\Model\VectorsDBCollection; use Appwrite\Utopia\Response\Model\Webhook; // General @@ -185,7 +193,7 @@ Response::setModel(new BaseList('Runtimes List', Response::MODEL_RUNTIME_LIST, ' Response::setModel(new BaseList('Deployments List', Response::MODEL_DEPLOYMENT_LIST, 'deployments', Response::MODEL_DEPLOYMENT)); Response::setModel(new BaseList('Executions List', Response::MODEL_EXECUTION_LIST, 'executions', Response::MODEL_EXECUTION)); Response::setModel(new BaseList('Projects List', Response::MODEL_PROJECT_LIST, 'projects', Response::MODEL_PROJECT, true, false)); -Response::setModel(new BaseList('Webhooks List', Response::MODEL_WEBHOOK_LIST, 'webhooks', Response::MODEL_WEBHOOK, true, false)); +Response::setModel(new BaseList('Webhooks List', Response::MODEL_WEBHOOK_LIST, 'webhooks', Response::MODEL_WEBHOOK, true, true)); Response::setModel(new BaseList('API Keys List', Response::MODEL_KEY_LIST, 'keys', Response::MODEL_KEY, true, true)); Response::setModel(new BaseList('Dev Keys List', Response::MODEL_DEV_KEY_LIST, 'devKeys', Response::MODEL_DEV_KEY, true, false)); Response::setModel(new BaseList('Auth Providers List', Response::MODEL_AUTH_PROVIDER_LIST, 'platforms', Response::MODEL_AUTH_PROVIDER, true, false)); @@ -211,9 +219,12 @@ Response::setModel(new BaseList('Migrations List', Response::MODEL_MIGRATION_LIS Response::setModel(new BaseList('Migrations Firebase Projects List', Response::MODEL_MIGRATION_FIREBASE_PROJECT_LIST, 'projects', Response::MODEL_MIGRATION_FIREBASE_PROJECT)); Response::setModel(new BaseList('Specifications List', Response::MODEL_SPECIFICATION_LIST, 'specifications', Response::MODEL_SPECIFICATION)); Response::setModel(new BaseList('VCS Content List', Response::MODEL_VCS_CONTENT_LIST, 'contents', Response::MODEL_VCS_CONTENT)); +Response::setModel(new BaseList('VectorsDB Collections List', Response::MODEL_VECTORSDB_COLLECTION_LIST, 'collections', Response::MODEL_VECTORSDB_COLLECTION)); +Response::setModel(new BaseList('Embedding list', Response::MODEL_EMBEDDING_LIST, 'embeddings', Response::MODEL_EMBEDDING)); // Entities Response::setModel(new Database()); +Response::setModel(new Embedding()); // Collection API Models Response::setModel(new Collection()); @@ -237,6 +248,17 @@ Response::setModel(new AttributeText()); Response::setModel(new AttributeMediumtext()); Response::setModel(new AttributeLongtext()); +// DocumentsDB API Models +Response::setModel(new UsageDocumentsDBs()); +Response::setModel(new UsageDocumentsDB()); + +// VectorsDB API Models +Response::setModel(new VectorsDBCollection()); +Response::setModel(new AttributeObject()); +Response::setModel(new AttributeVector()); +Response::setModel(new UsageVectorsDBs()); +Response::setModel(new UsageVectorsDB()); + // Table API Models Response::setModel(new Table()); Response::setModel(new Column()); diff --git a/app/init/registers.php b/app/init/registers.php index 7b68c2af9a..7c2f822fdd 100644 --- a/app/init/registers.php +++ b/app/init/registers.php @@ -160,7 +160,6 @@ $register->set('pools', function () { 'pass' => System::getEnv('_APP_DB_PASS', ''), 'path' => System::getEnv('_APP_DB_SCHEMA', ''), ]); - $fallbackForRedis = 'redis_main=' . AppwriteURL::unparse([ 'scheme' => 'redis', 'host' => System::getEnv('_APP_REDIS_HOST', 'redis'), @@ -169,6 +168,23 @@ $register->set('pools', function () { 'pass' => System::getEnv('_APP_REDIS_PASS', ''), ]); + $fallbackForDocumentsDB = 'db_main=' . AppwriteURL::unparse([ + 'scheme' => System::getEnv('_APP_DB_ADAPTER_DOCUMENTSDB', 'mongodb'), + 'host' => System::getEnv('_APP_DB_HOST_DOCUMENTSDB', 'mongodb'), + 'port' => System::getEnv('_APP_DB_PORT_DOCUMENTSDB', '27017'), + 'user' => System::getEnv('_APP_DB_USER', ''), + 'pass' => System::getEnv('_APP_DB_PASS', ''), + 'path' => System::getEnv('_APP_DB_SCHEMA', ''), + ]); + $fallbackForVectorsDB = 'db_main=' . AppwriteURL::unparse([ + 'scheme' => System::getEnv('_APP_DB_ADAPTER_VECTORSDB', 'postgresql'), + 'host' => System::getEnv('_APP_DB_HOST_VECTORSDB', 'postgresql'), + 'port' => System::getEnv('_APP_DB_PORT_VECTORSDB', '5432'), + 'user' => System::getEnv('_APP_DB_USER', ''), + 'pass' => System::getEnv('_APP_DB_PASS', ''), + 'path' => System::getEnv('_APP_DB_SCHEMA', ''), + ]); + $connections = [ 'console' => [ 'type' => 'database', @@ -180,13 +196,25 @@ $register->set('pools', function () { 'type' => 'database', 'dsns' => $fallbackForDB, 'multiple' => true, - 'schemes' => ['mariadb', 'mongodb', 'mysql', 'postgresql'], + 'schemes' => ['mongodb','mariadb', 'mysql','postgresql'], + ], + 'documentsdb' => [ + 'type' => 'database', + 'dsns' => System::getEnv('_APP_CONNECTIONS_DATABASE_DOCUMENTSDB', $fallbackForDocumentsDB), + 'multiple' => true, + 'schemes' => ['mongodb'], + ], + 'vectorsdb' => [ + 'type' => 'database', + 'dsns' => System::getEnv('_APP_CONNECTIONS_DATABASE_VECTORSDB', $fallbackForVectorsDB), + 'multiple' => true, + 'schemes' => ['postgresql'], ], 'logs' => [ 'type' => 'database', 'dsns' => System::getEnv('_APP_CONNECTIONS_DB_LOGS', $fallbackForDB), 'multiple' => false, - 'schemes' => ['mariadb', 'mongodb', 'mysql', 'postgresql'], + 'schemes' => ['mongodb','mariadb', 'mysql','postgresql'], ], 'publisher' => [ 'type' => 'publisher', diff --git a/app/init/resources.php b/app/init/resources.php index 4ebe705e73..c19ceadfee 100644 --- a/app/init/resources.php +++ b/app/init/resources.php @@ -32,6 +32,8 @@ use Appwrite\Utopia\Request; use Appwrite\Utopia\Response; use Executor\Executor; use Utopia\Abuse\Adapters\TimeLimit\Redis as TimeLimitRedis; +use Utopia\Agents\Adapters\Ollama; +use Utopia\Agents\Agent; use Utopia\Audit\Adapter\Database as AdapterDatabase; use Utopia\Audit\Audit; use Utopia\Auth\Hashes\Argon2; @@ -476,7 +478,7 @@ Http::setResource('user', function (string $mode, Document $project, Document $c return $user; }, ['mode', 'project', 'console', 'request', 'response', 'dbForProject', 'dbForPlatform', 'store', 'proofForToken', 'authorization']); -Http::setResource('project', function ($dbForPlatform, $request, $console, $authorization) { +Http::setResource('project', function ($dbForPlatform, $request, $console, $authorization, Http $utopia) { /** @var Appwrite\Utopia\Request $request */ /** @var Utopia\Database\Database $dbForPlatform */ /** @var Utopia\Database\Document $console */ @@ -486,6 +488,20 @@ Http::setResource('project', function ($dbForPlatform, $request, $console, $auth $projectId = $request->getHeader('x-appwrite-project', ''); } + // Backwards compatibility for new services, originally project resources + // These endpoints moved from /v1/projects/:projectId/ to /v1/ + // When accessed via the old alias path, extract projectId from the URI + $deprecatedProjectPathPrefix = '/v1/projects/'; + $route = $utopia->match($request); + if (!empty($route)) { + $isDeprecatedAlias = \str_starts_with($request->getURI(), $deprecatedProjectPathPrefix) && + !\str_starts_with($route->getPath(), $deprecatedProjectPathPrefix); + + if ($isDeprecatedAlias) { + $projectId = \explode('/', $request->getURI(), 5)[3] ?? ''; + } + } + if (empty($projectId) || $projectId === 'console') { return $console; } @@ -493,7 +509,7 @@ Http::setResource('project', function ($dbForPlatform, $request, $console, $auth $project = $authorization->skip(fn () => $dbForPlatform->getDocument('projects', $projectId)); return $project; -}, ['dbForPlatform', 'request', 'console', 'authorization']); +}, ['dbForPlatform', 'request', 'console', 'authorization', 'utopia']); Http::setResource('session', function (User $user, Store $store, Token $proofForToken) { if ($user->isEmpty()) { @@ -555,7 +571,7 @@ Http::setResource('authorization', function () { return new Authorization(); }, []); -Http::setResource('dbForProject', function (Group $pools, Database $dbForPlatform, Cache $cache, Document $project, Response $response, Publisher $publisher, Publisher $publisherFunctions, Publisher $publisherWebhooks, Event $queueForEvents, Func $queueForFunctions, Webhook $queueForWebhooks, Realtime $queueForRealtime, UsageContext $usage, Authorization $authorization) { +Http::setResource('dbForProject', function (Group $pools, Database $dbForPlatform, Cache $cache, Document $project, Response $response, Publisher $publisher, Publisher $publisherFunctions, Publisher $publisherWebhooks, Event $queueForEvents, Func $queueForFunctions, Webhook $queueForWebhooks, Realtime $queueForRealtime, UsageContext $usage, Authorization $authorization, Request $request) { if ($project->isEmpty() || $project->getId() === 'console') { return $dbForPlatform; } @@ -661,7 +677,31 @@ Http::setResource('dbForProject', function (Group $pools, Database $dbForPlatfor $dbForProject->getCache()->purge($cacheKey); }; - $usageDatabaseListener = function (string $event, Document $document, UsageContext $usage) { + /** + * Prefix metrics with database type when applicable. + * Avoids prefixing for legacy and tablesdb types to preserve historical metrics. + */ + $getDatabaseTypePrefixedMetric = function (string $databaseType, string $metric): string { + if ( + $databaseType === '' || + $databaseType === DATABASE_TYPE_LEGACY || + $databaseType === DATABASE_TYPE_TABLESDB + ) { + return $metric; + } + + return $databaseType . '.' . $metric; + }; + + // Determine database type from request path, similar to api.php + $path = $request->getURI(); + $databaseType = match (true) { + str_contains($path, '/documentsdb') => DATABASE_TYPE_DOCUMENTSDB, + str_contains($path, '/vectorsdb') => DATABASE_TYPE_VECTORSDB, + default => '', + }; + + $usageDatabaseListener = function (string $event, Document $document, UsageContext $usage) use ($getDatabaseTypePrefixedMetric, $databaseType) { $value = 1; switch ($event) { @@ -693,7 +733,8 @@ Http::setResource('dbForProject', function (Group $pools, Database $dbForPlatfor $usage->addMetric(METRIC_SESSIONS, $value); // per project break; case $document->getCollection() === 'databases': // databases - $usage->addMetric(METRIC_DATABASES, $value); // per project + $metric = $getDatabaseTypePrefixedMetric($databaseType, METRIC_DATABASES); + $usage->addMetric($metric, $value); // per project if ($event === Database::EVENT_DOCUMENT_DELETE) { $usage->addReduce($document); @@ -702,9 +743,11 @@ Http::setResource('dbForProject', function (Group $pools, Database $dbForPlatfor case str_starts_with($document->getCollection(), 'database_') && ! str_contains($document->getCollection(), 'collection'): // collections $parts = explode('_', $document->getCollection()); $databaseInternalId = $parts[1] ?? 0; + $collectionMetric = $getDatabaseTypePrefixedMetric($databaseType, METRIC_COLLECTIONS); + $databaseIdCollectionMetric = $getDatabaseTypePrefixedMetric($databaseType, METRIC_DATABASE_ID_COLLECTIONS); $usage - ->addMetric(METRIC_COLLECTIONS, $value) // per project - ->addMetric(str_replace('{databaseInternalId}', $databaseInternalId, METRIC_DATABASE_ID_COLLECTIONS), $value); + ->addMetric($collectionMetric, $value) // per project + ->addMetric(str_replace('{databaseInternalId}', $databaseInternalId, $databaseIdCollectionMetric), $value); if ($event === Database::EVENT_DOCUMENT_DELETE) { $usage->addReduce($document); @@ -714,10 +757,13 @@ Http::setResource('dbForProject', function (Group $pools, Database $dbForPlatfor $parts = explode('_', $document->getCollection()); $databaseInternalId = $parts[1] ?? 0; $collectionInternalId = $parts[3] ?? 0; + $documentsMetric = $getDatabaseTypePrefixedMetric($databaseType, METRIC_DOCUMENTS); + $databaseIdDocumentsMetric = $getDatabaseTypePrefixedMetric($databaseType, METRIC_DATABASE_ID_DOCUMENTS); + $databaseIdCollectionIdDocumentsMetric = $getDatabaseTypePrefixedMetric($databaseType, METRIC_DATABASE_ID_COLLECTION_ID_DOCUMENTS); $usage - ->addMetric(METRIC_DOCUMENTS, $value) // per project - ->addMetric(str_replace('{databaseInternalId}', $databaseInternalId, METRIC_DATABASE_ID_DOCUMENTS), $value) // per database - ->addMetric(str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$databaseInternalId, $collectionInternalId], METRIC_DATABASE_ID_COLLECTION_ID_DOCUMENTS), $value); // per collection + ->addMetric($documentsMetric, $value) // per project + ->addMetric(str_replace('{databaseInternalId}', $databaseInternalId, $databaseIdDocumentsMetric), $value) // per database + ->addMetric(str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$databaseInternalId, $collectionInternalId], $databaseIdCollectionIdDocumentsMetric), $value); // per collection break; case $document->getCollection() === 'buckets': // buckets $usage->addMetric(METRIC_BUCKETS, $value); // per project @@ -792,7 +838,7 @@ Http::setResource('dbForProject', function (Group $pools, Database $dbForPlatfor ->on(Database::EVENT_DOCUMENT_DELETE, 'purge-function-events-cache', fn ($event, $document) => $functionsEventsCacheListener($event, $document, $project, $database)); return $database; -}, ['pools', 'dbForPlatform', 'cache', 'project', 'response', 'publisher', 'publisherFunctions', 'publisherWebhooks', 'queueForEvents', 'queueForFunctions', 'queueForWebhooks', 'queueForRealtime', 'usage', 'authorization']); +}, ['pools', 'dbForPlatform', 'cache', 'project', 'response', 'publisher', 'publisherFunctions', 'publisherWebhooks', 'queueForEvents', 'queueForFunctions', 'queueForWebhooks', 'queueForRealtime', 'usage', 'authorization', 'request']); Http::setResource('dbForPlatform', function (Group $pools, Cache $cache, Authorization $authorization) { @@ -813,6 +859,138 @@ Http::setResource('dbForPlatform', function (Group $pools, Cache $cache, Authori return $database; }, ['pools', 'cache', 'authorization']); +Http::setResource('getDatabasesDB', function (Group $pools, Cache $cache, Document $project, Request $request, UsageContext $usage, Authorization $authorization) { + + return function (Document $database) use ($pools, $cache, $project, $request, $usage, $authorization): Database { + $databaseDSN = $database->getAttribute('database', $project->getAttribute('database', '')); + $databaseType = $database->getAttribute('type', ''); + + try { + $databaseDSN = new DSN($databaseDSN); + } catch (\InvalidArgumentException) { + // for old databases migrated through patch script + // databaseDSN determines the adapter + $databaseDSN = new DSN('mysql://'.$databaseDSN); + } + try { + $dsn = new DSN($project->getAttribute('database')); + } catch (\InvalidArgumentException) { + // TODO: Temporary until all projects are using shared tables + $dsn = new DSN('mysql://' . $project->getAttribute('database')); + } + + $pool = $pools->get($databaseDSN->getHost()); + + $adapter = new DatabasePool($pool); + $database = new Database($adapter, $cache); + $sharedTables = \explode(',', System::getEnv('_APP_DATABASE_SHARED_TABLES', '')); + + $database + ->setDatabase(APP_DATABASE) + ->setAuthorization($authorization) + ->setMetadata('host', \gethostname()) + ->setMetadata('project', $project->getId()) + ->setTimeout(APP_DATABASE_TIMEOUT_MILLISECONDS_API) + ->setMaxQueryValues(APP_DATABASE_QUERY_MAX_VALUES); + // inside pools authorization needs to be set first + $database->getAdapter()->setSupportForAttributes($databaseType !== DOCUMENTSDB); + if (\in_array($dsn->getHost(), $sharedTables)) { + $database + ->setSharedTables(true) + ->setTenant((int)$project->getSequence()) + ->setNamespace($dsn->getParam('namespace')); + } else { + $database + ->setSharedTables(false) + ->setTenant(null) + ->setNamespace('_' . $project->getSequence()); + } + $timeout = \intval($request->getHeader('x-appwrite-timeout')); + if (!empty($timeout) && Http::isDevelopment()) { + $database->setTimeout($timeout); + } + + // Register database event listeners for usage stats collection + $documentsMetric = METRIC_DOCUMENTS; + $databaseIdDocumentsMetric = METRIC_DATABASE_ID_DOCUMENTS; + $databaseIdCollectionIdDocumentsMetric = METRIC_DATABASE_ID_COLLECTION_ID_DOCUMENTS; + if ($databaseType !== DATABASE_TYPE_LEGACY && $databaseType !== DATABASE_TYPE_TABLESDB) { + $documentsMetric = $databaseType. '.' .$documentsMetric; + $databaseIdDocumentsMetric = $databaseType. '.' .$databaseIdDocumentsMetric; + $databaseIdCollectionIdDocumentsMetric = $databaseType . '.' .$databaseIdCollectionIdDocumentsMetric; + } + $database + ->on(Database::EVENT_DOCUMENT_CREATE, 'calculate-usage', function ($event, $document) use ($usage, $documentsMetric, $databaseIdDocumentsMetric, $databaseIdCollectionIdDocumentsMetric) { + $value = 1; + + if (str_starts_with($document->getCollection(), 'database_') && str_contains($document->getCollection(), '_collection_')) { + $parts = explode('_', $document->getCollection()); + $databaseInternalId = $parts[1] ?? 0; + $collectionInternalId = $parts[3] ?? 0; + $usage + ->addMetric($documentsMetric, $value) // per project + ->addMetric(str_replace('{databaseInternalId}', $databaseInternalId, $databaseIdDocumentsMetric), $value) // per database + ->addMetric(str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$databaseInternalId, $collectionInternalId], $databaseIdCollectionIdDocumentsMetric), $value); // per collection + } + }) + ->on(Database::EVENT_DOCUMENT_DELETE, 'calculate-usage', function ($event, $document) use ($usage, $documentsMetric, $databaseIdDocumentsMetric, $databaseIdCollectionIdDocumentsMetric) { + $value = -1; + + if (str_starts_with($document->getCollection(), 'database_') && str_contains($document->getCollection(), '_collection_')) { + $parts = explode('_', $document->getCollection()); + $databaseInternalId = $parts[1] ?? 0; + $collectionInternalId = $parts[3] ?? 0; + $usage + ->addMetric($documentsMetric, $value) // per project + ->addMetric(str_replace('{databaseInternalId}', $databaseInternalId, $databaseIdDocumentsMetric), $value) // per database + ->addMetric(str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$databaseInternalId, $collectionInternalId], $databaseIdCollectionIdDocumentsMetric), $value); // per collection + } + }) + ->on(Database::EVENT_DOCUMENTS_CREATE, 'calculate-usage', function ($event, $document) use ($usage, $documentsMetric, $databaseIdDocumentsMetric, $databaseIdCollectionIdDocumentsMetric) { + $value = $document->getAttribute('modified', 0); + + if (str_starts_with($document->getCollection(), 'database_') && str_contains($document->getCollection(), '_collection_')) { + $parts = explode('_', $document->getCollection()); + $databaseInternalId = $parts[1] ?? 0; + $collectionInternalId = $parts[3] ?? 0; + $usage + ->addMetric($documentsMetric, $value) // per project + ->addMetric(str_replace('{databaseInternalId}', $databaseInternalId, $databaseIdDocumentsMetric), $value) // per database + ->addMetric(str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$databaseInternalId, $collectionInternalId], $databaseIdCollectionIdDocumentsMetric), $value); // per collection + } + }) + ->on(Database::EVENT_DOCUMENTS_DELETE, 'calculate-usage', function ($event, $document) use ($usage, $documentsMetric, $databaseIdDocumentsMetric, $databaseIdCollectionIdDocumentsMetric) { + $value = -1 * $document->getAttribute('modified', 0); + + if (str_starts_with($document->getCollection(), 'database_') && str_contains($document->getCollection(), '_collection_')) { + $parts = explode('_', $document->getCollection()); + $databaseInternalId = $parts[1] ?? 0; + $collectionInternalId = $parts[3] ?? 0; + $usage + ->addMetric($documentsMetric, $value) // per project + ->addMetric(str_replace('{databaseInternalId}', $databaseInternalId, $databaseIdDocumentsMetric), $value) // per database + ->addMetric(str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$databaseInternalId, $collectionInternalId], $databaseIdCollectionIdDocumentsMetric), $value); // per collection + } + }) + ->on(Database::EVENT_DOCUMENTS_UPSERT, 'calculate-usage', function ($event, $document) use ($usage, $documentsMetric, $databaseIdDocumentsMetric, $databaseIdCollectionIdDocumentsMetric) { + $value = $document->getAttribute('created', 0); + + if (str_starts_with($document->getCollection(), 'database_') && str_contains($document->getCollection(), '_collection_')) { + $parts = explode('_', $document->getCollection()); + $databaseInternalId = $parts[1] ?? 0; + $collectionInternalId = $parts[3] ?? 0; + $usage + ->addMetric($documentsMetric, $value) // per project + ->addMetric(str_replace('{databaseInternalId}', $databaseInternalId, $databaseIdDocumentsMetric), $value) // per database + ->addMetric(str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$databaseInternalId, $collectionInternalId], $databaseIdCollectionIdDocumentsMetric), $value); // per collection + } + }); + + return $database; + }; + +}, ['pools','cache','project','request','usage','authorization']); + Http::setResource('getProjectDB', function (Group $pools, Database $dbForPlatform, $cache, Authorization $authorization) { $databases = []; @@ -1072,16 +1250,21 @@ function getDevice(string $root, string $connection = ''): Device } } -Http::setResource('mode', function ($request) { - /** @var Appwrite\Utopia\Request $request */ - +Http::setResource('mode', function (Request $request, Document $project) { /** * Defines the mode for the request: * - 'default' => Requests for Client and Server Side * - 'admin' => Request from the Console on non-console projects */ - return $request->getParam('mode', $request->getHeader('x-appwrite-mode', APP_MODE_DEFAULT)); -}, ['request']); + $mode = $request->getParam('mode', $request->getHeader('x-appwrite-mode', APP_MODE_DEFAULT)); + + $projectId = $request->getParam('project', $request->getHeader('x-appwrite-project', '')); + if (!empty($projectId) && $project->getId() !== $projectId) { + $mode = APP_MODE_ADMIN; + } + + return $mode; +}, ['request', 'project']); Http::setResource('geodb', function ($register) { /** @var Utopia\Registry\Registry $register */ @@ -1441,9 +1624,9 @@ Http::setResource('resourceToken', function ($project, $dbForProject, $request, return new Document([]); }, ['project', 'dbForProject', 'request', 'authorization']); -Http::setResource('transactionState', function (Database $dbForProject, Authorization $authorization) { - return new TransactionState($dbForProject, $authorization); -}, ['dbForProject', 'authorization']); +Http::setResource('transactionState', function (Database $dbForProject, Authorization $authorization, callable $getDatabasesDB) { + return new TransactionState($dbForProject, $authorization, $getDatabasesDB); +}, ['dbForProject', 'authorization', 'getDatabasesDB']); Http::setResource('executionsRetentionCount', function (Document $project, array $plan) { if ($project->getId() === 'console' || empty($plan)) { @@ -1452,3 +1635,10 @@ Http::setResource('executionsRetentionCount', function (Document $project, array return (int) ($plan['executionsRetentionCount'] ?? 100); }, ['project', 'plan']); + +Http::setResource('embeddingAgent', function ($register) { + $adapter = new Ollama(); + $adapter->setEndpoint(System::getEnv('_APP_EMBEDDING_ENDPOINT', 'http://ollama:11434/api/embed')); + $adapter->setTimeout((int) System::getEnv('_APP_EMBEDDING_TIMEOUT', '30000')); + return new Agent($adapter); +}, ['register']); diff --git a/app/views/install/installer.phtml b/app/views/install/installer.phtml index 3cf2b77aa9..05bc1b80ed 100644 --- a/app/views/install/installer.phtml +++ b/app/views/install/installer.phtml @@ -9,6 +9,7 @@ $defaultAppDomain = ($defaultAppDomain === 'traefik') ? 'localhost' : $defaultAp $defaultEmailCertificates = $defaultEmailCertificates ?? ''; $defaultDatabase = $vars['_APP_DB_ADAPTER']['default'] ?? 'mongodb'; $lockedDatabase = $isUpgrade && empty($lockedDatabase) ? $defaultDatabase : $lockedDatabase; +$enabledDatabases = $enabledDatabases ?? ['mongodb', 'mariadb', 'postgresql']; $isLocalInstall = $isLocalInstall ?? false; @@ -64,6 +65,7 @@ $installerVersion = @filemtime(__DIR__ . '/installer/js/installer.js') ?: time() data-default-secret-key="" data-default-assistant-openai-key="" data-default-database="" + data-enabled-databases="" data-dev-mode="true" diff --git a/app/views/install/installer/css/styles.css b/app/views/install/installer/css/styles.css index 4601cfcb67..b1d8fe5089 100644 --- a/app/views/install/installer/css/styles.css +++ b/app/views/install/installer/css/styles.css @@ -11,9 +11,13 @@ --neutral-300: rgba(173, 173, 176, 1); --neutral-400: rgba(151, 151, 155, 1); --neutral-500: rgba(129, 129, 134, 1); + --neutral-600: rgba(108, 108, 113, 1); --neutral-700: rgba(86, 86, 92, 1); + --neutral-750: rgba(65, 65, 70, 1); --neutral-800: rgba(45, 45, 49, 1); + --neutral-850: rgba(29, 29, 33, 1); --neutral-900: rgba(25, 25, 28, 1); + --neutral-250: rgba(195, 195, 198, 1); /* Warning colors */ --web-orange-200: rgba(255, 213, 194, 1); @@ -172,6 +176,44 @@ --fgColor-neutral-weak: var(--fgcolor-neutral-weak); --fgColor-accent: var(--fgcolor-accent); --fgColor-on-accent: var(--fgcolor-on-accent); + + color-scheme: light dark; +} + +@media (prefers-color-scheme: dark) { + :root { + --bgcolor-neutral-default: var(--neutral-900); + --bgcolor-neutral-primary: var(--neutral-850); + --bgcolor-neutral-secondary: var(--neutral-800); + --bgcolor-neutral-tertiary: var(--neutral-800); + --bgcolor-neutral-invert-weaker: var(--neutral-400); + --bgcolor-neutral-invert-weak: var(--neutral-300); + --bgcolor-success-weak: rgba(16, 185, 129, 0.12); + --bgcolor-warning-weaker: rgba(254, 124, 67, 0.08); + --bgcolor-warning-weak: rgba(254, 124, 67, 0.12); + --bgcolor-error-weaker: rgba(255, 69, 58, 0.08); + + --fgcolor-neutral-primary: var(--neutral-50); + --fgcolor-neutral-secondary: var(--neutral-250); + --fgcolor-neutral-tertiary: var(--neutral-500); + --fgcolor-neutral-weak: var(--neutral-600); + --fgcolor-on-accent: var(--neutral-0); + --fgcolor-on-invert: var(--neutral-900); + --fgcolor-on-success-weak: rgba(52, 211, 153, 1); + --fgcolor-warning: var(--web-orange-500); + --fgcolor-on-warning-weak: var(--web-orange-500); + --fgcolor-error: var(--web-red-500); + --fgcolor-on-error: var(--neutral-0); + + --border-neutral: var(--neutral-800); + --border-neutral-strong: var(--neutral-750); + --border-neutral-stronger: var(--neutral-600); + --border-focus: var(--neutral-600); + + --overlay-neutral-hover: rgba(255, 255, 255, 0.04); + --overlay-neutral-pressed: rgba(255, 255, 255, 0.08); + --overlay-scrim: rgba(0, 0, 0, 0.48); + } } .installer-toast-stack { @@ -1354,6 +1396,7 @@ body { align-items: center; justify-content: center; line-height: 0; + color: var(--fgcolor-neutral-weak); } .step-indicator svg { diff --git a/app/views/install/installer/icons/appwrite-logo.svg b/app/views/install/installer/icons/appwrite-logo.svg index f0f2fa5af0..af9378f8fc 100644 --- a/app/views/install/installer/icons/appwrite-logo.svg +++ b/app/views/install/installer/icons/appwrite-logo.svg @@ -1,13 +1,13 @@ diff --git a/app/views/install/installer/icons/indicator-inactive.svg b/app/views/install/installer/icons/indicator-inactive.svg index 78b5ef3ac0..550101da9b 100644 --- a/app/views/install/installer/icons/indicator-inactive.svg +++ b/app/views/install/installer/icons/indicator-inactive.svg @@ -1,3 +1,3 @@ diff --git a/app/views/install/installer/js/modules/context.js b/app/views/install/installer/js/modules/context.js index bc08a8c8df..c531ecddce 100644 --- a/app/views/install/installer/js/modules/context.js +++ b/app/views/install/installer/js/modules/context.js @@ -2,6 +2,11 @@ const getBodyDataset = () => document.body?.dataset ?? {}; const isUpgradeMode = () => getBodyDataset().upgrade === 'true'; const getLockedDatabase = () => getBodyDataset().lockedDatabase || ''; + const getEnabledDatabases = () => { + const raw = getBodyDataset().enabledDatabases; + if (!raw) return ['mongodb', 'mariadb', 'postgresql']; + try { return JSON.parse(raw); } catch (e) { return ['mongodb', 'mariadb', 'postgresql']; } + }; const STEP_IDS = Object.freeze({ CONFIG_FILES: 'config-files', @@ -95,6 +100,7 @@ getBodyDataset, isUpgradeMode, getLockedDatabase, + getEnabledDatabases, STEP_IDS, STATUS, SSE_EVENTS, diff --git a/app/views/install/installer/js/modules/progress.js b/app/views/install/installer/js/modules/progress.js index a6e784ada9..7f7b23e3fc 100644 --- a/app/views/install/installer/js/modules/progress.js +++ b/app/views/install/installer/js/modules/progress.js @@ -27,6 +27,7 @@ let activeInstall = null; let unloadGuard = null; + let sseSessionDetails = null; const csrfToken = document.querySelector('meta[name="appwrite-installer-csrf"]')?.getAttribute('content') || ''; const withCsrfHeader = (headers = {}) => { @@ -524,6 +525,9 @@ inProgress: payload.message || payload.step, done: payload.message || payload.step }; + if (step.id === STEP_IDS.ACCOUNT_SETUP && payload.details?.sessionSecret) { + sseSessionDetails = payload.details; + } progressState.set(step.id, { status: payload.status || STATUS.IN_PROGRESS, message: payload.message, @@ -599,7 +603,7 @@ }); if (!allDone) return; const accountState = progressState.get(STEP_IDS.ACCOUNT_SETUP); - const sessionDetails = accountState?.details; + const sessionDetails = sseSessionDetails || accountState?.details; finalizeInstall(); notifyInstallComplete(activeInstall?.installId, sessionDetails).finally(() => { setTimeout(() => redirectToApp(), TIMINGS?.redirectDelay ?? 0); @@ -740,7 +744,7 @@ } const accountState = progressState.get(STEP_IDS.ACCOUNT_SETUP); - const sessionDetails = accountState?.details; + const sessionDetails = sseSessionDetails || accountState?.details; finalizeInstall(); notifyInstallComplete(activeInstall?.installId, sessionDetails).finally(() => { setTimeout(() => redirectToApp(), TIMINGS?.redirectDelay ?? 0); diff --git a/app/views/install/installer/js/steps.js b/app/views/install/installer/js/steps.js index d8b0621761..2a71d075cc 100644 --- a/app/views/install/installer/js/steps.js +++ b/app/views/install/installer/js/steps.js @@ -9,7 +9,8 @@ const { INSTALLATION_STEPS, clampStep, - isUpgradeMode + isUpgradeMode, + getEnabledDatabases } = Context; const { @@ -79,6 +80,19 @@ } }; + const applyEnabledDatabases = (root) => { + const enabled = getEnabledDatabases?.() || []; + const radios = root.querySelectorAll('input[name="database"]'); + radios.forEach((radio) => { + if (!enabled.includes(radio.value)) { + const card = radio.closest('.selector-card'); + if (card) { + card.remove(); + } + } + }); + }; + const bindDatabaseSelection = (root) => { const radios = root.querySelectorAll('input[name="database"]'); radios.forEach((radio) => { @@ -139,6 +153,8 @@ return; } + applyEnabledDatabases(root); + const lockedDatabase = getLockedDatabase?.() || ''; if (lockedDatabase) { lockDatabaseSelection(root, lockedDatabase); diff --git a/app/views/install/installer/templates/steps/step-1.phtml b/app/views/install/installer/templates/steps/step-1.phtml index ae7674c3bf..8f4a726158 100644 --- a/app/views/install/installer/templates/steps/step-1.phtml +++ b/app/views/install/installer/templates/steps/step-1.phtml @@ -7,8 +7,12 @@ $defaultHttpsPort = $defaultHttpsPort ?? '443'; $defaultEmailCertificates = $defaultEmailCertificates ?? ''; $defaultAssistantOpenAIKey = $defaultAssistantOpenAIKey ?? ''; $defaultDatabase = $defaultDatabase ?? 'mongodb'; +$enabledDatabases = $enabledDatabases ?? ['mongodb', 'mariadb', 'postgresql']; $selectedDatabase = $lockedDatabase ?: $defaultDatabase; $isDatabaseLocked = !empty($lockedDatabase); +$mongoEnabled = in_array('mongodb', $enabledDatabases, true); +$mariaEnabled = in_array('mariadb', $enabledDatabases, true); +$postgresEnabled = in_array('postgresql', $enabledDatabases, true); $mongoDisabled = $isDatabaseLocked && $selectedDatabase !== 'mongodb'; $mariaDisabled = $isDatabaseLocked && $selectedDatabase !== 'mariadb'; $postgresDisabled = $isDatabaseLocked && $selectedDatabase !== 'postgresql'; @@ -43,6 +47,7 @@ $assistantOpenAIKeyValue = htmlspecialchars((string) $defaultAssistantOpenAIKey,
+