Merge pull request #11389 from appwrite/multitypedb-with-dedicated-db

chore: split sdk release artifacts from automation

.env

@@ -41,10 +41,19 @@ _APP_REDIS_USER=
 _APP_DB_ADAPTER=mariadb
 _APP_DB_HOST=mariadb
 _APP_DB_PORT=3306
+_APP_DB_ADAPTER=mariadb
 _APP_DB_SCHEMA=appwrite
 _APP_DB_USER=user
 _APP_DB_PASS=password
 _APP_DB_ROOT_PASS=rootsecretpassword
+_APP_DB_ADAPTER_DOCUMENTSDB=mongodb
+_APP_DB_HOST_DOCUMENTSDB=mongodb
+_APP_DB_PORT_DOCUMENTSDB=27017
+_APP_DB_ADAPTER_VECTORDB=postgresql
+_APP_DB_HOST_VECTORDB=postgresql
+_APP_DB_PORT_VECTORDB=5432
+_APP_EMBEDDING_MODELS=embeddinggemma
+_APP_EMBEDDING_ENDPOINT='http://ollama:11434/api/embed'
 _APP_STORAGE_DEVICE=Local
 _APP_STORAGE_S3_ACCESS_KEY=
 _APP_STORAGE_S3_SECRET=
@@ -133,4 +142,5 @@ _APP_PROJECT_REGIONS=default
 _APP_FUNCTIONS_CREATION_ABUSE_LIMIT=5000
 _APP_STATS_USAGE_DUAL_WRITING_DBS=database_db_main
 _APP_TRUSTED_HEADERS=x-forwarded-for
-_APP_POOL_ADAPTER=stack
+_APP_POOL_ADAPTER=stack
+_APP_WORKER_SCREENSHOTS_ROUTER=http://appwrite

.github/workflows/publish.yml

@@ -24,7 +24,7 @@ jobs:
       uses: docker/setup-buildx-action@v3
 
     - name: Login to Docker Hub
-      uses: docker/login-action@v2
+      uses: docker/login-action@v3
       with:
         username: ${{ vars.DOCKERHUB_USERNAME }}
         password: ${{ secrets.DOCKERHUB_TOKEN }}

.github/workflows/release.yml

@@ -26,7 +26,7 @@ jobs:
       uses: docker/setup-buildx-action@v3
 
     - name: Login to Docker Hub
-      uses: docker/login-action@v2
+      uses: docker/login-action@v3
       with:
         username: ${{ vars.DOCKERHUB_USERNAME }}
         password: ${{ secrets.DOCKERHUB_TOKEN }}

.github/workflows/sdk-preview.yml

@@ -4,9 +4,6 @@ env:
   COMPOSE_FILE: docker-compose.yml
 
 on:
-  pull_request:
-    paths:
-      - 'app/config/specs/*-latest-console.json'
   workflow_dispatch:
     inputs:
       platform:
@@ -42,6 +39,13 @@ jobs:
         run: |
           docker compose build
           docker compose up -d
+
+      - name: Generate Specs
+        run: |
+          docker compose exec appwrite specs --version=latest --mode=normal --git=no
+
+      - name: Generate SDK
+        run: |
           docker compose exec appwrite sdks --platform=${{ steps.set-sdk.outputs.platform }} --sdk=${{ steps.set-sdk.outputs.sdk_type }} --version=latest --git=no
           sudo chown -R $USER:$USER ./app/sdks/${{ steps.set-sdk.outputs.platform }}-${{ steps.set-sdk.outputs.sdk_type }}
           
@@ -50,7 +54,7 @@ jobs:
           node-version: 20
 
       - name: Build and Publish SDK
-        working-directory: ./app/sdks/console-web
+        working-directory: ./app/sdks/${{ steps.set-sdk.outputs.platform }}-${{ steps.set-sdk.outputs.sdk_type }}
         run: |
             npm install
             npm run build

.github/workflows/specs.yml

@@ -0,0 +1,113 @@
+name: "Generate Specs"
+
+env:
+  COMPOSE_FILE: docker-compose.yml
+
+on:
+  workflow_dispatch:
+    inputs:
+      version:
+        type: choice
+        description: "Appwrite version to generate specs for"
+        required: true
+        options:
+          - "1.8.x"
+          - "1.7.x"
+          - "1.6.x"
+          - "1.5.x"
+          - "latest"
+      push:
+        type: boolean
+        description: "Push specs to appwrite/specs repo and create PR"
+        default: true
+      message:
+        type: string
+        description: "Commit message for the specs PR"
+        default: "chore: update API specs and SDK examples"
+
+jobs:
+  generate:
+    name: Generate Specs
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v6
+        with:
+          submodules: recursive
+
+      - name: Build and start Appwrite
+        run: |
+          docker compose build
+          docker compose up -d
+
+      - name: Generate specs
+        run: |
+          docker compose exec appwrite specs --version=${{ inputs.version }} --mode=normal --git=no
+
+      - name: Generate SDK examples
+        if: inputs.push
+        run: |
+          docker compose exec appwrite sdks --platform=* --sdk=* --version=${{ inputs.version }} --git=no --mode=examples
+          sudo chown -R $USER:$USER ./docs/examples/
+
+      - name: Push to appwrite/specs and create PR
+        if: inputs.push
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          VERSION="${{ inputs.version }}"
+          MESSAGE="${{ inputs.message }}"
+          GIT_BRANCH="feat-${VERSION}-specs"
+          SPECS_DIR="./app/config/specs"
+          EXAMPLES_DIR="./docs/examples/${VERSION}"
+          TARGET="/tmp/specs-repo"
+
+          sudo chown -R $USER:$USER "${SPECS_DIR}"
+
+          # Clone the specs repo
+          git clone "https://x-access-token:${GH_TOKEN}@github.com/appwrite/specs.git" "${TARGET}"
+          cd "${TARGET}"
+          git config user.name "github-actions[bot]"
+          git config user.email "github-actions[bot]@users.noreply.github.com"
+
+          # Create or checkout the feature branch
+          git checkout -b "${GIT_BRANCH}" || git checkout "${GIT_BRANCH}"
+
+          # Copy spec files
+          mkdir -p "specs/${VERSION}"
+          cp ${GITHUB_WORKSPACE}/${SPECS_DIR}/*${VERSION}*.json "specs/${VERSION}/" 2>/dev/null || true
+
+          # Copy latest specs if version is latest
+          if [ "${VERSION}" = "latest" ]; then
+            cp ${GITHUB_WORKSPACE}/${SPECS_DIR}/*latest*.json "specs/latest/" 2>/dev/null || true
+          fi
+
+          # Copy SDK examples
+          if [ -d "${GITHUB_WORKSPACE}/docs/examples/${VERSION}" ]; then
+            mkdir -p "examples/${VERSION}"
+            cp -r "${GITHUB_WORKSPACE}/docs/examples/${VERSION}/." "examples/${VERSION}/"
+          fi
+
+          # Commit and push
+          git add -A
+          git diff --cached --quiet && echo "No changes to commit" && exit 0
+          git commit -m "${MESSAGE}"
+          git push -u origin "${GIT_BRANCH}" --force
+
+          # Create or update PR
+          EXISTING_PR=$(gh pr list --repo appwrite/specs --head "${GIT_BRANCH}" --json number --jq '.[0].number' 2>/dev/null || true)
+          if [ -n "${EXISTING_PR}" ]; then
+            echo "PR #${EXISTING_PR} already exists, updated branch"
+            echo "https://github.com/appwrite/specs/pull/${EXISTING_PR}"
+          else
+            gh pr create \
+              --repo "appwrite/specs" \
+              --title "feat: API specs update for version ${VERSION}" \
+              --body "This PR contains API specification updates and SDK examples for version ${VERSION}." \
+              --base "main" \
+              --head "${GIT_BRANCH}"
+          fi
+
+      - name: Stop containers
+        if: always()
+        run: docker compose down

.github/workflows/tests.yml

@@ -173,10 +173,8 @@ jobs:
       - name: Failure Logs
         if: failure()
         run: |
-          echo "=== Appwrite Worker Builds Logs ==="
-          docker compose logs appwrite-worker-builds
-          echo "=== OpenRuntimes Executor Logs ==="
-          docker compose logs openruntimes-executor
+          echo "=== Appwrite Logs ==="
+          docker compose logs
 
   e2e_service_test:
     name: E2E Service Test
@@ -190,15 +188,17 @@ jobs:
       matrix:
         db_adapter: [
           MARIADB,
-          POSTGRESQL
+          POSTGRESQL,
+          # MONGODB
         ]
-
         service: [
           Account,
           Avatars,
           Console,
           Databases/Legacy,
           Databases/TablesDB,
+          Databases/DocumentsDB,
+          Databases/VectorDB,
           Functions,
           FunctionsSchedule,
           GraphQL,
@@ -232,6 +232,18 @@ jobs:
         run: |
           docker load --input /tmp/${{ env.IMAGE }}.tar
           sed -i 's|^_APP_BROWSER_HOST=.*|_APP_BROWSER_HOST=http://invalid-browser/v1|' .env
+
+          # Set DB Adapter in .env before starting services
+          if [ "${{ matrix.db_adapter }}" = "MARIADB" ]; then
+            sed -i 's|^_APP_DB_ADAPTER=.*|_APP_DB_ADAPTER=mariadb|' .env
+            sed -i 's|^_APP_DB_HOST=.*|_APP_DB_HOST=mariadb|' .env
+            sed -i 's|^_APP_DB_PORT=.*|_APP_DB_PORT=3306|' .env
+          elif [ "${{ matrix.db_adapter }}" = "MONGODB" ]; then
+            sed -i 's|^_APP_DB_ADAPTER=.*|_APP_DB_ADAPTER=mongodb|' .env
+            sed -i 's|^_APP_DB_HOST=.*|_APP_DB_HOST=mongodb|' .env
+            sed -i 's|^_APP_DB_PORT=.*|_APP_DB_PORT=27017|' .env
+          fi
+
           docker compose up -d
           sleep 30
 

.gitignore

@@ -17,4 +17,5 @@ dev/yasd_init.php
 .phpunit.result.cache
 Makefile
 appwrite.config.json
-/.zed/
+/.zed/
+/app/config/specs/

Dockerfile

@@ -37,6 +37,9 @@ COPY ./app /usr/src/code/app
 COPY ./public /usr/src/code/public
 COPY ./bin /usr/local/bin
 COPY ./src /usr/src/code/src
+COPY ./dev /usr/src/code/dev
+COPY ./mongo-init.js /usr/src/code/mongo-init.js
+COPY ./mongo-entrypoint.sh /usr/src/code/mongo-entrypoint.sh
 
 # Set Volumes
 RUN mkdir -p /storage/uploads && \

app/cli.php

@@ -30,9 +30,6 @@ use Utopia\Pools\Group;
 use Utopia\Queue\Broker\Pool as BrokerPool;
 use Utopia\Queue\Publisher;
 use Utopia\Registry\Registry;
-use Utopia\Span\Exporter;
-use Utopia\Span\Span;
-use Utopia\Span\Storage;
 use Utopia\System\System;
 use Utopia\Telemetry\Adapter\None as NoTelemetry;
 
@@ -340,6 +337,5 @@ $cli
 $cli->shutdown()->action(fn () => Timer::clearAll());
 
 Runtime::enableCoroutine(SWOOLE_HOOK_ALL);
-Span::setStorage(new Storage\Coroutine());
-Span::addExporter(new Exporter\Stdout());
+require_once __DIR__ . '/init/span.php';
 run($cli->run(...));

app/config/collections.php

@@ -4,6 +4,7 @@
 $common = include __DIR__ . '/collections/common.php';
 $projects = include __DIR__ . '/collections/projects.php';
 $databases = include __DIR__ . '/collections/databases.php';
+$vectordb = include __DIR__ . '/collections/vectordb.php';
 $platform = include __DIR__ . '/collections/platform.php';
 $logs = include __DIR__ . '/collections/logs.php';
 
@@ -26,6 +27,7 @@ unset($common['files']);
 $collections = [
     'buckets' => $buckets,
     'databases' => $databases,
+    'vectordb' => $vectordb,
     'projects' => array_merge_recursive($projects, $common),
     'console' => array_merge_recursive($platform, $common),
     'logs' => $logs,

app/config/collections/common.php

@@ -1617,13 +1617,6 @@ return [
             ],
         ],
         'indexes' => [
-            [
-                '$id' => ID::custom('_fulltext_name'),
-                'type' => Database::INDEX_FULLTEXT,
-                'attributes' => ['name'],
-                'lengths' => [],
-                'orders' => [],
-            ],
             [
                 '$id' => ID::custom('_key_search'),
                 'type' => Database::INDEX_FULLTEXT,
@@ -1853,13 +1846,6 @@ return [
                 'lengths' => [],
                 'orders' => [Database::ORDER_ASC],
             ],
-            [
-                '$id' => ID::custom('_key_name'),
-                'type' => Database::INDEX_FULLTEXT,
-                'attributes' => ['name'],
-                'lengths' => [],
-                'orders' => [Database::ORDER_ASC],
-            ],
             [
                 '$id' => ID::custom('_key_type'),
                 'type' => Database::INDEX_KEY,
@@ -2127,14 +2113,8 @@ return [
                 'filters' => ['topicSearch'],
             ],
         ],
+
         'indexes' => [
-            [
-                '$id' => ID::custom('_key_name'),
-                'type' => Database::INDEX_FULLTEXT,
-                'attributes' => ['name'],
-                'lengths' => [],
-                'orders' => [],
-            ],
             [
                 '$id' => ID::custom('_key_search'),
                 'type' => Database::INDEX_FULLTEXT,

app/config/collections/projects.php

@@ -61,6 +61,15 @@ return [
                 'array' => false,
                 'filters' => [],
             ],
+            [
+                '$id' => ID::custom('database'),
+                'type' => Database::VAR_STRING,
+                'size' => 128,
+                'required' => true,
+                'signed' => true,
+                'array' => false,
+                'filters' => [],
+            ]
         ],
         'indexes' => [
             [

app/config/collections/vectordb.php

@@ -0,0 +1,165 @@
+<?php
+
+use Utopia\Database\Database;
+use Utopia\Database\Helpers\ID;
+
+return [
+    'collections' => [
+        '$collection' => ID::custom('databases'),
+        '$id' => ID::custom('collections'),
+        'name' => 'Collections',
+        'attributes' => [
+            [
+                '$id' => ID::custom('databaseInternalId'),
+                'type' => Database::VAR_STRING,
+                'format' => '',
+                'size' => Database::LENGTH_KEY,
+                'signed' => true,
+                'required' => true,
+                'default' => null,
+                'array' => false,
+                'filters' => [],
+            ],
+            [
+                '$id' => ID::custom('databaseId'),
+                'type' => Database::VAR_STRING,
+                'signed' => true,
+                'size' => Database::LENGTH_KEY,
+                'format' => '',
+                'filters' => [],
+                'required' => true,
+                'default' => null,
+                'array' => false,
+            ],
+            [
+                '$id' => ID::custom('name'),
+                'type' => Database::VAR_STRING,
+                'size' => 256,
+                'required' => true,
+                'signed' => true,
+                'array' => false,
+                'filters' => [],
+            ],
+            [
+                '$id' => ID::custom('dimension'),
+                'type' => Database::VAR_INTEGER,
+                'size' => 0,
+                'required' => true,
+                'signed' => false,
+                'array' => false,
+                'filters' => [],
+            ],
+            [
+                '$id' => ID::custom('enabled'),
+                'type' => Database::VAR_BOOLEAN,
+                'signed' => true,
+                'size' => 0,
+                'format' => '',
+                'filters' => [],
+                'required' => true,
+                'default' => null,
+                'array' => false,
+            ],
+            [
+                '$id' => ID::custom('documentSecurity'),
+                'type' => Database::VAR_BOOLEAN,
+                'signed' => true,
+                'size' => 0,
+                'format' => '',
+                'filters' => [],
+                'required' => true,
+                'default' => null,
+                'array' => false,
+            ],
+            [
+                '$id' => ID::custom('attributes'),
+                'type' => Database::VAR_STRING,
+                'size' => 1000000,
+                'required' => false,
+                'signed' => true,
+                'array' => false,
+                'filters' => ['subQueryAttributes'],
+            ],
+            [
+                '$id' => ID::custom('indexes'),
+                'type' => Database::VAR_STRING,
+                'size' => 1000000,
+                'required' => false,
+                'signed' => true,
+                'array' => false,
+                'filters' => ['subQueryIndexes'],
+            ],
+            [
+                '$id' => ID::custom('search'),
+                'type' => Database::VAR_STRING,
+                'format' => '',
+                'size' => 16384,
+                'signed' => true,
+                'required' => false,
+                'default' => null,
+                'array' => false,
+                'filters' => [],
+            ],
+        ],
+        'defaultAttributes' => [
+            [
+                '$id' => ID::custom('embeddings'),
+                'type' => Database::VAR_VECTOR,
+                'required' => true,
+                'signed' => false,
+                'array' => false,
+                'filters' => [],
+            ],
+            [
+                '$id' => ID::custom('metadata'),
+                'type' => Database::VAR_OBJECT,
+                'default' => [],
+                'required' => false,
+                'size' => 0,
+                'signed' => false,
+                'array' => false,
+                'filters' => [],
+            ],
+        ],
+        'indexes' => [
+            [
+                '$id' => ID::custom('_fulltext_search'),
+                'type' => Database::INDEX_FULLTEXT,
+                'attributes' => ['search'],
+                'lengths' => [],
+                'orders' => [],
+            ],
+            [
+                '$id' => ID::custom('_key_name'),
+                'type' => Database::INDEX_KEY,
+                'attributes' => ['name'],
+                'lengths' => [256],
+                'orders' => [Database::ORDER_ASC],
+            ],
+            [
+                '$id' => ID::custom('_key_enabled'),
+                'type' => Database::INDEX_KEY,
+                'attributes' => ['enabled'],
+                'lengths' => [],
+                'orders' => [Database::ORDER_ASC],
+            ],
+            [
+                '$id' => ID::custom('_key_documentSecurity'),
+                'type' => Database::INDEX_KEY,
+                'attributes' => ['documentSecurity'],
+                'lengths' => [],
+                'orders' => [Database::ORDER_ASC],
+            ],
+        ],
+        'defaultIndexes' => [
+            // not creating default indexes on the embeddings as it depends on the type of query users using the most
+            [
+                '$id' => ID::custom('_key_metadata'),
+                'type' => Database::INDEX_OBJECT,
+                'attributes' => ['metadata'],
+                'lengths' => [],
+                'orders' => [],
+            ],
+        ]
+    ]
+];

app/config/cors.php

@@ -0,0 +1,53 @@
+<?php
+
+/**
+ * CORS Configuration
+ *
+ * Centralised list of allowed methods, headers, and exposed headers for CORS responses.
+ */
+return [
+    'allowedMethods' => ['GET', 'POST', 'PUT', 'PATCH', 'DELETE'],
+    'allowedHeaders' => [
+        'Accept',
+        'Origin',
+        'Cookie',
+        'Set-Cookie',
+        // Content
+        'Content-Type',
+        'Content-Range',
+        // Appwrite
+        'X-Appwrite-Project',
+        'X-Appwrite-Key',
+        'X-Appwrite-Dev-Key',
+        'X-Appwrite-Locale',
+        'X-Appwrite-Mode',
+        'X-Appwrite-JWT',
+        'X-Appwrite-Response-Format',
+        'X-Appwrite-Timeout',
+        'X-Appwrite-ID',
+        'X-Appwrite-Timestamp',
+        'X-Appwrite-Session',
+        'X-Appwrite-Platform',
+        // SDK generator
+        'X-SDK-Version',
+        'X-SDK-Name',
+        'X-SDK-Language',
+        'X-SDK-Platform',
+        'X-SDK-GraphQL',
+        'X-SDK-Profile',
+        // Caching
+        'Range',
+        'Cache-Control',
+        'Expires',
+        'Pragma',
+        // Server to server
+        'X-Fallback-Cookies',
+        'X-Requested-With',
+        'X-Forwarded-For',
+        'X-Forwarded-User-Agent',
+    ],
+    'exposedHeaders' => [
+        'X-Appwrite-Session',
+        'X-Fallback-Cookies',
+    ],
+];

app/config/events.php

@@ -405,6 +405,14 @@ return [
             '$description' => 'This event triggers when a provider is deleted.'
         ],
     ],
+    'schedules' => [
+        '$model' => Response::MODEL_SCHEDULE,
+        '$resource' => true,
+        '$description' => 'This event triggers on any schedule event.',
+        'create' => [
+            '$description' => 'This event triggers when a schedule is created.',
+        ],
+    ],
     'rules' => [
         '$model' => Response::MODEL_PROXY_RULE,
         '$resource' => true,

app/config/roles.php

@@ -91,6 +91,8 @@ $admins = [
     'subscribers.read',
     'tokens.read',
     'tokens.write',
+    'schedules.read',
+    'schedules.write',
 ];
 
 return [

app/config/scopes/project.php

@@ -145,6 +145,12 @@ return [ // List of publicly visible scopes
     'rules.write' => [
         'description' => 'Access to create, update, and delete your project\'s proxy rules',
     ],
+    'schedules.read' => [
+        'description' => 'Access to read your project\'s schedules',
+    ],
+    'schedules.write' => [
+        'description' => 'Access to create, update, and delete your project\'s schedules',
+    ],
     'migrations.read' => [
         'description' => 'Access to read your project\'s migrations',
     ],

app/config/sdks.php

@@ -11,7 +11,7 @@ return [
             [
                 'key' => 'web',
                 'name' => 'Web',
-                'version' => '22.1.0',
+                'version' => '22.4.0',
                 'url' => 'https://github.com/appwrite/sdk-for-web',
                 'package' => 'https://www.npmjs.com/package/appwrite',
                 'enabled' => true,
@@ -60,7 +60,7 @@ return [
             [
                 'key' => 'flutter',
                 'name' => 'Flutter',
-                'version' => '21.1.0',
+                'version' => '21.4.0',
                 'url' => 'https://github.com/appwrite/sdk-for-flutter',
                 'package' => 'https://pub.dev/packages/appwrite',
                 'enabled' => true,
@@ -79,7 +79,7 @@ return [
             [
                 'key' => 'apple',
                 'name' => 'Apple',
-                'version' => '14.1.0',
+                'version' => '14.3.0',
                 'url' => 'https://github.com/appwrite/sdk-for-apple',
                 'package' => 'https://github.com/appwrite/sdk-for-apple',
                 'enabled' => true,
@@ -117,7 +117,7 @@ return [
                 'key' => 'android',
                 'name' => 'Android',
                 'namespace' => 'io.appwrite',
-                'version' => '12.1.0',
+                'version' => '12.2.0',
                 'url' => 'https://github.com/appwrite/sdk-for-android',
                 'package' => 'https://search.maven.org/artifact/io.appwrite/sdk-for-android',
                 'enabled' => true,
@@ -140,7 +140,7 @@ return [
             [
                 'key' => 'react-native',
                 'name' => 'React Native',
-                'version' => '0.21.0',
+                'version' => '0.24.0',
                 'url' => 'https://github.com/appwrite/sdk-for-react-native',
                 'package' => 'https://npmjs.com/package/react-native-appwrite',
                 'enabled' => true,
@@ -208,7 +208,7 @@ return [
             [
                 'key' => 'web',
                 'name' => 'Console',
-                'version' => '0.3.0',
+                'version' => '22.4.0',
                 'url' => '',
                 'package' => '',
                 'enabled' => true,
@@ -227,7 +227,7 @@ return [
             [
                 'key' => 'cli',
                 'name' => 'Command Line',
-                'version' => '13.4.0',
+                'version' => '13.5.0',
                 'url' => 'https://github.com/appwrite/sdk-for-cli',
                 'package' => 'https://www.npmjs.com/package/appwrite-cli',
                 'enabled' => true,