diff --git a/.github/workflows/ai-moderator.yml b/.github/workflows/ai-moderator.yml index d0b180985f..483f3dbeee 100644 --- a/.github/workflows/ai-moderator.yml +++ b/.github/workflows/ai-moderator.yml @@ -5,8 +5,6 @@ on: types: [opened, edited] issue_comment: types: [created, edited] - pull_request: - types: [opened, edited] pull_request_review: types: [submitted, edited] pull_request_review_comment: diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml deleted file mode 100644 index b7b4fa0d2f..0000000000 --- a/.github/workflows/benchmark.yml +++ /dev/null @@ -1,123 +0,0 @@ -name: Benchmark -concurrency: - group: '${{ github.workflow }}-${{ github.ref }}' - cancel-in-progress: true -env: - COMPOSE_FILE: docker-compose.yml - IMAGE: appwrite-dev - CACHE_KEY: 'appwrite-dev-${{ github.event.pull_request.head.sha }}' -'on': - - pull_request -jobs: - setup: - name: Setup & Build Appwrite Image - runs-on: ubuntu-latest - steps: - - name: Checkout repository - uses: actions/checkout@v6 - with: - submodules: recursive - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - name: Build Appwrite - uses: docker/build-push-action@v6 - with: - context: . - push: false - tags: '${{ env.IMAGE }}' - load: true - cache-from: type=gha - cache-to: 'type=gha,mode=max' - outputs: 'type=docker,dest=/tmp/${{ env.IMAGE }}.tar' - target: development - build-args: | - DEBUG=false - TESTING=true - VERSION=dev - - name: Cache Docker Image - uses: actions/cache@v4 - with: - key: '${{ env.CACHE_KEY }}' - path: '/tmp/${{ env.IMAGE }}.tar' - benchmarking: - name: Benchmark - runs-on: ubuntu-latest - needs: setup - permissions: - pull-requests: write - steps: - - name: Checkout repository - uses: actions/checkout@v6 - - name: Load Cache - uses: actions/cache@v4 - with: - key: '${{ env.CACHE_KEY }}' - path: '/tmp/${{ env.IMAGE }}.tar' - fail-on-cache-miss: true - - name: Load and Start Appwrite - run: | - sed -i 's/traefik/localhost/g' .env - docker load --input /tmp/${{ env.IMAGE }}.tar - docker compose up -d - sleep 10 - - name: Install Oha - run: | - echo "deb [signed-by=/usr/share/keyrings/azlux-archive-keyring.gpg] http://packages.azlux.fr/debian/ stable main" | sudo tee /etc/apt/sources.list.d/azlux.list - sudo wget -O /usr/share/keyrings/azlux-archive-keyring.gpg https://azlux.fr/repo.gpg - sudo apt update - sudo apt install oha - oha --version - - name: Benchmark PR - run: 'oha -z 180s http://localhost/v1/health/version --output-format json > benchmark.json' - - name: Cleaning - run: docker compose down -v - - name: Installing latest version - run: | - rm docker-compose.yml - rm .env - curl https://appwrite.io/install/compose -o docker-compose.yml - curl https://appwrite.io/install/env -o .env - sed -i 's/_APP_OPTIONS_ABUSE=enabled/_APP_OPTIONS_ABUSE=disabled/g' .env - docker compose up -d - sleep 10 - - name: Benchmark Latest - run: oha -z 180s http://localhost/v1/health/version --output-format json > benchmark-latest.json - - name: Prepare comment - run: | - echo '## :sparkles: Benchmark results' > benchmark.txt - echo ' ' >> benchmark.txt - echo "- Requests per second: $(jq -r '.summary.requestsPerSec|tonumber?|floor|tostring|[while(length>0;.[:-3])|.[-3:]]|reverse|join(",")' benchmark.json)" >> benchmark.txt - echo "- Requests with 200 status code: $(jq -r '.statusCodeDistribution."200"|tostring|[while(length>0;.[:-3])|.[-3:]]|reverse|join(",")' benchmark.json)" >> benchmark.txt - echo "- P99 latency: $(jq -r '.latencyPercentiles.p99' benchmark.json )" >> benchmark.txt - echo " " >> benchmark.txt - echo " " >> benchmark.txt - echo "## :zap: Benchmark Comparison" >> benchmark.txt - echo " " >> benchmark.txt - echo "| Metric | This PR | Latest version | " >> benchmark.txt - echo "| --- | --- | --- | " >> benchmark.txt - echo "| RPS | $(jq -r '.summary.requestsPerSec|tonumber?|floor|tostring|[while(length>0;.[:-3])|.[-3:]]|reverse|join(",")' benchmark.json) | $(jq -r '.summary.requestsPerSec|tonumber|floor|tostring|[while(length>0;.[:-3])|.[-3:]]|reverse|join(",")' benchmark-latest.json) | " >> benchmark.txt - echo "| 200 | $(jq -r '.statusCodeDistribution."200"|tostring|[while(length>0;.[:-3])|.[-3:]]|reverse|join(",")' benchmark.json) | $(jq -r '.statusCodeDistribution."200"|tostring|[while(length>0;.[:-3])|.[-3:]]|reverse|join(",")' benchmark-latest.json) | " >> benchmark.txt - echo "| P99 | $(jq -r '.latencyPercentiles.p99' benchmark.json ) | $(jq -r '.latencyPercentiles.p99' benchmark-latest.json ) | " >> benchmark.txt - - name: Save results - uses: actions/upload-artifact@v6 - if: '${{ !cancelled() }}' - with: - name: benchmark.json - path: benchmark.json - retention-days: 7 - - name: Find Comment - if: github.event.pull_request.head.repo.full_name == github.repository - uses: peter-evans/find-comment@v3 - id: fc - with: - issue-number: '${{ github.event.pull_request.number }}' - comment-author: 'github-actions[bot]' - body-includes: Benchmark results - - name: Comment on PR - if: github.event.pull_request.head.repo.full_name == github.repository - uses: peter-evans/create-or-update-comment@v4 - with: - comment-id: '${{ steps.fc.outputs.comment-id }}' - issue-number: '${{ github.event.pull_request.number }}' - body-path: benchmark.txt - edit-mode: replace diff --git a/.github/workflows/check-dependencies.yml b/.github/workflows/check-dependencies.yml deleted file mode 100644 index 17caf3aa6b..0000000000 --- a/.github/workflows/check-dependencies.yml +++ /dev/null @@ -1,19 +0,0 @@ -name: Check dependencies - -# Adapted from https://google.github.io/osv-scanner/github-action/#scan-on-pull-request - -on: - pull_request: - branches: [main, 1.*.x] - merge_group: - branches: [main, 1.*.x] - -permissions: - # Require writing security events to upload SARIF file to security tab - security-events: write - # Only need to read contents - contents: read - -jobs: - scan-pr: - uses: "google/osv-scanner-action/.github/workflows/osv-scanner-reusable.yml@v1.7.1" \ No newline at end of file diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000000..e59b14e550 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,651 @@ +name: CI + +concurrency: + group: ci-${{ github.ref }} + cancel-in-progress: true + +env: + COMPOSE_FILE: docker-compose.yml + IMAGE: appwrite-dev + CACHE_KEY: appwrite-dev-${{ github.event.pull_request.head.sha }} + +on: + pull_request: + workflow_dispatch: + inputs: + response_format: + description: 'Response format version to test (e.g., 1.5.0, 1.4.0)' + required: false + type: string + default: '' + +jobs: + dependencies: + name: Checks / Dependencies + if: github.event_name == 'pull_request' + permissions: + actions: read + security-events: write + contents: read + uses: "google/osv-scanner-action/.github/workflows/osv-scanner-reusable-pr.yml@v2.3.3" + + security: + name: Checks / Image + if: github.event_name == 'pull_request' + runs-on: ubuntu-latest + permissions: + contents: read + security-events: write + steps: + - name: Check out code + uses: actions/checkout@v6 + with: + fetch-depth: 0 + submodules: 'recursive' + + - name: Build the Docker image + uses: docker/build-push-action@v6 + with: + context: . + push: false + load: true + tags: pr_image:${{ github.sha }} + target: production + + - name: Run Trivy vulnerability scanner on image + uses: aquasecurity/trivy-action@0.35.0 + with: + image-ref: 'pr_image:${{ github.sha }}' + format: 'sarif' + output: 'trivy-image-results.sarif' + severity: 'CRITICAL,HIGH' + + - name: Run Trivy vulnerability scanner on source code + uses: aquasecurity/trivy-action@0.35.0 + with: + scan-type: 'fs' + scan-ref: '.' + format: 'sarif' + output: 'trivy-fs-results.sarif' + severity: 'CRITICAL,HIGH' + skip-setup-trivy: true + + - name: Upload image scan results + uses: github/codeql-action/upload-sarif@v4 + if: always() && hashFiles('trivy-image-results.sarif') != '' + with: + sarif_file: 'trivy-image-results.sarif' + category: 'trivy-image' + + - name: Upload source code scan results + uses: github/codeql-action/upload-sarif@v4 + if: always() && hashFiles('trivy-fs-results.sarif') != '' + with: + sarif_file: 'trivy-fs-results.sarif' + category: 'trivy-source' + + format: + name: Checks / Format + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v6 + with: + fetch-depth: 2 + + - run: git checkout HEAD^2 + if: github.event_name == 'pull_request' + + - name: Validate composer.json and composer.lock + run: | + docker run --rm -v $PWD:/app composer:2.8 sh -c \ + "composer validate" + + - name: Run Linter + run: | + docker run --rm -v $PWD:/app composer:2.8 sh -c \ + "composer install --profile --ignore-platform-reqs && composer lint" + + analyze: + name: Checks / Analyze + runs-on: ubuntu-latest + steps: + - name: Check out the repo + uses: actions/checkout@v6 + + - name: Run PHPStan + run: | + docker run --rm -v $PWD:/app composer:2.8 sh -c \ + "composer install --profile --ignore-platform-reqs && composer check" + + - name: Run Locale check + run: | + docker run --rm -v $PWD:/app node:24-alpine sh -c \ + "cd /app/.github/workflows/static-analysis/locale && node index.js" + + matrix: + name: Tests / Matrix + runs-on: ubuntu-latest + outputs: + databases: ${{ steps.generate.outputs.databases }} + modes: ${{ steps.generate.outputs.modes }} + steps: + - name: Generate matrix + id: generate + uses: actions/github-script@v8 + with: + script: | + const allDatabases = ['MariaDB', 'PostgreSQL', 'MongoDB']; + const allModes = ['dedicated', 'shared_v1', 'shared_v2']; + + const defaultDatabases = ['MongoDB']; + const defaultModes = ['dedicated']; + + const pr = context.payload.pull_request; + if (!pr) { + core.setOutput('databases', JSON.stringify(allDatabases)); + core.setOutput('modes', JSON.stringify(allModes)); + return; + } + + const files = await github.paginate(github.rest.pulls.listFiles, { + owner: context.repo.owner, + repo: context.repo.repo, + pull_number: pr.number, + }); + + const lockFile = files.find(f => f.filename === 'composer.lock'); + const databaseChanged = lockFile?.patch?.includes('"name": "utopia-php/database"') ?? false; + + core.setOutput('databases', JSON.stringify(databaseChanged ? allDatabases : defaultDatabases)); + core.setOutput('modes', JSON.stringify(databaseChanged ? allModes : defaultModes)); + + build: + name: Build + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v6 + with: + submodules: recursive + + - name: Login to Docker Hub + uses: docker/login-action@v4 + with: + username: ${{ vars.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v4 + + - name: Build Appwrite + uses: docker/build-push-action@v6 + with: + context: . + push: false + tags: ${{ env.IMAGE }} + load: true + cache-from: type=gha + cache-to: type=gha,mode=max + outputs: type=docker,dest=/tmp/${{ env.IMAGE }}.tar + target: development + build-args: | + DEBUG=false + TESTING=true + VERSION=dev + + - name: Cache Docker Image + uses: actions/cache@v5 + with: + key: ${{ env.CACHE_KEY }} + path: /tmp/${{ env.IMAGE }}.tar + + unit: + name: Tests / Unit + runs-on: ubuntu-latest + needs: build + permissions: + contents: read + pull-requests: write + steps: + - name: checkout + uses: actions/checkout@v6 + + - name: Load Cache + uses: actions/cache@v5 + with: + key: ${{ env.CACHE_KEY }} + path: /tmp/${{ env.IMAGE }}.tar + fail-on-cache-miss: true + + - name: Login to Docker Hub + uses: docker/login-action@v4 + with: + username: ${{ vars.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Load and Start Appwrite + timeout-minutes: 5 + run: | + docker load --input /tmp/${{ env.IMAGE }}.tar + docker compose pull --quiet --ignore-buildable + docker compose up -d --quiet-pull --wait + + - name: Environment Variables + run: docker compose exec -T appwrite vars + + - name: Run Unit Tests + uses: itznotabug/php-retry@v3 + with: + max_attempts: 2 + retry_wait_seconds: 60 + timeout_minutes: 15 + job_id: ${{ job.check_run_id }} + github_token: ${{ secrets.GITHUB_TOKEN }} + test_dir: tests/unit + command: >- + docker compose exec + -e _APP_E2E_RESPONSE_FORMAT="${{ github.event.inputs.response_format }}" + appwrite test /usr/src/code/tests/unit + + e2e_general: + name: Tests / E2E / General + runs-on: ubuntu-latest + needs: build + permissions: + contents: read + pull-requests: write + steps: + - name: checkout + uses: actions/checkout@v6 + + - name: Load Cache + uses: actions/cache@v5 + with: + key: ${{ env.CACHE_KEY }} + path: /tmp/${{ env.IMAGE }}.tar + fail-on-cache-miss: true + + - name: Login to Docker Hub + uses: docker/login-action@v4 + with: + username: ${{ vars.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Load and Start Appwrite + timeout-minutes: 5 + run: | + docker load --input /tmp/${{ env.IMAGE }}.tar + docker compose pull --quiet --ignore-buildable + docker compose up -d --quiet-pull --wait + + - name: Wait for Open Runtimes + timeout-minutes: 3 + run: | + while ! docker compose logs openruntimes-executor | grep -q "Executor is ready."; do + echo "Waiting for Executor to come online" + sleep 1 + done + + - name: Run General Tests + uses: itznotabug/php-retry@v3 + with: + max_attempts: 2 + retry_wait_seconds: 60 + timeout_minutes: 15 + job_id: ${{ job.check_run_id }} + github_token: ${{ secrets.GITHUB_TOKEN }} + test_dir: tests/e2e/General + command: >- + docker compose exec -T + -e _APP_E2E_RESPONSE_FORMAT="${{ github.event.inputs.response_format }}" + appwrite test /usr/src/code/tests/e2e/General + + - name: Failure Logs + if: failure() + run: | + echo "=== Appwrite Logs ===" + docker compose logs + + e2e_service: + name: Tests / E2E / ${{ matrix.database }} (${{ matrix.mode }}) / ${{ matrix.service }} + runs-on: ubuntu-latest + needs: [build, matrix] + permissions: + contents: read + pull-requests: write + strategy: + fail-fast: false + matrix: + database: ${{ fromJSON(needs.matrix.outputs.databases) }} + mode: ${{ fromJSON(needs.matrix.outputs.modes) }} + service: [ + Account, + Avatars, + Console, + Databases, + Functions, + FunctionsSchedule, + GraphQL, + Health, + Locale, + Projects, + Realtime, + Sites, + Proxy, + Storage, + Tokens, + Teams, + Users, + Webhooks, + VCS, + Messaging, + Migrations + ] + steps: + - name: Checkout repository + uses: actions/checkout@v6 + + - name: Load Cache + uses: actions/cache@v5 + with: + key: ${{ env.CACHE_KEY }} + path: /tmp/${{ env.IMAGE }}.tar + fail-on-cache-miss: true + + - name: Set database environment + run: | + if [ "${{ matrix.database }}" = "MariaDB" ]; then + echo "COMPOSE_PROFILES=mariadb" >> $GITHUB_ENV + echo "_APP_DB_ADAPTER=mariadb" >> $GITHUB_ENV + echo "_APP_DB_HOST=mariadb" >> $GITHUB_ENV + echo "_APP_DB_PORT=3306" >> $GITHUB_ENV + elif [ "${{ matrix.database }}" = "MongoDB" ]; then + echo "COMPOSE_PROFILES=mongodb" >> $GITHUB_ENV + echo "_APP_DB_ADAPTER=mongodb" >> $GITHUB_ENV + echo "_APP_DB_HOST=mongodb" >> $GITHUB_ENV + echo "_APP_DB_PORT=27017" >> $GITHUB_ENV + elif [ "${{ matrix.database }}" = "PostgreSQL" ]; then + echo "COMPOSE_PROFILES=postgresql" >> $GITHUB_ENV + echo "_APP_DB_ADAPTER=postgresql" >> $GITHUB_ENV + echo "_APP_DB_HOST=postgresql" >> $GITHUB_ENV + echo "_APP_DB_PORT=5432" >> $GITHUB_ENV + fi + + - name: Login to Docker Hub + uses: docker/login-action@v4 + with: + username: ${{ vars.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Load and Start Appwrite + timeout-minutes: 5 + env: + _APP_BROWSER_HOST: http://invalid-browser/v1 + _APP_DATABASE_SHARED_TABLES: ${{ matrix.mode != 'dedicated' && 'database_db_main' || '' }} + _APP_DATABASE_SHARED_TABLES_V1: ${{ matrix.mode == 'shared_v1' && 'database_db_main' || '' }} + run: | + docker load --input /tmp/${{ env.IMAGE }}.tar + docker compose pull --quiet --ignore-buildable + docker compose up -d --quiet-pull --wait + + - name: Wait for Open Runtimes + timeout-minutes: 3 + run: | + while ! docker compose logs openruntimes-executor | grep -q "Executor is ready."; do + echo "Waiting for Executor to come online" + sleep 1 + done + + - name: Run tests + uses: itznotabug/php-retry@v3 + with: + max_attempts: 2 + retry_wait_seconds: 60 + timeout_minutes: 20 + job_id: ${{ job.check_run_id }} + github_token: ${{ secrets.GITHUB_TOKEN }} + test_dir: tests/e2e/Services/${{ matrix.service }} + command: | + SERVICE_PATH="/usr/src/code/tests/e2e/Services/${{ matrix.service }}" + + # Services that rely on sequential test method execution (shared static state) + FUNCTIONAL_FLAG="--functional" + case "${{ matrix.service }}" in + Databases|Functions|Realtime) FUNCTIONAL_FLAG="" ;; + esac + + docker compose exec -T \ + -e _APP_E2E_RESPONSE_FORMAT="${{ github.event.inputs.response_format }}" \ + appwrite vendor/bin/paratest --processes $(nproc) $FUNCTIONAL_FLAG "$SERVICE_PATH" --exclude-group abuseEnabled --exclude-group screenshots --log-junit tests/e2e/Services/${{ matrix.service }}/junit.xml + + - name: Failure Logs + if: failure() + run: | + echo "=== Appwrite Logs ===" + docker compose logs + + e2e_abuse: + name: Tests / E2E / Abuse (${{ matrix.mode }}) + runs-on: ubuntu-latest + needs: [build, matrix] + permissions: + contents: read + pull-requests: write + strategy: + fail-fast: false + matrix: + mode: ${{ fromJSON(needs.matrix.outputs.modes) }} + steps: + - name: Checkout repository + uses: actions/checkout@v6 + + - name: Load Cache + uses: actions/cache@v5 + with: + key: ${{ env.CACHE_KEY }} + path: /tmp/${{ env.IMAGE }}.tar + fail-on-cache-miss: true + + - name: Login to Docker Hub + uses: docker/login-action@v4 + with: + username: ${{ vars.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Load and Start Appwrite + timeout-minutes: 5 + env: + _APP_OPTIONS_ABUSE: enabled + _APP_DATABASE_SHARED_TABLES: ${{ matrix.mode != 'dedicated' && 'database_db_main' || '' }} + _APP_DATABASE_SHARED_TABLES_V1: ${{ matrix.mode == 'shared_v1' && 'database_db_main' || '' }} + run: | + docker load --input /tmp/${{ env.IMAGE }}.tar + docker compose pull --quiet --ignore-buildable + docker compose up -d --quiet-pull --wait + + - name: Run tests + uses: itznotabug/php-retry@v3 + with: + max_attempts: 2 + retry_wait_seconds: 60 + timeout_minutes: 15 + job_id: ${{ job.check_run_id }} + github_token: ${{ secrets.GITHUB_TOKEN }} + test_dir: tests/e2e + command: >- + docker compose exec -T + -e _APP_E2E_RESPONSE_FORMAT="${{ github.event.inputs.response_format }}" + appwrite test /usr/src/code/tests/e2e --group=abuseEnabled + + - name: Failure Logs + if: failure() + run: | + echo "=== Appwrite Logs ===" + docker compose logs + + e2e_screenshots: + name: Tests / E2E / Screenshots (${{ matrix.mode }}) + runs-on: ubuntu-latest + needs: [build, matrix] + permissions: + contents: read + pull-requests: write + strategy: + fail-fast: false + matrix: + mode: ${{ fromJSON(needs.matrix.outputs.modes) }} + steps: + - name: Checkout repository + uses: actions/checkout@v6 + + - name: Load Cache + uses: actions/cache@v5 + with: + key: ${{ env.CACHE_KEY }} + path: /tmp/${{ env.IMAGE }}.tar + fail-on-cache-miss: true + + - name: Login to Docker Hub + uses: docker/login-action@v4 + with: + username: ${{ vars.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Load and Start Appwrite + timeout-minutes: 5 + env: + _APP_DATABASE_SHARED_TABLES: ${{ matrix.mode != 'dedicated' && 'database_db_main' || '' }} + _APP_DATABASE_SHARED_TABLES_V1: ${{ matrix.mode == 'shared_v1' && 'database_db_main' || '' }} + run: | + docker load --input /tmp/${{ env.IMAGE }}.tar + docker compose pull --quiet --ignore-buildable + docker compose up -d --quiet-pull --wait + + - name: Wait for Open Runtimes + timeout-minutes: 3 + run: | + while ! docker compose logs openruntimes-executor | grep -q "Executor is ready."; do + echo "Waiting for Executor to come online" + sleep 1 + done + + - name: Run tests + uses: itznotabug/php-retry@v3 + with: + max_attempts: 2 + retry_wait_seconds: 60 + timeout_minutes: 15 + job_id: ${{ job.check_run_id }} + github_token: ${{ secrets.GITHUB_TOKEN }} + test_dir: tests/e2e/Services/Sites + command: >- + docker compose exec -T + -e _APP_E2E_RESPONSE_FORMAT="${{ github.event.inputs.response_format }}" + appwrite test /usr/src/code/tests/e2e/Services/Sites --group=screenshots + + - name: Failure Logs + if: failure() + run: | + echo "=== Appwrite Logs ===" + docker compose logs + + benchmark: + name: Benchmark + runs-on: ubuntu-latest + needs: build + permissions: + pull-requests: write + steps: + - name: Checkout repository + uses: actions/checkout@v6 + + - name: Load Cache + uses: actions/cache@v5 + with: + key: ${{ env.CACHE_KEY }} + path: /tmp/${{ env.IMAGE }}.tar + fail-on-cache-miss: true + + - name: Login to Docker Hub + uses: docker/login-action@v4 + with: + username: ${{ vars.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Load and Start Appwrite + run: | + sed -i 's/traefik/localhost/g' .env + docker load --input /tmp/${{ env.IMAGE }}.tar + docker compose up -d + sleep 10 + + - name: Install Oha + run: | + echo "deb [signed-by=/usr/share/keyrings/azlux-archive-keyring.gpg] http://packages.azlux.fr/debian/ stable main" | sudo tee /etc/apt/sources.list.d/azlux.list + sudo wget -O /usr/share/keyrings/azlux-archive-keyring.gpg https://azlux.fr/repo.gpg + sudo apt update + sudo apt install oha + oha --version + + - name: Benchmark PR + run: 'oha -z 180s http://localhost/v1/health/version --output-format json > benchmark.json' + + - name: Cleaning + run: docker compose down -v + + - name: Installing latest version + run: | + rm docker-compose.yml + rm .env + curl https://appwrite.io/install/compose -o docker-compose.yml + curl https://appwrite.io/install/env -o .env + sed -i 's/_APP_OPTIONS_ABUSE=enabled/_APP_OPTIONS_ABUSE=disabled/g' .env + docker compose up -d + sleep 10 + + - name: Benchmark Latest + run: oha -z 180s http://localhost/v1/health/version --output-format json > benchmark-latest.json + + - name: Prepare comment + run: | + echo '## :sparkles: Benchmark results' > benchmark.txt + echo ' ' >> benchmark.txt + echo "- Requests per second: $(jq -r '.summary.requestsPerSec|tonumber?|floor|tostring|[while(length>0;.[:-3])|.[-3:]]|reverse|join(",")' benchmark.json)" >> benchmark.txt + echo "- Requests with 200 status code: $(jq -r '.statusCodeDistribution."200"|tostring|[while(length>0;.[:-3])|.[-3:]]|reverse|join(",")' benchmark.json)" >> benchmark.txt + echo "- P99 latency: $(jq -r '.latencyPercentiles.p99' benchmark.json )" >> benchmark.txt + echo " " >> benchmark.txt + echo " " >> benchmark.txt + echo "## :zap: Benchmark Comparison" >> benchmark.txt + echo " " >> benchmark.txt + echo "| Metric | This PR | Latest version | " >> benchmark.txt + echo "| --- | --- | --- | " >> benchmark.txt + echo "| RPS | $(jq -r '.summary.requestsPerSec|tonumber?|floor|tostring|[while(length>0;.[:-3])|.[-3:]]|reverse|join(",")' benchmark.json) | $(jq -r '.summary.requestsPerSec|tonumber|floor|tostring|[while(length>0;.[:-3])|.[-3:]]|reverse|join(",")' benchmark-latest.json) | " >> benchmark.txt + echo "| 200 | $(jq -r '.statusCodeDistribution."200"|tostring|[while(length>0;.[:-3])|.[-3:]]|reverse|join(",")' benchmark.json) | $(jq -r '.statusCodeDistribution."200"|tostring|[while(length>0;.[:-3])|.[-3:]]|reverse|join(",")' benchmark-latest.json) | " >> benchmark.txt + echo "| P99 | $(jq -r '.latencyPercentiles.p99' benchmark.json ) | $(jq -r '.latencyPercentiles.p99' benchmark-latest.json ) | " >> benchmark.txt + + - name: Save results + uses: actions/upload-artifact@v7 + if: ${{ !cancelled() }} + with: + name: benchmark.json + path: benchmark.json + retention-days: 7 + + - name: Find Comment + if: github.event.pull_request.head.repo.full_name == github.repository + uses: peter-evans/find-comment@v3 + id: fc + with: + issue-number: ${{ github.event.pull_request.number }} + comment-author: 'github-actions[bot]' + body-includes: Benchmark results + + - name: Comment on PR + if: github.event.pull_request.head.repo.full_name == github.repository + uses: peter-evans/create-or-update-comment@v4 + with: + comment-id: ${{ steps.fc.outputs.comment-id }} + issue-number: ${{ github.event.pull_request.number }} + body-path: benchmark.txt + edit-mode: replace diff --git a/.github/workflows/linter.yml b/.github/workflows/linter.yml deleted file mode 100644 index f4ae5df1ce..0000000000 --- a/.github/workflows/linter.yml +++ /dev/null @@ -1,28 +0,0 @@ -name: "Linter" - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -on: [pull_request] -jobs: - lint: - name: Linter - runs-on: ubuntu-latest - - steps: - - name: Checkout repository - uses: actions/checkout@v6 - with: - fetch-depth: 2 - - - run: git checkout HEAD^2 - - - name: Validate composer.json and composer.lock - run: | - docker run --rm -v $PWD:/app composer:2.8 sh -c \ - "composer validate" - - name: Run Linter - run: | - docker run --rm -v $PWD:/app composer:2.8 sh -c \ - "composer install --profile --ignore-platform-reqs && composer lint" diff --git a/.github/workflows/pr-scan.yml b/.github/workflows/pr-scan.yml deleted file mode 100644 index 51f3460d03..0000000000 --- a/.github/workflows/pr-scan.yml +++ /dev/null @@ -1,106 +0,0 @@ -name: PR Security Scan -on: - pull_request_target: - types: [opened, synchronize, reopened] - -jobs: - scan: - runs-on: ubuntu-latest - permissions: - contents: read - pull-requests: write - steps: - - name: Check out code - uses: actions/checkout@v6 - with: - ref: ${{ github.event.pull_request.head.sha }} - fetch-depth: 0 - submodules: 'recursive' - - - name: Build the Docker image - uses: docker/build-push-action@v6 - with: - context: . - push: false - load: true - tags: pr_image:${{ github.sha }} - target: production - - - name: Run Trivy vulnerability scanner on image - uses: aquasecurity/trivy-action@0.20.0 - with: - image-ref: 'pr_image:${{ github.sha }}' - format: 'json' - output: 'trivy-image-results.json' - severity: 'CRITICAL,HIGH' - - - name: Run Trivy vulnerability scanner on source code - uses: aquasecurity/trivy-action@0.20.0 - with: - scan-type: 'fs' - scan-ref: '.' - format: 'json' - output: 'trivy-fs-results.json' - severity: 'CRITICAL,HIGH' - - - name: Process Trivy scan results - id: process-results - uses: actions/github-script@v8 - with: - script: | - const fs = require('fs'); - let commentBody = '## Security Scan Results for PR\n\n'; - - function processResults(results, title) { - let sectionBody = `### ${title}\n\n`; - if (results.Results && results.Results.some(result => result.Vulnerabilities && result.Vulnerabilities.length > 0)) { - sectionBody += '| Package | Version | Vulnerability | Severity |\n'; - sectionBody += '|---------|---------|----------------|----------|\n'; - - const uniqueVulns = new Set(); - results.Results.forEach(result => { - if (result.Vulnerabilities) { - result.Vulnerabilities.forEach(vuln => { - const vulnKey = `${vuln.PkgName}-${vuln.InstalledVersion}-${vuln.VulnerabilityID}`; - if (!uniqueVulns.has(vulnKey)) { - uniqueVulns.add(vulnKey); - sectionBody += `| ${vuln.PkgName} | ${vuln.InstalledVersion} | [${vuln.VulnerabilityID}](https://nvd.nist.gov/vuln/detail/${vuln.VulnerabilityID}) | ${vuln.Severity} |\n`; - } - }); - } - }); - } else { - sectionBody += '🎉 No vulnerabilities found!\n'; - } - return sectionBody; - } - - try { - const imageResults = JSON.parse(fs.readFileSync('trivy-image-results.json', 'utf8')); - const fsResults = JSON.parse(fs.readFileSync('trivy-fs-results.json', 'utf8')); - - commentBody += processResults(imageResults, "Docker Image Scan Results"); - commentBody += '\n'; - commentBody += processResults(fsResults, "Source Code Scan Results"); - - } catch (error) { - commentBody += `There was an error while running the security scan: ${error.message}\n`; - commentBody += 'Please contact the core team for assistance.'; - } - - core.setOutput('comment-body', commentBody); - - name: Find Comment - uses: peter-evans/find-comment@v3 - id: fc - with: - issue-number: ${{ github.event.pull_request.number }} - comment-author: 'github-actions[bot]' - body-includes: Security Scan Results for PR - - - name: Create or update comment - uses: peter-evans/create-or-update-comment@v3 - with: - issue-number: ${{ github.event.pull_request.number }} - comment-id: ${{ steps.fc.outputs.comment-id }} - body: ${{ steps.process-results.outputs.comment-body }} - edit-mode: replace diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index 5987eeeb0c..6e4a8ba73b 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -9,7 +9,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/stale@v9 + - uses: actions/stale@v10 with: repo-token: ${{ secrets.GITHUB_TOKEN }} stale-issue-message: "This issue has been labeled as a 'question', indicating that it requires additional information from the requestor. It has been inactive for 7 days. If no further activity occurs, this issue will be closed in 14 days." diff --git a/.github/workflows/static-analysis.yml b/.github/workflows/static-analysis.yml deleted file mode 100644 index a0dc38b3b4..0000000000 --- a/.github/workflows/static-analysis.yml +++ /dev/null @@ -1,21 +0,0 @@ -name: "Static code analysis" - -on: [pull_request] -jobs: - lint: - name: CodeQL - runs-on: ubuntu-latest - - steps: - - name: Check out the repo - uses: actions/checkout@v6 - - - name: Run CodeQL - run: | - docker run --rm -v $PWD:/app composer:2.8 sh -c \ - "composer install --profile --ignore-platform-reqs && composer check" - - - name: Run Locale check - run: | - docker run --rm -v $PWD:/app node:24-alpine sh -c \ - "cd /app/.github/workflows/static-analysis/locale && node index.js" diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml deleted file mode 100644 index 0cff6288e2..0000000000 --- a/.github/workflows/tests.yml +++ /dev/null @@ -1,691 +0,0 @@ -name: "Tests" - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -env: - COMPOSE_FILE: docker-compose.yml - IMAGE: appwrite-dev - CACHE_KEY: appwrite-dev-${{ github.event.pull_request.head.sha }} - -on: - pull_request: - workflow_dispatch: - inputs: - response_format: - description: 'Response format version to test (e.g., 1.5.0, 1.4.0)' - required: false - type: string - default: '' - -jobs: - check_database_changes: - name: Check if utopia-php/database changed - runs-on: ubuntu-latest - outputs: - database_changed: ${{ steps.check.outputs.database_changed }} - steps: - - name: Checkout repository - uses: actions/checkout@v6 - - - name: Fetch base branch - run: git fetch origin ${{ github.event.pull_request.base.ref }} - - - name: Check for utopia-php/database changes - id: check - run: | - if git diff origin/${{ github.event.pull_request.base.ref }} HEAD -- composer.lock | grep -q '"name": "utopia-php/database"'; then - echo "Database version changed, going to run all mode tests." - echo "database_changed=true" >> "$GITHUB_ENV" - echo "database_changed=true" >> "$GITHUB_OUTPUT" - else - echo "database_changed=false" >> "$GITHUB_ENV" - echo "database_changed=false" >> "$GITHUB_OUTPUT" - fi - - setup: - name: Setup & Build Appwrite Image - runs-on: ubuntu-latest - steps: - - name: Checkout repository - uses: actions/checkout@v6 - with: - submodules: recursive - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Build Appwrite - uses: docker/build-push-action@v6 - with: - context: . - push: false - tags: ${{ env.IMAGE }} - load: true - cache-from: type=gha - cache-to: type=gha,mode=max - outputs: type=docker,dest=/tmp/${{ env.IMAGE }}.tar - target: development - build-args: | - DEBUG=false - TESTING=true - VERSION=dev - - - name: Cache Docker Image - uses: actions/cache@v4 - with: - key: ${{ env.CACHE_KEY }} - path: /tmp/${{ env.IMAGE }}.tar - - unit_test: - name: Unit Test - runs-on: ubuntu-latest - needs: setup - permissions: - contents: read - pull-requests: write - - steps: - - name: checkout - uses: actions/checkout@v6 - - - name: Load Cache - uses: actions/cache@v4 - with: - key: ${{ env.CACHE_KEY }} - path: /tmp/${{ env.IMAGE }}.tar - fail-on-cache-miss: true - - - name: Load and Start Appwrite - timeout-minutes: 3 - run: | - docker load --input /tmp/${{ env.IMAGE }}.tar - docker compose up -d - until docker compose exec -T appwrite doctor > /dev/null 2>&1; do - echo "Waiting for Appwrite to be ready..." - sleep 2 - done - - - name: Environment Variables - run: docker compose exec -T appwrite vars - - - name: Run Unit Tests - uses: itznotabug/php-retry@v3 - with: - max_attempts: 2 - retry_wait_seconds: 300 - timeout_minutes: 15 - job_id: ${{ job.check_run_id }} - github_token: ${{ secrets.GITHUB_TOKEN }} - test_dir: tests/unit - command: >- - docker compose exec - -e _APP_E2E_RESPONSE_FORMAT="${{ github.event.inputs.response_format }}" - appwrite test /usr/src/code/tests/unit - - e2e_general_test: - name: E2E General Test - runs-on: ubuntu-latest - needs: setup - permissions: - contents: read - pull-requests: write - steps: - - name: checkout - uses: actions/checkout@v6 - - - name: Load Cache - uses: actions/cache@v4 - with: - key: ${{ env.CACHE_KEY }} - path: /tmp/${{ env.IMAGE }}.tar - fail-on-cache-miss: true - - - name: Load and Start Appwrite - timeout-minutes: 3 - run: | - docker load --input /tmp/${{ env.IMAGE }}.tar - docker compose up -d - until docker compose exec -T appwrite doctor > /dev/null 2>&1; do - echo "Waiting for Appwrite to be ready..." - sleep 2 - done - - - name: Wait for Open Runtimes - timeout-minutes: 3 - run: | - while ! docker compose logs openruntimes-executor | grep -q "Executor is ready."; do - echo "Waiting for Executor to come online" - sleep 1 - done - - - name: Run General Tests - uses: itznotabug/php-retry@v3 - with: - max_attempts: 2 - retry_wait_seconds: 300 - timeout_minutes: 15 - job_id: ${{ job.check_run_id }} - github_token: ${{ secrets.GITHUB_TOKEN }} - test_dir: tests/e2e/General - command: >- - docker compose exec -T - -e _APP_E2E_RESPONSE_FORMAT="${{ github.event.inputs.response_format }}" - appwrite test /usr/src/code/tests/e2e/General --debug - - - name: Failure Logs - if: failure() - run: | - echo "=== Appwrite Logs ===" - docker compose logs - - e2e_service_test: - name: E2E Service Test - runs-on: ubuntu-latest - needs: setup - permissions: - contents: read - pull-requests: write - strategy: - fail-fast: false - matrix: - db_adapter: [ - MARIADB, - POSTGRESQL, - MONGODB - ] - service: [ - Account, - Avatars, - Console, - Databases, - Functions, - FunctionsSchedule, - GraphQL, - Health, - Locale, - Projects, - Realtime, - Sites, - Proxy, - Storage, - Tokens, - Teams, - Users, - Webhooks, - VCS, - Messaging, - Migrations - ] - steps: - - name: Checkout repository - uses: actions/checkout@v6 - - - name: Load Cache - uses: actions/cache@v4 - with: - key: ${{ env.CACHE_KEY }} - path: /tmp/${{ env.IMAGE }}.tar - fail-on-cache-miss: true - - - name: Set DB Adapter environment - id: set-db-env - run: | - DB_ADAPTER_LOWER=$(echo "${{ matrix.db_adapter }}" | tr 'A-Z' 'a-z') - echo "COMPOSE_PROFILES=${DB_ADAPTER_LOWER}" >> $GITHUB_ENV - - if [ "${{ matrix.db_adapter }}" = "MARIADB" ]; then - echo "_APP_DB_ADAPTER=mariadb" >> $GITHUB_ENV - echo "_APP_DB_HOST=mariadb" >> $GITHUB_ENV - echo "_APP_DB_PORT=3306" >> $GITHUB_ENV - elif [ "${{ matrix.db_adapter }}" = "MONGODB" ]; then - echo "_APP_DB_ADAPTER=mongodb" >> $GITHUB_ENV - echo "_APP_DB_HOST=mongodb" >> $GITHUB_ENV - echo "_APP_DB_PORT=27017" >> $GITHUB_ENV - elif [ "${{ matrix.db_adapter }}" = "POSTGRESQL" ]; then - echo "_APP_DB_ADAPTER=postgresql" >> $GITHUB_ENV - echo "_APP_DB_HOST=postgresql" >> $GITHUB_ENV - echo "_APP_DB_PORT=5432" >> $GITHUB_ENV - fi - - - name: Load and Start Appwrite - timeout-minutes: 3 - env: - _APP_BROWSER_HOST: http://invalid-browser/v1 - run: | - docker load --input /tmp/${{ env.IMAGE }}.tar - docker compose up -d - until docker compose exec -T appwrite doctor > /dev/null 2>&1; do - echo "Waiting for Appwrite to be ready..." - sleep 2 - done - - - name: Wait for Open Runtimes - timeout-minutes: 3 - run: | - while ! docker compose logs openruntimes-executor | grep -q "Executor is ready."; do - echo "Waiting for Executor to come online" - sleep 1 - done - - - name: Run ${{ matrix.service }} tests with Project table mode - uses: itznotabug/php-retry@v3 - with: - max_attempts: 2 - retry_wait_seconds: 300 - timeout_minutes: 20 - job_id: ${{ job.check_run_id }} - github_token: ${{ secrets.GITHUB_TOKEN }} - test_dir: tests/e2e/Services/${{ matrix.service }} - command: | - echo "Using project tables" - SERVICE_PATH="/usr/src/code/tests/e2e/Services/${{ matrix.service }}" - - # Services that rely on sequential test method execution (shared static state) - FUNCTIONAL_FLAG="--functional" - case "${{ matrix.service }}" in - Databases|Functions|Realtime) FUNCTIONAL_FLAG="" ;; - esac - - echo "Running with paratest (parallel) for: ${{ matrix.service }} ${FUNCTIONAL_FLAG:+(functional)}" - docker compose exec -T \ - -e _APP_DATABASE_SHARED_TABLES="" \ - -e _APP_DATABASE_SHARED_TABLES_V1="" \ - -e _APP_DB_ADAPTER="${{ env._APP_DB_ADAPTER }}" \ - -e _APP_DB_HOST="${{ env._APP_DB_HOST }}" \ - -e _APP_DB_PORT="${{ env._APP_DB_PORT }}" \ - -e _APP_DB_SCHEMA=appwrite \ - -e _APP_E2E_RESPONSE_FORMAT="${{ github.event.inputs.response_format }}" \ - appwrite vendor/bin/paratest --processes $(nproc) $FUNCTIONAL_FLAG "$SERVICE_PATH" --exclude-group abuseEnabled --exclude-group screenshots --exclude-group ciIgnore --log-junit tests/e2e/Services/${{ matrix.service }}/junit.xml - - - name: Failure Logs - if: failure() - run: | - echo "=== Appwrite Logs ===" - docker compose logs - - e2e_shared_mode_test: - name: E2E Shared Mode Service Test - runs-on: ubuntu-latest - needs: [ setup, check_database_changes ] - if: needs.check_database_changes.outputs.database_changed == 'true' - permissions: - contents: read - pull-requests: write - strategy: - fail-fast: false - matrix: - service: - [ - Account, - Avatars, - Console, - Databases, - Functions, - FunctionsSchedule, - GraphQL, - Health, - Locale, - Projects, - Realtime, - Sites, - Proxy, - Storage, - Teams, - Users, - Webhooks, - VCS, - Messaging, - Migrations, - Tokens - ] - tables-mode: [ - 'Shared V1', - 'Shared V2', - ] - - steps: - - name: checkout - uses: actions/checkout@v6 - - - name: Load Cache - uses: actions/cache@v4 - with: - key: ${{ env.CACHE_KEY }} - path: /tmp/${{ env.IMAGE }}.tar - fail-on-cache-miss: true - - - name: Load and Start Appwrite - timeout-minutes: 3 - run: | - docker load --input /tmp/${{ env.IMAGE }}.tar - docker compose up -d - until docker compose exec -T appwrite doctor > /dev/null 2>&1; do - echo "Waiting for Appwrite to be ready..." - sleep 2 - done - - - name: Wait for Open Runtimes - timeout-minutes: 3 - run: | - while ! docker compose logs openruntimes-executor | grep -q "Executor is ready."; do - echo "Waiting for Executor to come online" - sleep 1 - done - - - name: Run ${{ matrix.service }} tests with ${{ matrix.tables-mode }} table mode - uses: itznotabug/php-retry@v3 - with: - max_attempts: 2 - retry_wait_seconds: 300 - timeout_minutes: 20 - job_id: ${{ job.check_run_id }} - github_token: ${{ secrets.GITHUB_TOKEN }} - test_dir: tests/e2e/Services/${{ matrix.service }} - command: | - if [ "${{ matrix.tables-mode }}" == "Shared V1" ]; then - echo "Using shared tables V1" - export _APP_DATABASE_SHARED_TABLES=database_db_main - export _APP_DATABASE_SHARED_TABLES_V1=database_db_main - elif [ "${{ matrix.tables-mode }}" == "Shared V2" ]; then - echo "Using shared tables V2" - export _APP_DATABASE_SHARED_TABLES=database_db_main - export _APP_DATABASE_SHARED_TABLES_V1= - fi - - SERVICE_PATH="/usr/src/code/tests/e2e/Services/${{ matrix.service }}" - - # Services that rely on sequential test method execution (shared static state) - FUNCTIONAL_FLAG="--functional" - case "${{ matrix.service }}" in - Databases|Functions|Realtime) FUNCTIONAL_FLAG="" ;; - esac - - docker compose exec -T \ - -e _APP_DATABASE_SHARED_TABLES \ - -e _APP_DATABASE_SHARED_TABLES_V1 \ - -e _APP_E2E_RESPONSE_FORMAT="${{ github.event.inputs.response_format }}" \ - appwrite vendor/bin/paratest --processes $(nproc) $FUNCTIONAL_FLAG "$SERVICE_PATH" --exclude-group abuseEnabled --exclude-group screenshots --exclude-group ciIgnore --log-junit tests/e2e/Services/${{ matrix.service }}/junit.xml - - - name: Failure Logs - if: failure() - run: | - echo "=== Appwrite Worker Builds Logs ===" - docker compose logs appwrite-worker-builds - echo "=== OpenRuntimes Executor Logs ===" - docker compose logs openruntimes-executor - - e2e_abuse_enabled: - name: E2E Service Test (Abuse enabled) - runs-on: ubuntu-latest - needs: setup - permissions: - contents: read - pull-requests: write - steps: - - name: checkout - uses: actions/checkout@v6 - - - name: Load Cache - uses: actions/cache@v4 - with: - key: ${{ env.CACHE_KEY }} - path: /tmp/${{ env.IMAGE }}.tar - fail-on-cache-miss: true - - - name: Load and Start Appwrite - timeout-minutes: 3 - run: | - docker load --input /tmp/${{ env.IMAGE }}.tar - sed -i 's/_APP_OPTIONS_ABUSE=disabled/_APP_OPTIONS_ABUSE=enabled/' .env - docker compose up -d - until docker compose exec -T appwrite doctor > /dev/null 2>&1; do - echo "Waiting for Appwrite to be ready..." - sleep 2 - done - - - name: Run Projects tests in dedicated table mode - uses: itznotabug/php-retry@v3 - with: - max_attempts: 2 - retry_wait_seconds: 300 - timeout_minutes: 15 - job_id: ${{ job.check_run_id }} - github_token: ${{ secrets.GITHUB_TOKEN }} - test_dir: tests/e2e/Services/Projects - command: | - echo "Using project tables" - export _APP_DATABASE_SHARED_TABLES= - export _APP_DATABASE_SHARED_TABLES_V1= - - docker compose exec -T \ - -e _APP_DATABASE_SHARED_TABLES \ - -e _APP_DATABASE_SHARED_TABLES_V1 \ - -e _APP_E2E_RESPONSE_FORMAT="${{ github.event.inputs.response_format }}" \ - appwrite test /usr/src/code/tests/e2e/Services/Projects --debug --group=abuseEnabled - - - name: Failure Logs - if: failure() - run: | - echo "=== Appwrite Worker Builds Logs ===" - docker compose logs appwrite-worker-builds - echo "=== OpenRuntimes Executor Logs ===" - docker compose logs openruntimes-executor - - e2e_abuse_enabled_shared_mode: - name: E2E Shared Mode Service Test (Abuse enabled) - runs-on: ubuntu-latest - needs: [ setup, check_database_changes ] - if: needs.check_database_changes.outputs.database_changed == 'true' - permissions: - contents: read - pull-requests: write - strategy: - fail-fast: false - matrix: - tables-mode: [ - 'Shared V1', - 'Shared V2', - ] - steps: - - name: checkout - uses: actions/checkout@v6 - - - name: Load Cache - uses: actions/cache@v4 - with: - key: ${{ env.CACHE_KEY }} - path: /tmp/${{ env.IMAGE }}.tar - fail-on-cache-miss: true - - - name: Load and Start Appwrite - timeout-minutes: 3 - run: | - docker load --input /tmp/${{ env.IMAGE }}.tar - sed -i 's/_APP_OPTIONS_ABUSE=disabled/_APP_OPTIONS_ABUSE=enabled/' .env - docker compose up -d - until docker compose exec -T appwrite doctor > /dev/null 2>&1; do - echo "Waiting for Appwrite to be ready..." - sleep 2 - done - - - name: Run Projects tests in ${{ matrix.tables-mode }} table mode - uses: itznotabug/php-retry@v3 - with: - max_attempts: 2 - retry_wait_seconds: 300 - timeout_minutes: 15 - job_id: ${{ job.check_run_id }} - github_token: ${{ secrets.GITHUB_TOKEN }} - test_dir: tests/e2e/Services/Projects - command: | - if [ "${{ matrix.tables-mode }}" == "Shared V1" ]; then - echo "Using shared tables V1" - export _APP_DATABASE_SHARED_TABLES=database_db_main - export _APP_DATABASE_SHARED_TABLES_V1=database_db_main - elif [ "${{ matrix.tables-mode }}" == "Shared V2" ]; then - echo "Using shared tables V2" - export _APP_DATABASE_SHARED_TABLES=database_db_main - export _APP_DATABASE_SHARED_TABLES_V1= - fi - - docker compose exec -T \ - -e _APP_DATABASE_SHARED_TABLES \ - -e _APP_DATABASE_SHARED_TABLES_V1 \ - -e _APP_E2E_RESPONSE_FORMAT="${{ github.event.inputs.response_format }}" \ - appwrite test /usr/src/code/tests/e2e/Services/Projects --debug --group=abuseEnabled - - - name: Failure Logs - if: failure() - run: | - echo "=== Appwrite Worker Builds Logs ===" - docker compose logs appwrite-worker-builds - echo "=== OpenRuntimes Executor Logs ===" - docker compose logs openruntimes-executor - - e2e_screenshots: - name: E2E Service Test (Site Screenshots) - runs-on: ubuntu-latest - needs: setup - permissions: - contents: read - pull-requests: write - steps: - - name: checkout - uses: actions/checkout@v6 - - - name: Load Cache - uses: actions/cache@v4 - with: - key: ${{ env.CACHE_KEY }} - path: /tmp/${{ env.IMAGE }}.tar - fail-on-cache-miss: true - - - name: Load and Start Appwrite - timeout-minutes: 3 - run: | - docker load --input /tmp/${{ env.IMAGE }}.tar - sed -i 's/_APP_OPTIONS_ABUSE=disabled/_APP_OPTIONS_ABUSE=enabled/' .env - docker compose up -d - until docker compose exec -T appwrite doctor > /dev/null 2>&1; do - echo "Waiting for Appwrite to be ready..." - sleep 2 - done - - - name: Wait for Open Runtimes - timeout-minutes: 3 - run: | - while ! docker compose logs openruntimes-executor | grep -q "Executor is ready."; do - echo "Waiting for Executor to come online" - sleep 1 - done - - - name: Run Site tests with browser connected in dedicated table mode - uses: itznotabug/php-retry@v3 - with: - max_attempts: 2 - retry_wait_seconds: 300 - timeout_minutes: 15 - job_id: ${{ job.check_run_id }} - github_token: ${{ secrets.GITHUB_TOKEN }} - test_dir: tests/e2e/Services/Sites - command: | - echo "Keeping original value of _APP_BROWSER_HOST" - echo "Using project tables" - export _APP_DATABASE_SHARED_TABLES= - export _APP_DATABASE_SHARED_TABLES_V1= - - docker compose exec -T \ - -e _APP_DATABASE_SHARED_TABLES \ - -e _APP_DATABASE_SHARED_TABLES_V1 \ - -e _APP_E2E_RESPONSE_FORMAT="${{ github.event.inputs.response_format }}" \ - appwrite test /usr/src/code/tests/e2e/Services/Sites --debug --group=screenshots - - - name: Failure Logs - if: failure() - run: | - echo "=== Appwrite Worker Builds Logs ===" - docker compose logs appwrite-worker-builds - echo "=== OpenRuntimes Executor Logs ===" - docker compose logs openruntimes-executor - - e2e_screenshots_shared_mode: - name: E2E Shared Mode Service Test (Site Screenshots) - runs-on: ubuntu-latest - needs: [ setup, check_database_changes ] - if: needs.check_database_changes.outputs.database_changed == 'true' - permissions: - contents: read - pull-requests: write - strategy: - fail-fast: false - matrix: - tables-mode: [ - 'Shared V1', - 'Shared V2', - ] - steps: - - name: checkout - uses: actions/checkout@v6 - - - name: Load Cache - uses: actions/cache@v4 - with: - key: ${{ env.CACHE_KEY }} - path: /tmp/${{ env.IMAGE }}.tar - fail-on-cache-miss: true - - - name: Load and Start Appwrite - timeout-minutes: 3 - run: | - docker load --input /tmp/${{ env.IMAGE }}.tar - sed -i 's/_APP_OPTIONS_ABUSE=disabled/_APP_OPTIONS_ABUSE=enabled/' .env - docker compose up -d - until docker compose exec -T appwrite doctor > /dev/null 2>&1; do - echo "Waiting for Appwrite to be ready..." - sleep 2 - done - - - name: Wait for Open Runtimes - timeout-minutes: 3 - run: | - while ! docker compose logs openruntimes-executor | grep -q "Executor is ready."; do - echo "Waiting for Executor to come online" - sleep 1 - done - - - name: Run Site tests with browser connected in ${{ matrix.tables-mode }} table mode - uses: itznotabug/php-retry@v3 - with: - max_attempts: 2 - retry_wait_seconds: 300 - timeout_minutes: 15 - job_id: ${{ job.check_run_id }} - github_token: ${{ secrets.GITHUB_TOKEN }} - test_dir: tests/e2e/Services/Sites - command: | - echo "Keeping original value of _APP_BROWSER_HOST" - if [ "${{ matrix.tables-mode }}" == "Shared V1" ]; then - echo "Using shared tables V1" - export _APP_DATABASE_SHARED_TABLES=database_db_main - export _APP_DATABASE_SHARED_TABLES_V1=database_db_main - elif [ "${{ matrix.tables-mode }}" == "Shared V2" ]; then - echo "Using shared tables V2" - export _APP_DATABASE_SHARED_TABLES=database_db_main - export _APP_DATABASE_SHARED_TABLES_V1= - fi - - docker compose exec -T \ - -e _APP_DATABASE_SHARED_TABLES \ - -e _APP_DATABASE_SHARED_TABLES_V1 \ - -e _APP_E2E_RESPONSE_FORMAT="${{ github.event.inputs.response_format }}" \ - appwrite test /usr/src/code/tests/e2e/Services/Sites --debug --group=screenshots - - - name: Failure Logs - if: failure() - run: | - echo "=== Appwrite Worker Builds Logs ===" - docker compose logs appwrite-worker-builds - echo "=== OpenRuntimes Executor Logs ===" - docker compose logs openruntimes-executor diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 0ccc8e8372..5d7ab96a4f 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -409,14 +409,16 @@ Next follow the appropriate steps below depending on whether you're adding the m **API** -In file `app/controllers/shared/api.php` On the database listener, add to an existing or create a new switch case. Add a call to the usage worker with your new metric const like so: +In file `app/controllers/shared/api.php` On the database listener, add to an existing or create a new switch case. Accumulate metrics in the usage context like so: ```php case $document->getCollection() === 'teams': - $queueForStatsUsage - ->addMetric(METRIC_TEAMS, $value); // per project + $usage->addMetric(METRIC_TEAMS, $value); // per project break; ``` + +The metrics will be automatically published by the shutdown hook at the end of the request. There is no need to manually trigger or publish. + There are cases when you need to handle metric that has a parent entity, like buckets. Files are linked to a parent bucket, you should verify you remove the files stats when you delete a bucket. @@ -425,14 +427,13 @@ In that case you need also to handle children removal using addReduce() method c ```php case $document->getCollection() === 'buckets': //buckets - $queueForStatsUsage - ->addMetric(METRIC_BUCKETS, $value); // per project + $usage->addMetric(METRIC_BUCKETS, $value); // per project if ($event === Database::EVENT_DOCUMENT_DELETE) { - $queueForStatsUsage + $usage ->addReduce($document); } break; - + ``` In addition, you will also need to add some logic to the `reduce()` method of the Usage worker located in `/src/Appwrite/Platform/Workers/Usage.php`, like so: @@ -460,8 +461,12 @@ case $document->getCollection() === 'buckets': **Background worker** -You need to inject the usage queue in the desired worker on the constructor method +You need to inject the usage context and publisher in the desired worker on the constructor method ```php +use Appwrite\Usage\Context; +use Appwrite\Event\Publisher\Usage as UsagePublisher; +use Appwrite\Event\Message\Usage as UsageMessage; + /** * @throws Exception */ @@ -474,24 +479,32 @@ public function __construct() ->inject('dbForProject') ->inject('queueForFunctions') ->inject('queueForEvents') - ->inject('queueForStatsUsage') + ->inject('usage') + ->inject('publisherForUsage') ->inject('log') - ->callback(fn (Message $message, Database $dbForProject, Func $queueForFunctions, Event $queueForEvents, StatsUsage $queueForStatsUsage, Log $log) => $this->action($message, $dbForProject, $queueForFunctions, $queueForEvents, $queueForStatsUsage, $log)); + ->callback(fn (Message $message, Database $dbForProject, Func $queueForFunctions, Event $queueForEvents, Context $usage, UsagePublisher $publisherForUsage, Log $log) => $this->action($message, $dbForProject, $queueForFunctions, $queueForEvents, $usage, $publisherForUsage, $log)); } ``` -and then trigger the queue with the new metric like so: +and then accumulate metrics, create a message, and publish like so: ```php -$queueForStatsUsage +$usage ->addMetric(METRIC_BUILDS, 1) ->addMetric(METRIC_BUILDS_STORAGE, $build->getAttribute('size', 0)) ->addMetric(METRIC_BUILDS_COMPUTE, (int)$build->getAttribute('duration', 0) * 1000) - ->addMetric(str_replace('{functionInternalId}', $function->getSequence(), METRIC_FUNCTION_ID_BUILDS), 1) + ->addMetric(str_replace('{functionInternalId}', $function->getSequence(), METRIC_FUNCTION_ID_BUILDS), 1) ->addMetric(str_replace('{functionInternalId}', $function->getSequence(), METRIC_FUNCTION_ID_BUILDS_STORAGE), $build->getAttribute('size', 0)) - ->addMetric(str_replace('{functionInternalId}', $function->getSequence(), METRIC_FUNCTION_ID_BUILDS_COMPUTE), (int)$build->getAttribute('duration', 0) * 1000) - ->setProject($project) - ->trigger(); + ->addMetric(str_replace('{functionInternalId}', $function->getSequence(), METRIC_FUNCTION_ID_BUILDS_COMPUTE), (int)$build->getAttribute('duration', 0) * 1000); + +// Publish the accumulated metrics (workers don't have shutdown hooks) +$message = new UsageMessage( + project: $project, + metrics: $usage->getMetrics(), + reduce: $usage->getReduce() +); +$publisherForUsage->enqueue($message); +$usage->reset(); ``` diff --git a/Dockerfile b/Dockerfile index 266d4501d0..210c2bc3d9 100755 --- a/Dockerfile +++ b/Dockerfile @@ -70,7 +70,7 @@ RUN chmod +x /usr/local/bin/doctor && \ chmod +x /usr/local/bin/sdks && \ chmod +x /usr/local/bin/specs && \ chmod +x /usr/local/bin/ssl && \ - chmod +x /usr/local/bin/time-travel && \ + chmod +x /usr/local/bin/task-time-travel && \ chmod +x /usr/local/bin/screenshot && \ chmod +x /usr/local/bin/test && \ chmod +x /usr/local/bin/upgrade && \ diff --git a/app/cli.php b/app/cli.php index 052643f004..ee134b9487 100644 --- a/app/cli.php +++ b/app/cli.php @@ -4,11 +4,13 @@ require_once __DIR__ . '/init.php'; use Appwrite\Event\Certificate; use Appwrite\Event\Delete; +use Appwrite\Event\Event; use Appwrite\Event\Func; +use Appwrite\Event\Publisher\Usage as UsagePublisher; use Appwrite\Event\StatsResources; -use Appwrite\Event\StatsUsage; use Appwrite\Platform\Appwrite; use Appwrite\Runtimes\Runtimes; +use Appwrite\Usage\Context as UsageContext; use Appwrite\Utopia\Database\Documents\User; use Executor\Executor; use Swoole\Runtime; @@ -29,6 +31,7 @@ use Utopia\Platform\Service; use Utopia\Pools\Group; use Utopia\Queue\Broker\Pool as BrokerPool; use Utopia\Queue\Publisher; +use Utopia\Queue\Queue; use Utopia\Registry\Registry; use Utopia\System\System; use Utopia\Telemetry\Adapter\None as NoTelemetry; @@ -47,7 +50,7 @@ $platform = new Appwrite(); $args = $platform->getEnv('argv'); \array_shift($args); -if (!isset($args[0])) { +if (! isset($args[0])) { Console::error('Missing task name'); Console::exit(1); } @@ -85,6 +88,7 @@ $setResource('pools', function (Registry $register) { $setResource('authorization', function () { $authorization = new Authorization(); $authorization->disable(); + return $authorization; }, []); @@ -113,7 +117,7 @@ $setResource('dbForPlatform', function ($pools, $cache, $authorization) { $collections = Config::getParam('collections', [])['console']; $last = \array_key_last($collections); - if (!($dbForPlatform->exists($dbForPlatform->getDatabase(), $last))) { /** TODO cache ready variable using registry */ + if (! ($dbForPlatform->exists($dbForPlatform->getDatabase(), $last))) { /** TODO cache ready variable using registry */ throw new Exception('Tables not ready yet.'); } @@ -122,10 +126,10 @@ $setResource('dbForPlatform', function ($pools, $cache, $authorization) { Console::warning($err->getMessage()); sleep($sleep); } - } while ($attempts < $maxAttempts && !$ready); + } while ($attempts < $maxAttempts && ! $ready); - if (!$ready) { - throw new Exception("Console is not ready yet. Please try again later."); + if (! $ready) { + throw new Exception('Console is not ready yet. Please try again later.'); } return $dbForPlatform; @@ -163,7 +167,7 @@ $setResource('getProjectDB', function (Group $pools, Database $dbForPlatform, $c if (\in_array($dsn->getHost(), $sharedTables)) { $database ->setSharedTables(true) - ->setTenant((int)$project->getSequence()) + ->setTenant((int) $project->getSequence()) ->setNamespace($dsn->getParam('namespace')); } else { $database @@ -184,7 +188,7 @@ $setResource('getProjectDB', function (Group $pools, Database $dbForPlatform, $c if (\in_array($dsn->getHost(), $sharedTables)) { $database ->setSharedTables(true) - ->setTenant((int)$project->getSequence()) + ->setTenant((int) $project->getSequence()) ->setNamespace($dsn->getParam('namespace')); } else { $database @@ -207,8 +211,9 @@ $setResource('getLogsDB', function (Group $pools, Cache $cache, Authorization $a $database = null; return function (?Document $project = null) use ($pools, $cache, $database, $authorization) { - if ($database !== null && $project !== null && !$project->isEmpty() && $project->getId() !== 'console') { - $database->setTenant((int)$project->getSequence()); + if ($database !== null && $project !== null && ! $project->isEmpty() && $project->getId() !== 'console') { + $database->setTenant((int) $project->getSequence()); + return $database; } @@ -224,8 +229,8 @@ $setResource('getLogsDB', function (Group $pools, Cache $cache, Authorization $a ->setMaxQueryValues(APP_DATABASE_QUERY_MAX_VALUES); // set tenant - if ($project !== null && !$project->isEmpty() && $project->getId() !== 'console') { - $database->setTenant((int)$project->getSequence()); + if ($project !== null && ! $project->isEmpty() && $project->getId() !== 'console') { + $database->setTenant((int) $project->getSequence()); } return $database; @@ -243,15 +248,16 @@ $setResource('publisherFunctions', function (BrokerPool $publisher) { $setResource('publisherMigrations', function (BrokerPool $publisher) { return $publisher; }, ['publisher']); -$setResource('publisherStatsUsage', function (BrokerPool $publisher) { - return $publisher; -}, ['publisher']); $setResource('publisherMessaging', function (BrokerPool $publisher) { return $publisher; }, ['publisher']); -$setResource('queueForStatsUsage', function (Publisher $publisher) { - return new StatsUsage($publisher); -}, ['publisher']); +$setResource('usage', function () { + return new UsageContext(); +}, []); +$setResource('publisherForUsage', fn (Publisher $publisher) => new UsagePublisher( + $publisher, + new Queue(System::getEnv('_APP_STATS_USAGE_QUEUE_NAME', Event::STATS_USAGE_QUEUE_NAME)) +), ['publisher']); $setResource('queueForStatsResources', function (Publisher $publisher) { return new StatsResources($publisher); }, ['publisher']); diff --git a/app/config/collections/projects.php b/app/config/collections/projects.php index 6c417ae145..55dceb9b40 100644 --- a/app/config/collections/projects.php +++ b/app/config/collections/projects.php @@ -788,6 +788,17 @@ return [ 'default' => null, 'filters' => [], ], + [ + 'array' => false, + '$id' => ID::custom('specification'), + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => 128, + 'signed' => false, + 'required' => false, + 'default' => APP_COMPUTE_SPECIFICATION_DEFAULT, + 'filters' => [], + ], [ 'array' => false, '$id' => ID::custom('buildSpecification'), @@ -1245,6 +1256,17 @@ return [ 'array' => false, 'filters' => [], ], + [ + 'array' => false, + '$id' => ID::custom('specification'), + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => 128, + 'signed' => false, + 'required' => false, + 'default' => APP_COMPUTE_SPECIFICATION_DEFAULT, + 'filters' => [], + ], [ 'array' => false, '$id' => ID::custom('buildSpecification'), diff --git a/app/controllers/api/account.php b/app/controllers/api/account.php index b58a9b4185..a780bfdac3 100644 --- a/app/controllers/api/account.php +++ b/app/controllers/api/account.php @@ -14,7 +14,6 @@ use Appwrite\Event\Delete; use Appwrite\Event\Event; use Appwrite\Event\Mail; use Appwrite\Event\Messaging; -use Appwrite\Event\StatsUsage; use Appwrite\Extend\Exception; use Appwrite\Hooks\Hooks; use Appwrite\Network\Validator\Email as EmailValidator; @@ -28,6 +27,7 @@ use Appwrite\SDK\MethodType; use Appwrite\SDK\Response as SDKResponse; use Appwrite\Template\Template; use Appwrite\URL\URL as URLParser; +use Appwrite\Usage\Context; use Appwrite\Utopia\Database\Documents\User; use Appwrite\Utopia\Database\Validator\CustomId; use Appwrite\Utopia\Database\Validator\Queries\Identities; @@ -2801,12 +2801,12 @@ Http::post('/v1/account/tokens/phone') ->inject('queueForMessaging') ->inject('locale') ->inject('timelimit') - ->inject('queueForStatsUsage') + ->inject('usage') ->inject('plan') ->inject('store') ->inject('proofForCode') ->inject('authorization') - ->action(function (string $userId, string $phone, Request $request, Response $response, User $user, Document $project, array $platform, Database $dbForProject, Event $queueForEvents, Messaging $queueForMessaging, Locale $locale, callable $timelimit, StatsUsage $queueForStatsUsage, array $plan, Store $store, ProofsCode $proofForCode, Authorization $authorization) { + ->action(function (string $userId, string $phone, Request $request, Response $response, User $user, Document $project, array $platform, Database $dbForProject, Event $queueForEvents, Messaging $queueForMessaging, Locale $locale, callable $timelimit, Context $usage, array $plan, Store $store, ProofsCode $proofForCode, Authorization $authorization) { if (empty(System::getEnv('_APP_SMS_PROVIDER'))) { throw new Exception(Exception::GENERAL_PHONE_DISABLED, 'Phone provider not configured'); } @@ -2955,16 +2955,12 @@ Http::post('/v1/account/tokens/phone') $countryCode = $helper->parse($phone)->getCountryCode(); if (!empty($countryCode)) { - $queueForStatsUsage - ->addMetric(str_replace('{countryCode}', $countryCode, METRIC_AUTH_METHOD_PHONE_COUNTRY_CODE), 1); + $usage->addMetric(str_replace('{countryCode}', $countryCode, METRIC_AUTH_METHOD_PHONE_COUNTRY_CODE), 1); } } catch (NumberParseException $e) { // Ignore invalid phone number for country code stats } - $queueForStatsUsage - ->addMetric(METRIC_AUTH_METHOD_PHONE, 1) - ->setProject($project) - ->trigger(); + $usage->addMetric(METRIC_AUTH_METHOD_PHONE, 1); } $token->setAttribute('secret', $secret); @@ -4199,11 +4195,11 @@ Http::post('/v1/account/verifications/phone') ->inject('project') ->inject('locale') ->inject('timelimit') - ->inject('queueForStatsUsage') + ->inject('usage') ->inject('plan') ->inject('proofForCode') ->inject('authorization') - ->action(function (Request $request, Response $response, User $user, Database $dbForProject, Event $queueForEvents, Messaging $queueForMessaging, Document $project, Locale $locale, callable $timelimit, StatsUsage $queueForStatsUsage, array $plan, ProofsCode $proofForCode, Authorization $authorization) { + ->action(function (Request $request, Response $response, User $user, Database $dbForProject, Event $queueForEvents, Messaging $queueForMessaging, Document $project, Locale $locale, callable $timelimit, Context $usage, array $plan, ProofsCode $proofForCode, Authorization $authorization) { if (empty(System::getEnv('_APP_SMS_PROVIDER'))) { throw new Exception(Exception::GENERAL_PHONE_DISABLED, 'Phone provider not configured'); } @@ -4288,16 +4284,12 @@ Http::post('/v1/account/verifications/phone') $countryCode = $helper->parse($phone)->getCountryCode(); if (!empty($countryCode)) { - $queueForStatsUsage - ->addMetric(str_replace('{countryCode}', $countryCode, METRIC_AUTH_METHOD_PHONE_COUNTRY_CODE), 1); + $usage->addMetric(str_replace('{countryCode}', $countryCode, METRIC_AUTH_METHOD_PHONE_COUNTRY_CODE), 1); } } catch (NumberParseException $e) { // Ignore invalid phone number for country code stats } - $queueForStatsUsage - ->addMetric(METRIC_AUTH_METHOD_PHONE, 1) - ->setProject($project) - ->trigger(); + $usage->addMetric(METRIC_AUTH_METHOD_PHONE, 1); } $verification->setAttribute('secret', $secret); diff --git a/app/controllers/api/users.php b/app/controllers/api/users.php index d0e5e19a51..9d04018b10 100644 --- a/app/controllers/api/users.php +++ b/app/controllers/api/users.php @@ -1560,12 +1560,15 @@ Http::patch('/v1/users/:userId/phone') $oldPhone = $user->getAttribute('phone'); + // Store null instead of empty string so unique constraint allows multiple users without phone + $phoneValue = $number !== '' ? $number : null; + $user - ->setAttribute('phone', $number) + ->setAttribute('phone', $phoneValue) ->setAttribute('phoneVerification', false) ; - if (\strlen($number) !== 0) { + if ($number !== '') { $target = $dbForProject->findOne('targets', [ Query::equal('identifier', [$number]), ]); @@ -1577,7 +1580,7 @@ Http::patch('/v1/users/:userId/phone') try { $user = $dbForProject->updateDocument('users', $user->getId(), new Document([ - 'phone' => $user->getAttribute('phone'), + 'phone' => $phoneValue, 'phoneVerification' => $user->getAttribute('phoneVerification'), ])); /** @@ -1586,14 +1589,14 @@ Http::patch('/v1/users/:userId/phone') $oldTarget = $user->find('identifier', $oldPhone, 'targets'); if ($oldTarget instanceof Document && !$oldTarget->isEmpty()) { - if (\strlen($number) !== 0) { + if ($number !== '') { $dbForProject->updateDocument('targets', $oldTarget->getId(), new Document(['identifier' => $number])); $oldTarget->setAttribute('identifier', $number); } else { $dbForProject->deleteDocument('targets', $oldTarget->getId()); } } else { - if (\strlen($number) !== 0) { + if ($number !== '') { $target = $dbForProject->createDocument('targets', new Document([ '$permissions' => [ Permission::read(Role::user($user->getId())), diff --git a/app/controllers/general.php b/app/controllers/general.php index 57edd98bc4..f77aa3ec52 100644 --- a/app/controllers/general.php +++ b/app/controllers/general.php @@ -1399,6 +1399,9 @@ Http::error() $sdk = $route?->getLabel("sdk", false); $action = 'UNKNOWN_NAMESPACE.UNKNOWN.METHOD'; if (!empty($sdk)) { + if (\is_array($sdk)) { + $sdk = $sdk[0]; + } /** @var \Appwrite\SDK\Method $sdk */ $action = $sdk->getNamespace() . '.' . $sdk->getMethodName(); } elseif ($route === null) { diff --git a/app/controllers/shared/api.php b/app/controllers/shared/api.php index 29ccc90179..c8824d3708 100644 --- a/app/controllers/shared/api.php +++ b/app/controllers/shared/api.php @@ -10,14 +10,16 @@ use Appwrite\Event\Delete; use Appwrite\Event\Event; use Appwrite\Event\Func; use Appwrite\Event\Mail; +use Appwrite\Event\Message\Usage as UsageMessage; use Appwrite\Event\Messaging; +use Appwrite\Event\Publisher\Usage as UsagePublisher; use Appwrite\Event\Realtime; -use Appwrite\Event\StatsUsage; use Appwrite\Event\Webhook; use Appwrite\Extend\Exception; use Appwrite\Extend\Exception as AppwriteException; use Appwrite\Functions\EventProcessor; use Appwrite\SDK\Method; +use Appwrite\Usage\Context; use Appwrite\Utopia\Database\Documents\User; use Appwrite\Utopia\Request; use Appwrite\Utopia\Response; @@ -53,7 +55,7 @@ $parseLabel = function (string $label, array $responsePayload, array $requestPar $replace = $parts[1] ?? ''; $params = match ($namespace) { - 'user' => (array)$user, + 'user' => (array) $user, 'request' => $requestParams, default => $responsePayload, }; @@ -61,13 +63,13 @@ $parseLabel = function (string $label, array $responsePayload, array $requestPar if (array_key_exists($replace, $params)) { $replacement = $params[$replace]; // Convert to string if it's not already a string - if (!is_string($replacement)) { + if (! is_string($replacement)) { if (is_array($replacement)) { $replacement = json_encode($replacement); } elseif (is_object($replacement) && method_exists($replacement, '__toString')) { - $replacement = (string)$replacement; + $replacement = (string) $replacement; } elseif (is_scalar($replacement)) { - $replacement = (string)$replacement; + $replacement = (string) $replacement; } else { throw new Exception(Exception::GENERAL_SERVER_ERROR, "The server encountered an error while parsing the label: $label. Please create an issue on GitHub to allow us to investigate further https://github.com/appwrite/appwrite/issues/new/choose"); } @@ -75,6 +77,7 @@ $parseLabel = function (string $label, array $responsePayload, array $requestPar $label = \str_replace($find, $replacement, $label); } } + return $label; }; @@ -160,7 +163,7 @@ Http::init() $scopes = $roles[$role]['scopes']; // Step 5: API Key Authentication - if (!empty($apiKey)) { + if (! empty($apiKey)) { // Check if key is expired if ($apiKey->isExpired()) { throw new Exception(Exception::PROJECT_KEY_EXPIRED); @@ -170,7 +173,6 @@ Http::init() $role = $apiKey->getRole(); $scopes = $apiKey->getScopes(); - // Handle special app role case if ($apiKey->getRole() === User::ROLE_APPS) { // Disable authorization checks for project API keys @@ -193,19 +195,19 @@ Http::init() // For standard keys, update last accessed time if (\in_array($apiKey->getType(), [API_KEY_STANDARD, API_KEY_ORGANIZATION, API_KEY_ACCOUNT])) { $dbKey = null; - if (!empty($apiKey->getProjectId())) { + if (! empty($apiKey->getProjectId())) { $dbKey = $project->find( key: 'secret', find: $request->getHeader('x-appwrite-key', ''), subject: 'keys' ); - } elseif (!empty($apiKey->getUserId())) { + } elseif (! empty($apiKey->getUserId())) { $dbKey = $user->find( key: 'secret', find: $request->getHeader('x-appwrite-key', ''), subject: 'keys' ); - } elseif (!empty($apiKey->getTeamId())) { + } elseif (! empty($apiKey->getTeamId())) { $dbKey = $team->find( key: 'secret', find: $request->getHeader('x-appwrite-key', ''), @@ -213,9 +215,7 @@ Http::init() ); } - if (!$dbKey) { - \var_dump($apiKey); - \var_dump($request->getHeader('x-appwrite-key', '')); + if (! $dbKey) { throw new Exception(Exception::USER_UNAUTHORIZED); } @@ -233,7 +233,7 @@ Http::init() if ($sdk !== 'UNKNOWN' && $sdkValidator->isValid($sdk)) { $sdks = $dbKey->getAttribute('sdks', []); - if (!in_array($sdk, $sdks)) { + if (! in_array($sdk, $sdks)) { $sdks[] = $sdk; $updates->setAttribute('sdks', $sdks); @@ -241,14 +241,14 @@ Http::init() } } - if (!$updates->isEmpty()) { + if (! $updates->isEmpty()) { $dbForPlatform->getAuthorization()->skip(fn () => $dbForPlatform->updateDocument('keys', $dbKey->getId(), $updates)); - if (!empty($apiKey->getProjectId())) { + if (! empty($apiKey->getProjectId())) { $dbForPlatform->getAuthorization()->skip(fn () => $dbForPlatform->purgeCachedDocument('projects', $project->getId())); - } elseif (!empty($apiKey->getUserId())) { + } elseif (! empty($apiKey->getUserId())) { $dbForPlatform->getAuthorization()->skip(fn () => $dbForPlatform->purgeCachedDocument('users', $user->getId())); - } elseif (!empty($apiKey->getTeamId())) { + } elseif (! empty($apiKey->getTeamId())) { $dbForPlatform->getAuthorization()->skip(fn () => $dbForPlatform->purgeCachedDocument('teams', $team->getId())); } } @@ -285,7 +285,7 @@ Http::init() } } } // Admin User Authentication - elseif (($project->getId() === 'console' && !$team->isEmpty() && !$user->isEmpty()) || ($project->getId() !== 'console' && !$user->isEmpty() && $mode === APP_MODE_ADMIN)) { + elseif (($project->getId() === 'console' && ! $team->isEmpty() && ! $user->isEmpty()) || ($project->getId() !== 'console' && ! $user->isEmpty() && $mode === APP_MODE_ADMIN)) { $teamId = $team->getId(); $adminRoles = []; $memberships = $user->getAttribute('memberships', []); @@ -310,7 +310,7 @@ Http::init() // Useful for those who have project-specific roles but don't have team-wide role. $scopes = ['teams.read', 'projects.read']; foreach ($adminRoles as $adminRole) { - $isTeamWideRole = !str_starts_with($adminRole, 'project-'); + $isTeamWideRole = ! str_starts_with($adminRole, 'project-'); $isProjectSpecificRole = $projectId !== 'console' && str_starts_with($adminRole, 'project-' . $projectId); if ($isTeamWideRole || $isProjectSpecificRole) { @@ -348,18 +348,18 @@ Http::init() * But, for actions on resources (sites, functions, etc.) in a non-console project, we explicitly check * whether the admin user has necessary permission on the project (sites, functions, etc. don't have permissions associated to them). */ - if (empty($apiKey) && !$user->isEmpty() && $project->getId() !== 'console' && $mode === APP_MODE_ADMIN) { + if (empty($apiKey) && ! $user->isEmpty() && $project->getId() !== 'console' && $mode === APP_MODE_ADMIN) { $input = new Input(Database::PERMISSION_READ, $project->getPermissionsByType(Database::PERMISSION_READ)); $initialStatus = $authorization->getStatus(); $authorization->enable(); - if (!$authorization->isValid($input)) { + if (! $authorization->isValid($input)) { throw new Exception(Exception::PROJECT_NOT_FOUND); } $authorization->setStatus($initialStatus); } // Step 6: Update project and user last activity - if (!$project->isEmpty() && $project->getId() !== 'console') { + if (! $project->isEmpty() && $project->getId() !== 'console') { $accessedAt = $project->getAttribute('accessedAt', 0); if (DateTime::formatTz(DateTime::addSeconds(new \DateTime(), -APP_PROJECT_ACCESS)) > $accessedAt) { $authorization->skip(fn () => $dbForPlatform->updateDocument('projects', $project->getId(), new Document([ @@ -368,12 +368,12 @@ Http::init() } } - if (!empty($user->getId())) { + if (! empty($user->getId())) { $accessedAt = $user->getAttribute('accessedAt', 0); if (DateTime::formatTz(DateTime::addSeconds(new \DateTime(), -APP_USER_ACCESS)) > $accessedAt) { $user->setAttribute('accessedAt', DateTime::now()); - if ($project->getId() !== 'console' && APP_MODE_ADMIN !== $mode) { + if ($project->getId() !== 'console' && $mode !== APP_MODE_ADMIN) { $dbForProject->updateDocument('users', $user->getId(), new Document([ 'accessedAt' => $user->getAttribute('accessedAt') ])); @@ -397,26 +397,26 @@ Http::init() $method = $method[0]; } - if (!empty($method)) { + if (! empty($method)) { $namespace = $method->getNamespace(); if ( array_key_exists($namespace, $project->getAttribute('services', [])) - && !$project->getAttribute('services', [])[$namespace] - && !(User::isPrivileged($authorization->getRoles()) || User::isApp($authorization->getRoles())) + && ! $project->getAttribute('services', [])[$namespace] + && ! (User::isPrivileged($authorization->getRoles()) || User::isApp($authorization->getRoles())) ) { throw new Exception(Exception::GENERAL_SERVICE_DISABLED); } } // Step 9: Validate scope permissions - $allowed = (array)$route->getLabel('scope', 'none'); + $allowed = (array) $route->getLabel('scope', 'none'); if (empty(\array_intersect($allowed, $scopes))) { throw new Exception(Exception::GENERAL_UNAUTHORIZED_SCOPE, $user->getAttribute('email', 'User') . ' (role: ' . \strtolower($roles[$role]['label']) . ') missing scopes (' . \json_encode($allowed) . ')'); } // Step 10: Check if user is blocked - if (false === $user->getAttribute('status')) { // Account is blocked + if ($user->getAttribute('status') === false) { // Account is blocked throw new Exception(Exception::USER_BLOCKED); } @@ -434,7 +434,7 @@ Http::init() $minimumFactors = ($mfaEnabled && $hasMoreFactors) ? 2 : 1; // Step 13: Handle Multi-Factor Authentication - if (!in_array('mfa', $route->getGroups())) { + if (! in_array('mfa', $route->getGroups())) { if ($session && \count($session->getAttribute('factors', [])) < $minimumFactors) { throw new Exception(Exception::USER_MORE_FACTORS_REQUIRED); } @@ -454,7 +454,7 @@ Http::init() ->inject('queueForDeletes') ->inject('queueForDatabase') ->inject('queueForBuilds') - ->inject('queueForStatsUsage') + ->inject('usage') ->inject('queueForFunctions') ->inject('queueForMails') ->inject('dbForProject') @@ -467,14 +467,14 @@ Http::init() ->inject('telemetry') ->inject('platform') ->inject('authorization') - ->action(function (Http $utopia, Request $request, Response $response, Document $project, Document $user, Event $queueForEvents, Messaging $queueForMessaging, Audit $queueForAudits, Delete $queueForDeletes, EventDatabase $queueForDatabase, Build $queueForBuilds, StatsUsage $queueForStatsUsage, Func $queueForFunctions, Mail $queueForMails, Database $dbForProject, callable $timelimit, Document $resourceToken, string $mode, ?Key $apiKey, array $plan, Document $devKey, Telemetry $telemetry, array $platform, Authorization $authorization) { + ->action(function (Http $utopia, Request $request, Response $response, Document $project, Document $user, Event $queueForEvents, Messaging $queueForMessaging, Audit $queueForAudits, Delete $queueForDeletes, EventDatabase $queueForDatabase, Build $queueForBuilds, Context $usage, Func $queueForFunctions, Mail $queueForMails, Database $dbForProject, callable $timelimit, Document $resourceToken, string $mode, ?Key $apiKey, array $plan, Document $devKey, Telemetry $telemetry, array $platform, Authorization $authorization) { $route = $utopia->getRoute(); if ( array_key_exists('rest', $project->getAttribute('apis', [])) - && !$project->getAttribute('apis', [])['rest'] - && !(User::isPrivileged($authorization->getRoles()) || User::isApp($authorization->getRoles())) + && ! $project->getAttribute('apis', [])['rest'] + && ! (User::isPrivileged($authorization->getRoles()) || User::isApp($authorization->getRoles())) ) { throw new AppwriteException(AppwriteException::GENERAL_API_DISABLED); } @@ -486,7 +486,7 @@ Http::init() $abuseKeyLabel = $route->getLabel('abuse-key', 'url:{url},ip:{ip}'); $timeLimitArray = []; - $abuseKeyLabel = (!is_array($abuseKeyLabel)) ? [$abuseKeyLabel] : $abuseKeyLabel; + $abuseKeyLabel = (! is_array($abuseKeyLabel)) ? [$abuseKeyLabel] : $abuseKeyLabel; foreach ($abuseKeyLabel as $abuseKey) { $start = $request->getContentRangeStart(); @@ -499,7 +499,7 @@ Http::init() ->setParam('{ip}', $request->getIP()) ->setParam('{url}', $request->getHostname() . $route->getPath()) ->setParam('{method}', $request->getMethod()) - ->setParam('{chunkId}', (int)($start / ($end + 1 - $start))); + ->setParam('{chunkId}', (int) ($start / ($end + 1 - $start))); $timeLimitArray[] = $timeLimit; } @@ -511,7 +511,7 @@ Http::init() foreach ($timeLimitArray as $timeLimit) { foreach ($request->getParams() as $key => $value) { // Set request params as potential abuse keys - if (!empty($value)) { + if (! empty($value)) { $timeLimit->setParam('{param-' . $key . '}', (\is_array($value)) ? \json_encode($value) : $value); } } @@ -534,8 +534,8 @@ Http::init() if ( $enabled // Abuse is enabled - && !$isAppUser // User is not API key - && !$isPrivilegedUser // User is not an admin + && ! $isAppUser // User is not API key + && ! $isPrivilegedUser // User is not an admin && $devKey->isEmpty() // request doesn't not contain development key && $abuse->check() // Route is rate-limited ) { @@ -564,19 +564,13 @@ Http::init() ->setProject($project); /* If a session exists, use the user associated with the session */ - if (!$user->isEmpty()) { + if (! $user->isEmpty()) { $userClone = clone $user; // $user doesn't support `type` and can cause unintended effects. $userClone->setAttribute('type', ACTIVITY_TYPE_USER); $queueForAudits->setUser($userClone); } - if (!empty($apiKey) && !empty($apiKey->getDisabledMetrics())) { - foreach ($apiKey->getDisabledMetrics() as $key) { - $queueForStatsUsage->disableMetric($key); - } - } - /* Auto-set projects */ $queueForDeletes->setProject($project); $queueForDatabase->setProject($project); @@ -590,69 +584,64 @@ Http::init() $queueForBuilds->setPlatform($platform); $queueForMails->setPlatform($platform); - $useCache = $route->getLabel('cache', false); $storageCacheOperationsCounter = $telemetry->createCounter('storage.cache.operations.load'); if ($useCache) { $route = $utopia->match($request); $isImageTransformation = $route->getPath() === '/v1/storage/buckets/:bucketId/files/:fileId/preview'; - $isDisabled = isset($plan['imageTransformations']) && $plan['imageTransformations'] === -1 && !User::isPrivileged($authorization->getRoles()); + $isDisabled = isset($plan['imageTransformations']) && $plan['imageTransformations'] === -1 && ! User::isPrivileged($authorization->getRoles()); $key = $request->cacheIdentifier(); - $cacheLog = $authorization->skip(fn () => $dbForProject->getDocument('cache', $key)); + $cacheLog = $authorization->skip(fn () => $dbForProject->getDocument('cache', $key)); $cache = new Cache( new Filesystem(APP_STORAGE_CACHE . DIRECTORY_SEPARATOR . 'app-' . $project->getId()) ); $timestamp = 60 * 60 * 24 * 180; // Temporarily increase the TTL to 180 day to ensure files in the cache are still fetched. $data = $cache->load($key, $timestamp); - if (!empty($data) && !$cacheLog->isEmpty()) { - $usageMetric = $route->getLabel('usage.metric', null); - if ($usageMetric === METRIC_AVATARS_SCREENSHOTS_GENERATED) { - $queueForStatsUsage->disableMetric(METRIC_AVATARS_SCREENSHOTS_GENERATED); - } + if (! empty($data) && ! $cacheLog->isEmpty()) { $parts = explode('/', $cacheLog->getAttribute('resourceType', '')); $type = $parts[0] ?? null; - if ($type === 'bucket' && (!$isImageTransformation || !$isDisabled)) { + if ($type === 'bucket' && (! $isImageTransformation || ! $isDisabled)) { $bucketId = $parts[1] ?? null; $bucket = $authorization->skip(fn () => $dbForProject->getDocument('buckets', $bucketId)); - $isToken = !$resourceToken->isEmpty() && $resourceToken->getAttribute('bucketInternalId') === $bucket->getSequence(); + $isToken = ! $resourceToken->isEmpty() && $resourceToken->getAttribute('bucketInternalId') === $bucket->getSequence(); $isPrivilegedUser = User::isPrivileged($authorization->getRoles()); - if ($bucket->isEmpty() || (!$bucket->getAttribute('enabled') && !$isAppUser && !$isPrivilegedUser)) { + if ($bucket->isEmpty() || (! $bucket->getAttribute('enabled') && ! $isAppUser && ! $isPrivilegedUser)) { throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND); } - if (!$bucket->getAttribute('transformations', true) && !$isAppUser && !$isPrivilegedUser) { + if (! $bucket->getAttribute('transformations', true) && ! $isAppUser && ! $isPrivilegedUser) { throw new Exception(Exception::STORAGE_BUCKET_TRANSFORMATIONS_DISABLED); } $fileSecurity = $bucket->getAttribute('fileSecurity', false); $valid = $authorization->isValid(new Input(Database::PERMISSION_READ, $bucket->getRead())); - if (!$fileSecurity && !$valid && !$isToken) { + if (! $fileSecurity && ! $valid && ! $isToken) { throw new Exception(Exception::USER_UNAUTHORIZED); } $parts = explode('/', $cacheLog->getAttribute('resource')); $fileId = $parts[1] ?? null; - if ($fileSecurity && !$valid && !$isToken) { + if ($fileSecurity && ! $valid && ! $isToken) { $file = $dbForProject->getDocument('bucket_' . $bucket->getSequence(), $fileId); } else { $file = $authorization->skip(fn () => $dbForProject->getDocument('bucket_' . $bucket->getSequence(), $fileId)); } - if (!$resourceToken->isEmpty() && $resourceToken->getAttribute('fileInternalId') !== $file->getSequence()) { + if (! $resourceToken->isEmpty() && $resourceToken->getAttribute('fileInternalId') !== $file->getSequence()) { throw new Exception(Exception::USER_UNAUTHORIZED); } if ($file->isEmpty()) { throw new Exception(Exception::STORAGE_FILE_NOT_FOUND); } - //Do not update transformedAt if it's a console user - if (!User::isPrivileged($authorization->getRoles())) { + // Do not update transformedAt if it's a console user + if (! User::isPrivileged($authorization->getRoles())) { $transformedAt = $file->getAttribute('transformedAt', ''); if (DateTime::formatTz(DateTime::addSeconds(new \DateTime(), -APP_PROJECT_ACCESS)) > $transformedAt) { $file->setAttribute('transformedAt', DateTime::now()); @@ -668,7 +657,7 @@ Http::init() ->addHeader('X-Appwrite-Cache', 'hit') ->setContentType($cacheLog->getAttribute('mimeType')); $storageCacheOperationsCounter->add(1, ['result' => 'hit']); - if (!$isImageTransformation || !$isDisabled) { + if (! $isImageTransformation || ! $isDisabled) { $response->send($data); } } else { @@ -691,7 +680,7 @@ Http::init() return; } - if (!$user->isEmpty()) { + if (! $user->isEmpty()) { throw new Exception(Exception::USER_SESSION_ALREADY_EXISTS); } }); @@ -745,7 +734,8 @@ Http::shutdown() ->inject('user') ->inject('queueForEvents') ->inject('queueForAudits') - ->inject('queueForStatsUsage') + ->inject('usage') + ->inject('publisherForUsage') ->inject('queueForDeletes') ->inject('queueForDatabase') ->inject('queueForBuilds') @@ -758,11 +748,12 @@ Http::shutdown() ->inject('timelimit') ->inject('eventProcessor') ->inject('bus') - ->action(function (Http $utopia, Request $request, Response $response, Document $project, User $user, Event $queueForEvents, Audit $queueForAudits, StatsUsage $queueForStatsUsage, Delete $queueForDeletes, EventDatabase $queueForDatabase, Build $queueForBuilds, Messaging $queueForMessaging, Func $queueForFunctions, Event $queueForWebhooks, Realtime $queueForRealtime, Database $dbForProject, Authorization $authorization, callable $timelimit, EventProcessor $eventProcessor, Bus $bus) use ($parseLabel) { + ->inject('apiKey') + ->action(function (Http $utopia, Request $request, Response $response, Document $project, User $user, Event $queueForEvents, Audit $queueForAudits, Context $usage, UsagePublisher $publisherForUsage, Delete $queueForDeletes, EventDatabase $queueForDatabase, Build $queueForBuilds, Messaging $queueForMessaging, Func $queueForFunctions, Event $queueForWebhooks, Realtime $queueForRealtime, Database $dbForProject, Authorization $authorization, callable $timelimit, EventProcessor $eventProcessor, Bus $bus, ?Key $apiKey) use ($parseLabel) { $responsePayload = $response->getPayload(); - if (!empty($queueForEvents->getEvent())) { + if (! empty($queueForEvents->getEvent())) { if (empty($queueForEvents->getPayload())) { $queueForEvents->setPayload($responsePayload); } @@ -784,7 +775,7 @@ Http::shutdown() } // Only trigger functions if there are matching function events - if (!empty($functionsEvents)) { + if (! empty($functionsEvents)) { foreach ($generatedEvents as $event) { if (isset($functionsEvents[$event])) { $queueForFunctions @@ -796,7 +787,7 @@ Http::shutdown() } // Only trigger webhooks if there are matching webhook events - if (!empty($webhooksEvents)) { + if (! empty($webhooksEvents)) { foreach ($generatedEvents as $event) { if (isset($webhooksEvents[$event])) { $queueForWebhooks @@ -820,7 +811,7 @@ Http::shutdown() if ($abuseEnabled && \count($abuseResetCode) > 0 && \in_array($response->getStatusCode(), $abuseResetCode)) { $abuseKeyLabel = $route->getLabel('abuse-key', 'url:{url},ip:{ip}'); - $abuseKeyLabel = (!is_array($abuseKeyLabel)) ? [$abuseKeyLabel] : $abuseKeyLabel; + $abuseKeyLabel = (! is_array($abuseKeyLabel)) ? [$abuseKeyLabel] : $abuseKeyLabel; foreach ($abuseKeyLabel as $abuseKey) { $start = $request->getContentRangeStart(); @@ -833,10 +824,10 @@ Http::shutdown() ->setParam('{ip}', $request->getIP()) ->setParam('{url}', $request->getHostname() . $route->getPath()) ->setParam('{method}', $request->getMethod()) - ->setParam('{chunkId}', (int)($start / ($end + 1 - $start))); + ->setParam('{chunkId}', (int) ($start / ($end + 1 - $start))); foreach ($request->getParams() as $key => $value) { // Set request params as potential abuse keys - if (!empty($value)) { + if (! empty($value)) { $timeLimit->setParam('{param-' . $key . '}', (\is_array($value)) ? \json_encode($value) : $value); } } @@ -850,14 +841,14 @@ Http::shutdown() * Audit labels */ $pattern = $route->getLabel('audits.resource', null); - if (!empty($pattern)) { + if (! empty($pattern)) { $resource = $parseLabel($pattern, $responsePayload, $requestParams, $user); - if (!empty($resource) && $resource !== $pattern) { + if (! empty($resource) && $resource !== $pattern) { $queueForAudits->setResource($resource); } } - if (!$user->isEmpty()) { + if (! $user->isEmpty()) { $userClone = clone $user; // $user doesn't support `type` and can cause unintended effects. $userClone->setAttribute('type', ACTIVITY_TYPE_USER); @@ -883,13 +874,13 @@ Http::shutdown() $queueForAudits->setUser($user); } - if (!empty($queueForAudits->getResource()) && !$queueForAudits->getUser()->isEmpty()) { + if (! empty($queueForAudits->getResource()) && ! $queueForAudits->getUser()->isEmpty()) { /** * audits.payload is switched to default true * in order to auto audit payload for all endpoints */ $pattern = $route->getLabel('audits.payload', true); - if (!empty($pattern)) { + if (! empty($pattern)) { $queueForAudits->setPayload($responsePayload); } @@ -900,19 +891,19 @@ Http::shutdown() $queueForAudits->trigger(); } - if (!empty($queueForDeletes->getType())) { + if (! empty($queueForDeletes->getType())) { $queueForDeletes->trigger(); } - if (!empty($queueForDatabase->getType())) { + if (! empty($queueForDatabase->getType())) { $queueForDatabase->trigger(); } - if (!empty($queueForBuilds->getType())) { + if (! empty($queueForBuilds->getType())) { $queueForBuilds->trigger(); } - if (!empty($queueForMessaging->getType())) { + if (! empty($queueForMessaging->getType())) { $queueForMessaging->trigger(); } @@ -921,14 +912,14 @@ Http::shutdown() if ($useCache) { $resource = $resourceType = null; $data = $response->getPayload(); - if (!empty($data['payload'])) { + if (! empty($data['payload'])) { $pattern = $route->getLabel('cache.resource', null); - if (!empty($pattern)) { + if (! empty($pattern)) { $resource = $parseLabel($pattern, $responsePayload, $requestParams, $user); } $pattern = $route->getLabel('cache.resourceType', null); - if (!empty($pattern)) { + if (! empty($pattern)) { $resourceType = $parseLabel($pattern, $responsePayload, $requestParams, $user); } @@ -938,7 +929,7 @@ Http::shutdown() $key = $request->cacheIdentifier(); $signature = md5($data['payload']); - $cacheLog = $authorization->skip(fn () => $dbForProject->getDocument('cache', $key)); + $cacheLog = $authorization->skip(fn () => $dbForProject->getDocument('cache', $key)); $accessedAt = $cacheLog->getAttribute('accessedAt', 0); $now = DateTime::now(); if ($cacheLog->isEmpty()) { @@ -971,7 +962,7 @@ Http::shutdown() } if ($project->getId() !== 'console') { - if (!User::isPrivileged($authorization->getRoles())) { + if (! User::isPrivileged($authorization->getRoles())) { $bus->dispatch(new RequestCompleted( project: $project->getArrayCopy(), request: $request, @@ -979,9 +970,32 @@ Http::shutdown() )); } - $queueForStatsUsage - ->setProject($project) - ->trigger(); + // Publish usage metrics if context has data + if (! $usage->isEmpty()) { + $metrics = $usage->getMetrics(); + + // Filter out API key disabled metrics using suffix pattern matching + $disabledMetrics = $apiKey?->getDisabledMetrics() ?? []; + if (! empty($disabledMetrics)) { + $metrics = array_values(array_filter($metrics, function ($metric) use ($disabledMetrics) { + foreach ($disabledMetrics as $pattern) { + if (str_ends_with($metric['key'], $pattern)) { + return false; + } + } + + return true; + })); + } + + $message = new UsageMessage( + project: $project, + metrics: $metrics, + reduce: $usage->getReduce() + ); + + $publisherForUsage->enqueue($message); + } } }); diff --git a/app/http.php b/app/http.php index 7f771de130..1302940856 100644 --- a/app/http.php +++ b/app/http.php @@ -581,6 +581,9 @@ $http->on(Constant::EVENT_REQUEST, function (SwooleRequest $swooleRequest, Swool $action = 'UNKNOWN_NAMESPACE.UNKNOWN.METHOD'; if (!empty($sdk)) { + if (\is_array($sdk)) { + $sdk = $sdk[0]; + } /** @var Appwrite\SDK\Method $sdk */ $action = $sdk->getNamespace() . '.' . $sdk->getMethodName(); } elseif ($route === null) { diff --git a/app/init/constants.php b/app/init/constants.php index 0cfaf44920..7a484c7f4e 100644 --- a/app/init/constants.php +++ b/app/init/constants.php @@ -362,6 +362,12 @@ const METRIC_AVATARS_SCREENSHOTS_GENERATED = 'avatars.screenshotsGenerated'; const METRIC_FUNCTIONS_RUNTIME = 'functions.runtimes.{runtime}'; const METRIC_SITES_FRAMEWORK = 'sites.frameworks.{framework}'; +// Realtime metrics +const METRIC_REALTIME_CONNECTIONS = 'realtime.connections'; +const METRIC_REALTIME_CONNECTIONS_MESSAGES_SENT = 'realtime.messages.sent'; +const METRIC_REALTIME_INBOUND = 'realtime.inbound'; +const METRIC_REALTIME_OUTBOUND = 'realtime.outbound'; + // Resource types const RESOURCE_TYPE_PROJECTS = 'projects'; const RESOURCE_TYPE_FUNCTIONS = 'functions'; diff --git a/app/init/registers.php b/app/init/registers.php index 26a9329270..7b68c2af9a 100644 --- a/app/init/registers.php +++ b/app/init/registers.php @@ -420,6 +420,7 @@ $register->set('smtp', function () { $mail->Password = $password; $mail->SMTPSecure = System::getEnv('_APP_SMTP_SECURE', ''); $mail->SMTPAutoTLS = false; + $mail->SMTPKeepAlive = true; $mail->CharSet = 'UTF-8'; $mail->Timeout = 10; /* Connection timeout */ $mail->getSMTPInstance()->Timelimit = 30; /* Timeout for each individual SMTP command (e.g. HELO, EHLO, etc.) */ diff --git a/app/init/resources.php b/app/init/resources.php index d5486c2a49..1bab4491a4 100644 --- a/app/init/resources.php +++ b/app/init/resources.php @@ -14,10 +14,10 @@ use Appwrite\Event\Func; use Appwrite\Event\Mail; use Appwrite\Event\Messaging; use Appwrite\Event\Migration; +use Appwrite\Event\Publisher\Usage as UsagePublisher; use Appwrite\Event\Realtime; use Appwrite\Event\Screenshot; use Appwrite\Event\StatsResources; -use Appwrite\Event\StatsUsage; use Appwrite\Event\Webhook; use Appwrite\Extend\Exception; use Appwrite\Functions\EventProcessor; @@ -26,6 +26,7 @@ use Appwrite\Network\Cors; use Appwrite\Network\Platform; use Appwrite\Network\Validator\Origin; use Appwrite\Network\Validator\Redirect; +use Appwrite\Usage\Context as UsageContext; use Appwrite\Utopia\Database\Documents\User; use Appwrite\Utopia\Request; use Appwrite\Utopia\Response; @@ -57,6 +58,7 @@ use Utopia\Logger\Log; use Utopia\Pools\Group; use Utopia\Queue\Broker\Pool as BrokerPool; use Utopia\Queue\Publisher; +use Utopia\Queue\Queue; use Utopia\Storage\Device; use Utopia\Storage\Device\AWS; use Utopia\Storage\Device\Backblaze; @@ -88,6 +90,7 @@ Http::setResource('register', fn () => $register); Http::setResource('locale', function () { $locale = new Locale(System::getEnv('_APP_LOCALE', 'en')); $locale->setFallback(System::getEnv('_APP_LOCALE', 'en')); + return $locale; }); @@ -108,9 +111,6 @@ Http::setResource('publisherFunctions', function (Publisher $publisher) { Http::setResource('publisherMigrations', function (Publisher $publisher) { return $publisher; }, ['publisher']); -Http::setResource('publisherStatsUsage', function (Publisher $publisher) { - return $publisher; -}, ['publisher']); Http::setResource('publisherMails', function (Publisher $publisher) { return $publisher; }, ['publisher']); @@ -150,9 +150,13 @@ Http::setResource('queueForWebhooks', function (Publisher $publisher) { Http::setResource('queueForRealtime', function () { return new Realtime(); }, []); -Http::setResource('queueForStatsUsage', function (Publisher $publisher) { - return new StatsUsage($publisher); -}, ['publisher']); +Http::setResource('usage', function () { + return new UsageContext(); +}, []); +Http::setResource('publisherForUsage', fn (Publisher $publisher) => new UsagePublisher( + $publisher, + new Queue(System::getEnv('_APP_STATS_USAGE_QUEUE_NAME', Event::STATS_USAGE_QUEUE_NAME)) +), ['publisher']); Http::setResource('queueForAudits', function (Publisher $publisher) { return new AuditEvent($publisher); }, ['publisher']); @@ -186,14 +190,14 @@ Http::setResource('allowedHostnames', function (array $platform, Document $proje $allowed = [...($platform['hostnames'] ?? [])]; /* Add platform configured hostnames */ - if (!$project->isEmpty() && $project->getId() !== 'console') { + if (! $project->isEmpty() && $project->getId() !== 'console') { $platforms = $project->getAttribute('platforms', []); $hostnames = Platform::getHostnames($platforms); $allowed = [...$allowed, ...$hostnames]; } /* Add the request hostname if a dev key is found */ - if (!$devKey->isEmpty()) { + if (! $devKey->isEmpty()) { $allowed[] = $request->getHostname(); } @@ -211,12 +215,12 @@ Http::setResource('allowedHostnames', function (array $platform, Document $proje } /* Allow the request origin of rule */ - if (!$rule->isEmpty() && !empty($rule->getAttribute('domain', ''))) { + if (! $rule->isEmpty() && ! empty($rule->getAttribute('domain', ''))) { $allowed[] = $rule->getAttribute('domain', ''); } /* Allow the request origin if a dev key is found */ - if (!$devKey->isEmpty() && !empty($hostname)) { + if (! $devKey->isEmpty() && ! empty($hostname)) { $allowed[] = $hostname; } @@ -229,7 +233,7 @@ Http::setResource('allowedHostnames', function (array $platform, Document $proje Http::setResource('allowedSchemes', function (array $platform, Document $project) { $allowed = [...($platform['schemas'] ?? [])]; - if (!$project->isEmpty() && $project->getId() !== 'console') { + if (! $project->isEmpty() && $project->getId() !== 'console') { /* Add hardcoded schemes */ $allowed[] = 'exp'; $allowed[] = 'appwrite-callback-' . $project->getId(); @@ -273,7 +277,7 @@ Http::setResource('rule', function (Request $request, Database $dbForPlatform, D // Temporary implementation until custom wildcard domains are an official feature // Allow trusted projects; Used for Console (website) previews - if (!$permitsCurrentProject && !$rule->isEmpty() && !empty($rule->getAttribute('projectId', ''))) { + if (! $permitsCurrentProject && ! $rule->isEmpty() && ! empty($rule->getAttribute('projectId', ''))) { $trustedProjects = []; foreach (\explode(',', System::getEnv('_APP_CONSOLE_TRUSTED_PROJECTS', '')) as $trustedProject) { if (empty($trustedProject)) { @@ -286,7 +290,7 @@ Http::setResource('rule', function (Request $request, Database $dbForPlatform, D } } - if (!$permitsCurrentProject) { + if (! $permitsCurrentProject) { return new Document(); } @@ -309,16 +313,18 @@ Http::setResource('cors', function (array $allowedHostnames) { }, ['allowedHostnames']); Http::setResource('originValidator', function (Document $devKey, array $allowedHostnames, array $allowedSchemes) { - if (!$devKey->isEmpty()) { + if (! $devKey->isEmpty()) { return new URL(); } + return new Origin($allowedHostnames, $allowedSchemes); }, ['devKey', 'allowedHostnames', 'allowedSchemes']); Http::setResource('redirectValidator', function (Document $devKey, array $allowedHostnames, array $allowedSchemes) { - if (!$devKey->isEmpty()) { + if (! $devKey->isEmpty()) { return new URL(); } + return new Redirect($allowedHostnames, $allowedSchemes); }, ['devKey', 'allowedHostnames', 'allowedSchemes']); @@ -342,12 +348,11 @@ Http::setResource('user', function (string $mode, Document $project, Document $c * overwriting the previous value. * 7. If account API key is passed, use user of the account API key as long as user ID header matches too */ - $authorization->setDefaultStatus(true); $store->setKey('a_session_' . $project->getId()); - if (APP_MODE_ADMIN === $mode) { + if ($mode === APP_MODE_ADMIN) { $store->setKey('a_session_' . $console->getId()); } @@ -362,7 +367,7 @@ Http::setResource('user', function (string $mode, Document $project, Document $c if (empty($store->getProperty('id', '')) && empty($store->getProperty('secret', ''))) { $sessionHeader = $request->getHeader('x-appwrite-session', ''); - if (!empty($sessionHeader)) { + if (! empty($sessionHeader)) { $store->decode($sessionHeader); } } @@ -382,14 +387,14 @@ Http::setResource('user', function (string $mode, Document $project, Document $c } $user = null; - if (APP_MODE_ADMIN === $mode) { + if ($mode === APP_MODE_ADMIN) { /** @var User $user */ $user = $dbForPlatform->getDocument('users', $store->getProperty('id', '')); } else { if ($project->isEmpty()) { $user = new User([]); } else { - if (!empty($store->getProperty('id', ''))) { + if (! empty($store->getProperty('id', ''))) { if ($project->getId() === 'console') { /** @var User $user */ $user = $dbForPlatform->getDocument('users', $store->getProperty('id', '')); @@ -402,16 +407,16 @@ Http::setResource('user', function (string $mode, Document $project, Document $c } if ( - !$user || + ! $user || $user->isEmpty() // Check a document has been found in the DB - || !$user->sessionVerify($store->getProperty('secret', ''), $proofForToken) + || ! $user->sessionVerify($store->getProperty('secret', ''), $proofForToken) ) { // Validate user has valid login token $user = new User([]); } $authJWT = $request->getHeader('x-appwrite-jwt', ''); - if (!empty($authJWT) && !$project->isEmpty()) { // JWT authentication - if (!$user->isEmpty()) { + if (! empty($authJWT) && ! $project->isEmpty()) { // JWT authentication + if (! $user->isEmpty()) { throw new Exception(Exception::USER_JWT_AND_COOKIE_SET); } @@ -423,7 +428,7 @@ Http::setResource('user', function (string $mode, Document $project, Document $c } $jwtUserId = $payload['userId'] ?? ''; - if (!empty($jwtUserId)) { + if (! empty($jwtUserId)) { if ($mode === APP_MODE_ADMIN) { $user = $dbForPlatform->getDocument('users', $jwtUserId); } else { @@ -431,7 +436,7 @@ Http::setResource('user', function (string $mode, Document $project, Document $c } } $jwtSessionId = $payload['sessionId'] ?? ''; - if (!empty($jwtSessionId)) { + if (! empty($jwtSessionId)) { if (empty($user->find('$id', $jwtSessionId, 'sessions'))) { // Match JWT to active token $user = new User([]); } @@ -441,22 +446,22 @@ Http::setResource('user', function (string $mode, Document $project, Document $c // Account based on account API key $accountKey = $request->getHeader('x-appwrite-key', ''); $accountKeyUserId = $request->getHeader('x-appwrite-user', ''); - if (!empty($accountKeyUserId) && !empty($accountKey)) { - if (!$user->isEmpty()) { + if (! empty($accountKeyUserId) && ! empty($accountKey)) { + if (! $user->isEmpty()) { throw new Exception(Exception::USER_API_KEY_AND_SESSION_SET); } $accountKeyUser = $dbForPlatform->getAuthorization()->skip(fn () => $dbForPlatform->getDocument('users', $accountKeyUserId)); - if (!$accountKeyUser->isEmpty()) { + if (! $accountKeyUser->isEmpty()) { $key = $accountKeyUser->find( key: 'secret', find: $accountKey, subject: 'keys' ); - if (!empty($key)) { + if (! empty($key)) { $expire = $key->getAttribute('expire'); - if (!empty($expire) && $expire < DatabaseDateTime::formatTz(DatabaseDateTime::now())) { + if (! empty($expire) && $expire < DatabaseDateTime::formatTz(DatabaseDateTime::now())) { throw new Exception(Exception::ACCOUNT_KEY_EXPIRED); } @@ -475,10 +480,9 @@ Http::setResource('project', function ($dbForPlatform, $request, $console, $auth /** @var Appwrite\Utopia\Request $request */ /** @var Utopia\Database\Database $dbForPlatform */ /** @var Utopia\Database\Document $console */ - $projectId = $request->getParam('project', $request->getHeader('x-appwrite-project', '')); // Realtime channel "project" can send project=Query array - if (!\is_string($projectId)) { + if (! \is_string($projectId)) { $projectId = $request->getHeader('x-appwrite-project', ''); } @@ -499,7 +503,7 @@ Http::setResource('session', function (User $user, Store $store, Token $proofFor $sessions = $user->getAttribute('sessions', []); $sessionId = $user->sessionVerify($store->getProperty('secret', ''), $proofForToken); - if (!$sessionId) { + if (! $sessionId) { return; } foreach ($sessions as $session) { @@ -509,7 +513,6 @@ Http::setResource('session', function (User $user, Store $store, Token $proofFor } } - return; }, ['user', 'store', 'proofForToken']); Http::setResource('store', function (): Store { @@ -533,12 +536,14 @@ Http::setResource('proofForPassword', function (): Password { Http::setResource('proofForToken', function (): Token { $token = new Token(); $token->setHash(new Sha()); + return $token; }); Http::setResource('proofForCode', function (): Code { $code = new Code(); $code->setHash(new Sha()); + return $code; }); @@ -550,7 +555,7 @@ Http::setResource('authorization', function () { return new Authorization(); }, []); -Http::setResource('dbForProject', function (Group $pools, Database $dbForPlatform, Cache $cache, Document $project, Response $response, Publisher $publisher, Publisher $publisherFunctions, Publisher $publisherWebhooks, Event $queueForEvents, Func $queueForFunctions, Webhook $queueForWebhooks, Realtime $queueForRealtime, StatsUsage $queueForStatsUsage, Authorization $authorization) { +Http::setResource('dbForProject', function (Group $pools, Database $dbForPlatform, Cache $cache, Document $project, Response $response, Publisher $publisher, Publisher $publisherFunctions, Publisher $publisherWebhooks, Event $queueForEvents, Func $queueForFunctions, Webhook $queueForWebhooks, Realtime $queueForRealtime, UsageContext $usage, Authorization $authorization) { if ($project->isEmpty() || $project->getId() === 'console') { return $dbForPlatform; } @@ -615,9 +620,8 @@ Http::setResource('dbForProject', function (Group $pools, Database $dbForPlatfor ->from($queueForEvents) ->trigger(); - /** Trigger webhooks events only if a project has them enabled */ - if (!empty($project->getAttribute('webhooks'))) { + if (! empty($project->getAttribute('webhooks'))) { $queueForWebhooks ->from($queueForEvents) ->trigger(); @@ -636,7 +640,6 @@ Http::setResource('dbForProject', function (Group $pools, Database $dbForPlatfor */ $functionsEventsCacheListener = function (string $event, Document $document, Document $project, Database $dbForProject) { - if ($document->getCollection() !== 'functions') { return; } @@ -658,7 +661,7 @@ Http::setResource('dbForProject', function (Group $pools, Database $dbForPlatfor $dbForProject->getCache()->purge($cacheKey); }; - $usageDatabaseListener = function (string $event, Document $document, StatsUsage $queueForStatsUsage) { + $usageDatabaseListener = function (string $event, Document $document, UsageContext $usage) { $value = 1; switch ($event) { @@ -678,81 +681,78 @@ Http::setResource('dbForProject', function (Group $pools, Database $dbForPlatfor switch (true) { case $document->getCollection() === 'teams': - $queueForStatsUsage->addMetric(METRIC_TEAMS, $value); // per project + $usage->addMetric(METRIC_TEAMS, $value); // per project break; case $document->getCollection() === 'users': - $queueForStatsUsage->addMetric(METRIC_USERS, $value); // per project + $usage->addMetric(METRIC_USERS, $value); // per project if ($event === Database::EVENT_DOCUMENT_DELETE) { - $queueForStatsUsage->addReduce($document); + $usage->addReduce($document); } break; case $document->getCollection() === 'sessions': // sessions - $queueForStatsUsage->addMetric(METRIC_SESSIONS, $value); //per project + $usage->addMetric(METRIC_SESSIONS, $value); // per project break; case $document->getCollection() === 'databases': // databases - $queueForStatsUsage->addMetric(METRIC_DATABASES, $value); // per project + $usage->addMetric(METRIC_DATABASES, $value); // per project if ($event === Database::EVENT_DOCUMENT_DELETE) { - $queueForStatsUsage->addReduce($document); + $usage->addReduce($document); } break; - case str_starts_with($document->getCollection(), 'database_') && !str_contains($document->getCollection(), 'collection'): //collections + case str_starts_with($document->getCollection(), 'database_') && ! str_contains($document->getCollection(), 'collection'): // collections $parts = explode('_', $document->getCollection()); $databaseInternalId = $parts[1] ?? 0; - $queueForStatsUsage + $usage ->addMetric(METRIC_COLLECTIONS, $value) // per project ->addMetric(str_replace('{databaseInternalId}', $databaseInternalId, METRIC_DATABASE_ID_COLLECTIONS), $value); if ($event === Database::EVENT_DOCUMENT_DELETE) { - $queueForStatsUsage->addReduce($document); + $usage->addReduce($document); } break; - case str_starts_with($document->getCollection(), 'database_') && str_contains($document->getCollection(), '_collection_'): //documents + case str_starts_with($document->getCollection(), 'database_') && str_contains($document->getCollection(), '_collection_'): // documents $parts = explode('_', $document->getCollection()); - $databaseInternalId = $parts[1] ?? 0; + $databaseInternalId = $parts[1] ?? 0; $collectionInternalId = $parts[3] ?? 0; - $queueForStatsUsage + $usage ->addMetric(METRIC_DOCUMENTS, $value) // per project ->addMetric(str_replace('{databaseInternalId}', $databaseInternalId, METRIC_DATABASE_ID_DOCUMENTS), $value) // per database ->addMetric(str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$databaseInternalId, $collectionInternalId], METRIC_DATABASE_ID_COLLECTION_ID_DOCUMENTS), $value); // per collection break; - case $document->getCollection() === 'buckets': //buckets - $queueForStatsUsage - ->addMetric(METRIC_BUCKETS, $value); // per project + case $document->getCollection() === 'buckets': // buckets + $usage->addMetric(METRIC_BUCKETS, $value); // per project if ($event === Database::EVENT_DOCUMENT_DELETE) { - $queueForStatsUsage + $usage ->addReduce($document); } break; case str_starts_with($document->getCollection(), 'bucket_'): // files $parts = explode('_', $document->getCollection()); - $bucketInternalId = $parts[1]; - $queueForStatsUsage + $bucketInternalId = $parts[1]; + $usage ->addMetric(METRIC_FILES, $value) // per project ->addMetric(METRIC_FILES_STORAGE, $document->getAttribute('sizeOriginal') * $value) // per project ->addMetric(str_replace('{bucketInternalId}', $bucketInternalId, METRIC_BUCKET_ID_FILES), $value) // per bucket ->addMetric(str_replace('{bucketInternalId}', $bucketInternalId, METRIC_BUCKET_ID_FILES_STORAGE), $document->getAttribute('sizeOriginal') * $value); // per bucket break; case $document->getCollection() === 'functions': - $queueForStatsUsage - ->addMetric(METRIC_FUNCTIONS, $value); // per project + $usage->addMetric(METRIC_FUNCTIONS, $value); // per project if ($event === Database::EVENT_DOCUMENT_DELETE) { - $queueForStatsUsage + $usage ->addReduce($document); } break; case $document->getCollection() === 'sites': - $queueForStatsUsage - ->addMetric(METRIC_SITES, $value); // per project + $usage->addMetric(METRIC_SITES, $value); // per project if ($event === Database::EVENT_DOCUMENT_DELETE) { - $queueForStatsUsage + $usage ->addReduce($document); } break; case $document->getCollection() === 'deployments': - $queueForStatsUsage + $usage ->addMetric(METRIC_DEPLOYMENTS, $value) // per project ->addMetric(METRIC_DEPLOYMENTS_STORAGE, $document->getAttribute('size') * $value) // per project ->addMetric(str_replace(['{resourceType}'], [$document->getAttribute('resourceType')], METRIC_RESOURCE_TYPE_DEPLOYMENTS), $value) // per function @@ -772,30 +772,27 @@ Http::setResource('dbForProject', function (Group $pools, Database $dbForPlatfor $queueForWebhooks = new Webhook($publisherWebhooks); $queueForRealtime = new Realtime(); - $database - ->on(Database::EVENT_DOCUMENT_CREATE, 'calculate-usage', fn ($event, $document) => $usageDatabaseListener($event, $document, $queueForStatsUsage)) - ->on(Database::EVENT_DOCUMENT_DELETE, 'calculate-usage', fn ($event, $document) => $usageDatabaseListener($event, $document, $queueForStatsUsage)) - ->on(Database::EVENT_DOCUMENTS_CREATE, 'calculate-usage', fn ($event, $document) => $usageDatabaseListener($event, $document, $queueForStatsUsage)) - ->on(Database::EVENT_DOCUMENTS_DELETE, 'calculate-usage', fn ($event, $document) => $usageDatabaseListener($event, $document, $queueForStatsUsage)) - ->on(Database::EVENT_DOCUMENTS_UPSERT, 'calculate-usage', fn ($event, $document) => $usageDatabaseListener($event, $document, $queueForStatsUsage)) - ->on(Database::EVENT_DOCUMENT_CREATE, 'create-trigger-events', fn ($event, $document) => $eventDatabaseListener( - $project, - $document, - $response, - $queueForEventsClone->from($queueForEvents), - $queueForFunctions->from($queueForEvents), - $queueForWebhooks->from($queueForEvents), - $queueForRealtime->from($queueForEvents) - )) - ->on(Database::EVENT_DOCUMENT_CREATE, 'purge-function-events-cache', fn ($event, $document) => $functionsEventsCacheListener($event, $document, $project, $database)) - ->on(Database::EVENT_DOCUMENT_UPDATE, 'purge-function-events-cache', fn ($event, $document) => $functionsEventsCacheListener($event, $document, $project, $database)) - ->on(Database::EVENT_DOCUMENT_DELETE, 'purge-function-events-cache', fn ($event, $document) => $functionsEventsCacheListener($event, $document, $project, $database)) - ; - + ->on(Database::EVENT_DOCUMENT_CREATE, 'calculate-usage', fn ($event, $document) => $usageDatabaseListener($event, $document, $usage)) + ->on(Database::EVENT_DOCUMENT_DELETE, 'calculate-usage', fn ($event, $document) => $usageDatabaseListener($event, $document, $usage)) + ->on(Database::EVENT_DOCUMENTS_CREATE, 'calculate-usage', fn ($event, $document) => $usageDatabaseListener($event, $document, $usage)) + ->on(Database::EVENT_DOCUMENTS_DELETE, 'calculate-usage', fn ($event, $document) => $usageDatabaseListener($event, $document, $usage)) + ->on(Database::EVENT_DOCUMENTS_UPSERT, 'calculate-usage', fn ($event, $document) => $usageDatabaseListener($event, $document, $usage)) + ->on(Database::EVENT_DOCUMENT_CREATE, 'create-trigger-events', fn ($event, $document) => $eventDatabaseListener( + $project, + $document, + $response, + $queueForEventsClone->from($queueForEvents), + $queueForFunctions->from($queueForEvents), + $queueForWebhooks->from($queueForEvents), + $queueForRealtime->from($queueForEvents) + )) + ->on(Database::EVENT_DOCUMENT_CREATE, 'purge-function-events-cache', fn ($event, $document) => $functionsEventsCacheListener($event, $document, $project, $database)) + ->on(Database::EVENT_DOCUMENT_UPDATE, 'purge-function-events-cache', fn ($event, $document) => $functionsEventsCacheListener($event, $document, $project, $database)) + ->on(Database::EVENT_DOCUMENT_DELETE, 'purge-function-events-cache', fn ($event, $document) => $functionsEventsCacheListener($event, $document, $project, $database)); return $database; -}, ['pools', 'dbForPlatform', 'cache', 'project', 'response', 'publisher', 'publisherFunctions', 'publisherWebhooks', 'queueForEvents', 'queueForFunctions', 'queueForWebhooks', 'queueForRealtime', 'queueForStatsUsage', 'authorization']); +}, ['pools', 'dbForPlatform', 'cache', 'project', 'response', 'publisher', 'publisherFunctions', 'publisherWebhooks', 'queueForEvents', 'queueForFunctions', 'queueForWebhooks', 'queueForRealtime', 'usage', 'authorization']); Http::setResource('dbForPlatform', function (Group $pools, Cache $cache, Authorization $authorization) { @@ -844,8 +841,7 @@ Http::setResource('getProjectDB', function (Group $pools, Database $dbForPlatfor ->setMetadata('project', $project->getId()) ->setTimeout(APP_DATABASE_TIMEOUT_MILLISECONDS_API) ->setMaxQueryValues(APP_DATABASE_QUERY_MAX_VALUES) - ->setDocumentType('users', User::class) - ; + ->setDocumentType('users', User::class); $sharedTables = \explode(',', System::getEnv('_APP_DATABASE_SHARED_TABLES', '')); @@ -865,6 +861,7 @@ Http::setResource('getProjectDB', function (Group $pools, Database $dbForPlatfor if (isset($databases[$dsn->getHost()])) { $database = $databases[$dsn->getHost()]; $configure($database); + return $database; } @@ -881,8 +878,9 @@ Http::setResource('getLogsDB', function (Group $pools, Cache $cache, Authorizati $database = null; return function (?Document $project = null) use ($pools, $cache, $authorization, &$database) { - if ($database !== null && $project !== null && !$project->isEmpty() && $project->getId() !== 'console') { + if ($database !== null && $project !== null && ! $project->isEmpty() && $project->getId() !== 'console') { $database->setTenant((int) $project->getSequence()); + return $database; } @@ -898,7 +896,7 @@ Http::setResource('getLogsDB', function (Group $pools, Cache $cache, Authorizati ->setMaxQueryValues(APP_DATABASE_QUERY_MAX_VALUES); // set tenant - if ($project !== null && !$project->isEmpty() && $project->getId() !== 'console') { + if ($project !== null && ! $project->isEmpty() && $project->getId() !== 'console') { $database->setTenant((int) $project->getSequence()); } @@ -908,6 +906,7 @@ Http::setResource('getLogsDB', function (Group $pools, Cache $cache, Authorizati Http::setResource('audit', function ($dbForProject) { $adapter = new AdapterDatabase($dbForProject); + return new Audit($adapter); }, ['dbForProject']); @@ -923,6 +922,7 @@ Http::setResource('cache', function (Group $pools, Telemetry $telemetry) { $cache = new Cache(new Sharding($adapters)); $cache->setTelemetry($telemetry); + return $cache; }, ['pools', 'telemetry']); @@ -968,9 +968,9 @@ Http::setResource('deviceForBuilds', function ($project, Telemetry $telemetry) { function getDevice(string $root, string $connection = ''): Device { - $connection = !empty($connection) ? $connection : System::getEnv('_APP_CONNECTIONS_STORAGE', ''); + $connection = ! empty($connection) ? $connection : System::getEnv('_APP_CONNECTIONS_STORAGE', ''); - if (!empty($connection)) { + if (! empty($connection)) { $acl = 'private'; $device = Storage::DEVICE_LOCAL; $accessKey = ''; @@ -992,8 +992,9 @@ function getDevice(string $root, string $connection = ''): Device switch ($device) { case Storage::DEVICE_S3: - if (!empty($url)) { - $bucketRoot = (!empty($bucket) ? $bucket . '/' : '') . \ltrim($root, '/'); + if (! empty($url)) { + $bucketRoot = (! empty($bucket) ? $bucket . '/' : '') . \ltrim($root, '/'); + return new S3($bucketRoot, $accessKey, $accessSecret, $url, $region, $acl); } else { return new AWS($root, $accessKey, $accessSecret, $bucket, $region, $acl); @@ -1002,6 +1003,7 @@ function getDevice(string $root, string $connection = ''): Device case STORAGE::DEVICE_DO_SPACES: $device = new DOSpaces($root, $accessKey, $accessSecret, $bucket, $region, $acl); $device->setHttpVersion(S3::HTTP_VERSION_1_1); + return $device; case Storage::DEVICE_BACKBLAZE: return new Backblaze($root, $accessKey, $accessSecret, $bucket, $region, $acl); @@ -1025,8 +1027,9 @@ function getDevice(string $root, string $connection = ''): Device $s3Bucket = System::getEnv('_APP_STORAGE_S3_BUCKET', ''); $s3Acl = 'private'; $s3EndpointUrl = System::getEnv('_APP_STORAGE_S3_ENDPOINT', ''); - if (!empty($s3EndpointUrl)) { - $bucketRoot = (!empty($s3Bucket) ? $s3Bucket . '/' : '') . \ltrim($root, '/'); + if (! empty($s3EndpointUrl)) { + $bucketRoot = (! empty($s3Bucket) ? $s3Bucket . '/' : '') . \ltrim($root, '/'); + return new S3($bucketRoot, $s3AccessKey, $s3SecretKey, $s3EndpointUrl, $s3Region, $s3Acl); } else { return new AWS($root, $s3AccessKey, $s3SecretKey, $s3Bucket, $s3Region, $s3Acl); @@ -1040,6 +1043,7 @@ function getDevice(string $root, string $connection = ''): Device $doSpacesAcl = 'private'; $device = new DOSpaces($root, $doSpacesAccessKey, $doSpacesSecretKey, $doSpacesBucket, $doSpacesRegion, $doSpacesAcl); $device->setHttpVersion(S3::HTTP_VERSION_1_1); + return $device; case Storage::DEVICE_BACKBLAZE: $backblazeAccessKey = System::getEnv('_APP_STORAGE_BACKBLAZE_ACCESS_KEY', ''); @@ -1047,6 +1051,7 @@ function getDevice(string $root, string $connection = ''): Device $backblazeRegion = System::getEnv('_APP_STORAGE_BACKBLAZE_REGION', ''); $backblazeBucket = System::getEnv('_APP_STORAGE_BACKBLAZE_BUCKET', ''); $backblazeAcl = 'private'; + return new Backblaze($root, $backblazeAccessKey, $backblazeSecretKey, $backblazeBucket, $backblazeRegion, $backblazeAcl); case Storage::DEVICE_LINODE: $linodeAccessKey = System::getEnv('_APP_STORAGE_LINODE_ACCESS_KEY', ''); @@ -1054,6 +1059,7 @@ function getDevice(string $root, string $connection = ''): Device $linodeRegion = System::getEnv('_APP_STORAGE_LINODE_REGION', ''); $linodeBucket = System::getEnv('_APP_STORAGE_LINODE_BUCKET', ''); $linodeAcl = 'private'; + return new Linode($root, $linodeAccessKey, $linodeSecretKey, $linodeBucket, $linodeRegion, $linodeAcl); case Storage::DEVICE_WASABI: $wasabiAccessKey = System::getEnv('_APP_STORAGE_WASABI_ACCESS_KEY', ''); @@ -1061,6 +1067,7 @@ function getDevice(string $root, string $connection = ''): Device $wasabiRegion = System::getEnv('_APP_STORAGE_WASABI_REGION', ''); $wasabiBucket = System::getEnv('_APP_STORAGE_WASABI_BUCKET', ''); $wasabiAcl = 'private'; + return new Wasabi($root, $wasabiAccessKey, $wasabiSecretKey, $wasabiBucket, $wasabiRegion, $wasabiAcl); } } @@ -1087,7 +1094,6 @@ Http::setResource('passwordsDictionary', function ($register) { return $register->get('passwordsDictionary'); }, ['register']); - Http::setResource('servers', function () { $platforms = Config::getParam('sdks'); $server = $platforms[APP_SDK_PLATFORM_SERVER]; @@ -1195,16 +1201,17 @@ Http::setResource('gitHub', function (Cache $cache) { }, ['cache']); Http::setResource('requestTimestamp', function ($request) { - //TODO: Move this to the Request class itself + // TODO: Move this to the Request class itself $timestampHeader = $request->getHeader('x-appwrite-timestamp'); $requestTimestamp = null; - if (!empty($timestampHeader)) { + if (! empty($timestampHeader)) { try { $requestTimestamp = new \DateTime($timestampHeader); } catch (\Throwable $e) { throw new Exception(Exception::GENERAL_ARGUMENT_INVALID, 'Invalid X-Appwrite-Timestamp header value'); } } + return $requestTimestamp; }, ['request']); @@ -1221,13 +1228,13 @@ Http::setResource('devKey', function (Request $request, Document $project, array // Check if given key match project's development keys $key = $project->find('secret', $devKey, 'devKeys'); - if (!$key) { + if (! $key) { return new Document([]); } // check expiration $expire = $key->getAttribute('expire'); - if (!empty($expire) && $expire < DatabaseDateTime::formatTz(DatabaseDateTime::now())) { + if (! empty($expire) && $expire < DatabaseDateTime::formatTz(DatabaseDateTime::now())) { return new Document([]); } @@ -1248,7 +1255,7 @@ Http::setResource('devKey', function (Request $request, Document $project, array if ($sdk !== 'UNKNOWN' && $sdkValidator->isValid($sdk)) { $sdks = $key->getAttribute('sdks', []); - if (!in_array($sdk, $sdks)) { + if (! in_array($sdk, $sdks)) { $sdks[] = $sdk; $key->setAttribute('sdks', $sdks); @@ -1271,7 +1278,7 @@ Http::setResource('team', function (Document $project, Database $dbForPlatform, $teamInternalId = $project->getAttribute('teamInternalId', ''); } else { $route = $utopia->match($request); - $path = !empty($route) ? $route->getPath() : $request->getURI(); + $path = ! empty($route) ? $route->getPath() : $request->getURI(); $orgHeader = $request->getHeader('x-appwrite-organization', ''); if (str_starts_with($path, '/v1/projects/:projectId')) { $uri = $request->getURI(); @@ -1286,8 +1293,9 @@ Http::setResource('team', function (Document $project, Database $dbForPlatform, } $team = $authorization->skip(fn () => $dbForPlatform->getDocument('teams', $teamId)); + return $team; - } elseif (!empty($orgHeader)) { + } elseif (! empty($orgHeader)) { return $authorization->skip(fn () => $dbForPlatform->getDocument('teams', $orgHeader)); } } @@ -1317,13 +1325,13 @@ Http::setResource('previewHostname', function (Request $request, ?Key $apiKey) { if (Http::isDevelopment()) { $allowed = true; - } elseif (!\is_null($apiKey) && $apiKey->getHostnameOverride() === true) { + } elseif (! \is_null($apiKey) && $apiKey->getHostnameOverride() === true) { $allowed = true; } if ($allowed) { $host = $request->getQuery('appwrite-hostname', $request->getHeader('x-appwrite-hostname', '')) ?? ''; - if (!empty($host)) { + if (! empty($host)) { return $host; } } @@ -1344,19 +1352,19 @@ Http::setResource('apiKey', function (Request $request, Document $project, Docum $organizationHeader = $request->getHeader('x-appwrite-organization'); $projectHeader = $request->getHeader('x-appwrite-project'); - if (!empty($key->getProjectId())) { + if (! empty($key->getProjectId())) { if (empty($projectHeader) || $projectHeader !== $key->getProjectId()) { throw new Exception(Exception::PROJECT_ID_MISSING); } } - if (!empty($key->getUserId())) { + if (! empty($key->getUserId())) { if (empty($userHeader) || $userHeader !== $key->getUserId()) { throw new Exception(Exception::USER_ID_MISSING); } } - if (!empty($key->getTeamId())) { + if (! empty($key->getTeamId())) { if (empty($organizationHeader) || $organizationHeader !== $key->getTeamId()) { throw new Exception(Exception::ORGANIZATION_ID_MISSING); } @@ -1370,7 +1378,7 @@ Http::setResource('executor', fn () => new Executor()); Http::setResource('resourceToken', function ($project, $dbForProject, $request, Authorization $authorization) { $tokenJWT = $request->getParam('token'); - if (!empty($tokenJWT) && !$project->isEmpty()) { // JWT authentication + if (! empty($tokenJWT) && ! $project->isEmpty()) { // JWT authentication // Use a large but reasonable maxAge to avoid auto-exp when token has no expiry $jwt = new JWT(System::getEnv('_APP_OPENSSL_KEY_V1'), RESOURCE_TOKEN_ALGORITHM, RESOURCE_TOKEN_MAX_AGE, RESOURCE_TOKEN_LEEWAY); // Instantiate with key, algo, maxAge and leeway. @@ -1430,6 +1438,7 @@ Http::setResource('resourceToken', function ($project, $dbForProject, $request, default => throw new Exception(Exception::TOKEN_RESOURCE_TYPE_INVALID), }; } + return new Document([]); }, ['project', 'dbForProject', 'request', 'authorization']); diff --git a/app/init/span.php b/app/init/span.php index 76f37f5300..8afa01b2df 100644 --- a/app/init/span.php +++ b/app/init/span.php @@ -5,4 +5,9 @@ use Utopia\Span\Span; use Utopia\Span\Storage; Span::setStorage(new Storage\Coroutine()); -Span::addExporter(new Exporter\Pretty()); +Span::addExporter(new Exporter\Pretty(), function (Span $span): bool { + if (\str_starts_with($span->getAction(), 'listener.')) { + return $span->getError() !== null; + } + return true; +}); diff --git a/app/realtime.php b/app/realtime.php index e0591a2596..5addb2a78f 100644 --- a/app/realtime.php +++ b/app/realtime.php @@ -224,6 +224,13 @@ if (!function_exists('getTelemetry')) { } } +if (!function_exists('triggerStats')) { + function triggerStats(array $event, string $projectId): void + { + return; + } +} + $realtime = getRealtime(); /** @@ -548,20 +555,41 @@ $server->onWorkerStart(function (int $workerId) use ($server, $register, $stats, } $total = 0; + $outboundBytes = 0; + foreach ($groups as $group) { $data = $event['data']; $data['subscriptions'] = $group['subscriptions']; - $server->send($group['ids'], json_encode([ + $payloadJson = json_encode([ 'type' => 'event', 'data' => $data - ])); - $total += count($group['ids']); + ]); + + $server->send($group['ids'], $payloadJson); + + $count = count($group['ids']); + $total += $count; + $outboundBytes += strlen($payloadJson) * $count; } if ($total > 0) { $register->get('telemetry.messageSentCounter')->add($total); $stats->incr($event['project'], 'messages', $total); + + $projectId = $event['project'] ?? null; + + if (!empty($projectId)) { + $metrics = [ + METRIC_REALTIME_CONNECTIONS_MESSAGES_SENT => $total, + ]; + + if ($outboundBytes > 0) { + $metrics[METRIC_REALTIME_OUTBOUND] = $outboundBytes; + } + + triggerStats($metrics, $projectId); + } } }); } catch (Throwable $th) { @@ -638,6 +666,12 @@ $server->onOpen(function (int $connection, SwooleRequest $request) use ($server, throw new Exception(Exception::REALTIME_TOO_MANY_MESSAGES, 'Too many requests'); } + $rawSize = $request->getSize(); + + triggerStats([ + METRIC_REALTIME_INBOUND => $rawSize, + ], $project->getId()); + /* * Validate Client Domain - Check to avoid CSRF attack. * Adding Appwrite API domains to allow XDOMAIN communication. @@ -692,14 +726,16 @@ $server->onOpen(function (int $connection, SwooleRequest $request) use ($server, $user = empty($user->getId()) ? null : $response->output($user, Response::MODEL_ACCOUNT); - $server->send([$connection], json_encode([ + $connectedPayloadJson = json_encode([ 'type' => 'connected', 'data' => [ 'channels' => $names, 'subscriptions' => $mapping, 'user' => $user ] - ])); + ]); + + $server->send([$connection], $connectedPayloadJson); $register->get('telemetry.connectionCounter')->add(1); $register->get('telemetry.connectionCreatedCounter')->add(1); @@ -710,6 +746,12 @@ $server->onOpen(function (int $connection, SwooleRequest $request) use ($server, ]); $stats->incr($project->getId(), 'connections'); $stats->incr($project->getId(), 'connectionsTotal'); + + $connectedOutboundBytes = \strlen($connectedPayloadJson); + + triggerStats([METRIC_REALTIME_CONNECTIONS => 1, METRIC_REALTIME_OUTBOUND => $connectedOutboundBytes], $project->getId()); + + } catch (Throwable $th) { logError($th, 'realtime', project: $project, user: $logUser, authorization: $authorization); @@ -751,6 +793,7 @@ $server->onMessage(function (int $connection, string $message) use ($server, $re $authorization = null; try { + $rawSize = \strlen($message); $response = new Response(new SwooleResponse()); $projectId = $realtime->connections[$connection]['projectId'] ?? null; @@ -763,7 +806,7 @@ $server->onMessage(function (int $connection, string $message) use ($server, $re $database = getConsoleDB(); $database->setAuthorization($authorization); - if ($projectId !== 'console') { + if (!empty($projectId) && $projectId !== 'console') { $project = $authorization->skip(fn () => $database->getDocument('projects', $projectId)); $database = getProjectDB($project); @@ -789,17 +832,41 @@ $server->onMessage(function (int $connection, string $message) use ($server, $re throw new Exception(Exception::REALTIME_TOO_MANY_MESSAGES, 'Too many messages.'); } + // Record realtime inbound bytes for this project + if ($project !== null && !$project->isEmpty()) { + triggerStats([ + METRIC_REALTIME_INBOUND => $rawSize, + ], $project->getId()); + } + $message = json_decode($message, true); if (is_null($message) || (!array_key_exists('type', $message) && !array_key_exists('data', $message))) { throw new Exception(Exception::REALTIME_MESSAGE_FORMAT_INVALID, 'Message format is not valid.'); } + // Ping does not require project context; other messages do (e.g. after unsubscribe during auth) + if (empty($projectId) && ($message['type'] ?? '') !== 'ping') { + throw new Exception(Exception::REALTIME_POLICY_VIOLATION, 'Missing project context. Reconnect to the project first.'); + } + switch ($message['type']) { case 'ping': - $server->send([$connection], json_encode([ + $pongPayloadJson = json_encode([ 'type' => 'pong' - ])); + ]); + + $server->send([$connection], $pongPayloadJson); + + if ($project !== null && !$project->isEmpty()) { + $pongOutboundBytes = \strlen($pongPayloadJson); + + if ($pongOutboundBytes > 0) { + triggerStats([ + METRIC_REALTIME_OUTBOUND => $pongOutboundBytes, + ], $project->getId()); + } + } break; case 'authentication': @@ -860,14 +927,27 @@ $server->onMessage(function (int $connection, string $message) use ($server, $re } $user = $response->output($user, Response::MODEL_ACCOUNT); - $server->send([$connection], json_encode([ + + $authResponsePayloadJson = json_encode([ 'type' => 'response', 'data' => [ 'to' => 'authentication', 'success' => true, 'user' => $user ] - ])); + ]); + + $server->send([$connection], $authResponsePayloadJson); + + if ($project !== null && !$project->isEmpty()) { + $authOutboundBytes = \strlen($authResponsePayloadJson); + + if ($authOutboundBytes > 0) { + triggerStats([ + METRIC_REALTIME_OUTBOUND => $authOutboundBytes, + ], $project->getId()); + } + } break; @@ -908,6 +988,12 @@ $server->onClose(function (int $connection) use ($realtime, $stats, $register) { if (array_key_exists($connection, $realtime->connections)) { $stats->decr($realtime->connections[$connection]['projectId'], 'connectionsTotal'); $register->get('telemetry.connectionCounter')->add(-1); + + $projectId = $realtime->connections[$connection]['projectId']; + + triggerStats([ + METRIC_REALTIME_CONNECTIONS => -1, + ], $projectId); } $realtime->unsubscribe($connection); diff --git a/app/worker.php b/app/worker.php index 2ee1803ddc..db036b6a99 100644 --- a/app/worker.php +++ b/app/worker.php @@ -13,11 +13,12 @@ use Appwrite\Event\Func; use Appwrite\Event\Mail; use Appwrite\Event\Messaging; use Appwrite\Event\Migration; +use Appwrite\Event\Publisher\Usage as UsagePublisher; use Appwrite\Event\Realtime; use Appwrite\Event\Screenshot; -use Appwrite\Event\StatsUsage; use Appwrite\Event\Webhook; use Appwrite\Platform\Appwrite; +use Appwrite\Usage\Context; use Appwrite\Utopia\Database\Documents\User; use Executor\Executor; use Swoole\Runtime; @@ -42,6 +43,7 @@ use Utopia\Pools\Group; use Utopia\Queue\Broker\Pool as BrokerPool; use Utopia\Queue\Message; use Utopia\Queue\Publisher; +use Utopia\Queue\Queue; use Utopia\Queue\Server; use Utopia\Registry\Registry; use Utopia\Storage\Device\Telemetry as TelemetryDevice; @@ -58,7 +60,8 @@ Server::setResource('register', fn () => $register); Server::setResource('authorization', function () { $authorization = new Authorization(); $authorization->disable(); - return $authorization; + + return $authorization; }, []); Server::setResource('dbForPlatform', function (Cache $cache, Registry $register, Authorization $authorization) { @@ -70,9 +73,7 @@ Server::setResource('dbForPlatform', function (Cache $cache, Registry $register, ->setDatabase(APP_DATABASE) ->setAuthorization($authorization) ->setNamespace('_console') - ->setDocumentType('users', User::class) - ; - + ->setDocumentType('users', User::class); return $dbForPlatform; }, ['cache', 'register', 'authorization']); @@ -111,7 +112,7 @@ Server::setResource('dbForProject', function (Cache $cache, Registry $register, if (\in_array($dsn->getHost(), $sharedTables)) { $database ->setSharedTables(true) - ->setTenant((int)$project->getSequence()) + ->setTenant((int) $project->getSequence()) ->setNamespace($dsn->getParam('namespace')); } else { $database @@ -151,7 +152,7 @@ Server::setResource('getProjectDB', function (Group $pools, Database $dbForPlatf if (\in_array($dsn->getHost(), $sharedTables)) { $database ->setSharedTables(true) - ->setTenant((int)$project->getSequence()) + ->setTenant((int) $project->getSequence()) ->setNamespace($dsn->getParam('namespace')); } else { $database @@ -173,7 +174,7 @@ Server::setResource('getProjectDB', function (Group $pools, Database $dbForPlatf if (\in_array($dsn->getHost(), $sharedTables)) { $database ->setSharedTables(true) - ->setTenant((int)$project->getSequence()) + ->setTenant((int) $project->getSequence()) ->setNamespace($dsn->getParam('namespace')); } else { $database @@ -193,9 +194,11 @@ Server::setResource('getProjectDB', function (Group $pools, Database $dbForPlatf Server::setResource('getLogsDB', function (Group $pools, Cache $cache, Authorization $authorization) { $database = null; + return function (?Document $project = null) use ($pools, $cache, $database, $authorization) { - if ($database !== null && $project !== null && !$project->isEmpty() && $project->getId() !== 'console') { - $database->setTenant((int)$project->getSequence()); + if ($database !== null && $project !== null && ! $project->isEmpty() && $project->getId() !== 'console') { + $database->setTenant((int) $project->getSequence()); + return $database; } @@ -211,8 +214,8 @@ Server::setResource('getLogsDB', function (Group $pools, Cache $cache, Authoriza ->setMaxQueryValues(APP_DATABASE_QUERY_MAX_VALUES_WORKER); // set tenant - if ($project !== null && !$project->isEmpty() && $project->getId() !== 'console') { - $database->setTenant((int)$project->getSequence()); + if ($project !== null && ! $project->isEmpty() && $project->getId() !== 'console') { + $database->setTenant((int) $project->getSequence()); } return $database; @@ -227,6 +230,7 @@ Server::setResource('auditRetention', function (Document $project) { if ($project->getId() === 'console') { return DateTime::addSeconds(new \DateTime(), -1 * System::getEnv('_APP_MAINTENANCE_RETENTION_AUDIT_CONSOLE', 15778800)); // 6 months } + return DateTime::addSeconds(new \DateTime(), -1 * System::getEnv('_APP_MAINTENANCE_RETENTION_AUDIT', 1209600)); // 14 days }, ['project']); @@ -252,7 +256,7 @@ Server::setResource('redis', function () { $pass = System::getEnv('_APP_REDIS_PASS', ''); $redis = new \Redis(); - @$redis->pconnect($host, (int)$port); + @$redis->pconnect($host, (int) $port); if ($pass) { $redis->auth($pass); } @@ -269,7 +273,6 @@ Server::setResource('timelimit', function (\Redis $redis) { Server::setResource('log', fn () => new Log()); - Server::setResource('publisher', function (Group $pools) { return new BrokerPool(publisher: $pools->get('publisher')); }, ['pools']); @@ -286,10 +289,6 @@ Server::setResource('publisherMigrations', function (BrokerPool $publisher) { return $publisher; }, ['publisher']); -Server::setResource('publisherStatsUsage', function (BrokerPool $publisher) { - return $publisher; -}, ['publisher']); - Server::setResource('publisherMessaging', function (BrokerPool $publisher) { return $publisher; }, ['publisher']); @@ -310,9 +309,13 @@ Server::setResource('consumerStatsUsage', function (BrokerPool $consumer) { return $consumer; }, ['consumer']); -Server::setResource('queueForStatsUsage', function (Publisher $publisher) { - return new StatsUsage($publisher); -}, ['publisher']); +Server::setResource('usage', function () { + return new Context(); +}, []); +Server::setResource('publisherForUsage', fn (Publisher $publisher) => new UsagePublisher( + $publisher, + new Queue(System::getEnv('_APP_STATS_USAGE_QUEUE_NAME', Event::STATS_USAGE_QUEUE_NAME)) +), ['publisher']); Server::setResource('queueForDatabase', function (Publisher $publisher) { return new EventDatabase($publisher); @@ -354,7 +357,6 @@ Server::setResource('queueForFunctions', function (Publisher $publisher) { return new Func($publisher); }, ['publisher']); - Server::setResource('queueForRealtime', function () { return new Realtime(); }, []); @@ -484,11 +486,13 @@ Server::setResource('getAudit', function (Database $dbForPlatform, callable $get return function (Document $project) use ($dbForPlatform, $getProjectDB) { if ($project->isEmpty() || $project->getId() === 'console') { $adapter = new AdapterDatabase($dbForPlatform); + return new UtopiaAudit($adapter); } $dbForProject = $getProjectDB($project); $adapter = new AdapterDatabase($dbForProject); + return new UtopiaAudit($adapter); }; }, ['dbForPlatform', 'getProjectDB']); @@ -505,7 +509,7 @@ $pools = $register->get('pools'); $platform = new Appwrite(); $args = $platform->getEnv('argv'); -if (!isset($args[1])) { +if (! isset($args[1])) { Console::error('Missing worker name'); Console::exit(1); } @@ -530,10 +534,10 @@ try { 'workersNum' => System::getEnv('_APP_WORKERS_NUM', 1), 'connection' => $pools->get('consumer')->pop()->getResource(), 'workerName' => strtolower($workerName) ?? null, - 'queueName' => $queueName + 'queueName' => $queueName, ]); } catch (\Throwable $e) { - Console::error($e->getMessage() . ', File: ' . $e->getFile() . ', Line: ' . $e->getLine()); + Console::error($e->getMessage() . ', File: ' . $e->getFile() . ', Line: ' . $e->getLine()); } $worker = $platform->getWorker(); @@ -550,11 +554,11 @@ $worker ->inject('pools') ->inject('project') ->inject('authorization') - ->action(function (Throwable $error, ?Logger $logger, Log $log, Group $pools, Document $project, Authorization $authorization) use ($worker, $queueName) { + ->action(function (Throwable $error, ?Logger $logger, Log $log, Group $pools, Document $project, Authorization $authorization) use ($queueName) { $version = System::getEnv('_APP_VERSION', 'UNKNOWN'); if ($logger) { - $log->setNamespace("appwrite-worker"); + $log->setNamespace('appwrite-worker'); $log->setServer(System::getEnv('_APP_LOGGING_SERVICE_IDENTIFIER', \gethostname())); $log->setVersion($version); $log->setType(Log::TYPE_ERROR); diff --git a/bin/time-travel b/bin/task-time-travel similarity index 100% rename from bin/time-travel rename to bin/task-time-travel diff --git a/composer.json b/composer.json index d7f0ae66b5..19e1bb5c5a 100644 --- a/composer.json +++ b/composer.json @@ -70,12 +70,13 @@ "utopia-php/locale": "0.8.*", "utopia-php/logger": "0.6.*", "utopia-php/messaging": "0.20.*", - "utopia-php/migration": "1.6.*", + "utopia-php/migration": "1.7.*", "utopia-php/platform": "0.7.*", "utopia-php/pools": "1.*", "utopia-php/span": "1.1.*", "utopia-php/preloader": "0.2.*", "utopia-php/queue": "0.15.*", + "utopia-php/servers": "0.2.5", "utopia-php/registry": "0.5.*", "utopia-php/storage": "1.0.*", "utopia-php/system": "0.10.*", diff --git a/composer.lock b/composer.lock index 86093899d0..0e25d3bc5f 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "1cc64e07484256225f56bd525674c3b8", + "content-hash": "b99693284208ff3d006260a089a4f7b9", "packages": [ { "name": "adhocore/jwt", @@ -3850,16 +3850,16 @@ }, { "name": "utopia-php/database", - "version": "5.3.7", + "version": "5.3.8", "source": { "type": "git", "url": "https://github.com/utopia-php/database.git", - "reference": "438cc82af2981cd41ad200dd9b0df5bf00f3046a" + "reference": "4920bb60afb98d4bd81f4d331765716ae1d40255" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/database/zipball/438cc82af2981cd41ad200dd9b0df5bf00f3046a", - "reference": "438cc82af2981cd41ad200dd9b0df5bf00f3046a", + "url": "https://api.github.com/repos/utopia-php/database/zipball/4920bb60afb98d4bd81f4d331765716ae1d40255", + "reference": "4920bb60afb98d4bd81f4d331765716ae1d40255", "shasum": "" }, "require": { @@ -3902,9 +3902,9 @@ ], "support": { "issues": "https://github.com/utopia-php/database/issues", - "source": "https://github.com/utopia-php/database/tree/5.3.7" + "source": "https://github.com/utopia-php/database/tree/5.3.8" }, - "time": "2026-03-09T04:28:56+00:00" + "time": "2026-03-11T01:03:34+00:00" }, { "name": "utopia-php/detector", @@ -4058,16 +4058,16 @@ }, { "name": "utopia-php/domains", - "version": "1.0.5", + "version": "1.0.2", "source": { "type": "git", "url": "https://github.com/utopia-php/domains.git", - "reference": "0edf6bb2b07f30db849a267027077bf5abb994c6" + "reference": "b4896a6746f0fbe29dfd5e32f7790bd94c1af1e6" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/domains/zipball/0edf6bb2b07f30db849a267027077bf5abb994c6", - "reference": "0edf6bb2b07f30db849a267027077bf5abb994c6", + "url": "https://api.github.com/repos/utopia-php/domains/zipball/b4896a6746f0fbe29dfd5e32f7790bd94c1af1e6", + "reference": "b4896a6746f0fbe29dfd5e32f7790bd94c1af1e6", "shasum": "" }, "require": { @@ -4114,9 +4114,9 @@ ], "support": { "issues": "https://github.com/utopia-php/domains/issues", - "source": "https://github.com/utopia-php/domains/tree/1.0.5" + "source": "https://github.com/utopia-php/domains/tree/1.0.2" }, - "time": "2026-03-03T09:20:50+00:00" + "time": "2026-02-25T08:18:25+00:00" }, { "name": "utopia-php/dsn", @@ -4517,16 +4517,16 @@ }, { "name": "utopia-php/migration", - "version": "1.6.3", + "version": "1.7.0", "source": { "type": "git", "url": "https://github.com/utopia-php/migration.git", - "reference": "c2d016944cb029fa5ff822ceee704785a06ef289" + "reference": "97583ae502e40621ea91a71de19d053c5ae2e558" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/migration/zipball/c2d016944cb029fa5ff822ceee704785a06ef289", - "reference": "c2d016944cb029fa5ff822ceee704785a06ef289", + "url": "https://api.github.com/repos/utopia-php/migration/zipball/97583ae502e40621ea91a71de19d053c5ae2e558", + "reference": "97583ae502e40621ea91a71de19d053c5ae2e558", "shasum": "" }, "require": { @@ -4566,9 +4566,9 @@ ], "support": { "issues": "https://github.com/utopia-php/migration/issues", - "source": "https://github.com/utopia-php/migration/tree/1.6.3" + "source": "https://github.com/utopia-php/migration/tree/1.7.0" }, - "time": "2026-03-04T07:08:22+00:00" + "time": "2026-03-10T06:36:27+00:00" }, { "name": "utopia-php/mongo", @@ -5215,16 +5215,16 @@ }, { "name": "utopia-php/vcs", - "version": "2.0.1", + "version": "2.0.0", "source": { "type": "git", "url": "https://github.com/utopia-php/vcs.git", - "reference": "92a1650824ba0c5e6a1bc46e622ac87c50a08920" + "reference": "058049326e04a2a0c2f0ce8ad00c7e84825aba14" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/vcs/zipball/92a1650824ba0c5e6a1bc46e622ac87c50a08920", - "reference": "92a1650824ba0c5e6a1bc46e622ac87c50a08920", + "url": "https://api.github.com/repos/utopia-php/vcs/zipball/058049326e04a2a0c2f0ce8ad00c7e84825aba14", + "reference": "058049326e04a2a0c2f0ce8ad00c7e84825aba14", "shasum": "" }, "require": { @@ -5258,9 +5258,9 @@ ], "support": { "issues": "https://github.com/utopia-php/vcs/issues", - "source": "https://github.com/utopia-php/vcs/tree/2.0.1" + "source": "https://github.com/utopia-php/vcs/tree/2.0.0" }, - "time": "2026-02-27T12:18:49+00:00" + "time": "2026-02-25T11:36:45+00:00" }, { "name": "utopia-php/websocket", @@ -5438,16 +5438,16 @@ "packages-dev": [ { "name": "appwrite/sdk-generator", - "version": "1.11.6", + "version": "1.11.1", "source": { "type": "git", "url": "https://github.com/appwrite/sdk-generator.git", - "reference": "f80e302d000cdc2f98b4bb5ff2fc3bd0bdff7b38" + "reference": "6ff411f26f2750eea05c7598c14bb3a2ada898cb" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/appwrite/sdk-generator/zipball/f80e302d000cdc2f98b4bb5ff2fc3bd0bdff7b38", - "reference": "f80e302d000cdc2f98b4bb5ff2fc3bd0bdff7b38", + "url": "https://api.github.com/repos/appwrite/sdk-generator/zipball/6ff411f26f2750eea05c7598c14bb3a2ada898cb", + "reference": "6ff411f26f2750eea05c7598c14bb3a2ada898cb", "shasum": "" }, "require": { @@ -5483,22 +5483,22 @@ "description": "Appwrite PHP library for generating API SDKs for multiple programming languages and platforms", "support": { "issues": "https://github.com/appwrite/sdk-generator/issues", - "source": "https://github.com/appwrite/sdk-generator/tree/1.11.6" + "source": "https://github.com/appwrite/sdk-generator/tree/1.11.1" }, - "time": "2026-03-09T07:12:51+00:00" + "time": "2026-02-25T07:15:19+00:00" }, { "name": "brianium/paratest", - "version": "v7.19.1", + "version": "v7.19.0", "source": { "type": "git", "url": "https://github.com/paratestphp/paratest.git", - "reference": "95b03194f4cdf5c83175ceead673e21cb66465e7" + "reference": "7c6c29af7c4b406b49ce0c6b0a3a81d3684474e6" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/paratestphp/paratest/zipball/95b03194f4cdf5c83175ceead673e21cb66465e7", - "reference": "95b03194f4cdf5c83175ceead673e21cb66465e7", + "url": "https://api.github.com/repos/paratestphp/paratest/zipball/7c6c29af7c4b406b49ce0c6b0a3a81d3684474e6", + "reference": "7c6c29af7c4b406b49ce0c6b0a3a81d3684474e6", "shasum": "" }, "require": { @@ -5512,7 +5512,7 @@ "phpunit/php-code-coverage": "^12.5.3 || ^13.0.1", "phpunit/php-file-iterator": "^6.0.1 || ^7", "phpunit/php-timer": "^8 || ^9", - "phpunit/phpunit": "^12.5.14 || ^13.0.5", + "phpunit/phpunit": "^12.5.9 || ^13", "sebastian/environment": "^8.0.3 || ^9", "symfony/console": "^7.4.4 || ^8.0.4", "symfony/process": "^7.4.5 || ^8.0.5" @@ -5522,10 +5522,10 @@ "ext-pcntl": "*", "ext-pcov": "*", "ext-posix": "*", - "phpstan/phpstan": "^2.1.40", - "phpstan/phpstan-deprecation-rules": "^2.0.4", - "phpstan/phpstan-phpunit": "^2.0.16", - "phpstan/phpstan-strict-rules": "^2.0.10", + "phpstan/phpstan": "^2.1.38", + "phpstan/phpstan-deprecation-rules": "^2.0.3", + "phpstan/phpstan-phpunit": "^2.0.12", + "phpstan/phpstan-strict-rules": "^2.0.8", "symfony/filesystem": "^7.4.0 || ^8.0.1" }, "bin": [ @@ -5566,7 +5566,7 @@ ], "support": { "issues": "https://github.com/paratestphp/paratest/issues", - "source": "https://github.com/paratestphp/paratest/tree/v7.19.1" + "source": "https://github.com/paratestphp/paratest/tree/v7.19.0" }, "funding": [ { @@ -5578,7 +5578,7 @@ "type": "paypal" } ], - "time": "2026-02-25T14:53:45+00:00" + "time": "2026-02-06T10:53:26+00:00" }, { "name": "czproject/git-php", @@ -6398,16 +6398,16 @@ }, { "name": "phpbench/phpbench", - "version": "1.5.1", + "version": "1.4.3", "source": { "type": "git", "url": "https://github.com/phpbench/phpbench.git", - "reference": "9a28fd0833f11171b949843c6fd663eb69b6d14c" + "reference": "b641dde59d969ea42eed70a39f9b51950bc96878" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/phpbench/phpbench/zipball/9a28fd0833f11171b949843c6fd663eb69b6d14c", - "reference": "9a28fd0833f11171b949843c6fd663eb69b6d14c", + "url": "https://api.github.com/repos/phpbench/phpbench/zipball/b641dde59d969ea42eed70a39f9b51950bc96878", + "reference": "b641dde59d969ea42eed70a39f9b51950bc96878", "shasum": "" }, "require": { @@ -6418,7 +6418,7 @@ "ext-reflection": "*", "ext-spl": "*", "ext-tokenizer": "*", - "php": "^8.2", + "php": "^8.1", "phpbench/container": "^2.2", "psr/log": "^1.1 || ^2.0 || ^3.0", "seld/jsonlint": "^1.1", @@ -6438,9 +6438,8 @@ "phpstan/extension-installer": "^1.1", "phpstan/phpstan": "^1.0", "phpstan/phpstan-phpunit": "^1.0", - "phpunit/phpunit": "^11.5", + "phpunit/phpunit": "^10.4 || ^11.0", "rector/rector": "^1.2", - "sebastian/exporter": "^6.3.2", "symfony/error-handler": "^6.1 || ^7.0 || ^8.0", "symfony/var-dumper": "^6.1 || ^7.0 || ^8.0" }, @@ -6485,7 +6484,7 @@ ], "support": { "issues": "https://github.com/phpbench/phpbench/issues", - "source": "https://github.com/phpbench/phpbench/tree/1.5.1" + "source": "https://github.com/phpbench/phpbench/tree/1.4.3" }, "funding": [ { @@ -6493,15 +6492,15 @@ "type": "github" } ], - "time": "2026-03-05T08:18:58+00:00" + "time": "2025-11-06T19:07:31+00:00" }, { "name": "phpstan/phpstan", - "version": "1.12.33", + "version": "1.12.32", "dist": { "type": "zip", - "url": "https://api.github.com/repos/phpstan/phpstan/zipball/37982d6fc7cbb746dda7773530cda557cdf119e1", - "reference": "37982d6fc7cbb746dda7773530cda557cdf119e1", + "url": "https://api.github.com/repos/phpstan/phpstan/zipball/2770dcdf5078d0b0d53f94317e06affe88419aa8", + "reference": "2770dcdf5078d0b0d53f94317e06affe88419aa8", "shasum": "" }, "require": { @@ -6546,7 +6545,7 @@ "type": "github" } ], - "time": "2026-02-28T20:30:03+00:00" + "time": "2025-09-30T10:16:31+00:00" }, { "name": "phpunit/php-code-coverage", @@ -8096,16 +8095,16 @@ }, { "name": "symfony/console", - "version": "v8.0.7", + "version": "v8.0.4", "source": { "type": "git", "url": "https://github.com/symfony/console.git", - "reference": "15ed9008a4ebe2d6a78e4937f74e0c13ef2e618a" + "reference": "ace03c4cf9805080ff40cbeec69fca180c339a3b" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/console/zipball/15ed9008a4ebe2d6a78e4937f74e0c13ef2e618a", - "reference": "15ed9008a4ebe2d6a78e4937f74e0c13ef2e618a", + "url": "https://api.github.com/repos/symfony/console/zipball/ace03c4cf9805080ff40cbeec69fca180c339a3b", + "reference": "ace03c4cf9805080ff40cbeec69fca180c339a3b", "shasum": "" }, "require": { @@ -8162,7 +8161,7 @@ "terminal" ], "support": { - "source": "https://github.com/symfony/console/tree/v8.0.7" + "source": "https://github.com/symfony/console/tree/v8.0.4" }, "funding": [ { @@ -8182,20 +8181,20 @@ "type": "tidelift" } ], - "time": "2026-03-06T14:06:22+00:00" + "time": "2026-01-13T13:06:50+00:00" }, { "name": "symfony/filesystem", - "version": "v8.0.6", + "version": "v8.0.1", "source": { "type": "git", "url": "https://github.com/symfony/filesystem.git", - "reference": "7bf9162d7a0dff98d079b72948508fa48018a770" + "reference": "d937d400b980523dc9ee946bb69972b5e619058d" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/filesystem/zipball/7bf9162d7a0dff98d079b72948508fa48018a770", - "reference": "7bf9162d7a0dff98d079b72948508fa48018a770", + "url": "https://api.github.com/repos/symfony/filesystem/zipball/d937d400b980523dc9ee946bb69972b5e619058d", + "reference": "d937d400b980523dc9ee946bb69972b5e619058d", "shasum": "" }, "require": { @@ -8232,7 +8231,7 @@ "description": "Provides basic utilities for the filesystem", "homepage": "https://symfony.com", "support": { - "source": "https://github.com/symfony/filesystem/tree/v8.0.6" + "source": "https://github.com/symfony/filesystem/tree/v8.0.1" }, "funding": [ { @@ -8252,20 +8251,20 @@ "type": "tidelift" } ], - "time": "2026-02-25T16:59:43+00:00" + "time": "2025-12-01T09:13:36+00:00" }, { "name": "symfony/finder", - "version": "v8.0.6", + "version": "v8.0.5", "source": { "type": "git", "url": "https://github.com/symfony/finder.git", - "reference": "441404f09a54de6d1bd6ad219e088cdf4c91f97c" + "reference": "8bd576e97c67d45941365bf824e18dc8538e6eb0" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/finder/zipball/441404f09a54de6d1bd6ad219e088cdf4c91f97c", - "reference": "441404f09a54de6d1bd6ad219e088cdf4c91f97c", + "url": "https://api.github.com/repos/symfony/finder/zipball/8bd576e97c67d45941365bf824e18dc8538e6eb0", + "reference": "8bd576e97c67d45941365bf824e18dc8538e6eb0", "shasum": "" }, "require": { @@ -8300,7 +8299,7 @@ "description": "Finds files and directories via an intuitive fluent interface", "homepage": "https://symfony.com", "support": { - "source": "https://github.com/symfony/finder/tree/v8.0.6" + "source": "https://github.com/symfony/finder/tree/v8.0.5" }, "funding": [ { @@ -8320,7 +8319,7 @@ "type": "tidelift" } ], - "time": "2026-01-29T09:41:02+00:00" + "time": "2026-01-26T15:08:38+00:00" }, { "name": "symfony/options-resolver", @@ -8790,16 +8789,16 @@ }, { "name": "symfony/string", - "version": "v8.0.6", + "version": "v8.0.4", "source": { "type": "git", "url": "https://github.com/symfony/string.git", - "reference": "6c9e1108041b5dce21a9a4984b531c4923aa9ec4" + "reference": "758b372d6882506821ed666032e43020c4f57194" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/string/zipball/6c9e1108041b5dce21a9a4984b531c4923aa9ec4", - "reference": "6c9e1108041b5dce21a9a4984b531c4923aa9ec4", + "url": "https://api.github.com/repos/symfony/string/zipball/758b372d6882506821ed666032e43020c4f57194", + "reference": "758b372d6882506821ed666032e43020c4f57194", "shasum": "" }, "require": { @@ -8856,7 +8855,7 @@ "utf8" ], "support": { - "source": "https://github.com/symfony/string/tree/v8.0.6" + "source": "https://github.com/symfony/string/tree/v8.0.4" }, "funding": [ { @@ -8876,7 +8875,7 @@ "type": "tidelift" } ], - "time": "2026-02-09T10:14:57+00:00" + "time": "2026-01-12T12:37:40+00:00" }, { "name": "textalk/websocket", diff --git a/docker-compose.override.yml b/docker-compose.override.yml new file mode 100644 index 0000000000..29182f01d2 --- /dev/null +++ b/docker-compose.override.yml @@ -0,0 +1,144 @@ +# Dev tools for local development only. +# This file is automatically loaded by `docker compose` alongside docker-compose.yml. +# CI sets COMPOSE_FILE=docker-compose.yml explicitly, so these services are excluded from test runs. + +services: + appwrite-mongo-express: + profiles: ["mongodb"] + image: mongo-express + container_name: appwrite-mongo-express + networks: + - appwrite + ports: + - "8082:8081" + environment: + ME_CONFIG_MONGODB_URL: "mongodb://root:${_APP_DB_ROOT_PASS}@appwrite-mongodb:27017/?replicaSet=rs0&directConnection=true" + ME_CONFIG_BASICAUTH_USERNAME: ${_APP_DB_USER} + ME_CONFIG_BASICAUTH_PASSWORD: ${_APP_DB_PASS} + depends_on: + - mongodb + + adminer: + image: adminer + container_name: appwrite-adminer + restart: always + ports: + - 9506:8080 + networks: + - appwrite + - gateway + environment: + - ADMINER_DESIGN=pepa-linha + - ADMINER_DEFAULT_SERVER=mariadb + - ADMINER_DEFAULT_USERNAME=root + - ADMINER_DEFAULT_PASSWORD=rootsecretpassword + - ADMINER_DEFAULT_DB=appwrite + configs: + - source: adminer-index.php + target: /var/www/html/index.php + mode: 0755 + labels: + - "traefik.enable=true" + - "traefik.constraint-label-stack=appwrite" + - "traefik.docker.network=gateway" + - "traefik.http.services.appwrite_adminer.loadbalancer.server.port=8080" + - "traefik.http.routers.appwrite_adminer_http.entrypoints=appwrite_web" + - "traefik.http.routers.appwrite_adminer_http.rule=Host(`mysql.localhost`)" + - "traefik.http.routers.appwrite_adminer_http.service=appwrite_adminer" + - "traefik.http.routers.appwrite_adminer_https.entrypoints=appwrite_websecure" + - "traefik.http.routers.appwrite_adminer_https.rule=Host(`mysql.localhost`)" + - "traefik.http.routers.appwrite_adminer_https.service=appwrite_adminer" + - "traefik.http.routers.appwrite_adminer_https.tls=true" + + redis-insight: + image: redis/redisinsight:latest + restart: unless-stopped + networks: + - appwrite + - gateway + environment: + - RI_PRE_SETUP_DATABASES_PATH=/mnt/connections.json + configs: + - source: redisinsight-connections.json + target: /mnt/connections.json + mode: 0755 + labels: + - "traefik.enable=true" + - "traefik.constraint-label-stack=appwrite" + - "traefik.docker.network=gateway" + - "traefik.http.services.appwrite_redisinsight.loadbalancer.server.port=5540" + - "traefik.http.routers.appwrite_redisinsight_http.entrypoints=appwrite_web" + - "traefik.http.routers.appwrite_redisinsight_http.rule=Host(`redis.localhost`)" + - "traefik.http.routers.appwrite_redisinsight_http.service=appwrite_redisinsight" + - "traefik.http.routers.appwrite_redisinsight_https.entrypoints=appwrite_websecure" + - "traefik.http.routers.appwrite_redisinsight_https.rule=Host(`redis.localhost`)" + - "traefik.http.routers.appwrite_redisinsight_https.service=appwrite_redisinsight" + - "traefik.http.routers.appwrite_redisinsight_https.tls=true" + ports: + - "8081:5540" + + graphql-explorer: + container_name: appwrite-graphql-explorer + image: appwrite/altair:0.3.0 + restart: unless-stopped + networks: + - appwrite + ports: + - "9509:3000" + environment: + - SERVER_URL=http://localhost/v1/graphql + +configs: + redisinsight-connections.json: + content: | + [ + { + "compressor": "NONE", + "id": "104dc90a-21ef-4d5e-8912-b30baabb152f", + "host": "redis", + "port": 6379, + "name": "redis:6379", + "db": 0, + "username": "default", + "password": null, + "connectionType": "STANDALONE", + "nameFromProvider": null, + "provider": "REDIS", + "lastConnection": "2025-10-16T09:22:02.591Z", + "modules": [ + { + "name": "ReJSON", + "version": 20808, + "semanticVersion": "2.8.8" + }, + { + "name": "search", + "version": 21015, + "semanticVersion": "2.10.15" + } + ], + "tls": false, + "tlsServername": null, + "verifyServerCert": null, + "caCert": null, + "clientCert": null, + "ssh": false, + "sshOptions": null, + "forceStandalone": false, + "tags": [] + } + ] + + adminer-index.php: + content: | + $$_ENV['ADMINER_DEFAULT_SERVER'], + 'driver' => 'server', /* seems to autodetect the driver from server settings */ + 'username' => $$_ENV['ADMINER_DEFAULT_USERNAME'], + 'password' => $$_ENV['ADMINER_DEFAULT_PASSWORD'], + 'db' => $$_ENV['ADMINER_DEFAULT_DB'], + ]; + } + include './adminer.php'; diff --git a/docker-compose.yml b/docker-compose.yml index 4a38757737..c0b0560a7f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -10,6 +10,15 @@ x-logging: &x-logging max-file: "5" max-size: "10m" +x-build: &x-build + build: + context: . + target: development + args: + DEBUG: false + TESTING: true + VERSION: dev + services: traefik: image: traefik:3.6 @@ -50,15 +59,13 @@ services: appwrite: container_name: appwrite - <<: *x-logging + <<: [*x-logging, *x-build] image: appwrite-dev - build: - context: . - target: development - args: - DEBUG: false - TESTING: true - VERSION: dev + healthcheck: + test: ["CMD", "doctor"] + interval: 5s + timeout: 5s + retries: 12 ports: - 9501:80 networks: @@ -101,10 +108,10 @@ services: - ./dev:/usr/src/code/dev depends_on: - - ${_APP_DB_HOST:-mongodb} - - redis - - coredns - # - clamav + redis: + condition: service_healthy + coredns: + condition: service_started entrypoint: - php - -e @@ -260,7 +267,7 @@ services: appwrite-realtime: entrypoint: realtime - <<: *x-logging + <<: [*x-logging, *x-build] container_name: appwrite-realtime image: appwrite-dev restart: unless-stopped @@ -312,7 +319,7 @@ services: appwrite-worker-audits: entrypoint: worker-audits - <<: *x-logging + <<: [*x-logging, *x-build] container_name: appwrite-worker-audits image: appwrite-dev networks: @@ -343,7 +350,7 @@ services: appwrite-worker-webhooks: entrypoint: worker-webhooks - <<: *x-logging + <<: [*x-logging, *x-build] container_name: appwrite-worker-webhooks image: appwrite-dev networks: @@ -378,7 +385,7 @@ services: appwrite-worker-deletes: entrypoint: worker-deletes - <<: *x-logging + <<: [*x-logging, *x-build] container_name: appwrite-worker-deletes image: appwrite-dev networks: @@ -443,7 +450,7 @@ services: appwrite-worker-databases: entrypoint: worker-databases - <<: *x-logging + <<: [*x-logging, *x-build] container_name: appwrite-worker-databases image: appwrite-dev networks: @@ -476,7 +483,7 @@ services: appwrite-worker-builds: entrypoint: worker-builds - <<: *x-logging + <<: [*x-logging, *x-build] container_name: appwrite-worker-builds image: appwrite-dev networks: @@ -551,7 +558,7 @@ services: appwrite-worker-screenshots: entrypoint: worker-screenshots - <<: *x-logging + <<: [*x-logging, *x-build] container_name: appwrite-worker-screenshots image: appwrite-dev networks: @@ -614,7 +621,7 @@ services: appwrite-worker-certificates: entrypoint: worker-certificates - <<: *x-logging + <<: [*x-logging, *x-build] container_name: appwrite-worker-certificates image: appwrite-dev networks: @@ -656,7 +663,7 @@ services: appwrite-worker-executions: entrypoint: worker-executions - <<: *x-logging + <<: [*x-logging, *x-build] container_name: appwrite-worker-executions image: appwrite-dev networks: @@ -686,7 +693,7 @@ services: appwrite-worker-functions: entrypoint: worker-functions - <<: *x-logging + <<: [*x-logging, *x-build] container_name: appwrite-worker-functions image: appwrite-dev networks: @@ -730,7 +737,7 @@ services: appwrite-worker-mails: entrypoint: worker-mails - <<: *x-logging + <<: [*x-logging, *x-build] container_name: appwrite-worker-mails image: appwrite-dev networks: @@ -772,7 +779,7 @@ services: appwrite-worker-messaging: entrypoint: worker-messaging - <<: *x-logging + <<: [*x-logging, *x-build] container_name: appwrite-worker-messaging restart: unless-stopped image: appwrite-dev @@ -829,7 +836,7 @@ services: appwrite-worker-migrations: entrypoint: worker-migrations - <<: *x-logging + <<: [*x-logging, *x-build] container_name: appwrite-worker-migrations restart: unless-stopped image: appwrite-dev @@ -874,7 +881,7 @@ services: appwrite-task-maintenance: entrypoint: maintenance - <<: *x-logging + <<: [*x-logging, *x-build] container_name: appwrite-task-maintenance image: appwrite-dev networks: @@ -920,7 +927,7 @@ services: appwrite-task-interval: entrypoint: interval - <<: *x-logging + <<: [*x-logging, *x-build] container_name: appwrite-task-interval image: appwrite-dev networks: @@ -961,7 +968,7 @@ services: appwrite-task-stats-resources: container_name: appwrite-task-stats-resources entrypoint: stats-resources - <<: *x-logging + <<: [*x-logging, *x-build] image: appwrite-dev networks: - appwrite @@ -993,7 +1000,7 @@ services: appwrite-worker-stats-resources: entrypoint: worker-stats-resources - <<: *x-logging + <<: [*x-logging, *x-build] container_name: appwrite-worker-stats-resources image: appwrite-dev networks: @@ -1026,7 +1033,7 @@ services: appwrite-worker-stats-usage: entrypoint: worker-stats-usage - <<: *x-logging + <<: [*x-logging, *x-build] container_name: appwrite-worker-stats-usage image: appwrite-dev networks: @@ -1059,7 +1066,7 @@ services: appwrite-task-scheduler-functions: entrypoint: schedule-functions - <<: *x-logging + <<: [*x-logging, *x-build] container_name: appwrite-task-scheduler-functions image: appwrite-dev networks: @@ -1089,7 +1096,7 @@ services: appwrite-task-scheduler-executions: entrypoint: schedule-executions - <<: *x-logging + <<: [*x-logging, *x-build] container_name: appwrite-task-scheduler-executions image: appwrite-dev networks: @@ -1118,7 +1125,7 @@ services: appwrite-task-scheduler-messages: entrypoint: schedule-messages - <<: *x-logging + <<: [*x-logging, *x-build] container_name: appwrite-task-scheduler-messages image: appwrite-dev networks: @@ -1237,6 +1244,11 @@ services: - MYSQL_PASSWORD=${_APP_DB_PASS} - MARIADB_AUTO_UPGRADE=1 command: "mysqld --innodb-flush-method=fsync" + healthcheck: + test: ["CMD", "healthcheck.sh", "--connect", "--innodb_initialized"] + interval: 5s + timeout: 5s + retries: 12 mongodb: profiles: ["mongodb"] @@ -1275,20 +1287,7 @@ services: retries: 10 start_period: 30s - appwrite-mongo-express: - profiles: ["mongodb"] - image: mongo-express - container_name: appwrite-mongo-express - networks: - - appwrite - ports: - - "8082:8081" - environment: - ME_CONFIG_MONGODB_URL: "mongodb://root:${_APP_DB_ROOT_PASS}@appwrite-mongodb:27017/?replicaSet=rs0&directConnection=true" - ME_CONFIG_BASICAUTH_USERNAME: ${_APP_DB_USER} - ME_CONFIG_BASICAUTH_PASSWORD: ${_APP_DB_PASS} - depends_on: - - mongodb + postgresql: profiles: ["postgresql"] @@ -1309,6 +1308,11 @@ services: - POSTGRES_USER=${_APP_DB_USER} - POSTGRES_PASSWORD=${_APP_DB_PASS} command: "postgres" + healthcheck: + test: ["CMD-SHELL", "pg_isready -U ${_APP_DB_USER}"] + interval: 5s + timeout: 5s + retries: 12 redis: image: redis:7.4.7-alpine @@ -1325,6 +1329,11 @@ services: - appwrite volumes: - appwrite-redis:/data:rw + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 5s + timeout: 5s + retries: 12 coredns: # DNS server for testing purposes (Proxy APIs) image: coredns/coredns:1.12.4 @@ -1396,78 +1405,8 @@ services: networks: - appwrite - adminer: - image: adminer - container_name: appwrite-adminer - <<: *x-logging - restart: always - ports: - - 9506:8080 - networks: - - appwrite - - gateway - environment: - - ADMINER_DESIGN=pepa-linha - - ADMINER_DEFAULT_SERVER=mariadb - - ADMINER_DEFAULT_USERNAME=root - - ADMINER_DEFAULT_PASSWORD=rootsecretpassword - - ADMINER_DEFAULT_DB=appwrite - configs: - - source: adminer-index.php - target: /var/www/html/index.php - mode: 0755 - labels: - - "traefik.enable=true" - - "traefik.constraint-label-stack=appwrite" - - "traefik.docker.network=gateway" - - "traefik.http.services.appwrite_adminer.loadbalancer.server.port=8080" - - "traefik.http.routers.appwrite_adminer_http.entrypoints=appwrite_web" - - "traefik.http.routers.appwrite_adminer_http.rule=Host(`mysql.localhost`)" - - "traefik.http.routers.appwrite_adminer_http.service=appwrite_adminer" - - "traefik.http.routers.appwrite_adminer_https.entrypoints=appwrite_websecure" - - "traefik.http.routers.appwrite_adminer_https.rule=Host(`mysql.localhost`)" - - "traefik.http.routers.appwrite_adminer_https.service=appwrite_adminer" - - "traefik.http.routers.appwrite_adminer_https.tls=true" - - redis-insight: - image: redis/redisinsight:latest - restart: unless-stopped - networks: - - appwrite - - gateway - environment: - - RI_PRE_SETUP_DATABASES_PATH=/mnt/connections.json - configs: - - source: redisinsight-connections.json - target: /mnt/connections.json - mode: 0755 - labels: - - "traefik.enable=true" - - "traefik.constraint-label-stack=appwrite" - - "traefik.docker.network=gateway" - - "traefik.http.services.appwrite_redisinsight.loadbalancer.server.port=5540" - - "traefik.http.routers.appwrite_redisinsight_http.entrypoints=appwrite_web" - - "traefik.http.routers.appwrite_redisinsight_http.rule=Host(`redis.localhost`)" - - "traefik.http.routers.appwrite_redisinsight_http.service=appwrite_redisinsight" - - "traefik.http.routers.appwrite_redisinsight_https.entrypoints=appwrite_websecure" - - "traefik.http.routers.appwrite_redisinsight_https.rule=Host(`redis.localhost`)" - - "traefik.http.routers.appwrite_redisinsight_https.service=appwrite_redisinsight" - - "traefik.http.routers.appwrite_redisinsight_https.tls=true" - ports: - - "8081:5540" - - graphql-explorer: - container_name: appwrite-graphql-explorer - image: appwrite/altair:0.3.0 - restart: unless-stopped - networks: - - appwrite - ports: - - "9509:3000" - environment: - - SERVER_URL=http://localhost/v1/graphql - - # Dev Tools End ------------------------------------------------------------------------------------------ + # Dev tools (adminer, redis-insight, mongo-express, graphql-explorer) + # are defined in docker-compose.override.yml networks: gateway: @@ -1480,60 +1419,7 @@ networks: runtimes: name: runtimes -configs: - redisinsight-connections.json: - content: | - [ - { - "compressor": "NONE", - "id": "104dc90a-21ef-4d5e-8912-b30baabb152f", - "host": "redis", - "port": 6379, - "name": "redis:6379", - "db": 0, - "username": "default", - "password": null, - "connectionType": "STANDALONE", - "nameFromProvider": null, - "provider": "REDIS", - "lastConnection": "2025-10-16T09:22:02.591Z", - "modules": [ - { - "name": "ReJSON", - "version": 20808, - "semanticVersion": "2.8.8" - }, - { - "name": "search", - "version": 21015, - "semanticVersion": "2.10.15" - } - ], - "tls": false, - "tlsServername": null, - "verifyServerCert": null, - "caCert": null, - "clientCert": null, - "ssh": false, - "sshOptions": null, - "forceStandalone": false, - "tags": [] - } - ] - adminer-index.php: - content: | - $$_ENV['ADMINER_DEFAULT_SERVER'], - 'driver' => 'server', /* seems to autodetect the driver from server settings */ - 'username' => $$_ENV['ADMINER_DEFAULT_USERNAME'], - 'password' => $$_ENV['ADMINER_DEFAULT_PASSWORD'], - 'db' => $$_ENV['ADMINER_DEFAULT_DB'], - ]; - } - include './adminer.php'; volumes: appwrite-mariadb: diff --git a/src/Appwrite/Bus/Listeners/Usage.php b/src/Appwrite/Bus/Listeners/Usage.php index 219287033d..48178f1dee 100644 --- a/src/Appwrite/Bus/Listeners/Usage.php +++ b/src/Appwrite/Bus/Listeners/Usage.php @@ -4,11 +4,12 @@ namespace Appwrite\Bus\Listeners; use Appwrite\Bus\Events\ExecutionCompleted; use Appwrite\Bus\Events\RequestCompleted; -use Appwrite\Event\StatsUsage; +use Appwrite\Event\Message\Usage as UsageMessage; +use Appwrite\Event\Publisher\Usage as Publisher; +use Appwrite\Usage\Context; use Utopia\Bus\Event; use Utopia\Bus\Listener; use Utopia\Database\Document; -use Utopia\Queue\Publisher; class Usage extends Listener { @@ -29,20 +30,21 @@ class Usage extends Listener { $this ->desc('Records usage metrics') - ->inject('publisherStatsUsage') + ->inject('publisherForUsage') + ->inject('usage') ->callback($this->handle(...)); } - public function handle(Event $event, Publisher $publisher): void + public function handle(Event $event, Publisher $publisherForUsage, Context $usage): void { match (true) { - $event instanceof ExecutionCompleted => $this->handleExecutionCompleted($event, $publisher), - $event instanceof RequestCompleted => $this->handleRequestCompleted($event, $publisher), + $event instanceof ExecutionCompleted => $this->handleExecutionCompleted($event, $publisherForUsage), + $event instanceof RequestCompleted => $this->handleRequestCompleted($event, $usage), default => null, }; } - private function handleExecutionCompleted(ExecutionCompleted $event, Publisher $publisher): void + private function handleExecutionCompleted(ExecutionCompleted $event, Publisher $publisherForUsage): void { $execution = new Document($event->execution); $resource = new Document($event->resource); @@ -61,9 +63,7 @@ class Usage extends Listener $compute = (int)($duration * 1000); $mbSeconds = (int)(($spec['memory'] ?? APP_COMPUTE_MEMORY_DEFAULT) * $duration * ($spec['cpus'] ?? APP_COMPUTE_CPUS_DEFAULT)); - $queueForStatsUsage = new StatsUsage($publisher); - $queueForStatsUsage - ->setProject($project) + $context = (new Context()) ->addMetric(METRIC_EXECUTIONS, 1) ->addMetric(str_replace(['{resourceType}'], [$resourceType], METRIC_RESOURCE_TYPE_EXECUTIONS), 1) ->addMetric(str_replace(['{resourceType}', '{resourceInternalId}'], [$resourceType, $resourceInternalId], METRIC_RESOURCE_TYPE_ID_EXECUTIONS), 1) @@ -72,11 +72,18 @@ class Usage extends Listener ->addMetric(str_replace(['{resourceType}', '{resourceInternalId}'], [$resourceType, $resourceInternalId], METRIC_RESOURCE_TYPE_ID_EXECUTIONS_COMPUTE), $compute) ->addMetric(METRIC_EXECUTIONS_MB_SECONDS, $mbSeconds) ->addMetric(str_replace(['{resourceType}'], [$resourceType], METRIC_RESOURCE_TYPE_EXECUTIONS_MB_SECONDS), $mbSeconds) - ->addMetric(str_replace(['{resourceType}', '{resourceInternalId}'], [$resourceType, $resourceInternalId], METRIC_RESOURCE_TYPE_ID_EXECUTIONS_MB_SECONDS), $mbSeconds) - ->trigger(); + ->addMetric(str_replace(['{resourceType}', '{resourceInternalId}'], [$resourceType, $resourceInternalId], METRIC_RESOURCE_TYPE_ID_EXECUTIONS_MB_SECONDS), $mbSeconds); + + $message = new UsageMessage( + project: $project, + metrics: $context->getMetrics(), + reduce: $context->getReduce() + ); + + $publisherForUsage->enqueue($message); } - private function handleRequestCompleted(RequestCompleted $event, Publisher $publisher): void + private function handleRequestCompleted(RequestCompleted $event, Context $usage): void { $fileSize = 0; $file = $event->request->getFiles('file'); @@ -84,18 +91,14 @@ class Usage extends Listener $fileSize = (\is_array($file['size']) && isset($file['size'][0])) ? $file['size'][0] : $file['size']; } - $project = new Document($event->project); $deployment = new Document($event->deployment); - $queueForStatsUsage = new StatsUsage($publisher); $inbound = $event->request->getSize() + $fileSize; $outbound = $event->response->getSize(); - $queueForStatsUsage->setProject($project); - if ($deployment->getAttribute('resourceType') === 'sites') { $siteInternalId = $deployment->getAttribute('resourceInternalId', ''); - $queueForStatsUsage + $usage ->addMetric(METRIC_SITES_REQUESTS, 1) ->addMetric(METRIC_SITES_INBOUND, $inbound) ->addMetric(METRIC_SITES_OUTBOUND, $outbound) @@ -103,12 +106,10 @@ class Usage extends Listener ->addMetric(str_replace('{siteInternalId}', $siteInternalId, METRIC_SITES_ID_INBOUND), $inbound) ->addMetric(str_replace('{siteInternalId}', $siteInternalId, METRIC_SITES_ID_OUTBOUND), $outbound); } else { - $queueForStatsUsage + $usage ->addMetric(METRIC_NETWORK_REQUESTS, 1) ->addMetric(METRIC_NETWORK_INBOUND, $inbound) ->addMetric(METRIC_NETWORK_OUTBOUND, $outbound); } - - $queueForStatsUsage->trigger(); } } diff --git a/src/Appwrite/Event/Message/Base.php b/src/Appwrite/Event/Message/Base.php new file mode 100644 index 0000000000..38b6d5edee --- /dev/null +++ b/src/Appwrite/Event/Message/Base.php @@ -0,0 +1,21 @@ + $metrics + * @param array $reduce + */ + public function __construct( + public readonly Document $project, + public readonly array $metrics, + public readonly array $reduce = [], + ) { + } + + /** + * @return array + */ + public function toArray(): array + { + return [ + 'project' => [ + '$id' => $this->project->getId(), + '$sequence' => $this->project->getSequence(), + 'database' => $this->project->getAttribute('database', ''), + ], + 'metrics' => $this->metrics, + 'reduce' => array_map(fn (Document $doc) => $doc->getArrayCopy(), $this->reduce), + ]; + } + + /** + * @param array $data + * @return static + */ + public static function fromArray(array $data): static + { + return new self( + project: new Document($data['project'] ?? []), + metrics: $data['metrics'] ?? [], + reduce: array_map(fn (array $doc) => new Document($doc), $data['reduce'] ?? []), + ); + } +} diff --git a/src/Appwrite/Event/Publisher/Base.php b/src/Appwrite/Event/Publisher/Base.php new file mode 100644 index 0000000000..2063864723 --- /dev/null +++ b/src/Appwrite/Event/Publisher/Base.php @@ -0,0 +1,33 @@ +toArray(); + + return $this->publisher->enqueue($queue, $payload); + } + + /** + * Get the size of a queue + */ + public function getQueueSize(Queue $queue, bool $failed = false): int + { + return $this->publisher->getQueueSize($queue, $failed); + } +} diff --git a/src/Appwrite/Event/Publisher/Usage.php b/src/Appwrite/Event/Publisher/Usage.php new file mode 100644 index 0000000000..104690671b --- /dev/null +++ b/src/Appwrite/Event/Publisher/Usage.php @@ -0,0 +1,39 @@ +publish($this->queue, $message); + } catch (\Throwable $th) { + Console::error('[Usage] Failed to publish usage message: ' . $th->getMessage()); + return false; + } + } + + /** + * Get the size of the usage queue + */ + public function getSize(bool $failed = false): int + { + return $this->getQueueSize($this->queue, $failed); + } +} diff --git a/src/Appwrite/Event/StatsUsage.php b/src/Appwrite/Event/StatsUsage.php deleted file mode 100644 index a944d70c94..0000000000 --- a/src/Appwrite/Event/StatsUsage.php +++ /dev/null @@ -1,96 +0,0 @@ -setQueue(System::getEnv('_APP_STATS_USAGE_QUEUE_NAME', Event::STATS_USAGE_QUEUE_NAME)) - ->setClass(System::getEnv('_APP_STATS_USAGE_CLASS_NAME', Event::STATS_USAGE_CLASS_NAME)); - } - - /** - * Add reduce. - * - * @param Document $document - * @return self - */ - public function addReduce(Document $document): self - { - $this->reduce[] = $document; - - return $this; - } - - /** - * Add metric. - * - * @param string $key - * @param int $value - * @return self - */ - public function addMetric(string $key, int $value): self - { - $this->metrics[] = [ - 'key' => $key, - 'value' => $value, - ]; - - return $this; - } - - /** - * Set disabled metrics. - * - * @param string $key - * @return self - */ - public function disableMetric(string $key): self - { - $this->disabled[] = $key; - - return $this; - } - - /** - * Prepare the payload for the event - * - * @return array - */ - protected function preparePayload(): array - { - return [ - 'project' => $this->getProject(), - 'reduce' => $this->reduce, - 'metrics' => \array_filter($this->metrics, function ($metric) { - foreach ($this->disabled as $disabledMetric) { - if (\str_ends_with($metric['key'], $disabledMetric)) { - return false; - } - } - return true; - }), - ]; - } - - public function reset(): Event - { - $this->metrics = []; - parent::reset(); - return $this; - } -} diff --git a/src/Appwrite/Platform/Modules/Account/Http/Account/MFA/Challenges/Create.php b/src/Appwrite/Platform/Modules/Account/Http/Account/MFA/Challenges/Create.php index cde6b90fd6..20a6afed2e 100644 --- a/src/Appwrite/Platform/Modules/Account/Http/Account/MFA/Challenges/Create.php +++ b/src/Appwrite/Platform/Modules/Account/Http/Account/MFA/Challenges/Create.php @@ -7,7 +7,6 @@ use Appwrite\Detector\Detector; use Appwrite\Event\Event; use Appwrite\Event\Mail; use Appwrite\Event\Messaging; -use Appwrite\Event\StatsUsage; use Appwrite\Extend\Exception; use Appwrite\SDK\AuthType; use Appwrite\SDK\ContentType; @@ -15,6 +14,7 @@ use Appwrite\SDK\Deprecated; use Appwrite\SDK\Method; use Appwrite\SDK\Response as SDKResponse; use Appwrite\Template\Template; +use Appwrite\Usage\Context; use Appwrite\Utopia\Request; use Appwrite\Utopia\Response; use libphonenumber\NumberParseException; @@ -104,7 +104,7 @@ class Create extends Action ->inject('queueForMessaging') ->inject('queueForMails') ->inject('timelimit') - ->inject('queueForStatsUsage') + ->inject('usage') ->inject('plan') ->inject('proofForToken') ->inject('proofForCode') @@ -124,7 +124,7 @@ class Create extends Action Messaging $queueForMessaging, Mail $queueForMails, callable $timelimit, - StatsUsage $queueForStatsUsage, + Context $usage, array $plan, ProofsToken $proofForToken, ProofsCode $proofForCode @@ -201,16 +201,12 @@ class Create extends Action $countryCode = $helper->parse($phone)->getCountryCode(); if (!empty($countryCode)) { - $queueForStatsUsage - ->addMetric(str_replace('{countryCode}', $countryCode, METRIC_AUTH_METHOD_PHONE_COUNTRY_CODE), 1); + $usage->addMetric(str_replace('{countryCode}', $countryCode, METRIC_AUTH_METHOD_PHONE_COUNTRY_CODE), 1); } } catch (NumberParseException $e) { // Ignore invalid phone number for country code stats } - $queueForStatsUsage - ->addMetric(METRIC_AUTH_METHOD_PHONE, 1) - ->setProject($project) - ->trigger(); + $usage->addMetric(METRIC_AUTH_METHOD_PHONE, 1); break; case Type::EMAIL: if (empty(System::getEnv('_APP_SMTP_HOST'))) { diff --git a/src/Appwrite/Platform/Modules/Avatars/Http/Screenshots/Get.php b/src/Appwrite/Platform/Modules/Avatars/Http/Screenshots/Get.php index b6fd354ee3..2df12b17d1 100644 --- a/src/Appwrite/Platform/Modules/Avatars/Http/Screenshots/Get.php +++ b/src/Appwrite/Platform/Modules/Avatars/Http/Screenshots/Get.php @@ -2,7 +2,6 @@ namespace Appwrite\Platform\Modules\Avatars\Http\Screenshots; -use Appwrite\Event\StatsUsage; use Appwrite\Extend\Exception; use Appwrite\Platform\Modules\Avatars\Http\Action; use Appwrite\SDK\AuthType; @@ -10,6 +9,7 @@ use Appwrite\SDK\ContentType; use Appwrite\SDK\Method; use Appwrite\SDK\MethodType; use Appwrite\SDK\Response as SDKResponse; +use Appwrite\Usage\Context; use Appwrite\Utopia\Response; use Utopia\Config\Config; use Utopia\Domains\Domain; @@ -84,11 +84,11 @@ class Get extends Action ->param('quality', -1, new Range(-1, 100), 'Screenshot quality. Pass an integer between 0 to 100. Defaults to keep existing image quality.', true, example: '85') ->param('output', '', new WhiteList(\array_keys(Config::getParam('storage-outputs')), true), 'Output format type (jpeg, jpg, png, gif and webp).', true, example: 'jpeg') ->inject('response') - ->inject('queueForStatsUsage') + ->inject('usage') ->callback($this->action(...)); } - public function action(string $url, array $headers, int $viewportWidth, int $viewportHeight, float $scale, string $theme, string $userAgent, bool $fullpage, string $locale, string $timezone, float $latitude, float $longitude, float $accuracy, bool $touch, array $permissions, int $sleep, int $width, int $height, int $quality, string $output, Response $response, StatsUsage $queueForStatsUsage) + public function action(string $url, array $headers, int $viewportWidth, int $viewportHeight, float $scale, string $theme, string $userAgent, bool $fullpage, string $locale, string $timezone, float $latitude, float $longitude, float $accuracy, bool $touch, array $permissions, int $sleep, int $width, int $height, int $quality, string $output, Response $response, Context $usage) { if (!\extension_loaded('imagick')) { throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Imagick extension is missing'); @@ -210,7 +210,7 @@ class Get extends Action $outputs = Config::getParam('storage-outputs'); $contentType = $outputs[$output] ?? $outputs['png']; - $queueForStatsUsage->addMetric(METRIC_AVATARS_SCREENSHOTS_GENERATED, 1); + $usage->addMetric(METRIC_AVATARS_SCREENSHOTS_GENERATED, 1); $response ->addHeader('Cache-Control', 'private, max-age=2592000') // 30 days diff --git a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Attribute/Decrement.php b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Attribute/Decrement.php index 85813b2354..54557eaac0 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Attribute/Decrement.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Attribute/Decrement.php @@ -3,7 +3,6 @@ namespace Appwrite\Platform\Modules\Databases\Http\Databases\Collections\Documents\Attribute; use Appwrite\Event\Event; -use Appwrite\Event\StatsUsage; use Appwrite\Extend\Exception; use Appwrite\Platform\Modules\Databases\Http\Databases\Collections\Documents\Action; use Appwrite\SDK\AuthType; @@ -11,6 +10,7 @@ use Appwrite\SDK\ContentType; use Appwrite\SDK\Deprecated; use Appwrite\SDK\Method; use Appwrite\SDK\Response as SDKResponse; +use Appwrite\Usage\Context; use Appwrite\Utopia\Database\Documents\User; use Appwrite\Utopia\Response as UtopiaResponse; use InvalidArgumentException; @@ -83,13 +83,13 @@ class Decrement extends Action ->inject('response') ->inject('dbForProject') ->inject('queueForEvents') - ->inject('queueForStatsUsage') + ->inject('usage') ->inject('plan') ->inject('authorization') ->callback($this->action(...)); } - public function action(string $databaseId, string $collectionId, string $documentId, string $attribute, int|float $value, int|float|null $min, ?string $transactionId, UtopiaResponse $response, Database $dbForProject, Event $queueForEvents, StatsUsage $queueForStatsUsage, array $plan, Authorization $authorization): void + public function action(string $databaseId, string $collectionId, string $documentId, string $attribute, int|float $value, int|float|null $min, ?string $transactionId, UtopiaResponse $response, Database $dbForProject, Event $queueForEvents, Context $usage, array $plan, Authorization $authorization): void { $isAPIKey = User::isApp($authorization->getRoles()); $isPrivilegedUser = User::isPrivileged($authorization->getRoles()); @@ -200,7 +200,7 @@ class Decrement extends Action ) ); - $queueForStatsUsage + $usage ->addMetric(METRIC_DATABASES_OPERATIONS_WRITES, 1) ->addMetric(str_replace('{databaseInternalId}', $database->getSequence(), METRIC_DATABASE_ID_OPERATIONS_WRITES), 1); diff --git a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Attribute/Increment.php b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Attribute/Increment.php index 031b5abcc6..b9c19b2d06 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Attribute/Increment.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Attribute/Increment.php @@ -3,7 +3,6 @@ namespace Appwrite\Platform\Modules\Databases\Http\Databases\Collections\Documents\Attribute; use Appwrite\Event\Event; -use Appwrite\Event\StatsUsage; use Appwrite\Extend\Exception; use Appwrite\Platform\Modules\Databases\Http\Databases\Collections\Documents\Action; use Appwrite\SDK\AuthType; @@ -11,6 +10,7 @@ use Appwrite\SDK\ContentType; use Appwrite\SDK\Deprecated; use Appwrite\SDK\Method; use Appwrite\SDK\Response as SDKResponse; +use Appwrite\Usage\Context; use Appwrite\Utopia\Database\Documents\User; use Appwrite\Utopia\Response as UtopiaResponse; use InvalidArgumentException; @@ -83,13 +83,13 @@ class Increment extends Action ->inject('response') ->inject('dbForProject') ->inject('queueForEvents') - ->inject('queueForStatsUsage') + ->inject('usage') ->inject('plan') ->inject('authorization') ->callback($this->action(...)); } - public function action(string $databaseId, string $collectionId, string $documentId, string $attribute, int|float $value, int|float|null $max, ?string $transactionId, UtopiaResponse $response, Database $dbForProject, Event $queueForEvents, StatsUsage $queueForStatsUsage, array $plan, Authorization $authorization): void + public function action(string $databaseId, string $collectionId, string $documentId, string $attribute, int|float $value, int|float|null $max, ?string $transactionId, UtopiaResponse $response, Database $dbForProject, Event $queueForEvents, Context $usage, array $plan, Authorization $authorization): void { $isAPIKey = User::isApp($authorization->getRoles()); $isPrivilegedUser = User::isPrivileged($authorization->getRoles()); @@ -200,7 +200,7 @@ class Increment extends Action ) ); - $queueForStatsUsage + $usage ->addMetric(METRIC_DATABASES_OPERATIONS_WRITES, 1) ->addMetric(str_replace('{databaseInternalId}', $database->getSequence(), METRIC_DATABASE_ID_OPERATIONS_WRITES), 1); diff --git a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Bulk/Delete.php b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Bulk/Delete.php index 6ab67318c7..f45b126f16 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Bulk/Delete.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Bulk/Delete.php @@ -3,7 +3,6 @@ namespace Appwrite\Platform\Modules\Databases\Http\Databases\Collections\Documents\Bulk; use Appwrite\Event\Event; -use Appwrite\Event\StatsUsage; use Appwrite\Extend\Exception; use Appwrite\Functions\EventProcessor; use Appwrite\Platform\Modules\Databases\Http\Databases\Collections\Documents\Action; @@ -12,6 +11,7 @@ use Appwrite\SDK\ContentType; use Appwrite\SDK\Deprecated; use Appwrite\SDK\Method; use Appwrite\SDK\Response as SDKResponse; +use Appwrite\Usage\Context; use Appwrite\Utopia\Response as UtopiaResponse; use Utopia\Database\Database; use Utopia\Database\Document; @@ -76,7 +76,7 @@ class Delete extends Action ->param('transactionId', null, fn (Database $dbForProject) => new Nullable(new UID($dbForProject->getAdapter()->getMaxUIDLength())), 'Transaction ID for staging the operation.', true, ['dbForProject']) ->inject('response') ->inject('dbForProject') - ->inject('queueForStatsUsage') + ->inject('usage') ->inject('queueForEvents') ->inject('queueForRealtime') ->inject('queueForFunctions') @@ -86,7 +86,7 @@ class Delete extends Action ->callback($this->action(...)); } - public function action(string $databaseId, string $collectionId, array $queries, ?string $transactionId, UtopiaResponse $response, Database $dbForProject, StatsUsage $queueForStatsUsage, Event $queueForEvents, Event $queueForRealtime, Event $queueForFunctions, Event $queueForWebhooks, array $plan, EventProcessor $eventProcessor): void + public function action(string $databaseId, string $collectionId, array $queries, ?string $transactionId, UtopiaResponse $response, Database $dbForProject, Context $usage, Event $queueForEvents, Event $queueForRealtime, Event $queueForFunctions, Event $queueForWebhooks, array $plan, EventProcessor $eventProcessor): void { $database = $dbForProject->getDocument('databases', $databaseId); if ($database->isEmpty()) { @@ -185,10 +185,10 @@ class Delete extends Action foreach ($documents as $document) { $document->setAttribute('$databaseId', $database->getId()); - $document->setAttribute('$'.$this->getCollectionsEventsContext().'Id', $collection->getId()); + $document->setAttribute('$' . $this->getCollectionsEventsContext() . 'Id', $collection->getId()); } - $queueForStatsUsage + $usage ->addMetric(METRIC_DATABASES_OPERATIONS_WRITES, \max(1, $modified)) ->addMetric(str_replace('{databaseInternalId}', $database->getSequence(), METRIC_DATABASE_ID_OPERATIONS_WRITES), \max(1, $modified)); diff --git a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Bulk/Update.php b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Bulk/Update.php index d306414a89..000b59ff07 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Bulk/Update.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Bulk/Update.php @@ -3,7 +3,6 @@ namespace Appwrite\Platform\Modules\Databases\Http\Databases\Collections\Documents\Bulk; use Appwrite\Event\Event; -use Appwrite\Event\StatsUsage; use Appwrite\Extend\Exception; use Appwrite\Functions\EventProcessor; use Appwrite\Platform\Modules\Databases\Http\Databases\Collections\Documents\Action; @@ -12,6 +11,7 @@ use Appwrite\SDK\ContentType; use Appwrite\SDK\Deprecated; use Appwrite\SDK\Method; use Appwrite\SDK\Response as SDKResponse; +use Appwrite\Usage\Context; use Appwrite\Utopia\Response as UtopiaResponse; use Utopia\Database\Database; use Utopia\Database\Document; @@ -80,7 +80,7 @@ class Update extends Action ->param('transactionId', null, fn (Database $dbForProject) => new Nullable(new UID($dbForProject->getAdapter()->getMaxUIDLength())), 'Transaction ID for staging the operation.', true, ['dbForProject']) ->inject('response') ->inject('dbForProject') - ->inject('queueForStatsUsage') + ->inject('usage') ->inject('queueForEvents') ->inject('queueForRealtime') ->inject('queueForFunctions') @@ -90,7 +90,7 @@ class Update extends Action ->callback($this->action(...)); } - public function action(string $databaseId, string $collectionId, string|array $data, array $queries, ?string $transactionId, UtopiaResponse $response, Database $dbForProject, StatsUsage $queueForStatsUsage, Event $queueForEvents, Event $queueForRealtime, Event $queueForFunctions, Event $queueForWebhooks, array $plan, EventProcessor $eventProcessor): void + public function action(string $databaseId, string $collectionId, string|array $data, array $queries, ?string $transactionId, UtopiaResponse $response, Database $dbForProject, Context $usage, Event $queueForEvents, Event $queueForRealtime, Event $queueForFunctions, Event $queueForWebhooks, array $plan, EventProcessor $eventProcessor): void { $data = \is_string($data) ? \json_decode($data, true) @@ -216,10 +216,10 @@ class Update extends Action foreach ($documents as $document) { $document->setAttribute('$databaseId', $database->getId()); - $document->setAttribute('$'.$this->getCollectionsEventsContext().'Id', $collection->getId()); + $document->setAttribute('$' . $this->getCollectionsEventsContext() . 'Id', $collection->getId()); } - $queueForStatsUsage + $usage ->addMetric(METRIC_DATABASES_OPERATIONS_WRITES, \max(1, $modified)) ->addMetric(str_replace('{databaseInternalId}', $database->getSequence(), METRIC_DATABASE_ID_OPERATIONS_WRITES), \max(1, $modified)); diff --git a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Bulk/Upsert.php b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Bulk/Upsert.php index cf30fee173..564b5ee7b6 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Bulk/Upsert.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Bulk/Upsert.php @@ -3,7 +3,6 @@ namespace Appwrite\Platform\Modules\Databases\Http\Databases\Collections\Documents\Bulk; use Appwrite\Event\Event; -use Appwrite\Event\StatsUsage; use Appwrite\Extend\Exception; use Appwrite\Functions\EventProcessor; use Appwrite\Platform\Modules\Databases\Http\Databases\Collections\Documents\Action; @@ -12,6 +11,7 @@ use Appwrite\SDK\ContentType; use Appwrite\SDK\Deprecated; use Appwrite\SDK\Method; use Appwrite\SDK\Response as SDKResponse; +use Appwrite\Usage\Context; use Appwrite\Utopia\Response as UtopiaResponse; use Utopia\Database\Database; use Utopia\Database\Document; @@ -78,7 +78,7 @@ class Upsert extends Action ->param('transactionId', null, fn (Database $dbForProject) => new Nullable(new UID($dbForProject->getAdapter()->getMaxUIDLength())), 'Transaction ID for staging the operation.', true, ['dbForProject']) ->inject('response') ->inject('dbForProject') - ->inject('queueForStatsUsage') + ->inject('usage') ->inject('queueForEvents') ->inject('queueForRealtime') ->inject('queueForFunctions') @@ -88,7 +88,7 @@ class Upsert extends Action ->callback($this->action(...)); } - public function action(string $databaseId, string $collectionId, array $documents, ?string $transactionId, UtopiaResponse $response, Database $dbForProject, StatsUsage $queueForStatsUsage, Event $queueForEvents, Event $queueForRealtime, Event $queueForFunctions, Event $queueForWebhooks, array $plan, EventProcessor $eventProcessor): void + public function action(string $databaseId, string $collectionId, array $documents, ?string $transactionId, UtopiaResponse $response, Database $dbForProject, Context $usage, Event $queueForEvents, Event $queueForRealtime, Event $queueForFunctions, Event $queueForWebhooks, array $plan, EventProcessor $eventProcessor): void { $database = $dbForProject->getDocument('databases', $databaseId); if ($database->isEmpty()) { @@ -106,7 +106,7 @@ class Upsert extends Action ); if ($hasRelationships) { - throw new Exception(Exception::GENERAL_BAD_REQUEST, 'Bulk upsert is not supported for ' . $this->getSDKNamespace() . ' with relationship attributes'); + throw new Exception(Exception::GENERAL_BAD_REQUEST, 'Bulk upsert is not supported for ' . $this->getSDKNamespace() . ' with relationship attributes'); } foreach ($documents as $key => $document) { @@ -191,10 +191,10 @@ class Upsert extends Action foreach ($upserted as $document) { $document->setAttribute('$databaseId', $database->getId()); - $document->setAttribute('$'.$this->getCollectionsEventsContext().'Id', $collection->getId()); + $document->setAttribute('$' . $this->getCollectionsEventsContext() . 'Id', $collection->getId()); } - $queueForStatsUsage + $usage ->addMetric(METRIC_DATABASES_OPERATIONS_WRITES, \max(1, $modified)) ->addMetric(str_replace('{databaseInternalId}', $database->getSequence(), METRIC_DATABASE_ID_OPERATIONS_WRITES), \max(1, $modified)); diff --git a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Create.php b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Create.php index 9b14122abf..0bbe7c75cf 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Create.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Create.php @@ -3,7 +3,6 @@ namespace Appwrite\Platform\Modules\Databases\Http\Databases\Collections\Documents; use Appwrite\Event\Event; -use Appwrite\Event\StatsUsage; use Appwrite\Extend\Exception; use Appwrite\Functions\EventProcessor; use Appwrite\SDK\AuthType; @@ -12,6 +11,7 @@ use Appwrite\SDK\Deprecated; use Appwrite\SDK\Method; use Appwrite\SDK\Parameter; use Appwrite\SDK\Response as SDKResponse; +use Appwrite\Usage\Context; use Appwrite\Utopia\Database\Documents\User; use Appwrite\Utopia\Database\Validator\CustomId; use Appwrite\Utopia\Response as UtopiaResponse; @@ -129,7 +129,7 @@ class Create extends Action ->inject('dbForProject') ->inject('user') ->inject('queueForEvents') - ->inject('queueForStatsUsage') + ->inject('usage') ->inject('queueForRealtime') ->inject('queueForFunctions') ->inject('queueForWebhooks') @@ -138,7 +138,7 @@ class Create extends Action ->inject('eventProcessor') ->callback($this->action(...)); } - public function action(string $databaseId, string $documentId, string $collectionId, string|array $data, ?array $permissions, ?array $documents, ?string $transactionId, UtopiaResponse $response, Database $dbForProject, Document $user, Event $queueForEvents, StatsUsage $queueForStatsUsage, Event $queueForRealtime, Event $queueForFunctions, Event $queueForWebhooks, array $plan, Authorization $authorization, EventProcessor $eventProcessor): void + public function action(string $databaseId, string $documentId, string $collectionId, string|array $data, ?array $permissions, ?array $documents, ?string $transactionId, UtopiaResponse $response, Database $dbForProject, Document $user, Event $queueForEvents, Context $usage, Event $queueForRealtime, Event $queueForFunctions, Event $queueForWebhooks, array $plan, Authorization $authorization, EventProcessor $eventProcessor): void { $data = \is_string($data) ? \json_decode($data, true) @@ -205,7 +205,7 @@ class Create extends Action ); if ($isBulk && $hasRelationships) { - throw new Exception(Exception::GENERAL_BAD_REQUEST, 'Bulk create is not supported for ' . $this->getSDKNamespace() .' with relationship ' . $this->getStructureContext()); + throw new Exception(Exception::GENERAL_BAD_REQUEST, 'Bulk create is not supported for ' . $this->getSDKNamespace() . ' with relationship ' . $this->getStructureContext()); } $setPermissions = function (Document $document, ?array $permissions) use ($user, $isAPIKey, $isPrivilegedUser, $isBulk, $dbForProject, $authorization) { @@ -489,7 +489,7 @@ class Create extends Action ); } - $queueForStatsUsage + $usage ->addMetric(METRIC_DATABASES_OPERATIONS_WRITES, \max(1, $operations)) ->addMetric(str_replace('{databaseInternalId}', $database->getSequence(), METRIC_DATABASE_ID_OPERATIONS_WRITES), \max(1, $operations)); // per collection diff --git a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Delete.php b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Delete.php index 3171fe7aaf..0996fa24ab 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Delete.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Delete.php @@ -4,13 +4,13 @@ namespace Appwrite\Platform\Modules\Databases\Http\Databases\Collections\Documen use Appwrite\Databases\TransactionState; use Appwrite\Event\Event; -use Appwrite\Event\StatsUsage; use Appwrite\Extend\Exception; use Appwrite\SDK\AuthType; use Appwrite\SDK\ContentType; use Appwrite\SDK\Deprecated; use Appwrite\SDK\Method; use Appwrite\SDK\Response as SDKResponse; +use Appwrite\Usage\Context; use Appwrite\Utopia\Database\Documents\User; use Appwrite\Utopia\Response as UtopiaResponse; use Utopia\Database\Database; @@ -80,7 +80,7 @@ class Delete extends Action ->inject('response') ->inject('dbForProject') ->inject('queueForEvents') - ->inject('queueForStatsUsage') + ->inject('usage') ->inject('transactionState') ->inject('plan') ->inject('authorization') @@ -96,7 +96,7 @@ class Delete extends Action UtopiaResponse $response, Database $dbForProject, Event $queueForEvents, - StatsUsage $queueForStatsUsage, + Context $usage, TransactionState $transactionState, array $plan, Authorization $authorization @@ -210,7 +210,7 @@ class Delete extends Action authorization: $authorization ); - $queueForStatsUsage + $usage ->addMetric(METRIC_DATABASES_OPERATIONS_WRITES, 1) ->addMetric(str_replace('{databaseInternalId}', $database->getSequence(), METRIC_DATABASE_ID_OPERATIONS_WRITES), 1); // per collection diff --git a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Get.php b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Get.php index 515b7029e6..8784d30667 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Get.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Get.php @@ -3,13 +3,13 @@ namespace Appwrite\Platform\Modules\Databases\Http\Databases\Collections\Documents; use Appwrite\Databases\TransactionState; -use Appwrite\Event\StatsUsage; use Appwrite\Extend\Exception; use Appwrite\SDK\AuthType; use Appwrite\SDK\ContentType; use Appwrite\SDK\Deprecated; use Appwrite\SDK\Method; use Appwrite\SDK\Response as SDKResponse; +use Appwrite\Usage\Context; use Appwrite\Utopia\Database\Documents\User; use Appwrite\Utopia\Response as UtopiaResponse; use Utopia\Database\Database; @@ -68,13 +68,13 @@ class Get extends Action ->param('transactionId', null, fn (Database $dbForProject) => new Nullable(new UID($dbForProject->getAdapter()->getMaxUIDLength())), 'Transaction ID to read uncommitted changes within the transaction.', true, ['dbForProject']) ->inject('response') ->inject('dbForProject') - ->inject('queueForStatsUsage') + ->inject('usage') ->inject('transactionState') ->inject('authorization') ->callback($this->action(...)); } - public function action(string $databaseId, string $collectionId, string $documentId, array $queries, ?string $transactionId, UtopiaResponse $response, Database $dbForProject, StatsUsage $queueForStatsUsage, TransactionState $transactionState, Authorization $authorization): void + public function action(string $databaseId, string $collectionId, string $documentId, array $queries, ?string $transactionId, UtopiaResponse $response, Database $dbForProject, Context $usage, TransactionState $transactionState, Authorization $authorization): void { $isAPIKey = User::isApp($authorization->getRoles()); $isPrivilegedUser = User::isPrivileged($authorization->getRoles()); @@ -130,7 +130,7 @@ class Get extends Action operations: $operations ); - $queueForStatsUsage + $usage ->addMetric(METRIC_DATABASES_OPERATIONS_READS, max($operations, 1)) ->addMetric(str_replace('{databaseInternalId}', $database->getSequence(), METRIC_DATABASE_ID_OPERATIONS_READS), $operations); diff --git a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Update.php b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Update.php index 06eca79dad..ca7935dfbd 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Update.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Update.php @@ -4,13 +4,13 @@ namespace Appwrite\Platform\Modules\Databases\Http\Databases\Collections\Documen use Appwrite\Databases\TransactionState; use Appwrite\Event\Event; -use Appwrite\Event\StatsUsage; use Appwrite\Extend\Exception; use Appwrite\SDK\AuthType; use Appwrite\SDK\ContentType; use Appwrite\SDK\Deprecated; use Appwrite\SDK\Method; use Appwrite\SDK\Response as SDKResponse; +use Appwrite\Usage\Context; use Appwrite\Utopia\Database\Documents\User; use Appwrite\Utopia\Response as UtopiaResponse; use Utopia\Database\Database; @@ -84,14 +84,14 @@ class Update extends Action ->inject('response') ->inject('dbForProject') ->inject('queueForEvents') - ->inject('queueForStatsUsage') + ->inject('usage') ->inject('transactionState') ->inject('plan') ->inject('authorization') ->callback($this->action(...)); } - public function action(string $databaseId, string $collectionId, string $documentId, string|array $data, ?array $permissions, ?string $transactionId, ?\DateTime $requestTimestamp, UtopiaResponse $response, Database $dbForProject, Event $queueForEvents, StatsUsage $queueForStatsUsage, TransactionState $transactionState, array $plan, Authorization $authorization): void + public function action(string $databaseId, string $collectionId, string $documentId, string|array $data, ?array $permissions, ?string $transactionId, ?\DateTime $requestTimestamp, UtopiaResponse $response, Database $dbForProject, Event $queueForEvents, Context $usage, TransactionState $transactionState, array $plan, Authorization $authorization): void { $data = (\is_string($data)) ? \json_decode($data, true) : $data; // Cast to JSON array @@ -246,7 +246,7 @@ class Update extends Action $setCollection($collection, $newDocument); - $queueForStatsUsage + $usage ->addMetric(METRIC_DATABASES_OPERATIONS_WRITES, max($operations, 1)) ->addMetric(str_replace('{databaseInternalId}', $database->getSequence(), METRIC_DATABASE_ID_OPERATIONS_WRITES), $operations); diff --git a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Upsert.php b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Upsert.php index ef76ebe7cd..dc6655dfd3 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Upsert.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Upsert.php @@ -4,13 +4,13 @@ namespace Appwrite\Platform\Modules\Databases\Http\Databases\Collections\Documen use Appwrite\Databases\TransactionState; use Appwrite\Event\Event; -use Appwrite\Event\StatsUsage; use Appwrite\Extend\Exception; use Appwrite\SDK\AuthType; use Appwrite\SDK\ContentType; use Appwrite\SDK\Deprecated; use Appwrite\SDK\Method; use Appwrite\SDK\Response as SDKResponse; +use Appwrite\Usage\Context; use Appwrite\Utopia\Database\Documents\User; use Appwrite\Utopia\Database\Validator\CustomId; use Appwrite\Utopia\Response as UtopiaResponse; @@ -88,14 +88,14 @@ class Upsert extends Action ->inject('user') ->inject('dbForProject') ->inject('queueForEvents') - ->inject('queueForStatsUsage') + ->inject('usage') ->inject('transactionState') ->inject('plan') ->inject('authorization') ->callback($this->action(...)); } - public function action(string $databaseId, string $collectionId, string $documentId, string|array $data, ?array $permissions, ?string $transactionId, ?\DateTime $requestTimestamp, UtopiaResponse $response, Document $user, Database $dbForProject, Event $queueForEvents, StatsUsage $queueForStatsUsage, TransactionState $transactionState, array $plan, Authorization $authorization): void + public function action(string $databaseId, string $collectionId, string $documentId, string|array $data, ?array $permissions, ?string $transactionId, ?\DateTime $requestTimestamp, UtopiaResponse $response, Document $user, Database $dbForProject, Event $queueForEvents, Context $usage, TransactionState $transactionState, array $plan, Authorization $authorization): void { $data = (\is_string($data)) ? \json_decode($data, true) : $data; // Cast to JSON array @@ -256,7 +256,7 @@ class Upsert extends Action $setCollection($collection, $newDocument); - $queueForStatsUsage + $usage ->addMetric(METRIC_DATABASES_OPERATIONS_WRITES, \max(1, $operations)) ->addMetric(str_replace('{databaseInternalId}', $database->getSequence(), METRIC_DATABASE_ID_OPERATIONS_WRITES), \max(1, $operations)); diff --git a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/XList.php b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/XList.php index 0c93eaf105..3384153971 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/XList.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/XList.php @@ -3,13 +3,13 @@ namespace Appwrite\Platform\Modules\Databases\Http\Databases\Collections\Documents; use Appwrite\Databases\TransactionState; -use Appwrite\Event\StatsUsage; use Appwrite\Extend\Exception; use Appwrite\SDK\AuthType; use Appwrite\SDK\ContentType; use Appwrite\SDK\Deprecated; use Appwrite\SDK\Method; use Appwrite\SDK\Response as SDKResponse; +use Appwrite\Usage\Context; use Appwrite\Utopia\Database\Documents\User; use Appwrite\Utopia\Response as UtopiaResponse; use Utopia\Database\Database; @@ -75,13 +75,13 @@ class XList extends Action ->inject('response') ->inject('dbForProject') ->inject('user') - ->inject('queueForStatsUsage') + ->inject('usage') ->inject('transactionState') ->inject('authorization') ->callback($this->action(...)); } - public function action(string $databaseId, string $collectionId, array $queries, ?string $transactionId, bool $includeTotal, int $ttl, UtopiaResponse $response, Database $dbForProject, Document $user, StatsUsage $queueForStatsUsage, TransactionState $transactionState, Authorization $authorization): void + public function action(string $databaseId, string $collectionId, array $queries, ?string $transactionId, bool $includeTotal, int $ttl, UtopiaResponse $response, Database $dbForProject, Document $user, Context $usage, TransactionState $transactionState, Authorization $authorization): void { $isAPIKey = User::isApp($authorization->getRoles()); $isPrivilegedUser = User::isPrivileged($authorization->getRoles()); @@ -228,7 +228,7 @@ class XList extends Action ); } - $queueForStatsUsage + $usage ->addMetric(METRIC_DATABASES_OPERATIONS_READS, max($operations, 1)) ->addMetric(str_replace('{databaseInternalId}', $database->getSequence(), METRIC_DATABASE_ID_OPERATIONS_READS), $operations); diff --git a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Delete.php b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Delete.php index f849de94c1..1046d7e566 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Delete.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Delete.php @@ -60,7 +60,6 @@ class Delete extends Action ->inject('dbForProject') ->inject('queueForDatabase') ->inject('queueForEvents') - ->inject('queueForStatsUsage') ->callback($this->action(...)); } diff --git a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Transactions/Update.php b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Transactions/Update.php index a73ad70786..9a5a63ea91 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Transactions/Update.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Transactions/Update.php @@ -5,13 +5,13 @@ namespace Appwrite\Platform\Modules\Databases\Http\Databases\Transactions; use Appwrite\Databases\TransactionState; use Appwrite\Event\Delete; use Appwrite\Event\Event; -use Appwrite\Event\StatsUsage; use Appwrite\Extend\Exception; use Appwrite\Functions\EventProcessor; use Appwrite\SDK\AuthType; use Appwrite\SDK\ContentType; use Appwrite\SDK\Method; use Appwrite\SDK\Response as SDKResponse; +use Appwrite\Usage\Context; use Appwrite\Utopia\Database\Documents\User; use Appwrite\Utopia\Response as UtopiaResponse; use Utopia\Database\Database; @@ -73,7 +73,7 @@ class Update extends Action ->inject('transactionState') ->inject('queueForDeletes') ->inject('queueForEvents') - ->inject('queueForStatsUsage') + ->inject('usage') ->inject('queueForRealtime') ->inject('queueForFunctions') ->inject('queueForWebhooks') @@ -92,7 +92,7 @@ class Update extends Action * @param TransactionState $transactionState * @param Delete $queueForDeletes * @param Event $queueForEvents - * @param StatsUsage $queueForStatsUsage + * @param Context $usage * @param Event $queueForRealtime * @param Event $queueForFunctions * @param Event $queueForWebhooks @@ -106,7 +106,7 @@ class Update extends Action * @throws Structure * @throws \Utopia\Http\Exception */ - public function action(string $transactionId, bool $commit, bool $rollback, UtopiaResponse $response, Database $dbForProject, Document $user, TransactionState $transactionState, Delete $queueForDeletes, Event $queueForEvents, StatsUsage $queueForStatsUsage, Event $queueForRealtime, Event $queueForFunctions, Event $queueForWebhooks, Authorization $authorization, EventProcessor $eventProcessor): void + public function action(string $transactionId, bool $commit, bool $rollback, UtopiaResponse $response, Database $dbForProject, Document $user, TransactionState $transactionState, Delete $queueForDeletes, Event $queueForEvents, Context $usage, Event $queueForRealtime, Event $queueForFunctions, Event $queueForWebhooks, Authorization $authorization, EventProcessor $eventProcessor): void { if (!$commit && !$rollback) { throw new Exception(Exception::GENERAL_BAD_REQUEST, 'Either commit or rollback must be true'); @@ -142,7 +142,7 @@ class Update extends Action $currentDocumentId = null; try { - $dbForProject->withTransaction(function () use ($dbForProject, $transactionState, $queueForDeletes, $transactionId, &$transaction, &$operations, &$totalOperations, &$databaseOperations, &$currentDocumentId, $queueForEvents, $queueForStatsUsage, $queueForRealtime, $queueForFunctions, $queueForWebhooks, $authorization) { + $dbForProject->withTransaction(function () use ($dbForProject, $transactionState, $queueForDeletes, $transactionId, &$transaction, &$operations, &$totalOperations, &$databaseOperations, &$currentDocumentId, $queueForEvents, $usage, $queueForRealtime, $queueForFunctions, $queueForWebhooks, $authorization) { $authorization->skip(fn () => $dbForProject->updateDocument('transactions', $transactionId, new Document([ 'status' => 'committing', ]))); @@ -279,11 +279,10 @@ class Update extends Action throw new Exception(Exception::GENERAL_QUERY_INVALID, $e->getMessage()); } - $queueForStatsUsage - ->addMetric(METRIC_DATABASES_OPERATIONS_WRITES, $totalOperations); + $usage->addMetric(METRIC_DATABASES_OPERATIONS_WRITES, $totalOperations); foreach ($databaseOperations as $sequence => $count) { - $queueForStatsUsage->addMetric( + $usage->addMetric( str_replace('{databaseInternalId}', $sequence, METRIC_DATABASE_ID_OPERATIONS_WRITES), $count ); diff --git a/src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Delete.php b/src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Delete.php index c6cd0c6999..7873d369e6 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Delete.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Delete.php @@ -50,7 +50,6 @@ class Delete extends DatabaseDelete ->inject('dbForProject') ->inject('queueForDatabase') ->inject('queueForEvents') - ->inject('queueForStatsUsage') ->callback($this->action(...)); } } diff --git a/src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Tables/Rows/Bulk/Delete.php b/src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Tables/Rows/Bulk/Delete.php index a1695bdbc6..adaf83ccf1 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Tables/Rows/Bulk/Delete.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Tables/Rows/Bulk/Delete.php @@ -61,7 +61,7 @@ class Delete extends DocumentsDelete ->param('transactionId', null, fn (Database $dbForProject) => new Nullable(new UID($dbForProject->getAdapter()->getMaxUIDLength())), 'Transaction ID for staging the operation.', true, ['dbForProject']) ->inject('response') ->inject('dbForProject') - ->inject('queueForStatsUsage') + ->inject('usage') ->inject('queueForEvents') ->inject('queueForRealtime') ->inject('queueForFunctions') diff --git a/src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Tables/Rows/Bulk/Update.php b/src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Tables/Rows/Bulk/Update.php index a6bc78b3e9..d706d1f28b 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Tables/Rows/Bulk/Update.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Tables/Rows/Bulk/Update.php @@ -63,7 +63,7 @@ class Update extends DocumentsUpdate ->param('transactionId', null, fn (Database $dbForProject) => new Nullable(new UID($dbForProject->getAdapter()->getMaxUIDLength())), 'Transaction ID for staging the operation.', true, ['dbForProject']) ->inject('response') ->inject('dbForProject') - ->inject('queueForStatsUsage') + ->inject('usage') ->inject('queueForEvents') ->inject('queueForRealtime') ->inject('queueForFunctions') diff --git a/src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Tables/Rows/Bulk/Upsert.php b/src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Tables/Rows/Bulk/Upsert.php index 6c0815312d..58da5064f9 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Tables/Rows/Bulk/Upsert.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Tables/Rows/Bulk/Upsert.php @@ -63,7 +63,7 @@ class Upsert extends DocumentsUpsert ->param('transactionId', null, fn (Database $dbForProject) => new Nullable(new UID($dbForProject->getAdapter()->getMaxUIDLength())), 'Transaction ID for staging the operation.', true, ['dbForProject']) ->inject('response') ->inject('dbForProject') - ->inject('queueForStatsUsage') + ->inject('usage') ->inject('queueForEvents') ->inject('queueForRealtime') ->inject('queueForFunctions') diff --git a/src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Tables/Rows/Column/Decrement.php b/src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Tables/Rows/Column/Decrement.php index 63d70b40e2..e1e717e9b1 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Tables/Rows/Column/Decrement.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Tables/Rows/Column/Decrement.php @@ -66,7 +66,7 @@ class Decrement extends DecrementDocumentAttribute ->inject('response') ->inject('dbForProject') ->inject('queueForEvents') - ->inject('queueForStatsUsage') + ->inject('usage') ->inject('plan') ->inject('authorization') ->callback($this->action(...)); diff --git a/src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Tables/Rows/Column/Increment.php b/src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Tables/Rows/Column/Increment.php index 5beb8468d9..0b20450254 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Tables/Rows/Column/Increment.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Tables/Rows/Column/Increment.php @@ -66,7 +66,7 @@ class Increment extends IncrementDocumentAttribute ->inject('response') ->inject('dbForProject') ->inject('queueForEvents') - ->inject('queueForStatsUsage') + ->inject('usage') ->inject('plan') ->inject('authorization') ->callback($this->action(...)); diff --git a/src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Tables/Rows/Create.php b/src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Tables/Rows/Create.php index 4385303ffa..fde8005d2b 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Tables/Rows/Create.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Tables/Rows/Create.php @@ -106,7 +106,7 @@ class Create extends DocumentCreate ->inject('dbForProject') ->inject('user') ->inject('queueForEvents') - ->inject('queueForStatsUsage') + ->inject('usage') ->inject('queueForRealtime') ->inject('queueForFunctions') ->inject('queueForWebhooks') diff --git a/src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Tables/Rows/Delete.php b/src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Tables/Rows/Delete.php index bee4dc1093..1845edc307 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Tables/Rows/Delete.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Tables/Rows/Delete.php @@ -68,7 +68,7 @@ class Delete extends DocumentDelete ->inject('response') ->inject('dbForProject') ->inject('queueForEvents') - ->inject('queueForStatsUsage') + ->inject('usage') ->inject('transactionState') ->inject('plan') ->inject('authorization') diff --git a/src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Tables/Rows/Get.php b/src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Tables/Rows/Get.php index f0a7fcbbc2..43b799e5b1 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Tables/Rows/Get.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Tables/Rows/Get.php @@ -57,7 +57,7 @@ class Get extends DocumentGet ->param('transactionId', null, fn (Database $dbForProject) => new Nullable(new UID($dbForProject->getAdapter()->getMaxUIDLength())), 'Transaction ID to read uncommitted changes within the transaction.', true, ['dbForProject']) ->inject('response') ->inject('dbForProject') - ->inject('queueForStatsUsage') + ->inject('usage') ->inject('transactionState') ->inject('authorization') ->callback($this->action(...)); diff --git a/src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Tables/Rows/Update.php b/src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Tables/Rows/Update.php index 71abb5d167..c0d90f9531 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Tables/Rows/Update.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Tables/Rows/Update.php @@ -66,7 +66,7 @@ class Update extends DocumentUpdate ->inject('response') ->inject('dbForProject') ->inject('queueForEvents') - ->inject('queueForStatsUsage') + ->inject('usage') ->inject('transactionState') ->inject('plan') ->inject('authorization') diff --git a/src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Tables/Rows/Upsert.php b/src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Tables/Rows/Upsert.php index 0bcf9f9a63..7f0aa0ad7d 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Tables/Rows/Upsert.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Tables/Rows/Upsert.php @@ -69,7 +69,7 @@ class Upsert extends DocumentUpsert ->inject('user') ->inject('dbForProject') ->inject('queueForEvents') - ->inject('queueForStatsUsage') + ->inject('usage') ->inject('transactionState') ->inject('plan') ->inject('authorization') diff --git a/src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Tables/Rows/XList.php b/src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Tables/Rows/XList.php index 3b8ac0a70e..6e5dcd9370 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Tables/Rows/XList.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Tables/Rows/XList.php @@ -61,7 +61,7 @@ class XList extends DocumentXList ->inject('response') ->inject('dbForProject') ->inject('user') - ->inject('queueForStatsUsage') + ->inject('usage') ->inject('transactionState') ->inject('authorization') ->callback($this->action(...)); diff --git a/src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Transactions/Update.php b/src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Transactions/Update.php index 7da389b265..68ea2b8901 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Transactions/Update.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Transactions/Update.php @@ -57,7 +57,7 @@ class Update extends TransactionsUpdate ->inject('transactionState') ->inject('queueForDeletes') ->inject('queueForEvents') - ->inject('queueForStatsUsage') + ->inject('usage') ->inject('queueForRealtime') ->inject('queueForFunctions') ->inject('queueForWebhooks') diff --git a/src/Appwrite/Platform/Modules/Functions/Http/Executions/Create.php b/src/Appwrite/Platform/Modules/Functions/Http/Executions/Create.php index bc506c654a..ee33abe9e1 100644 --- a/src/Appwrite/Platform/Modules/Functions/Http/Executions/Create.php +++ b/src/Appwrite/Platform/Modules/Functions/Http/Executions/Create.php @@ -6,7 +6,6 @@ use Ahc\Jwt\JWT; use Appwrite\Event\Delete as DeleteEvent; use Appwrite\Event\Event; use Appwrite\Event\Func; -use Appwrite\Event\StatsUsage; use Appwrite\Extend\Exception; use Appwrite\Extend\Exception as AppwriteException; use Appwrite\Functions\Validator\Headers; @@ -15,6 +14,7 @@ use Appwrite\SDK\AuthType; use Appwrite\SDK\ContentType; use Appwrite\SDK\Method; use Appwrite\SDK\Response as SDKResponse; +use Appwrite\Usage\Context; use Appwrite\Utopia\Database\Documents\User; use Appwrite\Utopia\Response; use Executor\Executor; @@ -93,7 +93,7 @@ class Create extends Base ->inject('dbForPlatform') ->inject('user') ->inject('queueForEvents') - ->inject('queueForStatsUsage') + ->inject('usage') ->inject('queueForFunctions') ->inject('geodb') ->inject('store') @@ -121,7 +121,7 @@ class Create extends Base Database $dbForPlatform, Document $user, Event $queueForEvents, - StatsUsage $queueForStatsUsage, + Context $usage, Func $queueForFunctions, Reader $geodb, Store $store, @@ -499,7 +499,7 @@ class Create extends Base throw $th; } } finally { - $queueForStatsUsage + $usage ->addMetric(METRIC_EXECUTIONS, 1) ->addMetric(str_replace(['{resourceType}'], [RESOURCE_TYPE_FUNCTIONS], METRIC_RESOURCE_TYPE_EXECUTIONS), 1) ->addMetric(str_replace(['{resourceType}', '{resourceInternalId}'], [RESOURCE_TYPE_FUNCTIONS, $function->getSequence()], METRIC_RESOURCE_TYPE_ID_EXECUTIONS), 1) diff --git a/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php b/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php index 2c4043878e..c7465fd88f 100644 --- a/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php +++ b/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php @@ -5,11 +5,13 @@ namespace Appwrite\Platform\Modules\Functions\Workers; use Ahc\Jwt\JWT; use Appwrite\Event\Event; use Appwrite\Event\Func; +use Appwrite\Event\Message\Usage as UsageMessage; +use Appwrite\Event\Publisher\Usage as UsagePublisher; use Appwrite\Event\Realtime; use Appwrite\Event\Screenshot; -use Appwrite\Event\StatsUsage; use Appwrite\Event\Webhook; use Appwrite\Filter\BranchDomain as BranchDomainFilter; +use Appwrite\Usage\Context; use Appwrite\Utopia\Response\Model\Deployment; use Appwrite\Vcs\Comment; use Exception; @@ -60,7 +62,8 @@ class Builds extends Action ->inject('queueForWebhooks') ->inject('queueForFunctions') ->inject('queueForRealtime') - ->inject('queueForStatsUsage') + ->inject('usage') + ->inject('publisherForUsage') ->inject('cache') ->inject('dbForProject') ->inject('deviceForFunctions') @@ -74,24 +77,6 @@ class Builds extends Action } /** - * @param Message $message - * @param Document $project - * @param Database $dbForPlatform - * @param Event $queueForEvents - * @param Screenshot $queueForScreenshots - * @param Webhook $queueForWebhooks - * @param Func $queueForFunctions - * @param Realtime $queueForRealtime - * @param StatsUsage $queueForStatsUsage - * @param Cache $cache - * @param Database $dbForProject - * @param Device $deviceForFunctions - * @param Device $deviceForSites - * @param Device $deviceForFiles - * @param Log $log - * @param Executor $executor - * @param array $plan - * @return void * @throws \Utopia\Database\Exception */ public function action( @@ -103,7 +88,8 @@ class Builds extends Action Webhook $queueForWebhooks, Func $queueForFunctions, Realtime $queueForRealtime, - StatsUsage $queueForStatsUsage, + Context $usage, + UsagePublisher $publisherForUsage, Cache $cache, Database $dbForProject, Device $deviceForFunctions, @@ -145,7 +131,8 @@ class Builds extends Action $queueForFunctions, $queueForRealtime, $queueForEvents, - $queueForStatsUsage, + $usage, + $publisherForUsage, $dbForPlatform, $dbForProject, $github, @@ -167,28 +154,7 @@ class Builds extends Action } /** - * @param Device $deviceForFunctions - * @param Device $deviceForSites - * @param Device $deviceForFiles - * @param Screenshot $queueForScreenshots - * @param Webhook $queueForWebhooks - * @param Func $queueForFunctions - * @param Realtime $queueForRealtime - * @param Event $queueForEvents - * @param StatsUsage $queueForStatsUsage - * @param Database $dbForPlatform - * @param Database $dbForProject - * @param GitHub $github - * @param Document $project - * @param Document $resource - * @param Document $deployment - * @param Document $template - * @param Log $log - * @param Executor $executor - * @param array $plan - * @return void * @throws \Utopia\Database\Exception - * * @throws Exception */ protected function buildDeployment( @@ -200,7 +166,8 @@ class Builds extends Action Func $queueForFunctions, Realtime $queueForRealtime, Event $queueForEvents, - StatsUsage $queueForStatsUsage, + Context $usage, + UsagePublisher $publisherForUsage, Database $dbForPlatform, Database $dbForProject, GitHub $github, @@ -299,6 +266,7 @@ class Builds extends Action if ($deployment->getAttribute('status') === 'canceled') { $this->cancelDeployment($deployment->getId(), $dbForProject, $queueForRealtime); + return; } @@ -325,7 +293,7 @@ class Builds extends Action $installationId = $deployment->getAttribute('installationId', ''); $providerRepositoryId = $deployment->getAttribute('providerRepositoryId', ''); $providerCommitHash = $deployment->getAttribute('providerCommitHash', ''); - $isVcsEnabled = !empty($providerRepositoryId); + $isVcsEnabled = ! empty($providerRepositoryId); $owner = ''; $repositoryName = ''; @@ -339,7 +307,7 @@ class Builds extends Action } try { - if (!$isVcsEnabled) { + if (! $isVcsEnabled) { // Non-VCS + Template $templateRepositoryName = $template->getAttribute('repositoryName', ''); $templateOwnerName = $template->getAttribute('ownerName', ''); @@ -351,7 +319,7 @@ class Builds extends Action $templateRootDirectory = \ltrim($templateRootDirectory, '.'); $templateRootDirectory = \ltrim($templateRootDirectory, '/'); - if (!empty($templateRepositoryName) && !empty($templateOwnerName) && !empty($templateReferenceType) && !empty($templateReferenceValue)) { + if (! empty($templateRepositoryName) && ! empty($templateOwnerName) && ! empty($templateReferenceType) && ! empty($templateReferenceValue)) { $stdout = ''; $stderr = ''; @@ -385,8 +353,8 @@ class Builds extends Action $source = $device->getPath($deployment->getId() . '.' . \pathinfo('code.tar.gz', PATHINFO_EXTENSION)); $result = $localDevice->transfer($tmpPathFile, $source, $device); - if (!$result) { - throw new \Exception("Unable to move file"); + if (! $result) { + throw new \Exception('Unable to move file'); } Console::execute('rm -rf ' . \escapeshellarg($tmpTemplateDirectory), '', $stdout, $stderr); @@ -427,7 +395,7 @@ class Builds extends Action $cloneVersion = $branchName; $cloneType = GitHub::CLONE_TYPE_BRANCH; - if (!empty($commitHash)) { + if (! empty($commitHash)) { $cloneVersion = $commitHash; $cloneType = GitHub::CLONE_TYPE_COMMIT; } @@ -440,6 +408,7 @@ class Builds extends Action if ($dbForProject->getDocument('deployments', $deploymentId)->getAttribute('status') === 'canceled') { $this->cancelDeployment($deployment->getId(), $dbForProject, $queueForRealtime); + return; } @@ -456,7 +425,7 @@ class Builds extends Action $rootDirectoryWithoutSpaces = str_replace(' ', '', $rootDirectory); $from = $tmpDirectory . '/' . $rootDirectory; $to = $tmpDirectory . '/' . $rootDirectoryWithoutSpaces; - $exit = Console::execute('mv "' . \escapeshellarg($from) . '" "' . \escapeshellarg($to) . '"', '', $stdout, $stderr); + $exit = Console::execute('mv ' . \escapeshellarg($from) . ' ' . \escapeshellarg($to), '', $stdout, $stderr); if ($exit !== 0) { throw new \Exception('Unable to move function with spaces' . $stderr); @@ -464,7 +433,6 @@ class Builds extends Action $rootDirectory = $rootDirectoryWithoutSpaces; } - // Build from template $templateRepositoryName = $template->getAttribute('repositoryName', ''); $templateOwnerName = $template->getAttribute('ownerName', ''); @@ -476,7 +444,7 @@ class Builds extends Action $templateRootDirectory = \ltrim($templateRootDirectory, '.'); $templateRootDirectory = \ltrim($templateRootDirectory, '/'); - if (!empty($templateRepositoryName) && !empty($templateOwnerName) && !empty($templateReferenceType) && !empty($templateReferenceValue)) { + if (! empty($templateRepositoryName) && ! empty($templateOwnerName) && ! empty($templateReferenceType) && ! empty($templateReferenceValue)) { // Clone template repo $tmpTemplateDirectory = '/tmp/builds/' . $deploymentId . '/template'; @@ -495,7 +463,7 @@ class Builds extends Action Console::execute('rsync -av --exclude \'.git\' ' . \escapeshellarg($tmpTemplateDirectory . '/' . $templateRootDirectory . '/') . ' ' . \escapeshellarg($tmpDirectory . '/' . $rootDirectory), '', $stdout, $stderr); // Commit and push - $exit = Console::execute('git config --global user.email '. \escapeshellarg(APP_VCS_GITHUB_EMAIL) .' && git config --global user.name '. \escapeshellarg(APP_VCS_GITHUB_USERNAME) .' && cd ' . \escapeshellarg($tmpDirectory) . ' && git checkout -b ' . \escapeshellarg($branchName) . ' && git add . && git commit -m "Create ' . \escapeshellarg($resource->getAttribute('name', '')) . ' function" && git push origin ' . \escapeshellarg($branchName), '', $stdout, $stderr); + $exit = Console::execute('git config --global user.email ' . \escapeshellarg(APP_VCS_GITHUB_EMAIL) . ' && git config --global user.name ' . \escapeshellarg(APP_VCS_GITHUB_USERNAME) . ' && cd ' . \escapeshellarg($tmpDirectory) . ' && git checkout -b ' . \escapeshellarg($branchName) . ' && git add . && git commit -m "Create ' . \escapeshellarg($resource->getAttribute('name', '')) . ' function" && git push origin ' . \escapeshellarg($branchName), '', $stdout, $stderr); if ($exit !== 0) { throw new \Exception('Unable to push code repository: ' . $stderr); @@ -538,7 +506,7 @@ class Builds extends Action } $directorySize = $localDevice->getDirectorySize($tmpDirectory); - $sizeLimit = (int)System::getEnv('_APP_COMPUTE_SIZE_LIMIT', '30000000'); + $sizeLimit = (int) System::getEnv('_APP_COMPUTE_SIZE_LIMIT', '30000000'); if (isset($plan['deploymentSize'])) { $sizeLimit = (int) $plan['deploymentSize'] * 1000 * 1000; @@ -556,8 +524,8 @@ class Builds extends Action $source = $device->getPath($deployment->getId() . '.' . \pathinfo('code.tar.gz', PATHINFO_EXTENSION)); $result = $localDevice->transfer($tmpPathFile, $source, $device); - if (!$result) { - throw new \Exception("Unable to move file"); + if (! $result) { + throw new \Exception('Unable to move file'); } Console::execute('rm -rf ' . \escapeshellarg($tmpPath), '', $stdout, $stderr); @@ -650,16 +618,15 @@ class Builds extends Action } $cpus = $spec['cpus'] ?? APP_COMPUTE_CPUS_DEFAULT; - $memory = max($spec['memory'] ?? APP_COMPUTE_MEMORY_DEFAULT, $minMemory); + $memory = max($spec['memory'] ?? APP_COMPUTE_MEMORY_DEFAULT, $minMemory); $timeout = (int) System::getEnv('_APP_COMPUTE_BUILD_TIMEOUT', 900); - - $jwtExpiry = (int)System::getEnv('_APP_COMPUTE_BUILD_TIMEOUT', 900); + $jwtExpiry = (int) System::getEnv('_APP_COMPUTE_BUILD_TIMEOUT', 900); $jwtObj = new JWT(System::getEnv('_APP_OPENSSL_KEY_V1'), 'HS256', $jwtExpiry, 0); $apiKey = $jwtObj->encode([ 'projectId' => $project->getId(), - 'scopes' => $resource->getAttribute('scopes', []) + 'scopes' => $resource->getAttribute('scopes', []), ]); // Appwrite vars @@ -727,6 +694,7 @@ class Builds extends Action if ($dbForProject->getDocument('deployments', $deploymentId)->getAttribute('status') === 'canceled') { $this->cancelDeployment($deployment->getId(), $dbForProject, $queueForRealtime); + return; } @@ -748,7 +716,7 @@ class Builds extends Action $listFilesCommand .= 'echo "{APPWRITE_DETECTION_SEPARATOR_START}" && cd /usr/local/build'; // Enter output directory, if set - if (!empty($outputDirectory)) { + if (! empty($outputDirectory)) { $listFilesCommand .= ' && cd ' . \escapeshellarg($outputDirectory); } @@ -775,7 +743,7 @@ class Builds extends Action cpus: $cpus, memory: $memory, timeout: $timeout, - remove: true, + remove: true, entrypoint: $deployment->getAttribute('entrypoint', ''), destination: APP_STORAGE_BUILDS . "/app-{$project->getId()}", variables: $vars, @@ -809,6 +777,7 @@ class Builds extends Action if ($deployment->getAttribute('status') === 'canceled') { $isCanceled = true; Console::info('Ignoring realtime logs because build has been canceled'); + return; } @@ -816,7 +785,7 @@ class Builds extends Action $logs = \mb_substr($logs, 0, null, 'UTF-8'); // Do not stream logs added for SSR detection - if (!$insideSeparation) { + if (! $insideSeparation) { $separator = \strpos($logs, '{APPWRITE_DETECTION_SEPARATOR_START}'); if ($separator !== false) { $logs = \substr($logs, 0, $separator); @@ -846,19 +815,19 @@ class Builds extends Action $currentLogs = $deployment->getAttribute('buildLogs', ''); $affected = false; - $streamLogs = \str_replace("\\n", "{APPWRITE_LINEBREAK_PLACEHOLDER}", $logs); + $streamLogs = \str_replace('\\n', '{APPWRITE_LINEBREAK_PLACEHOLDER}', $logs); foreach (\explode("\n", $streamLogs) as $streamLog) { if (empty($streamLog)) { continue; } - $streamLog = \str_replace("{APPWRITE_LINEBREAK_PLACEHOLDER}", "\n", $streamLog); - $streamParts = \explode(" ", $streamLog, 2); + $streamLog = \str_replace('{APPWRITE_LINEBREAK_PLACEHOLDER}', "\n", $streamLog); + $streamParts = \explode(' ', $streamLog, 2); // TODO: use part[0] as timestamp when switching to dbForLogs for build logs $currentLogs .= $streamParts[1]; - if (!empty($streamParts[1])) { + if (! empty($streamParts[1])) { $affected = true; } } @@ -890,6 +859,7 @@ class Builds extends Action if ($dbForProject->getDocument('deployments', $deploymentId)->getAttribute('status') === 'canceled') { $this->cancelDeployment($deployment->getId(), $dbForProject, $queueForRealtime); + return; } @@ -897,7 +867,7 @@ class Builds extends Action throw $err; } - $buildSizeLimit = (int)System::getEnv('_APP_COMPUTE_BUILD_SIZE_LIMIT', '2000000000'); + $buildSizeLimit = (int) System::getEnv('_APP_COMPUTE_BUILD_SIZE_LIMIT', '2000000000'); if (isset($plan['buildSize'])) { $buildSizeLimit = $plan['buildSize'] * 1000 * 1000; } @@ -925,7 +895,7 @@ class Builds extends Action $deployment->setAttribute('buildLogs', $logs); $adapter = null; - if ($resource->getCollection() === 'sites' && !empty($detectionLogs)) { + if ($resource->getCollection() === 'sites' && ! empty($detectionLogs)) { $files = \explode("\n", $detectionLogs); // Parse output $files = \array_filter($files); // Remove empty $files = \array_map(fn ($file) => \trim($file), $files); // Remove whitepsaces @@ -997,9 +967,9 @@ class Builds extends Action // Check if current active deployment started later than this deployment $resource = $dbForProject->getDocument($resource->getCollection(), $resource->getId()); $currentActiveDeploymentId = $resource->getAttribute('deploymentId', ''); - if (!empty($currentActiveDeploymentId)) { + if (! empty($currentActiveDeploymentId)) { $currentActiveDeployment = $dbForProject->getDocument('deployments', $currentActiveDeploymentId); - if (!$currentActiveDeployment->isEmpty()) { + if (! $currentActiveDeployment->isEmpty()) { $currentActiveStartTime = $currentActiveDeployment->getCreatedAt(); $deploymentStartTime = $deployment->getCreatedAt(); @@ -1085,7 +1055,7 @@ class Builds extends Action if ($resource->getCollection() === 'sites') { // VCS branch $branchName = $deployment->getAttribute('providerBranch'); - if (!empty($branchName)) { + if (! empty($branchName)) { $domain = (new BranchDomainFilter())->apply([ 'branch' => $branchName, 'resourceId' => $resource->getId(), @@ -1112,7 +1082,7 @@ class Builds extends Action 'certificateId' => '', 'search' => implode(' ', [$ruleId, $domain]), 'owner' => 'Appwrite', - 'region' => $project->getAttribute('region') + 'region' => $project->getAttribute('region'), ])); } catch (Duplicate $err) { $rule = $dbForPlatform->updateDocument('rules', $ruleId, new Document([ @@ -1153,6 +1123,7 @@ class Builds extends Action if ($dbForProject->getDocument('deployments', $deploymentId)->getAttribute('status') === 'canceled') { $this->cancelDeployment($deployment->getId(), $dbForProject, $queueForRealtime); + return; } @@ -1166,7 +1137,7 @@ class Builds extends Action $schedule ->setAttribute('resourceUpdatedAt', DateTime::now()) ->setAttribute('schedule', $resource->getAttribute('schedule')) - ->setAttribute('active', !empty($resource->getAttribute('schedule')) && !empty($resource->getAttribute('deploymentId'))); + ->setAttribute('active', ! empty($resource->getAttribute('schedule')) && ! empty($resource->getAttribute('deploymentId'))); $dbForPlatform->updateDocument('schedules', $schedule->getId(), new Document([ 'resourceUpdatedAt' => $schedule->getAttribute('resourceUpdatedAt'), 'schedule' => $schedule->getAttribute('schedule'), @@ -1194,13 +1165,14 @@ class Builds extends Action if ($dbForProject->getDocument('deployments', $deploymentId)->getAttribute('status') === 'canceled') { $this->cancelDeployment($deployment->getId(), $dbForProject, $queueForRealtime); + return; } // Color message red $message = $th->getMessage(); - if (!\str_contains($message, '')) { - $message = "" . $message; + if (! \str_contains($message, '')) { + $message = '' . $message; } $message = \str_replace('{APPWRITE_DETECTION_SEPARATOR_START}', '', $message); @@ -1208,9 +1180,9 @@ class Builds extends Action // Combine with previous logs if deployment got past build process $previousLogs = ''; - if (!is_null($deployment->getAttribute('buildSize', null))) { + if (! is_null($deployment->getAttribute('buildSize', null))) { $previousLogs = $deployment->getAttribute('buildLogs', ''); - if (!empty($previousLogs)) { + if (! empty($previousLogs)) { $message = $previousLogs . "\n" . $message; } } @@ -1246,102 +1218,102 @@ class Builds extends Action ->trigger(); $this->sendUsage( - resource:$resource, + resource: $resource, deployment: $deployment, project: $project, - queue: $queueForStatsUsage + usage: $usage, + publisherForUsage: $publisherForUsage ); } } - protected function sendUsage(Document $resource, Document $deployment, Document $project, StatsUsage $queue): void + protected function sendUsage(Document $resource, Document $deployment, Document $project, Context $usage, UsagePublisher $publisherForUsage): void { $spec = Config::getParam('specifications')[$resource->getAttribute('buildSpecification', APP_COMPUTE_SPECIFICATION_DEFAULT)]; switch ($deployment->getAttribute('status')) { case 'ready': - $queue + $usage ->addMetric(METRIC_BUILDS_SUCCESS, 1) // per project - ->addMetric(METRIC_BUILDS_COMPUTE_SUCCESS, (int)$deployment->getAttribute('buildDuration', 0) * 1000) + ->addMetric(METRIC_BUILDS_COMPUTE_SUCCESS, (int) $deployment->getAttribute('buildDuration', 0) * 1000) ->addMetric(str_replace(['{resourceType}'], [$deployment->getAttribute('resourceType')], METRIC_RESOURCE_TYPE_BUILDS_SUCCESS), 1) // per function - ->addMetric(str_replace(['{resourceType}'], [$deployment->getAttribute('resourceType')], METRIC_RESOURCE_TYPE_BUILDS_COMPUTE_SUCCESS), (int)$deployment->getAttribute('buildDuration', 0) * 1000) + ->addMetric(str_replace(['{resourceType}'], [$deployment->getAttribute('resourceType')], METRIC_RESOURCE_TYPE_BUILDS_COMPUTE_SUCCESS), (int) $deployment->getAttribute('buildDuration', 0) * 1000) ->addMetric(str_replace(['{resourceType}', '{resourceInternalId}'], [$deployment->getAttribute('resourceType'), $resource->getSequence()], METRIC_RESOURCE_TYPE_ID_BUILDS_SUCCESS), 1) // per function - ->addMetric(str_replace(['{resourceType}', '{resourceInternalId}'], [$deployment->getAttribute('resourceType'), $resource->getSequence()], METRIC_RESOURCE_TYPE_ID_BUILDS_COMPUTE_SUCCESS), (int)$deployment->getAttribute('buildDuration', 0) * 1000); + ->addMetric(str_replace(['{resourceType}', '{resourceInternalId}'], [$deployment->getAttribute('resourceType'), $resource->getSequence()], METRIC_RESOURCE_TYPE_ID_BUILDS_COMPUTE_SUCCESS), (int) $deployment->getAttribute('buildDuration', 0) * 1000); break; case 'failed': - $queue + $usage ->addMetric(METRIC_BUILDS_FAILED, 1) // per project - ->addMetric(METRIC_BUILDS_COMPUTE_FAILED, (int)$deployment->getAttribute('buildDuration', 0) * 1000) + ->addMetric(METRIC_BUILDS_COMPUTE_FAILED, (int) $deployment->getAttribute('buildDuration', 0) * 1000) ->addMetric(str_replace(['{resourceType}'], [$deployment->getAttribute('resourceType')], METRIC_RESOURCE_TYPE_BUILDS_FAILED), 1) // per function - ->addMetric(str_replace(['{resourceType}'], [$deployment->getAttribute('resourceType')], METRIC_RESOURCE_TYPE_BUILDS_COMPUTE_FAILED), (int)$deployment->getAttribute('buildDuration', 0) * 1000) + ->addMetric(str_replace(['{resourceType}'], [$deployment->getAttribute('resourceType')], METRIC_RESOURCE_TYPE_BUILDS_COMPUTE_FAILED), (int) $deployment->getAttribute('buildDuration', 0) * 1000) ->addMetric(str_replace(['{resourceType}', '{resourceInternalId}'], [$deployment->getAttribute('resourceType'), $resource->getSequence()], METRIC_RESOURCE_TYPE_ID_BUILDS_FAILED), 1) // per function - ->addMetric(str_replace(['{resourceType}', '{resourceInternalId}'], [$deployment->getAttribute('resourceType'), $resource->getSequence()], METRIC_RESOURCE_TYPE_ID_BUILDS_COMPUTE_FAILED), (int)$deployment->getAttribute('buildDuration', 0) * 1000); + ->addMetric(str_replace(['{resourceType}', '{resourceInternalId}'], [$deployment->getAttribute('resourceType'), $resource->getSequence()], METRIC_RESOURCE_TYPE_ID_BUILDS_COMPUTE_FAILED), (int) $deployment->getAttribute('buildDuration', 0) * 1000); break; } - $queue + $usage ->addMetric(METRIC_BUILDS, 1) // per project ->addMetric(METRIC_BUILDS_STORAGE, $deployment->getAttribute('buildSize', 0)) - ->addMetric(METRIC_BUILDS_COMPUTE, (int)$deployment->getAttribute('buildDuration', 0) * 1000) - ->addMetric(METRIC_BUILDS_MB_SECONDS, (int)(($spec['memory'] ?? APP_COMPUTE_MEMORY_DEFAULT) * $deployment->getAttribute('buildDuration', 0) * ($spec['cpus'] ?? APP_COMPUTE_CPUS_DEFAULT))) + ->addMetric(METRIC_BUILDS_COMPUTE, (int) $deployment->getAttribute('buildDuration', 0) * 1000) + ->addMetric(METRIC_BUILDS_MB_SECONDS, (int) ($memory * $deployment->getAttribute('buildDuration', 0) * $cpus)) ->addMetric(str_replace(['{resourceType}'], [$deployment->getAttribute('resourceType')], METRIC_RESOURCE_TYPE_BUILDS), 1) // per function ->addMetric(str_replace(['{resourceType}'], [$deployment->getAttribute('resourceType')], METRIC_RESOURCE_TYPE_BUILDS_STORAGE), $deployment->getAttribute('buildSize', 0)) - ->addMetric(str_replace(['{resourceType}'], [$deployment->getAttribute('resourceType')], METRIC_RESOURCE_TYPE_BUILDS_COMPUTE), (int)$deployment->getAttribute('buildDuration', 0) * 1000) - ->addMetric(str_replace(['{resourceType}'], [$deployment->getAttribute('resourceType')], METRIC_RESOURCE_TYPE_BUILDS_MB_SECONDS), (int)(($spec['memory'] ?? APP_COMPUTE_MEMORY_DEFAULT) * $deployment->getAttribute('buildDuration', 0) * ($spec['cpus'] ?? APP_COMPUTE_CPUS_DEFAULT))) + ->addMetric(str_replace(['{resourceType}'], [$deployment->getAttribute('resourceType')], METRIC_RESOURCE_TYPE_BUILDS_COMPUTE), (int) $deployment->getAttribute('buildDuration', 0) * 1000) + ->addMetric(str_replace(['{resourceType}'], [$deployment->getAttribute('resourceType')], METRIC_RESOURCE_TYPE_BUILDS_MB_SECONDS), (int) ($memory * $deployment->getAttribute('buildDuration', 0) * $cpus)) ->addMetric(str_replace(['{resourceType}', '{resourceInternalId}'], [$deployment->getAttribute('resourceType'), $resource->getSequence()], METRIC_RESOURCE_TYPE_ID_BUILDS), 1) // per function ->addMetric(str_replace(['{resourceType}', '{resourceInternalId}'], [$deployment->getAttribute('resourceType'), $resource->getSequence()], METRIC_RESOURCE_TYPE_ID_BUILDS_STORAGE), $deployment->getAttribute('buildSize', 0)) - ->addMetric(str_replace(['{resourceType}', '{resourceInternalId}'], [$deployment->getAttribute('resourceType'), $resource->getSequence()], METRIC_RESOURCE_TYPE_ID_BUILDS_COMPUTE), (int)$deployment->getAttribute('buildDuration', 0) * 1000) - ->addMetric(str_replace(['{resourceType}', '{resourceInternalId}'], [$deployment->getAttribute('resourceType'), $resource->getSequence()], METRIC_RESOURCE_TYPE_ID_BUILDS_MB_SECONDS), (int)(($spec['memory'] ?? APP_COMPUTE_MEMORY_DEFAULT) * $deployment->getAttribute('buildDuration', 0) * ($spec['cpus'] ?? APP_COMPUTE_CPUS_DEFAULT))) - ->setProject($project) - ->trigger(); + ->addMetric(str_replace(['{resourceType}', '{resourceInternalId}'], [$deployment->getAttribute('resourceType'), $resource->getSequence()], METRIC_RESOURCE_TYPE_ID_BUILDS_COMPUTE), (int) $deployment->getAttribute('buildDuration', 0) * 1000) + ->addMetric(str_replace(['{resourceType}', '{resourceInternalId}'], [$deployment->getAttribute('resourceType'), $resource->getSequence()], METRIC_RESOURCE_TYPE_ID_BUILDS_MB_SECONDS), (int) ($memory * $deployment->getAttribute('buildDuration', 0) * $cpus)); + + // Publish usage metrics + if (! $usage->isEmpty()) { + $message = new UsageMessage( + project: $project, + metrics: $usage->getMetrics(), + reduce: $usage->getReduce() + ); + $publisherForUsage->enqueue($message); + $usage->reset(); + } } /** * Hook to run after build success * - * @param Realtime $queueForRealtime - * @param Database $dbForProject - * @param Document $deployment - * @param array $runtime - * @param string|null $adapter - * @return void * @throws Exception */ protected function afterBuildSuccess(Realtime $queueForRealtime, Database $dbForProject, Document &$deployment, array $runtime, ?string $adapter): void { - if (!($queueForRealtime instanceof Realtime)) { + if (! ($queueForRealtime instanceof Realtime)) { throw new Exception('queueForRealtime must be an instance of Realtime'); } - if (!($dbForProject instanceof Database)) { + if (! ($dbForProject instanceof Database)) { throw new Exception('dbForProject must be an instance of Database'); } - if (!($deployment instanceof Document)) { + if (! ($deployment instanceof Document)) { throw new Exception('deployment must be an instance of Document'); } - if (!is_array($runtime)) { + if (! is_array($runtime)) { throw new Exception('runtime must be an array'); } - if (!is_string($adapter) && !is_null($adapter)) { + if (! is_string($adapter) && ! is_null($adapter)) { throw new Exception('adapter must be a string or null'); } } /** * Hook to run after deployment is activated - * - * @param Document $project - * @param Document $deployment - * @return void */ protected function afterDeploymentSuccess( Document $project, Document $deployment, ): void { - if (!($project instanceof Document)) { + if (! ($project instanceof Document)) { throw new Exception('project must be an instance of Document'); } - if (!($deployment instanceof Document)) { + if (! ($deployment instanceof Document)) { throw new Exception('deployment must be an instance of Document'); } } @@ -1352,6 +1324,8 @@ class Builds extends Action protected function getRuntime(Document $resource, string $version): ?array { $runtimes = Config::getParam($version === 'v2' ? 'runtimes-v2' : 'runtimes', []); + $key = $resource->getAttribute('runtime'); + $runtime = match ($resource->getCollection()) { 'functions' => $runtimes[$resource->getAttribute('runtime')] ?? null, 'sites' => $runtimes[$resource->getAttribute('buildRuntime')] ?? null, @@ -1437,7 +1411,7 @@ class Builds extends Action $envCommand = ''; $bundleCommand = ''; - if (!is_null($framework)) { + if (! is_null($framework)) { $envCommand = $framework['envCommand'] ?? ''; $bundleCommand = $framework['bundleCommand'] ?? ''; } @@ -1446,7 +1420,7 @@ class Builds extends Action $commands[] = $deployment->getAttribute('buildCommands', ''); $commands[] = $bundleCommand; - $commands = array_filter($commands, fn ($command) => !empty($command)); + $commands = array_filter($commands, fn ($command) => ! empty($command)); return implode(' && ', $commands); } @@ -1455,19 +1429,6 @@ class Builds extends Action } /** - * @param string $status - * @param GitHub $github - * @param string $providerCommitHash - * @param string $owner - * @param string $repositoryName - * @param Document $project - * @param Document $resource - * @param string $deploymentId - * @param Database $dbForProject - * @param Database $dbForPlatform - * @param Realtime $queueForRealtime - * @param array $platform - * @return void * @throws Structure * @throws \Utopia\Database\Exception * @throws Conflict @@ -1495,7 +1456,7 @@ class Builds extends Action $deployment = $dbForProject->getDocument('deployments', $deploymentId); $commentId = $deployment->getAttribute('providerCommentId', ''); - if (!empty($providerCommitHash)) { + if (! empty($providerCommitHash)) { $message = match ($status) { 'ready' => 'Build succeeded.', 'failed' => 'Build failed.', @@ -1530,7 +1491,7 @@ class Builds extends Action $github->updateCommitStatus($repositoryName, $providerCommitHash, $owner, $state, $message, $providerTargetUrl, $name); } - if (!empty($commentId)) { + if (! empty($commentId)) { $retries = 0; while (true) { @@ -1538,7 +1499,7 @@ class Builds extends Action try { $dbForPlatform->createDocument('vcsCommentLocks', new Document([ - '$id' => $commentId + '$id' => $commentId, ])); break; } catch (\Throwable $err) { @@ -1552,22 +1513,22 @@ class Builds extends Action // Wrap in try/finally to ensure lock file gets deleted try { - $resourceType = match($resource->getCollection()) { + $resourceType = match ($resource->getCollection()) { 'functions' => 'function', 'sites' => 'site', default => throw new \Exception('Invalid resource type') }; $rule = $dbForPlatform->findOne('rules', [ - Query::equal("projectInternalId", [$project->getSequence()]), - Query::equal("type", ["deployment"]), - Query::equal("deploymentInternalId", [$deployment->getSequence()]), + Query::equal('projectInternalId', [$project->getSequence()]), + Query::equal('type', ['deployment']), + Query::equal('deploymentInternalId', [$deployment->getSequence()]), ]); $protocol = System::getEnv('_APP_OPTIONS_FORCE_HTTPS') == 'disabled' ? 'http' : 'https'; - $previewUrl = match($resource->getCollection()) { + $previewUrl = match ($resource->getCollection()) { 'functions' => '', - 'sites' => !empty($rule) ? ("{$protocol}://" . $rule->getAttribute('domain', '')) : '', + 'sites' => ! empty($rule) ? ("{$protocol}://" . $rule->getAttribute('domain', '')) : '', default => throw new \Exception('Invalid resource type') }; @@ -1580,7 +1541,7 @@ class Builds extends Action } } } catch (\Throwable $th) { - Console::warning("Git action failed:"); + Console::warning('Git action failed:'); Console::warning($th->getMessage()); Console::warning($th->getTraceAsString()); diff --git a/src/Appwrite/Platform/Modules/Health/Http/Health/Queue/Failed/Get.php b/src/Appwrite/Platform/Modules/Health/Http/Health/Queue/Failed/Get.php index dc84d0ee37..cb3640746f 100644 --- a/src/Appwrite/Platform/Modules/Health/Http/Health/Queue/Failed/Get.php +++ b/src/Appwrite/Platform/Modules/Health/Http/Health/Queue/Failed/Get.php @@ -12,9 +12,9 @@ use Appwrite\Event\Func; use Appwrite\Event\Mail; use Appwrite\Event\Messaging; use Appwrite\Event\Migration; +use Appwrite\Event\Publisher\Usage as UsagePublisher; use Appwrite\Event\Screenshot; use Appwrite\Event\StatsResources; -use Appwrite\Event\StatsUsage; use Appwrite\Event\Webhook; use Appwrite\Platform\Modules\Health\Http\Health\Queue\Base; use Appwrite\SDK\AuthType; @@ -79,7 +79,7 @@ class Get extends Base ->inject('queueForMails') ->inject('queueForFunctions') ->inject('queueForStatsResources') - ->inject('queueForStatsUsage') + ->inject('publisherForUsage') ->inject('queueForWebhooks') ->inject('queueForCertificates') ->inject('queueForBuilds') @@ -99,7 +99,7 @@ class Get extends Base Mail $queueForMails, Func $queueForFunctions, StatsResources $queueForStatsResources, - StatsUsage $queueForStatsUsage, + UsagePublisher $publisherForUsage, Webhook $queueForWebhooks, Certificate $queueForCertificates, Build $queueForBuilds, @@ -116,7 +116,7 @@ class Get extends Base System::getEnv('_APP_MAILS_QUEUE_NAME', Event::MAILS_QUEUE_NAME) => $queueForMails, System::getEnv('_APP_FUNCTIONS_QUEUE_NAME', Event::FUNCTIONS_QUEUE_NAME) => $queueForFunctions, System::getEnv('_APP_STATS_RESOURCES_QUEUE_NAME', Event::STATS_RESOURCES_QUEUE_NAME) => $queueForStatsResources, - System::getEnv('_APP_STATS_USAGE_QUEUE_NAME', Event::STATS_USAGE_QUEUE_NAME) => $queueForStatsUsage, + System::getEnv('_APP_STATS_USAGE_QUEUE_NAME', Event::STATS_USAGE_QUEUE_NAME) => $publisherForUsage, System::getEnv('_APP_WEBHOOK_QUEUE_NAME', Event::WEBHOOK_QUEUE_NAME) => $queueForWebhooks, System::getEnv('_APP_CERTIFICATES_QUEUE_NAME', Event::CERTIFICATES_QUEUE_NAME) => $queueForCertificates, System::getEnv('_APP_BUILDS_QUEUE_NAME', Event::BUILDS_QUEUE_NAME) => $queueForBuilds, diff --git a/src/Appwrite/Platform/Modules/Health/Http/Health/Queue/StatsUsage/Get.php b/src/Appwrite/Platform/Modules/Health/Http/Health/Queue/StatsUsage/Get.php index 10678efbc3..65b3d228a6 100644 --- a/src/Appwrite/Platform/Modules/Health/Http/Health/Queue/StatsUsage/Get.php +++ b/src/Appwrite/Platform/Modules/Health/Http/Health/Queue/StatsUsage/Get.php @@ -2,7 +2,7 @@ namespace Appwrite\Platform\Modules\Health\Http\Health\Queue\StatsUsage; -use Appwrite\Event\StatsUsage; +use Appwrite\Event\Publisher\Usage as UsagePublisher; use Appwrite\Platform\Modules\Health\Http\Health\Queue\Base; use Appwrite\SDK\AuthType; use Appwrite\SDK\ContentType; @@ -42,16 +42,16 @@ class Get extends Base contentType: ContentType::JSON )) ->param('threshold', 5000, new Integer(true), 'Queue size threshold. When hit (equal or higher), endpoint returns server error. Default value is 5000.', true) - ->inject('queueForStatsUsage') + ->inject('publisherForUsage') ->inject('response') ->callback($this->action(...)); } - public function action(int|string $threshold, StatsUsage $queueForStatsUsage, Response $response): void + public function action(int|string $threshold, UsagePublisher $publisherForUsage, Response $response): void { $threshold = (int) $threshold; - $size = $queueForStatsUsage->getSize(); + $size = $publisherForUsage->getSize(); $this->assertQueueThreshold($size, $threshold); diff --git a/src/Appwrite/Platform/Modules/Health/Http/Health/Storage/Get.php b/src/Appwrite/Platform/Modules/Health/Http/Health/Storage/Get.php index 52468cab5a..93c9483959 100644 --- a/src/Appwrite/Platform/Modules/Health/Http/Health/Storage/Get.php +++ b/src/Appwrite/Platform/Modules/Health/Http/Health/Storage/Get.php @@ -2,7 +2,6 @@ namespace Appwrite\Platform\Modules\Health\Http\Health\Storage; -use Appwrite\Extend\Exception; use Appwrite\SDK\AuthType; use Appwrite\SDK\ContentType; use Appwrite\SDK\Method; @@ -58,33 +57,21 @@ class Get extends Action $checkStart = \microtime(true); foreach ($devices as $device) { - $uniqueFileName = \uniqid('health', true); - $filePath = $device->getPath($uniqueFileName); + $path = $device->getPath(\uniqid('health', true)); - if (!$device->write($filePath, 'test', 'text/plain')) { - throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Failed writing test file to ' . $device->getRoot()); - } - - $readError = null; try { - if ($device->read($filePath) !== 'test') { - $readError = new Exception(Exception::GENERAL_SERVER_ERROR, 'Failed reading test file from ' . $device->getRoot()); + if (!$device->write($path, 'test', 'text/plain')) { + throw new \Exception("Failed writing test file to {$device->getRoot()}"); + } + + $content = $device->read($path); + if ($content !== 'test') { + throw new \Exception("Failed reading test file from {$device->getRoot()}: content mismatch"); } - } catch (\Throwable $e) { - $readError = $e; } finally { - // Always attempt to clean up test file - if (!$device->delete($filePath)) { - if ($readError !== null) { - // If read already failed, wrap delete error but preserve original - \error_log('Failed deleting test file from ' . $device->getRoot() . ' during read error recovery'); - } else { - throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Failed deleting test file from ' . $device->getRoot()); - } - } - // Re-throw read error if it occurred - if ($readError !== null) { - throw $readError; + try { + $device->delete($path); + } catch (\Throwable) { } } } diff --git a/src/Appwrite/Platform/Modules/Proxy/Http/Rules/Site/Create.php b/src/Appwrite/Platform/Modules/Proxy/Http/Rules/Site/Create.php index 47195a3eb5..ba99cefb42 100644 --- a/src/Appwrite/Platform/Modules/Proxy/Http/Rules/Site/Create.php +++ b/src/Appwrite/Platform/Modules/Proxy/Http/Rules/Site/Create.php @@ -75,7 +75,7 @@ class Create extends Action ->callback($this->action(...)); } - public function action(string $domain, string $siteId, string $branch, Response $response, Document $project, Certificate $queueForCertificates, Event $queueForEvents, Database $dbForPlatform, Database $dbForProject, array $platform, Log $log) + public function action(string $domain, string $siteId, ?string $branch, Response $response, Document $project, Certificate $queueForCertificates, Event $queueForEvents, Database $dbForPlatform, Database $dbForProject, array $platform, Log $log) { $this->validateDomainRestrictions($domain, $platform); diff --git a/src/Appwrite/Platform/Modules/Teams/Http/Memberships/Create.php b/src/Appwrite/Platform/Modules/Teams/Http/Memberships/Create.php index 3bf597eaca..221c3aa521 100644 --- a/src/Appwrite/Platform/Modules/Teams/Http/Memberships/Create.php +++ b/src/Appwrite/Platform/Modules/Teams/Http/Memberships/Create.php @@ -6,7 +6,6 @@ use Appwrite\Auth\Validator\Phone; use Appwrite\Event\Event; use Appwrite\Event\Mail; use Appwrite\Event\Messaging; -use Appwrite\Event\StatsUsage; use Appwrite\Extend\Exception; use Appwrite\Network\Validator\Email as EmailValidator; use Appwrite\Platform\Action; @@ -14,6 +13,7 @@ use Appwrite\SDK\AuthType; use Appwrite\SDK\Method; use Appwrite\SDK\Response as SDKResponse; use Appwrite\Template\Template; +use Appwrite\Usage\Context; use Appwrite\Utopia\Database\Documents\User; use Appwrite\Utopia\Response; use libphonenumber\NumberParseException; @@ -70,7 +70,7 @@ class Create extends Action new SDKResponse( code: Response::STATUS_CODE_CREATED, model: Response::MODEL_MEMBERSHIP, - ) + ), ] )) ->label('abuse-limit', 10) @@ -91,20 +91,20 @@ class Create extends Action ->inject('queueForMessaging') ->inject('queueForEvents') ->inject('timelimit') - ->inject('queueForStatsUsage') + ->inject('usage') ->inject('plan') ->inject('proofForPassword') ->inject('proofForToken') ->callback($this->action(...)); } - public function action(string $teamId, string $email, string $userId, string $phone, array $roles, string $url, string $name, Response $response, Document $project, Document $user, Database $dbForProject, Authorization $authorization, Locale $locale, Mail $queueForMails, Messaging $queueForMessaging, Event $queueForEvents, callable $timelimit, StatsUsage $queueForStatsUsage, array $plan, Password $proofForPassword, Token $proofForToken) + public function action(string $teamId, string $email, string $userId, string $phone, array $roles, string $url, string $name, Response $response, Document $project, Document $user, Database $dbForProject, Authorization $authorization, Locale $locale, Mail $queueForMails, Messaging $queueForMessaging, Event $queueForEvents, callable $timelimit, Context $usage, array $plan, Password $proofForPassword, Token $proofForToken) { $isAppUser = User::isApp($authorization->getRoles()); $isPrivilegedUser = User::isPrivileged($authorization->getRoles()); if (empty($url)) { - if (!$isAppUser && !$isPrivilegedUser) { + if (! $isAppUser && ! $isPrivilegedUser) { throw new Exception(Exception::GENERAL_ARGUMENT_INVALID, 'URL is required'); } } @@ -113,7 +113,7 @@ class Create extends Action throw new Exception(Exception::GENERAL_ARGUMENT_INVALID, 'At least one of userId, email, or phone is required'); } - if (!$isPrivilegedUser && !$isAppUser && empty(System::getEnv('_APP_SMTP_HOST'))) { + if (! $isPrivilegedUser && ! $isAppUser && empty(System::getEnv('_APP_SMTP_HOST'))) { throw new Exception(Exception::GENERAL_SMTP_DISABLED); } @@ -124,28 +124,28 @@ class Create extends Action if ($team->isEmpty()) { throw new Exception(Exception::TEAM_NOT_FOUND); } - if (!empty($userId)) { + if (! empty($userId)) { $invitee = $dbForProject->getDocument('users', $userId); if ($invitee->isEmpty()) { throw new Exception(Exception::USER_NOT_FOUND, 'User with given userId doesn\'t exist.', 404); } - if (!empty($email) && $invitee->getAttribute('email', '') !== $email) { + if (! empty($email) && $invitee->getAttribute('email', '') !== $email) { throw new Exception(Exception::USER_ALREADY_EXISTS, 'Given userId and email doesn\'t match', 409); } - if (!empty($phone) && $invitee->getAttribute('phone', '') !== $phone) { + if (! empty($phone) && $invitee->getAttribute('phone', '') !== $phone) { throw new Exception(Exception::USER_ALREADY_EXISTS, 'Given userId and phone doesn\'t match', 409); } $email = $invitee->getAttribute('email', ''); $phone = $invitee->getAttribute('phone', ''); $name = $invitee->getAttribute('name', '') ?: $name; - } elseif (!empty($email)) { + } elseif (! empty($email)) { $invitee = $dbForProject->findOne('users', [Query::equal('email', [$email])]); // Get user by email address - if (!$invitee->isEmpty() && !empty($phone) && $invitee->getAttribute('phone', '') !== $phone) { + if (! $invitee->isEmpty() && ! empty($phone) && $invitee->getAttribute('phone', '') !== $phone) { throw new Exception(Exception::USER_ALREADY_EXISTS, 'Given email and phone doesn\'t match', 409); } - } elseif (!empty($phone)) { + } elseif (! empty($phone)) { $invitee = $dbForProject->findOne('users', [Query::equal('phone', [$phone])]); - if (!$invitee->isEmpty() && !empty($email) && $invitee->getAttribute('email', '') !== $email) { + if (! $invitee->isEmpty() && ! empty($email) && $invitee->getAttribute('email', '') !== $email) { throw new Exception(Exception::USER_ALREADY_EXISTS, 'Given phone and email doesn\'t match', 409); } } @@ -153,7 +153,7 @@ class Create extends Action if ($invitee->isEmpty()) { // Create new user if no user with same email found $limit = $project->getAttribute('auths', [])['limit'] ?? 0; - if (!$isPrivilegedUser && !$isAppUser && $limit !== 0 && $project->getId() !== 'console') { // check users limit, console invites are allways allowed. + if (! $isPrivilegedUser && ! $isAppUser && $limit !== 0 && $project->getId() !== 'console') { // check users limit, console invites are allways allowed. $total = $dbForProject->count('users', [], APP_LIMIT_USERS); if ($total >= $limit) { @@ -165,7 +165,7 @@ class Create extends Action $identityWithMatchingEmail = $dbForProject->findOne('identities', [ Query::equal('providerEmail', [$email]), ]); - if (!$identityWithMatchingEmail->isEmpty()) { + if (! $identityWithMatchingEmail->isEmpty()) { throw new Exception(Exception::USER_EMAIL_ALREADY_EXISTS); } @@ -225,7 +225,7 @@ class Create extends Action $isOwner = $authorization->hasRole('team:' . $team->getId() . '/owner'); - if (!$isOwner && !$isPrivilegedUser && !$isAppUser) { // Not owner, not admin, not app (server) + if (! $isOwner && ! $isPrivilegedUser && ! $isAppUser) { // Not owner, not admin, not app (server) throw new Exception(Exception::USER_UNAUTHORIZED, 'User is not allowed to send invitations for this team'); } @@ -255,7 +255,7 @@ class Create extends Action 'joined' => ($isPrivilegedUser || $isAppUser) ? DateTime::now() : null, 'confirm' => ($isPrivilegedUser || $isAppUser), 'secret' => $proofForToken->hash($secret), - 'search' => implode(' ', [$membershipId, $invitee->getId()]) + 'search' => implode(' ', [$membershipId, $invitee->getId()]), ]); $membership = ($isPrivilegedUser || $isAppUser) ? @@ -292,22 +292,22 @@ class Create extends Action $url = Template::parseURL($url); $url['query'] = Template::mergeQuery(((isset($url['query'])) ? $url['query'] : ''), ['membershipId' => $membership->getId(), 'userId' => $invitee->getId(), 'secret' => $secret, 'teamId' => $teamId, 'teamName' => $team->getAttribute('name')]); $url = Template::unParseURL($url); - if (!empty($email)) { + if (! empty($email)) { $projectName = $project->isEmpty() ? 'Console' : $project->getAttribute('name', '[APP-NAME]'); - $body = $locale->getText("emails.invitation.body"); - $preview = $locale->getText("emails.invitation.preview"); - $subject = $locale->getText("emails.invitation.subject"); + $body = $locale->getText('emails.invitation.body'); + $preview = $locale->getText('emails.invitation.preview'); + $subject = $locale->getText('emails.invitation.subject'); $customTemplate = $project->getAttribute('templates', [])['email.invitation-' . $locale->default] ?? []; $message = Template::fromFile(APP_CE_CONFIG_DIR . '/locale/templates/email-inner-base.tpl'); $message ->setParam('{{body}}', $body, escapeHtml: false) - ->setParam('{{hello}}', $locale->getText("emails.invitation.hello")) - ->setParam('{{footer}}', $locale->getText("emails.invitation.footer")) - ->setParam('{{thanks}}', $locale->getText("emails.invitation.thanks")) - ->setParam('{{buttonText}}', $locale->getText("emails.invitation.buttonText")) - ->setParam('{{signature}}', $locale->getText("emails.invitation.signature")); + ->setParam('{{hello}}', $locale->getText('emails.invitation.hello')) + ->setParam('{{footer}}', $locale->getText('emails.invitation.footer')) + ->setParam('{{thanks}}', $locale->getText('emails.invitation.thanks')) + ->setParam('{{buttonText}}', $locale->getText('emails.invitation.buttonText')) + ->setParam('{{signature}}', $locale->getText('emails.invitation.signature')); $body = $message->render(); $smtp = $project->getAttribute('smtp', []); @@ -315,16 +315,16 @@ class Create extends Action $senderEmail = System::getEnv('_APP_SYSTEM_EMAIL_ADDRESS', APP_EMAIL_TEAM); $senderName = System::getEnv('_APP_SYSTEM_EMAIL_NAME', APP_NAME . ' Server'); - $replyTo = ""; + $replyTo = ''; if ($smtpEnabled) { - if (!empty($smtp['senderEmail'])) { + if (! empty($smtp['senderEmail'])) { $senderEmail = $smtp['senderEmail']; } - if (!empty($smtp['senderName'])) { + if (! empty($smtp['senderName'])) { $senderName = $smtp['senderName']; } - if (!empty($smtp['replyTo'])) { + if (! empty($smtp['replyTo'])) { $replyTo = $smtp['replyTo']; } @@ -335,14 +335,14 @@ class Create extends Action ->setSmtpPassword($smtp['password'] ?? '') ->setSmtpSecure($smtp['secure'] ?? ''); - if (!empty($customTemplate)) { - if (!empty($customTemplate['senderEmail'])) { + if (! empty($customTemplate)) { + if (! empty($customTemplate['senderEmail'])) { $senderEmail = $customTemplate['senderEmail']; } - if (!empty($customTemplate['senderName'])) { + if (! empty($customTemplate['senderName'])) { $senderName = $customTemplate['senderName']; } - if (!empty($customTemplate['replyTo'])) { + if (! empty($customTemplate['replyTo'])) { $replyTo = $customTemplate['replyTo']; } @@ -363,7 +363,7 @@ class Create extends Action 'user' => $name, 'team' => $team->getAttribute('name'), 'redirect' => $url, - 'project' => $projectName + 'project' => $projectName, ]; $queueForMails @@ -374,7 +374,7 @@ class Create extends Action ->setName($invitee->getAttribute('name', '')) ->appendVariables($emailVariables) ->trigger(); - } elseif (!empty($phone)) { + } elseif (! empty($phone)) { if (empty(System::getEnv('_APP_SMS_PROVIDER'))) { throw new Exception(Exception::GENERAL_PHONE_DISABLED, 'Phone provider not configured'); } @@ -382,7 +382,7 @@ class Create extends Action $message = Template::fromFile(APP_CE_CONFIG_DIR . '/locale/templates/sms-base.tpl'); $customTemplate = $project->getAttribute('templates', [])['sms.invitation-' . $locale->default] ?? []; - if (!empty($customTemplate)) { + if (! empty($customTemplate)) { $message = $customTemplate['message']; } @@ -406,25 +406,20 @@ class Create extends Action try { $countryCode = $helper->parse($phone)->getCountryCode(); - if (!empty($countryCode)) { - $queueForStatsUsage - ->addMetric(str_replace('{countryCode}', $countryCode, METRIC_AUTH_METHOD_PHONE_COUNTRY_CODE), 1); + if (! empty($countryCode)) { + $usage->addMetric(str_replace('{countryCode}', $countryCode, METRIC_AUTH_METHOD_PHONE_COUNTRY_CODE), 1); } } catch (NumberParseException $e) { // Ignore invalid phone number for country code stats } - $queueForStatsUsage - ->addMetric(METRIC_AUTH_METHOD_PHONE, 1) - ->setProject($project) - ->trigger(); + $usage->addMetric(METRIC_AUTH_METHOD_PHONE, 1); } } $queueForEvents ->setParam('userId', $invitee->getId()) ->setParam('teamId', $team->getId()) - ->setParam('membershipId', $membership->getId()) - ; + ->setParam('membershipId', $membership->getId()); $response ->setStatusCode(Response::STATUS_CODE_CREATED) diff --git a/src/Appwrite/Platform/Workers/Mails.php b/src/Appwrite/Platform/Workers/Mails.php index 72f7cddd06..f144c58e1b 100644 --- a/src/Appwrite/Platform/Workers/Mails.php +++ b/src/Appwrite/Platform/Workers/Mails.php @@ -214,6 +214,7 @@ class Mails extends Action $mail->Password = $password; $mail->SMTPSecure = $smtp['secure']; $mail->SMTPAutoTLS = false; + $mail->SMTPKeepAlive = true; $mail->CharSet = 'UTF-8'; $mail->Timeout = 10; /* Connection timeout */ $mail->getSMTPInstance()->Timelimit = 30; /* Timeout for each individual SMTP command (e.g. HELO, EHLO, etc.) */ diff --git a/src/Appwrite/Platform/Workers/Messaging.php b/src/Appwrite/Platform/Workers/Messaging.php index d866cc2bd0..af7d2027e3 100644 --- a/src/Appwrite/Platform/Workers/Messaging.php +++ b/src/Appwrite/Platform/Workers/Messaging.php @@ -2,8 +2,10 @@ namespace Appwrite\Platform\Workers; -use Appwrite\Event\StatsUsage; +use Appwrite\Event\Message\Usage; +use Appwrite\Event\Publisher\Usage as UsagePublisher; use Appwrite\Messaging\Status as MessageStatus; +use Appwrite\Usage\Context as UsageContext; use libphonenumber\NumberParseException; use libphonenumber\PhoneNumberUtil; use Swoole\Runtime; @@ -71,7 +73,7 @@ class Messaging extends Action ->inject('log') ->inject('dbForProject') ->inject('deviceForFiles') - ->inject('queueForStatsUsage') + ->inject('publisherForUsage') ->callback($this->action(...)); } @@ -81,7 +83,7 @@ class Messaging extends Action * @param Log $log * @param Database $dbForProject * @param Device $deviceForFiles - * @param StatsUsage $queueForStatsUsage + * @param UsagePublisher $publisherForUsage * @return void * @throws \Exception */ @@ -91,7 +93,7 @@ class Messaging extends Action Log $log, Database $dbForProject, Device $deviceForFiles, - StatsUsage $queueForStatsUsage + UsagePublisher $publisherForUsage ): void { Runtime::setHookFlags(SWOOLE_HOOK_ALL ^ SWOOLE_HOOK_TCP); $payload = $message->getPayload() ?? []; @@ -115,7 +117,7 @@ class Messaging extends Action case MESSAGE_SEND_TYPE_EXTERNAL: $message = $dbForProject->getDocument('messages', $payload['messageId']); - $this->sendExternalMessage($dbForProject, $message, $deviceForFiles, $project, $queueForStatsUsage); + $this->sendExternalMessage($dbForProject, $message, $deviceForFiles, $project, $publisherForUsage); break; default: throw new \Exception('Unknown message type: ' . $type); @@ -133,7 +135,7 @@ class Messaging extends Action Document $message, Device $deviceForFiles, Document $project, - StatsUsage $queueForStatsUsage + UsagePublisher $publisherForUsage ): void { $topicIds = $message->getAttribute('topics', []); $targetIds = $message->getAttribute('targets', []); @@ -239,8 +241,8 @@ class Messaging extends Action /** * @var array $results */ - $results = batch(\array_map(function ($providerId) use ($identifiers, &$providers, $default, $message, $dbForProject, $deviceForFiles, $project, $queueForStatsUsage) { - return function () use ($providerId, $identifiers, &$providers, $default, $message, $dbForProject, $deviceForFiles, $project, $queueForStatsUsage) { + $results = batch(\array_map(function ($providerId) use ($identifiers, &$providers, $default, $message, $dbForProject, $deviceForFiles, $project, $publisherForUsage) { + return function () use ($providerId, $identifiers, &$providers, $default, $message, $dbForProject, $deviceForFiles, $project, $publisherForUsage) { if (\array_key_exists($providerId, $providers)) { $provider = $providers[$providerId]; } else { @@ -267,8 +269,8 @@ class Messaging extends Action $adapter->getMaxMessagesPerRequest() ); - return batch(\array_map(function ($batch) use ($message, $provider, $adapter, $dbForProject, $deviceForFiles, $project, $queueForStatsUsage) { - return function () use ($batch, $message, $provider, $adapter, $dbForProject, $deviceForFiles, $project, $queueForStatsUsage) { + return batch(\array_map(function ($batch) use ($message, $provider, $adapter, $dbForProject, $deviceForFiles, $project, $publisherForUsage) { + return function () use ($batch, $message, $provider, $adapter, $dbForProject, $deviceForFiles, $project, $publisherForUsage) { $deliveredTotal = 0; $deliveryErrors = []; $messageData = clone $message; @@ -308,8 +310,8 @@ class Messaging extends Action $deliveryErrors[] = 'Failed sending to targets with error: ' . $e->getMessage(); } finally { $errorTotal = \count($deliveryErrors); - $queueForStatsUsage - ->setProject($project) + $usage = new UsageContext(); + $usage ->addMetric(METRIC_MESSAGES, ($deliveredTotal + $errorTotal)) ->addMetric(METRIC_MESSAGES_SENT, $deliveredTotal) ->addMetric(METRIC_MESSAGES_FAILED, $errorTotal) @@ -318,8 +320,12 @@ class Messaging extends Action ->addMetric(str_replace('{type}', $provider->getAttribute('type'), METRIC_MESSAGES_TYPE_FAILED), $errorTotal) ->addMetric(str_replace(['{type}', '{provider}'], [$provider->getAttribute('type'), $provider->getAttribute('provider')], METRIC_MESSAGES_TYPE_PROVIDER), ($deliveredTotal + $errorTotal)) ->addMetric(str_replace(['{type}', '{provider}'], [$provider->getAttribute('type'), $provider->getAttribute('provider')], METRIC_MESSAGES_TYPE_PROVIDER_SENT), $deliveredTotal) - ->addMetric(str_replace(['{type}', '{provider}'], [$provider->getAttribute('type'), $provider->getAttribute('provider')], METRIC_MESSAGES_TYPE_PROVIDER_FAILED), $errorTotal) - ->trigger(); + ->addMetric(str_replace(['{type}', '{provider}'], [$provider->getAttribute('type'), $provider->getAttribute('provider')], METRIC_MESSAGES_TYPE_PROVIDER_FAILED), $errorTotal); + + $publisherForUsage->enqueue(new Usage( + project: $project, + metrics: $usage->getMetrics(), + )); return [ 'deliveredTotal' => $deliveredTotal, diff --git a/src/Appwrite/Platform/Workers/Migrations.php b/src/Appwrite/Platform/Workers/Migrations.php index d87edaf788..ce20358626 100644 --- a/src/Appwrite/Platform/Workers/Migrations.php +++ b/src/Appwrite/Platform/Workers/Migrations.php @@ -4,10 +4,12 @@ namespace Appwrite\Platform\Workers; use Ahc\Jwt\JWT; use Appwrite\Event\Mail; +use Appwrite\Event\Message\Usage as UsageMessage; +use Appwrite\Event\Publisher\Usage as UsagePublisher; use Appwrite\Event\Realtime; -use Appwrite\Event\StatsUsage; use Appwrite\Extend\Exception; use Appwrite\Template\Template; +use Appwrite\Usage\Context; use Utopia\Compression\Compression; use Utopia\Config\Config; use Utopia\Console; @@ -84,7 +86,8 @@ class Migrations extends Action ->inject('deviceForMigrations') ->inject('deviceForFiles') ->inject('queueForMails') - ->inject('queueForStatsUsage') + ->inject('usage') + ->inject('publisherForUsage') ->inject('plan') ->inject('authorization') ->callback($this->action(...)); @@ -103,7 +106,8 @@ class Migrations extends Action Device $deviceForMigrations, Device $deviceForFiles, Mail $queueForMails, - StatsUsage $queueForStatsUsage, + Context $usage, + UsagePublisher $publisherForUsage, array $plan, Authorization $authorization, ): void { @@ -147,7 +151,8 @@ class Migrations extends Action $migration, $queueForRealtime, $queueForMails, - $queueForStatsUsage, + $usage, + $publisherForUsage, $platform, $authorization ); @@ -317,6 +322,16 @@ class Migrations extends Action 'sites.write', 'tokens.read', 'tokens.write', + 'providers.read', + 'providers.write', + 'topics.read', + 'topics.write', + 'subscribers.read', + 'subscribers.write', + 'messages.read', + 'messages.write', + 'targets.read', + 'targets.write', ] ]); @@ -335,7 +350,8 @@ class Migrations extends Action Document $migration, Realtime $queueForRealtime, Mail $queueForMails, - StatsUsage $queueForStatsUsage, + Context $usage, + UsagePublisher $publisherForUsage, array $platform, Authorization $authorization, ): void { @@ -350,7 +366,7 @@ class Migrations extends Action throw new \Exception('_APP_MIGRATION_HOST is not set'); } - $endpoint = 'http://'.$host.'/v1'; + $endpoint = 'http://' . $host . '/v1'; try { $credentials = $migration->getAttribute('credentials', []); @@ -453,7 +469,7 @@ class Migrations extends Action $migration->setAttribute('status', 'failed'); $migration->setAttribute('stage', 'finished'); - call_user_func($this->logError, $th, 'appwrite-worker', 'appwrite-queue-'.self::getName(), [ + call_user_func($this->logError, $th, 'appwrite-worker', 'appwrite-queue-' . self::getName(), [ 'migrationId' => $migration->getId(), 'source' => $migration->getAttribute('source') ?? '', 'destination' => $migration->getAttribute('destination') ?? '', @@ -464,7 +480,7 @@ class Migrations extends Action $this->updateMigrationDocument($migration, $project, $queueForRealtime); if ($migration->getAttribute('status', '') === 'failed') { - Console::error('Migration('.$migration->getSequence().':'.$migration->getId().') failed, Project('.$this->project->getSequence().':'.$this->project->getId().')'); + Console::error('Migration(' . $migration->getSequence() . ':' . $migration->getId() . ') failed, Project(' . $this->project->getSequence() . ':' . $this->project->getId() . ')'); $sourceErrors = $source?->getErrors() ?? []; $destinationErrors = $destination?->getErrors() ?? []; @@ -490,8 +506,9 @@ class Migrations extends Action foreach ($aggregatedResources as $resource) { $this->processMigrationResourceStats( $resource, - $queueForStatsUsage, + $usage, $project, + $publisherForUsage, $migration->getAttribute('source'), $authorization, $migration->getAttribute('resourceId') @@ -792,7 +809,7 @@ class Migrations extends Action return $errors; } - private function processMigrationResourceStats(array $resources, StatsUsage $queueForStatsUsage, Document $projectDocument, string $source, Authorization $authorization, ?string $resourceId) + private function processMigrationResourceStats(array $resources, Context $usage, Document $projectDocument, UsagePublisher $publisherForUsage, string $source, Authorization $authorization, ?string $resourceId) { $resourceName = $resources['name']; $count = $resources['count']; @@ -809,11 +826,11 @@ class Migrations extends Action switch ($resourceName) { case ResourceDatabase::getName(): - $queueForStatsUsage->addMetric(METRIC_DATABASES, $count); + $usage->addMetric(METRIC_DATABASES, $count); break; case ResourceTable::getName(): - $queueForStatsUsage + $usage ->addMetric(METRIC_COLLECTIONS, $count) ->addMetric( str_replace('{databaseInternalId}', $databaseInternalId, METRIC_DATABASE_ID_COLLECTIONS), @@ -822,7 +839,7 @@ class Migrations extends Action break; case ResourceRow::getName(): - $queueForStatsUsage + $usage ->addMetric( str_replace( ['{databaseInternalId}','{collectionInternalId}'], @@ -842,7 +859,12 @@ class Migrations extends Action break; } - $queueForStatsUsage->setProject($projectDocument)->trigger(); - $queueForStatsUsage->reset(); + $message = new UsageMessage( + project: $projectDocument, + metrics: $usage->getMetrics(), + reduce: $usage->getReduce() + ); + $publisherForUsage->enqueue($message); + $usage->reset(); } } diff --git a/src/Appwrite/Platform/Workers/StatsUsage.php b/src/Appwrite/Platform/Workers/StatsUsage.php index 56298a0dcd..07051d1f15 100644 --- a/src/Appwrite/Platform/Workers/StatsUsage.php +++ b/src/Appwrite/Platform/Workers/StatsUsage.php @@ -160,7 +160,7 @@ class StatsUsage extends Action } $this->stats[$projectId]['project'] = $project; - $this->stats[$projectId]['receivedAt'] = DateTime::now(); + $this->stats[$projectId]['receivedAt'] = DateTime::format(new \DateTime('@' . $message->getTimestamp())); foreach ($payload['metrics'] ?? [] as $metric) { $this->keys++; if (!isset($this->stats[$projectId]['keys'][$metric['key']])) { diff --git a/src/Appwrite/Platform/Workers/Webhooks.php b/src/Appwrite/Platform/Workers/Webhooks.php index 4855a1d4d8..fce3c7b149 100644 --- a/src/Appwrite/Platform/Workers/Webhooks.php +++ b/src/Appwrite/Platform/Workers/Webhooks.php @@ -3,8 +3,10 @@ namespace Appwrite\Platform\Workers; use Appwrite\Event\Mail; -use Appwrite\Event\StatsUsage; +use Appwrite\Event\Message\Usage as UsageMessage; +use Appwrite\Event\Publisher\Usage as UsagePublisher; use Appwrite\Template\Template; +use Appwrite\Usage\Context as UsageContext; use Exception; use Utopia\Database\Database; use Utopia\Database\Document; @@ -35,7 +37,7 @@ class Webhooks extends Action ->inject('project') ->inject('dbForPlatform') ->inject('queueForMails') - ->inject('queueForStatsUsage') + ->inject('publisherForUsage') ->inject('log') ->inject('plan') ->callback($this->action(...)); @@ -46,13 +48,13 @@ class Webhooks extends Action * @param Document $project * @param Database $dbForPlatform * @param Mail $queueForMails - * @param StatsUsage $queueForStatsUsage + * @param UsagePublisher $publisherForUsage * @param Log $log * @param array $plan * @return void * @throws Exception */ - public function action(Message $message, Document $project, Database $dbForPlatform, Mail $queueForMails, StatsUsage $queueForStatsUsage, Log $log, array $plan): void + public function action(Message $message, Document $project, Database $dbForPlatform, Mail $queueForMails, UsagePublisher $publisherForUsage, Log $log, array $plan): void { $this->errors = []; $payload = $message->getPayload() ?? []; @@ -71,7 +73,7 @@ class Webhooks extends Action foreach ($project->getAttribute('webhooks', []) as $webhook) { if (array_intersect($webhook->getAttribute('events', []), $events)) { - $this->execute($events, $webhookPayload, $webhook, $user, $project, $dbForPlatform, $queueForMails, $queueForStatsUsage, $plan); + $this->execute($events, $webhookPayload, $webhook, $user, $project, $dbForPlatform, $queueForMails, $publisherForUsage, $plan); } } @@ -91,7 +93,7 @@ class Webhooks extends Action * @param array $plan * @return void */ - private function execute(array $events, string $payload, Document $webhook, Document $user, Document $project, Database $dbForPlatform, Mail $queueForMails, StatsUsage $queueForStatsUsage, array $plan): void + private function execute(array $events, string $payload, Document $webhook, Document $user, Document $project, Database $dbForPlatform, Mail $queueForMails, UsagePublisher $publisherForUsage, array $plan): void { if ($webhook->getAttribute('enabled') !== true) { return; @@ -180,26 +182,23 @@ class Webhooks extends Action $dbForPlatform->purgeCachedDocument('projects', $project->getId()); $this->errors[] = $logs; - $queueForStatsUsage + $usage = (new UsageContext()) ->addMetric(METRIC_WEBHOOKS_FAILED, 1) - ->addMetric(str_replace('{webhookInternalId}', $webhook->getSequence(), METRIC_WEBHOOK_ID_FAILED), 1) - ; - - + ->addMetric(str_replace('{webhookInternalId}', $webhook->getSequence(), METRIC_WEBHOOK_ID_FAILED), 1); } else { $dbForPlatform->updateDocument('webhooks', $webhook->getId(), new Document([ 'attempts' => 0, ])); $dbForPlatform->purgeCachedDocument('projects', $project->getId()); - $queueForStatsUsage + $usage = (new UsageContext()) ->addMetric(METRIC_WEBHOOKS_SENT, 1) - ->addMetric(str_replace('{webhookInternalId}', $webhook->getSequence(), METRIC_WEBHOOK_ID_SENT), 1) - ; + ->addMetric(str_replace('{webhookInternalId}', $webhook->getSequence(), METRIC_WEBHOOK_ID_SENT), 1); } - $queueForStatsUsage - ->setProject($project) - ->trigger(); + $publisherForUsage->enqueue(new UsageMessage( + project: $project, + metrics: $usage->getMetrics(), + )); } /** diff --git a/src/Appwrite/Usage/Context.php b/src/Appwrite/Usage/Context.php new file mode 100644 index 0000000000..7283cff836 --- /dev/null +++ b/src/Appwrite/Usage/Context.php @@ -0,0 +1,74 @@ +metrics[] = [ + 'key' => $key, + 'value' => $value, + ]; + + return $this; + } + + /** + * Add a document to reduce + */ + public function addReduce(Document $document): self + { + $this->reduce[] = $document; + + return $this; + } + + /** + * Get all metrics + * + * @return array + */ + public function getMetrics(): array + { + return $this->metrics; + } + + /** + * Get all reduce documents + * + * @return array + */ + public function getReduce(): array + { + return $this->reduce; + } + + /** + * Check if context is empty + */ + public function isEmpty(): bool + { + return empty($this->metrics) && empty($this->reduce); + } + + /** + * Reset the context + */ + public function reset(): self + { + $this->metrics = []; + $this->reduce = []; + + return $this; + } +} diff --git a/src/Appwrite/Utopia/Response/Model/MigrationReport.php b/src/Appwrite/Utopia/Response/Model/MigrationReport.php index 7ebc22d22e..850e4b5ae9 100644 --- a/src/Appwrite/Utopia/Response/Model/MigrationReport.php +++ b/src/Appwrite/Utopia/Response/Model/MigrationReport.php @@ -59,6 +59,30 @@ class MigrationReport extends Model 'default' => 0, 'example' => 5, ]) + ->addRule(Resource::TYPE_PROVIDER, [ + 'type' => self::TYPE_INTEGER, + 'description' => 'Number of providers to be migrated.', + 'default' => 0, + 'example' => 5, + ]) + ->addRule(Resource::TYPE_TOPIC, [ + 'type' => self::TYPE_INTEGER, + 'description' => 'Number of topics to be migrated.', + 'default' => 0, + 'example' => 10, + ]) + ->addRule(Resource::TYPE_SUBSCRIBER, [ + 'type' => self::TYPE_INTEGER, + 'description' => 'Number of subscribers to be migrated.', + 'default' => 0, + 'example' => 100, + ]) + ->addRule(Resource::TYPE_MESSAGE, [ + 'type' => self::TYPE_INTEGER, + 'description' => 'Number of messages to be migrated.', + 'default' => 0, + 'example' => 50, + ]) ->addRule('size', [ 'type' => self::TYPE_INTEGER, 'description' => 'Size of files to be migrated in mb.', diff --git a/src/Appwrite/Vcs/Comment.php b/src/Appwrite/Vcs/Comment.php index 90f9c8f95d..e6d6996748 100644 --- a/src/Appwrite/Vcs/Comment.php +++ b/src/Appwrite/Vcs/Comment.php @@ -251,7 +251,7 @@ class Comment $json = \base64_decode($state); $builds = \json_decode($json, true); - $this->builds = $builds; + $this->builds = \is_array($builds) ? $builds : []; return $this; } diff --git a/tests/e2e/Services/Functions/FunctionsConsoleClientTest.php b/tests/e2e/Services/Functions/FunctionsConsoleClientTest.php index cc18d14a8e..06044d9984 100644 --- a/tests/e2e/Services/Functions/FunctionsConsoleClientTest.php +++ b/tests/e2e/Services/Functions/FunctionsConsoleClientTest.php @@ -686,7 +686,7 @@ class FunctionsConsoleClientTest extends Scope $stdout = ''; $stderr = ''; - $code = Console::execute("docker exec appwrite time-travel --projectId={$this->getProject()['$id']} --resourceType=deployment --resourceId={$deploymentIdInactiveOld} --createdAt=2020-01-01T00:00:00Z", '', $stdout, $stderr); + $code = Console::execute("docker exec appwrite task-time-travel --projectId={$this->getProject()['$id']} --resourceType=deployment --resourceId={$deploymentIdInactiveOld} --createdAt=2020-01-01T00:00:00Z", '', $stdout, $stderr); $this->assertSame(0, $code, "Time-travel command failed with code $code: $stderr ($stdout)"); $stdout = ''; diff --git a/tests/e2e/Services/Migrations/MigrationsBase.php b/tests/e2e/Services/Migrations/MigrationsBase.php index 606f9e8127..0d992c472e 100644 --- a/tests/e2e/Services/Migrations/MigrationsBase.php +++ b/tests/e2e/Services/Migrations/MigrationsBase.php @@ -1651,9 +1651,9 @@ trait MigrationsBase }, 30_000, 500); // Check that email was sent with download link - $lastEmail = $this->getLastEmail(); - $this->assertNotEmpty($lastEmail); - $this->assertEquals('Your CSV export is ready', $lastEmail['subject']); + $lastEmail = $this->getLastEmail(probe: function ($email) { + $this->assertEquals('Your CSV export is ready', $email['subject']); + }); $this->assertStringContainsStringIgnoringCase('Your data export has been completed successfully', $lastEmail['text']); // Extract download URL from email HTML @@ -1694,4 +1694,842 @@ trait MigrationsBase 'x-appwrite-key' => $this->getProject()['apiKey'] ]); } + + /** + * Messaging + */ + public function testAppwriteMigrationMessagingProvider(): void + { + $provider = $this->client->call(Client::METHOD_POST, '/messaging/providers/sendgrid', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ], [ + 'providerId' => ID::unique(), + 'name' => 'Migration Sendgrid', + 'apiKey' => 'my-apikey', + 'from' => 'migration@test.com', + ]); + + $this->assertEquals(201, $provider['headers']['status-code']); + $this->assertNotEmpty($provider['body']['$id']); + + $providerId = $provider['body']['$id']; + + $result = $this->performMigrationSync([ + 'resources' => [ + Resource::TYPE_PROVIDER, + ], + 'endpoint' => $this->webEndpoint, + 'projectId' => $this->getProject()['$id'], + 'apiKey' => $this->getProject()['apiKey'], + ]); + + $this->assertEquals('completed', $result['status']); + $this->assertEquals([Resource::TYPE_PROVIDER], $result['resources']); + $this->assertArrayHasKey(Resource::TYPE_PROVIDER, $result['statusCounters']); + $this->assertEquals(0, $result['statusCounters'][Resource::TYPE_PROVIDER]['error']); + $this->assertEquals(0, $result['statusCounters'][Resource::TYPE_PROVIDER]['pending']); + $this->assertGreaterThanOrEqual(1, $result['statusCounters'][Resource::TYPE_PROVIDER]['success']); + $this->assertEquals(0, $result['statusCounters'][Resource::TYPE_PROVIDER]['processing']); + $this->assertEquals(0, $result['statusCounters'][Resource::TYPE_PROVIDER]['warning']); + + $response = $this->client->call(Client::METHOD_GET, '/messaging/providers/' . $providerId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getDestinationProject()['$id'], + 'x-appwrite-key' => $this->getDestinationProject()['apiKey'], + ]); + + $this->assertEquals(200, $response['headers']['status-code']); + $this->assertEquals($providerId, $response['body']['$id']); + $this->assertEquals('Migration Sendgrid', $response['body']['name']); + $this->assertEquals('email', $response['body']['type']); + + // Cleanup + $this->client->call(Client::METHOD_DELETE, '/messaging/providers/' . $providerId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ]); + + $this->client->call(Client::METHOD_DELETE, '/messaging/providers/' . $providerId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getDestinationProject()['$id'], + 'x-appwrite-key' => $this->getDestinationProject()['apiKey'], + ]); + } + + public function testAppwriteMigrationMessagingProviderSMTP(): void + { + $provider = $this->client->call(Client::METHOD_POST, '/messaging/providers/smtp', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ], [ + 'providerId' => ID::unique(), + 'name' => 'Migration SMTP', + 'host' => 'smtp.test.com', + 'port' => 587, + 'from' => 'migration-smtp@test.com', + ]); + + $this->assertEquals(201, $provider['headers']['status-code']); + $providerId = $provider['body']['$id']; + + $result = $this->performMigrationSync([ + 'resources' => [ + Resource::TYPE_PROVIDER, + ], + 'endpoint' => $this->webEndpoint, + 'projectId' => $this->getProject()['$id'], + 'apiKey' => $this->getProject()['apiKey'], + ]); + + $this->assertEquals('completed', $result['status']); + $this->assertArrayHasKey(Resource::TYPE_PROVIDER, $result['statusCounters']); + $this->assertEquals(0, $result['statusCounters'][Resource::TYPE_PROVIDER]['error']); + $this->assertGreaterThanOrEqual(1, $result['statusCounters'][Resource::TYPE_PROVIDER]['success']); + + $response = $this->client->call(Client::METHOD_GET, '/messaging/providers/' . $providerId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getDestinationProject()['$id'], + 'x-appwrite-key' => $this->getDestinationProject()['apiKey'], + ]); + + $this->assertEquals(200, $response['headers']['status-code']); + $this->assertEquals($providerId, $response['body']['$id']); + $this->assertEquals('Migration SMTP', $response['body']['name']); + $this->assertEquals('email', $response['body']['type']); + $this->assertEquals('smtp', $response['body']['provider']); + + // Cleanup + $this->client->call(Client::METHOD_DELETE, '/messaging/providers/' . $providerId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ]); + + $this->client->call(Client::METHOD_DELETE, '/messaging/providers/' . $providerId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getDestinationProject()['$id'], + 'x-appwrite-key' => $this->getDestinationProject()['apiKey'], + ]); + } + + public function testAppwriteMigrationMessagingProviderTwilio(): void + { + $provider = $this->client->call(Client::METHOD_POST, '/messaging/providers/twilio', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ], [ + 'providerId' => ID::unique(), + 'name' => 'Migration Twilio', + 'from' => '+15551234567', + 'accountSid' => 'test-account-sid', + 'authToken' => 'test-auth-token', + ]); + + $this->assertEquals(201, $provider['headers']['status-code']); + $providerId = $provider['body']['$id']; + + $result = $this->performMigrationSync([ + 'resources' => [ + Resource::TYPE_PROVIDER, + ], + 'endpoint' => $this->webEndpoint, + 'projectId' => $this->getProject()['$id'], + 'apiKey' => $this->getProject()['apiKey'], + ]); + + $this->assertEquals('completed', $result['status']); + $this->assertArrayHasKey(Resource::TYPE_PROVIDER, $result['statusCounters']); + $this->assertEquals(0, $result['statusCounters'][Resource::TYPE_PROVIDER]['error']); + $this->assertGreaterThanOrEqual(1, $result['statusCounters'][Resource::TYPE_PROVIDER]['success']); + + $response = $this->client->call(Client::METHOD_GET, '/messaging/providers/' . $providerId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getDestinationProject()['$id'], + 'x-appwrite-key' => $this->getDestinationProject()['apiKey'], + ]); + + $this->assertEquals(200, $response['headers']['status-code']); + $this->assertEquals($providerId, $response['body']['$id']); + $this->assertEquals('Migration Twilio', $response['body']['name']); + $this->assertEquals('sms', $response['body']['type']); + $this->assertEquals('twilio', $response['body']['provider']); + + // Cleanup + $this->client->call(Client::METHOD_DELETE, '/messaging/providers/' . $providerId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ]); + + $this->client->call(Client::METHOD_DELETE, '/messaging/providers/' . $providerId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getDestinationProject()['$id'], + 'x-appwrite-key' => $this->getDestinationProject()['apiKey'], + ]); + } + + public function testAppwriteMigrationMessagingTopic(): void + { + $provider = $this->client->call(Client::METHOD_POST, '/messaging/providers/sendgrid', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ], [ + 'providerId' => ID::unique(), + 'name' => 'Migration Sendgrid Topic', + 'apiKey' => 'my-apikey', + 'from' => 'migration-topic@test.com', + ]); + + $this->assertEquals(201, $provider['headers']['status-code']); + $providerId = $provider['body']['$id']; + + $topic = $this->client->call(Client::METHOD_POST, '/messaging/topics', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ], [ + 'topicId' => ID::unique(), + 'name' => 'Migration Topic', + ]); + + $this->assertEquals(201, $topic['headers']['status-code']); + $this->assertNotEmpty($topic['body']['$id']); + + $topicId = $topic['body']['$id']; + + $result = $this->performMigrationSync([ + 'resources' => [ + Resource::TYPE_PROVIDER, + Resource::TYPE_TOPIC, + ], + 'endpoint' => $this->webEndpoint, + 'projectId' => $this->getProject()['$id'], + 'apiKey' => $this->getProject()['apiKey'], + ]); + + $this->assertEquals('completed', $result['status']); + $this->assertArrayHasKey(Resource::TYPE_TOPIC, $result['statusCounters']); + $this->assertEquals(0, $result['statusCounters'][Resource::TYPE_TOPIC]['error']); + $this->assertEquals(0, $result['statusCounters'][Resource::TYPE_TOPIC]['pending']); + $this->assertGreaterThanOrEqual(1, $result['statusCounters'][Resource::TYPE_TOPIC]['success']); + $this->assertEquals(0, $result['statusCounters'][Resource::TYPE_TOPIC]['processing']); + $this->assertEquals(0, $result['statusCounters'][Resource::TYPE_TOPIC]['warning']); + + $response = $this->client->call(Client::METHOD_GET, '/messaging/topics/' . $topicId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getDestinationProject()['$id'], + 'x-appwrite-key' => $this->getDestinationProject()['apiKey'], + ]); + + $this->assertEquals(200, $response['headers']['status-code']); + $this->assertEquals($topicId, $response['body']['$id']); + $this->assertEquals('Migration Topic', $response['body']['name']); + + // Cleanup + $this->client->call(Client::METHOD_DELETE, '/messaging/topics/' . $topicId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ]); + + $this->client->call(Client::METHOD_DELETE, '/messaging/topics/' . $topicId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getDestinationProject()['$id'], + 'x-appwrite-key' => $this->getDestinationProject()['apiKey'], + ]); + + $this->client->call(Client::METHOD_DELETE, '/messaging/providers/' . $providerId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ]); + + $this->client->call(Client::METHOD_DELETE, '/messaging/providers/' . $providerId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getDestinationProject()['$id'], + 'x-appwrite-key' => $this->getDestinationProject()['apiKey'], + ]); + } + + public function testAppwriteMigrationMessagingSubscriber(): void + { + $user = $this->client->call(Client::METHOD_POST, '/users', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ], [ + 'userId' => ID::unique(), + 'email' => uniqid() . '-migration-sub@test.com', + 'password' => 'password', + ]); + + $this->assertEquals(201, $user['headers']['status-code']); + $userId = $user['body']['$id']; + $this->assertEquals(1, \count($user['body']['targets'])); + $targetId = $user['body']['targets'][0]['$id']; + + $provider = $this->client->call(Client::METHOD_POST, '/messaging/providers/sendgrid', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ], [ + 'providerId' => ID::unique(), + 'name' => 'Migration Sendgrid Subscriber', + 'apiKey' => 'my-apikey', + 'from' => uniqid() . '-migration-sub@test.com', + ]); + + $this->assertEquals(201, $provider['headers']['status-code']); + $providerId = $provider['body']['$id']; + + $topic = $this->client->call(Client::METHOD_POST, '/messaging/topics', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ], [ + 'topicId' => ID::unique(), + 'name' => 'Migration Subscriber Topic', + ]); + + $this->assertEquals(201, $topic['headers']['status-code']); + $topicId = $topic['body']['$id']; + + $subscriber = $this->client->call(Client::METHOD_POST, '/messaging/topics/' . $topicId . '/subscribers', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ], [ + 'subscriberId' => ID::unique(), + 'targetId' => $targetId, + ]); + + $this->assertEquals(201, $subscriber['headers']['status-code']); + + $result = $this->performMigrationSync([ + 'resources' => [ + Resource::TYPE_USER, + Resource::TYPE_PROVIDER, + Resource::TYPE_TOPIC, + Resource::TYPE_SUBSCRIBER, + ], + 'endpoint' => $this->webEndpoint, + 'projectId' => $this->getProject()['$id'], + 'apiKey' => $this->getProject()['apiKey'], + ]); + + $this->assertEquals('completed', $result['status']); + $this->assertArrayHasKey(Resource::TYPE_SUBSCRIBER, $result['statusCounters']); + $this->assertEquals(0, $result['statusCounters'][Resource::TYPE_SUBSCRIBER]['error']); + $this->assertEquals(0, $result['statusCounters'][Resource::TYPE_SUBSCRIBER]['pending']); + $this->assertGreaterThanOrEqual(1, $result['statusCounters'][Resource::TYPE_SUBSCRIBER]['success']); + $this->assertEquals(0, $result['statusCounters'][Resource::TYPE_SUBSCRIBER]['processing']); + $this->assertEquals(0, $result['statusCounters'][Resource::TYPE_SUBSCRIBER]['warning']); + + $response = $this->client->call(Client::METHOD_GET, '/messaging/topics/' . $topicId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getDestinationProject()['$id'], + 'x-appwrite-key' => $this->getDestinationProject()['apiKey'], + ]); + + $this->assertEquals(200, $response['headers']['status-code']); + $this->assertEquals($topicId, $response['body']['$id']); + $this->assertGreaterThanOrEqual(1, $response['body']['emailTotal']); + + // Cleanup + $this->client->call(Client::METHOD_DELETE, '/messaging/topics/' . $topicId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ]); + + $this->client->call(Client::METHOD_DELETE, '/messaging/topics/' . $topicId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getDestinationProject()['$id'], + 'x-appwrite-key' => $this->getDestinationProject()['apiKey'], + ]); + + $this->client->call(Client::METHOD_DELETE, '/messaging/providers/' . $providerId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ]); + + $this->client->call(Client::METHOD_DELETE, '/messaging/providers/' . $providerId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getDestinationProject()['$id'], + 'x-appwrite-key' => $this->getDestinationProject()['apiKey'], + ]); + + $this->client->call(Client::METHOD_DELETE, '/users/' . $userId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ]); + + $this->client->call(Client::METHOD_DELETE, '/users/' . $userId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getDestinationProject()['$id'], + 'x-appwrite-key' => $this->getDestinationProject()['apiKey'], + ]); + } + + public function testAppwriteMigrationMessagingMessage(): void + { + $this->getDestinationProject(true); + + $user = $this->client->call(Client::METHOD_POST, '/users', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ], [ + 'userId' => ID::unique(), + 'email' => uniqid() . '-migration-msg@test.com', + 'password' => 'password', + ]); + + $this->assertEquals(201, $user['headers']['status-code']); + $userId = $user['body']['$id']; + $this->assertEquals(1, \count($user['body']['targets'])); + $targetId = $user['body']['targets'][0]['$id']; + + $provider = $this->client->call(Client::METHOD_POST, '/messaging/providers/sendgrid', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ], [ + 'providerId' => ID::unique(), + 'name' => 'Migration Sendgrid Message', + 'apiKey' => 'my-apikey', + 'from' => 'migration-msg@test.com', + ]); + + $this->assertEquals(201, $provider['headers']['status-code']); + $providerId = $provider['body']['$id']; + + $topic = $this->client->call(Client::METHOD_POST, '/messaging/topics', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ], [ + 'topicId' => ID::unique(), + 'name' => 'Migration Message Topic', + ]); + + $this->assertEquals(201, $topic['headers']['status-code']); + $topicId = $topic['body']['$id']; + + $message = $this->client->call(Client::METHOD_POST, '/messaging/messages/email', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ], [ + 'messageId' => ID::unique(), + 'targets' => [$targetId], + 'topics' => [$topicId], + 'subject' => 'Migration Test Email', + 'content' => 'This is a migration test email', + 'draft' => true, + ]); + + $this->assertEquals(201, $message['headers']['status-code']); + $this->assertNotEmpty($message['body']['$id']); + + $messageId = $message['body']['$id']; + + $result = $this->performMigrationSync([ + 'resources' => [ + Resource::TYPE_USER, + Resource::TYPE_PROVIDER, + Resource::TYPE_TOPIC, + Resource::TYPE_SUBSCRIBER, + Resource::TYPE_MESSAGE, + ], + 'endpoint' => $this->webEndpoint, + 'projectId' => $this->getProject()['$id'], + 'apiKey' => $this->getProject()['apiKey'], + ]); + + $this->assertEquals('completed', $result['status']); + $this->assertArrayHasKey(Resource::TYPE_MESSAGE, $result['statusCounters']); + $this->assertEquals(0, $result['statusCounters'][Resource::TYPE_MESSAGE]['error']); + $this->assertEquals(0, $result['statusCounters'][Resource::TYPE_MESSAGE]['pending']); + $this->assertGreaterThanOrEqual(1, $result['statusCounters'][Resource::TYPE_MESSAGE]['success']); + $this->assertEquals(0, $result['statusCounters'][Resource::TYPE_MESSAGE]['processing']); + $this->assertEquals(0, $result['statusCounters'][Resource::TYPE_MESSAGE]['warning']); + + $response = $this->client->call(Client::METHOD_GET, '/messaging/messages/' . $messageId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getDestinationProject()['$id'], + 'x-appwrite-key' => $this->getDestinationProject()['apiKey'], + ]); + + $this->assertEquals(200, $response['headers']['status-code']); + $this->assertEquals($messageId, $response['body']['$id']); + $this->assertEquals('draft', $response['body']['status']); + $this->assertEquals('Migration Test Email', $response['body']['data']['subject']); + $this->assertEquals('This is a migration test email', $response['body']['data']['content']); + $this->assertContains($topicId, $response['body']['topics']); + + // Cleanup + $this->client->call(Client::METHOD_DELETE, '/messaging/messages/' . $messageId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ]); + + $this->client->call(Client::METHOD_DELETE, '/messaging/messages/' . $messageId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getDestinationProject()['$id'], + 'x-appwrite-key' => $this->getDestinationProject()['apiKey'], + ]); + + $this->client->call(Client::METHOD_DELETE, '/messaging/topics/' . $topicId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ]); + + $this->client->call(Client::METHOD_DELETE, '/messaging/topics/' . $topicId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getDestinationProject()['$id'], + 'x-appwrite-key' => $this->getDestinationProject()['apiKey'], + ]); + + $this->client->call(Client::METHOD_DELETE, '/messaging/providers/' . $providerId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ]); + + $this->client->call(Client::METHOD_DELETE, '/messaging/providers/' . $providerId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getDestinationProject()['$id'], + 'x-appwrite-key' => $this->getDestinationProject()['apiKey'], + ]); + + $this->client->call(Client::METHOD_DELETE, '/users/' . $userId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ]); + + $this->client->call(Client::METHOD_DELETE, '/users/' . $userId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getDestinationProject()['$id'], + 'x-appwrite-key' => $this->getDestinationProject()['apiKey'], + ]); + } + + public function testAppwriteMigrationMessagingSmsMessage(): void + { + $this->getDestinationProject(true); + + $user = $this->client->call(Client::METHOD_POST, '/users', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ], [ + 'userId' => ID::unique(), + 'email' => uniqid() . '-migration-sms@test.com', + 'phone' => '+1' . str_pad((string) rand(200000000, 999999999), 10, '0', STR_PAD_LEFT), + 'password' => 'password', + ]); + + $this->assertEquals(201, $user['headers']['status-code']); + $userId = $user['body']['$id']; + $this->assertGreaterThanOrEqual(1, \count($user['body']['targets'])); + + $smsTarget = null; + foreach ($user['body']['targets'] as $target) { + if ($target['providerType'] === 'sms') { + $smsTarget = $target; + break; + } + } + $this->assertNotNull($smsTarget); + $targetId = $smsTarget['$id']; + + $provider = $this->client->call(Client::METHOD_POST, '/messaging/providers/twilio', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ], [ + 'providerId' => ID::unique(), + 'name' => 'Migration Twilio SMS Msg', + 'from' => '+15559876543', + 'accountSid' => 'test-account-sid', + 'authToken' => 'test-auth-token', + ]); + + $this->assertEquals(201, $provider['headers']['status-code']); + $providerId = $provider['body']['$id']; + + $topic = $this->client->call(Client::METHOD_POST, '/messaging/topics', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ], [ + 'topicId' => ID::unique(), + 'name' => 'Migration SMS Topic', + ]); + + $this->assertEquals(201, $topic['headers']['status-code']); + $topicId = $topic['body']['$id']; + + $message = $this->client->call(Client::METHOD_POST, '/messaging/messages/sms', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ], [ + 'messageId' => ID::unique(), + 'targets' => [$targetId], + 'topics' => [$topicId], + 'content' => 'Migration SMS test content', + 'draft' => true, + ]); + + $this->assertEquals(201, $message['headers']['status-code']); + $messageId = $message['body']['$id']; + + $result = $this->performMigrationSync([ + 'resources' => [ + Resource::TYPE_USER, + Resource::TYPE_PROVIDER, + Resource::TYPE_TOPIC, + Resource::TYPE_SUBSCRIBER, + Resource::TYPE_MESSAGE, + ], + 'endpoint' => $this->webEndpoint, + 'projectId' => $this->getProject()['$id'], + 'apiKey' => $this->getProject()['apiKey'], + ]); + + $this->assertEquals('completed', $result['status']); + $this->assertArrayHasKey(Resource::TYPE_MESSAGE, $result['statusCounters']); + $this->assertEquals(0, $result['statusCounters'][Resource::TYPE_MESSAGE]['error']); + $this->assertGreaterThanOrEqual(1, $result['statusCounters'][Resource::TYPE_MESSAGE]['success']); + + $response = $this->client->call(Client::METHOD_GET, '/messaging/messages/' . $messageId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getDestinationProject()['$id'], + 'x-appwrite-key' => $this->getDestinationProject()['apiKey'], + ]); + + $this->assertEquals(200, $response['headers']['status-code']); + $this->assertEquals($messageId, $response['body']['$id']); + $this->assertEquals('draft', $response['body']['status']); + $this->assertEquals('Migration SMS test content', $response['body']['data']['content']); + + // Cleanup + $this->client->call(Client::METHOD_DELETE, '/messaging/messages/' . $messageId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ]); + + $this->client->call(Client::METHOD_DELETE, '/messaging/messages/' . $messageId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getDestinationProject()['$id'], + 'x-appwrite-key' => $this->getDestinationProject()['apiKey'], + ]); + + $this->client->call(Client::METHOD_DELETE, '/messaging/topics/' . $topicId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ]); + + $this->client->call(Client::METHOD_DELETE, '/messaging/topics/' . $topicId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getDestinationProject()['$id'], + 'x-appwrite-key' => $this->getDestinationProject()['apiKey'], + ]); + + $this->client->call(Client::METHOD_DELETE, '/messaging/providers/' . $providerId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ]); + + $this->client->call(Client::METHOD_DELETE, '/messaging/providers/' . $providerId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getDestinationProject()['$id'], + 'x-appwrite-key' => $this->getDestinationProject()['apiKey'], + ]); + + $this->client->call(Client::METHOD_DELETE, '/users/' . $userId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ]); + + $this->client->call(Client::METHOD_DELETE, '/users/' . $userId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getDestinationProject()['$id'], + 'x-appwrite-key' => $this->getDestinationProject()['apiKey'], + ]); + } + + public function testAppwriteMigrationMessagingScheduledMessage(): void + { + $this->getDestinationProject(true); + + $user = $this->client->call(Client::METHOD_POST, '/users', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ], [ + 'userId' => ID::unique(), + 'email' => uniqid() . '-migration-sched@test.com', + 'password' => 'password', + ]); + + $this->assertEquals(201, $user['headers']['status-code']); + $userId = $user['body']['$id']; + $targetId = $user['body']['targets'][0]['$id']; + + $provider = $this->client->call(Client::METHOD_POST, '/messaging/providers/sendgrid', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ], [ + 'providerId' => ID::unique(), + 'name' => 'Migration Sendgrid Scheduled', + 'apiKey' => 'my-apikey', + 'from' => 'migration-sched@test.com', + ]); + + $this->assertEquals(201, $provider['headers']['status-code']); + $providerId = $provider['body']['$id']; + + $topic = $this->client->call(Client::METHOD_POST, '/messaging/topics', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ], [ + 'topicId' => ID::unique(), + 'name' => 'Migration Scheduled Topic', + ]); + + $this->assertEquals(201, $topic['headers']['status-code']); + $topicId = $topic['body']['$id']; + + $subscriber = $this->client->call(Client::METHOD_POST, '/messaging/topics/' . $topicId . '/subscribers', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ], [ + 'subscriberId' => ID::unique(), + 'targetId' => $targetId, + ]); + + $this->assertEquals(201, $subscriber['headers']['status-code']); + + // Create a scheduled message with a future date using topics only + // Direct targets use source IDs which won't resolve in the destination via API + $futureDate = (new \DateTime('+1 year'))->format(\DateTime::ATOM); + $message = $this->client->call(Client::METHOD_POST, '/messaging/messages/email', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ], [ + 'messageId' => ID::unique(), + 'topics' => [$topicId], + 'subject' => 'Migration Scheduled Email', + 'content' => 'This is a scheduled migration test email', + 'scheduledAt' => $futureDate, + ]); + + $this->assertEquals(201, $message['headers']['status-code']); + $messageId = $message['body']['$id']; + $this->assertEquals('scheduled', $message['body']['status']); + + $result = $this->performMigrationSync([ + 'resources' => [ + Resource::TYPE_USER, + Resource::TYPE_PROVIDER, + Resource::TYPE_TOPIC, + Resource::TYPE_SUBSCRIBER, + Resource::TYPE_MESSAGE, + ], + 'endpoint' => $this->webEndpoint, + 'projectId' => $this->getProject()['$id'], + 'apiKey' => $this->getProject()['apiKey'], + ]); + + $this->assertEquals('completed', $result['status']); + $this->assertArrayHasKey(Resource::TYPE_MESSAGE, $result['statusCounters']); + $this->assertEquals(0, $result['statusCounters'][Resource::TYPE_MESSAGE]['error']); + $this->assertGreaterThanOrEqual(1, $result['statusCounters'][Resource::TYPE_MESSAGE]['success']); + + $response = $this->client->call(Client::METHOD_GET, '/messaging/messages/' . $messageId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getDestinationProject()['$id'], + 'x-appwrite-key' => $this->getDestinationProject()['apiKey'], + ]); + + $this->assertEquals(200, $response['headers']['status-code']); + $this->assertEquals($messageId, $response['body']['$id']); + $this->assertEquals('scheduled', $response['body']['status']); + $this->assertEquals('Migration Scheduled Email', $response['body']['data']['subject']); + $this->assertEquals( + (new \DateTime($futureDate))->getTimestamp(), + (new \DateTime($response['body']['scheduledAt']))->getTimestamp(), + ); + + // Cleanup + $this->client->call(Client::METHOD_DELETE, '/messaging/messages/' . $messageId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ]); + + $this->client->call(Client::METHOD_DELETE, '/messaging/messages/' . $messageId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getDestinationProject()['$id'], + 'x-appwrite-key' => $this->getDestinationProject()['apiKey'], + ]); + + $this->client->call(Client::METHOD_DELETE, '/messaging/topics/' . $topicId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ]); + + $this->client->call(Client::METHOD_DELETE, '/messaging/topics/' . $topicId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getDestinationProject()['$id'], + 'x-appwrite-key' => $this->getDestinationProject()['apiKey'], + ]); + + $this->client->call(Client::METHOD_DELETE, '/messaging/providers/' . $providerId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ]); + + $this->client->call(Client::METHOD_DELETE, '/messaging/providers/' . $providerId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getDestinationProject()['$id'], + 'x-appwrite-key' => $this->getDestinationProject()['apiKey'], + ]); + + $this->client->call(Client::METHOD_DELETE, '/users/' . $userId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ]); + + $this->client->call(Client::METHOD_DELETE, '/users/' . $userId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getDestinationProject()['$id'], + 'x-appwrite-key' => $this->getDestinationProject()['apiKey'], + ]); + } } diff --git a/tests/e2e/Services/Sites/SitesConsoleClientTest.php b/tests/e2e/Services/Sites/SitesConsoleClientTest.php index 2a94dded5f..2e0e1a892d 100644 --- a/tests/e2e/Services/Sites/SitesConsoleClientTest.php +++ b/tests/e2e/Services/Sites/SitesConsoleClientTest.php @@ -180,12 +180,12 @@ class SitesConsoleClientTest extends Scope $stdout = ''; $stderr = ''; - $code = Console::execute("docker exec appwrite-task-maintenance time-travel --projectId={$this->getProject()['$id']} --resourceType=deployment --resourceId={$deploymentIdInactiveOld} --createdAt=2020-01-01T00:00:00Z", '', $stdout, $stderr); + $code = Console::execute("docker exec appwrite task-time-travel --projectId={$this->getProject()['$id']} --resourceType=deployment --resourceId={$deploymentIdInactiveOld} --createdAt=2020-01-01T00:00:00Z", '', $stdout, $stderr); $this->assertSame(0, $code, "Time-travel command failed with code $code: $stderr ($stdout)"); $stdout = ''; $stderr = ''; - $code = Console::execute("docker exec appwrite-task-maintenance maintenance --type=trigger", '', $stdout, $stderr); + $code = Console::execute("docker exec appwrite maintenance --type=trigger", '', $stdout, $stderr); $this->assertSame(0, $code, "Maintenance command failed with code $code: $stderr ($stdout)"); $this->assertEventually(function () use ($siteId) { diff --git a/tests/e2e/Services/Users/UsersBase.php b/tests/e2e/Services/Users/UsersBase.php index 5c7b289722..866ee591a2 100644 --- a/tests/e2e/Services/Users/UsersBase.php +++ b/tests/e2e/Services/Users/UsersBase.php @@ -1596,7 +1596,7 @@ trait UsersBase ]); $this->assertEquals($user['headers']['status-code'], 200); - $this->assertEquals($user['body']['phone'], $updatedNumber); + $this->assertEmpty($user['body']['phone'] ?? ''); $user = $this->client->call(Client::METHOD_GET, '/users/' . $data['userId'], array_merge([ 'content-type' => 'application/json', @@ -1604,7 +1604,7 @@ trait UsersBase ], $this->getHeaders())); $this->assertEquals($user['headers']['status-code'], 200); - $this->assertEquals($user['body']['phone'], $updatedNumber); + $this->assertEmpty($user['body']['phone'] ?? ''); $updatedNumber = "+910000000000"; //dummy number $user = $this->client->call(Client::METHOD_PATCH, '/users/' . $data['userId'] . '/phone', array_merge([ @@ -1648,6 +1648,58 @@ trait UsersBase static::$userNumberUpdated = true; } + public function testUpdateTwoUsersPhoneToEmpty(): void + { + $projectId = $this->getProject()['$id']; + $headers = array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $projectId, + ], $this->getHeaders()); + + // Create two users with distinct valid phone numbers + $user1 = $this->client->call(Client::METHOD_POST, '/users', $headers, [ + 'userId' => ID::unique(), + 'email' => 'user1-phone-empty-test@appwrite.io', + 'password' => 'password', + 'name' => 'User One', + 'phone' => '+16175551201', + ]); + $this->assertEquals(201, $user1['headers']['status-code']); + $this->assertEquals('+16175551201', $user1['body']['phone']); + + $user2 = $this->client->call(Client::METHOD_POST, '/users', $headers, [ + 'userId' => ID::unique(), + 'email' => 'user2-phone-empty-test@appwrite.io', + 'password' => 'password', + 'name' => 'User Two', + 'phone' => '+16175551202', + ]); + $this->assertEquals(201, $user2['headers']['status-code']); + $this->assertEquals('+16175551202', $user2['body']['phone']); + + // Update first user's phone to empty - must succeed + $response1 = $this->client->call(Client::METHOD_PATCH, '/users/' . $user1['body']['$id'] . '/phone', $headers, [ + 'number' => '', + ]); + $this->assertEquals(200, $response1['headers']['status-code'], 'First user phone should update to empty'); + $this->assertEmpty($response1['body']['phone'] ?? ''); + + // Update second user's phone to empty - must succeed (would fail with duplicate if empty was stored as '') + $response2 = $this->client->call(Client::METHOD_PATCH, '/users/' . $user2['body']['$id'] . '/phone', $headers, [ + 'number' => '', + ]); + $this->assertEquals(200, $response2['headers']['status-code'], 'Second user phone should update to empty without duplicate error'); + $this->assertEmpty($response2['body']['phone'] ?? ''); + + // Verify both users have empty phone via GET + $get1 = $this->client->call(Client::METHOD_GET, '/users/' . $user1['body']['$id'], $headers); + $get2 = $this->client->call(Client::METHOD_GET, '/users/' . $user2['body']['$id'], $headers); + $this->assertEquals(200, $get1['headers']['status-code']); + $this->assertEquals(200, $get2['headers']['status-code']); + $this->assertEmpty($get1['body']['phone'] ?? ''); + $this->assertEmpty($get2['body']['phone'] ?? ''); + } + public function testUpdateUserNumberSearch(): void { $data = $this->ensureUserNumberUpdated();