From dd2109dda3551596643aa9ba0691d99c0c922d29 Mon Sep 17 00:00:00 2001 From: Levi van Noort <73097785+levivannoort@users.noreply.github.com> Date: Mon, 14 Sep 2026 11:29:32 +0200 Subject: [PATCH 01/37] refactor(ci): rename the nightly security scan to security-scan The Trivy scan owned the nightly.yml file name, which the nightly build channel needs. Its schedule also moves to 01:30 UTC so it runs after the build rather than alongside it. --- .github/workflows/{nightly.yml => security-scan.yml} | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) rename .github/workflows/{nightly.yml => security-scan.yml} (94%) diff --git a/.github/workflows/nightly.yml b/.github/workflows/security-scan.yml similarity index 94% rename from .github/workflows/nightly.yml rename to .github/workflows/security-scan.yml index 0a49f658ac..5147215253 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/security-scan.yml @@ -1,7 +1,7 @@ -name: Nightly Security Scan +name: Security Scan on: schedule: - - cron: '0 0 * * *' # 12am UTC daily runtime + - cron: '30 1 * * *' # 1:30am UTC, after the nightly build has taken 00:00 workflow_dispatch: jobs: From d3228beae5657e3151b8bb979ecd42bec190044f Mon Sep 17 00:00:00 2001 From: Levi van Noort <73097785+levivannoort@users.noreply.github.com> Date: Mon, 14 Sep 2026 11:29:32 +0200 Subject: [PATCH 02/37] feat(releases): publish a nightly channel from the release branch Builds one image a day from the newest CI-green commit on the active release branch, so a fix can reach self-hosters the night it lands instead of waiting for the next patch. Resolves the build rather than trusting the tip: walks back through recent green CI runs, skips when nothing has moved since the last nightly, and derives the version from the branch's own APP_VERSION_STABLE so the tag is a prerelease of the patch it is heading for. The bare nightly tag only follows the newest line, and a guard refuses any tag outside appwrite/appwrite -- Cloud's appwrite/ce pipeline is not part of this channel. --- .github/workflows/nightly.yml | 249 ++++++++++++++++++++++++++++++++++ 1 file changed, 249 insertions(+) create mode 100644 .github/workflows/nightly.yml diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml new file mode 100644 index 0000000000..44b2c6e6cb --- /dev/null +++ b/.github/workflows/nightly.yml @@ -0,0 +1,249 @@ +name: "Nightly" + +# Publishes the nightly channel for self-hosting: one build a day from the newest +# CI-green commit on the active release branch. Never publishes to appwrite/ce -- +# Cloud builds from cl-* tags through publish.yml and is not part of this channel. +# +# Tags pushed (see the Releases section of AGENTS.md): +# X.Y.Z-nightly. immutable, the rollback anchor. X.Y.Z is the NEXT patch +# on the line, so the build sorts below the release it is +# heading for. +# X.Y-nightly rolling, per minor line. The form to recommend. +# nightly rolling, newest line only. + +on: + schedule: + - cron: '0 0 * * *' # 12am UTC daily + workflow_dispatch: + inputs: + branch: + description: 'Release branch to build (default: newest X.Y.x branch)' + required: false + type: string + +concurrency: + group: nightly + cancel-in-progress: false + +env: + # Invariant 4 in AGENTS.md: the nightly channel publishes here and nowhere else. + IMAGE: appwrite/appwrite + +permissions: + contents: read + +jobs: + resolve: + name: Resolve build + runs-on: ubuntu-latest + outputs: + skip: ${{ steps.resolve.outputs.skip }} + branch: ${{ steps.resolve.outputs.branch }} + sha: ${{ steps.resolve.outputs.sha }} + version: ${{ steps.resolve.outputs.version }} + tags: ${{ steps.resolve.outputs.tags }} + steps: + - name: Checkout repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + fetch-depth: 0 + + - name: Resolve branch, commit and version + id: resolve + env: + GH_TOKEN: ${{ github.token }} + REQUESTED_BRANCH: ${{ inputs.branch }} + run: | + set -euo pipefail + + # The active release branch, unless workflow_dispatch named one. + newest=$(git branch -r --format='%(refname:short)' \ + | sed -n 's#^origin/##p' \ + | grep -E '^[0-9]+\.[0-9]+\.x$' \ + | sort -V \ + | tail -1) + + if [ -z "$newest" ]; then + echo "::error::No X.Y.x release branch found on origin." + exit 1 + fi + + branch="${REQUESTED_BRANCH:-$newest}" + + if ! git rev-parse --verify "origin/${branch}" >/dev/null 2>&1; then + echo "::error::Branch ${branch} does not exist on origin." + exit 1 + fi + + echo "branch=${branch}" >> "$GITHUB_OUTPUT" + + # Invariant 2: build the newest CI-green commit, not the tip, so a red + # branch delays the channel instead of shipping through it. gh returns + # newest first; a run's commit can have been force-pushed away, so the + # candidate must still be on the branch. + sha="" + while read -r candidate; do + [ -n "$candidate" ] || continue + if git merge-base --is-ancestor "$candidate" "origin/${branch}" 2>/dev/null; then + sha="$candidate" + break + fi + done < <(gh run list \ + --workflow=ci.yml \ + --branch="${branch}" \ + --event=push \ + --status=success \ + --limit=30 \ + --json headSha \ + --jq '.[].headSha') + + if [ -z "$sha" ]; then + echo "::warning::No green CI run found on ${branch} in the last 30 pushes; nothing to publish." + echo "skip=true" >> "$GITHUB_OUTPUT" + exit 0 + fi + + echo "sha=${sha}" >> "$GITHUB_OUTPUT" + + # Nothing new since the last nightly. The marker tag is moved by the + # build job below and is the only state this workflow keeps. + if git rev-parse --verify "refs/tags/nightly-${branch}" >/dev/null 2>&1; then + previous=$(git rev-parse "refs/tags/nightly-${branch}^{commit}") + if [ "$previous" = "$sha" ]; then + echo "::notice::${branch} has not moved since the last nightly (${sha}); skipping." + echo "skip=true" >> "$GITHUB_OUTPUT" + exit 0 + fi + fi + + # A nightly is a prerelease of the next patch on the line, so read the + # branch's own stable version and bump the patch. + stable=$(git show "${sha}:app/init/constants.php" \ + | sed -n "s/^const APP_VERSION_STABLE = '\([^']*\)';.*/\1/p") + + if ! printf '%s' "$stable" | grep -qE '^[0-9]+\.[0-9]+\.[0-9]+$'; then + echo "::error::APP_VERSION_STABLE on ${branch} is '${stable}', which is not a released X.Y.Z. Refusing to guess a nightly version." + exit 1 + fi + + major=${stable%%.*} + rest=${stable#*.} + minor=${rest%%.*} + patch=${rest#*.} + next="${major}.${minor}.$((patch + 1))" + + if [ "${branch}" != "${major}.${minor}.x" ]; then + echo "::warning::${branch} carries APP_VERSION_STABLE=${stable}; tagging as ${next} from the constant rather than the branch name." + fi + + # Published nightlies are immutable, so a second green commit on the + # same day takes its own suffix rather than overwriting the day's tag. + build="$(date -u +%Y%m%d)" + status=$(curl --silent --output /dev/null --write-out '%{http_code}' \ + --retry 3 --retry-all-errors --max-time 30 \ + "https://hub.docker.com/v2/repositories/${IMAGE}/tags/${next}-nightly.${build}" || true) + + case "$status" in + 404) ;; + 200) + build="${build}.${GITHUB_RUN_NUMBER}" + echo "::notice::${next}-nightly.$(date -u +%Y%m%d) is already published; using ${build}." + ;; + *) + echo "::error::Unexpected response ${status} from Docker Hub while checking for an existing nightly tag. Refusing to publish." + exit 1 + ;; + esac + + version="${next}-nightly.${build}" + echo "version=${version}" >> "$GITHUB_OUTPUT" + + tags="${IMAGE}:${version}" + tags="${tags},${IMAGE}:${major}.${minor}-nightly" + + # The bare tag follows the newest line only, so a dispatched build of an + # older branch cannot drag it backwards. + if [ "${branch}" = "${newest}" ]; then + tags="${tags},${IMAGE}:nightly" + fi + + echo "tags=${tags}" >> "$GITHUB_OUTPUT" + echo "skip=false" >> "$GITHUB_OUTPUT" + + { + echo "### Nightly" + echo + echo "| | |" + echo "|---|---|" + echo "| Branch | \`${branch}\` |" + echo "| Commit | \`${sha}\` |" + echo "| Version | \`${version}\` |" + echo "| Tags | \`${tags//,/\`, \`}\` |" + } >> "$GITHUB_STEP_SUMMARY" + + build: + name: Build and publish + needs: resolve + if: needs.resolve.outputs.skip == 'false' + runs-on: ubuntu-latest + timeout-minutes: 120 + permissions: + contents: write # moves the nightly- marker tag + steps: + - name: Checkout the resolved commit + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + ref: ${{ needs.resolve.outputs.sha }} + submodules: recursive + + - name: Refuse to publish anywhere but appwrite/appwrite + env: + TAGS: ${{ needs.resolve.outputs.tags }} + run: | + set -euo pipefail + echo "$TAGS" | tr ',' '\n' | while read -r tag; do + [ -n "$tag" ] || continue + case "$tag" in + appwrite/appwrite:*) ;; + *) + echo "::error::Refusing to publish ${tag}: the nightly channel is appwrite/appwrite only." + exit 1 + ;; + esac + done + + - name: Set up QEMU + uses: docker/setup-qemu-action@ce360397dd3f832beb865e1373c09c0e9f86d70a # v4.0.0 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0 + + - name: Login to Docker Hub + uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0 + with: + username: ${{ vars.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Build and push + uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7.1.0 + with: + context: . + platforms: linux/amd64,linux/arm64 + target: production + build-args: | + VERSION=${{ needs.resolve.outputs.version }} + push: true + tags: ${{ needs.resolve.outputs.tags }} + labels: | + org.opencontainers.image.revision=${{ needs.resolve.outputs.sha }} + org.opencontainers.image.version=${{ needs.resolve.outputs.version }} + io.appwrite.channel=nightly + + - name: Move the marker tag + env: + BRANCH: ${{ needs.resolve.outputs.branch }} + SHA: ${{ needs.resolve.outputs.sha }} + run: | + set -euo pipefail + git tag --force "nightly-${BRANCH}" "${SHA}" + git push --force origin "nightly-${BRANCH}" From 5b8e9db3c05142ea7299d5821e693a5db0c7ed87 Mon Sep 17 00:00:00 2001 From: Levi van Noort <73097785+levivannoort@users.noreply.github.com> Date: Mon, 14 Sep 2026 11:29:44 +0200 Subject: [PATCH 03/37] feat(ci): gate releases on a fresh install and an upgrade AGENTS.md blocks a release on a fresh install and an upgrade from the previous stable with realistic data, and CI covered neither -- there was no migrate step anywhere in the pipeline. That gate was a person's day per release, which is a direct incentive to batch fixes instead of shipping them. Both now run on release branches and on demand. The upgrade job starts the previous published stable from the compose file and env that version shipped, seeds through the public API, switches to the candidate on the same volumes, migrates, and asserts the data survived -- then migrates a second time to prove idempotency. Seeding covers the shapes a migration is most likely to mangle: empty strings, 4000-character values, unicode, relationships, and row permissions that differ from the table's. The baseline comes from Docker Hub rather than git tags, because an image nobody can pull is not an upgrade anyone can perform. --- .github/scripts/previous-stable.py | 89 ++++++++++++ .github/scripts/upgrade-lib.sh | 109 +++++++++++++++ .github/scripts/upgrade-seed.sh | 159 ++++++++++++++++++++++ .github/scripts/upgrade-verify.sh | 120 +++++++++++++++++ .github/workflows/ci.yml | 210 ++++++++++++++++++++++++++++- 5 files changed, 686 insertions(+), 1 deletion(-) create mode 100755 .github/scripts/previous-stable.py create mode 100644 .github/scripts/upgrade-lib.sh create mode 100755 .github/scripts/upgrade-seed.sh create mode 100755 .github/scripts/upgrade-verify.sh diff --git a/.github/scripts/previous-stable.py b/.github/scripts/previous-stable.py new file mode 100755 index 0000000000..19221f8c27 --- /dev/null +++ b/.github/scripts/previous-stable.py @@ -0,0 +1,89 @@ +#!/usr/bin/env python3 +"""Print the published stable version directly below a target version. + +The upgrade gate needs a baseline to upgrade *from*. Docker Hub is the source of +truth rather than git tags, because the image has to exist for the gate to run at +all -- a tag with no published image is not an upgrade anyone can perform. + + ./previous-stable.py 2.0.0 -> 1.9.6 + +Release candidates and channel tags are ignored: only X.Y.Z counts as a stable a +self-hoster could be upgrading from. +""" + +from __future__ import annotations + +import json +import sys +import urllib.error +import urllib.request + +REPOSITORY = "appwrite/appwrite" +TAGS_URL = f"https://hub.docker.com/v2/repositories/{REPOSITORY}/tags/" +PAGES = 3 +PAGE_SIZE = 100 + + +def parse(version: str) -> tuple[int, int, int] | None: + parts = version.split(".") + + if len(parts) != 3 or not all(part.isdigit() for part in parts): + return None + + major, minor, patch = (int(part) for part in parts) + + return major, minor, patch + + +def published() -> list[tuple[int, int, int]]: + versions: list[tuple[int, int, int]] = [] + url: str | None = f"{TAGS_URL}?page_size={PAGE_SIZE}&ordering=last_updated" + + for _ in range(PAGES): + if url is None: + break + + request = urllib.request.Request(url, headers={"User-Agent": "appwrite-upgrade-gate"}) + + with urllib.request.urlopen(request, timeout=60) as response: + payload = json.load(response) + + versions.extend( + version + for version in (parse(tag["name"]) for tag in payload.get("results", [])) + if version is not None + ) + + url = payload.get("next") + + return versions + + +def main() -> int: + if len(sys.argv) != 2: + print(f"usage: {sys.argv[0]} ", file=sys.stderr) + return 2 + + target = parse(sys.argv[1]) + + if target is None: + print(f"'{sys.argv[1]}' is not an X.Y.Z version.", file=sys.stderr) + return 2 + + try: + candidates = sorted(version for version in published() if version < target) + except (urllib.error.URLError, TimeoutError, json.JSONDecodeError) as error: + print(f"Could not read tags from Docker Hub: {error}", file=sys.stderr) + return 1 + + if not candidates: + print(f"No published stable below {sys.argv[1]}.", file=sys.stderr) + return 1 + + print(".".join(str(part) for part in candidates[-1])) + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/scripts/upgrade-lib.sh b/.github/scripts/upgrade-lib.sh new file mode 100644 index 0000000000..055cff6627 --- /dev/null +++ b/.github/scripts/upgrade-lib.sh @@ -0,0 +1,109 @@ +#!/usr/bin/env bash +# Shared helpers for the upgrade gate scripts. Sourced, not executed. +# +# The upgrade gate talks to Appwrite the way a self-hoster does: over the public +# HTTP API, with no access to the container internals. That is the point -- it +# proves the data survives a version change through the surface users depend on. + +set -euo pipefail + +: "${ENDPOINT:?ENDPOINT must be set, e.g. http://localhost:8080/v1}" + +MANIFEST="${MANIFEST:-upgrade-seed.json}" +COOKIE_JAR="${COOKIE_JAR:-$(dirname "$MANIFEST")/upgrade-cookies.txt}" + +fail() { + echo "::error::$*" >&2 + exit 1 +} + +# Waits for the stack to answer on the public port. A fresh install spends a +# while on migrations and worker boot, so this is generous. +wait_for_appwrite() { + local attempts="${1:-90}" i + + for ((i = 1; i <= attempts; i++)); do + if curl --silent --fail --max-time 5 "${ENDPOINT}/health/version" >/dev/null 2>&1; then + echo "Appwrite answering at ${ENDPOINT} after ${i}s." + return 0 + fi + sleep 1 + done + + fail "Appwrite did not answer at ${ENDPOINT}/health/version within ${attempts}s." +} + +server_version() { + curl --silent --fail --max-time 10 "${ENDPOINT}/health/version" | jq -r '.version' +} + +# Console calls: cookie session, browser origin. Used to create the org and +# project a self-hoster starts with. +console() { + local method="$1" path="$2" body="${3:-}" + local args=( + --silent --show-error --fail-with-body --max-time 30 + -X "$method" + -H 'content-type: application/json' + -H 'origin: http://localhost' + -H 'x-appwrite-project: console' + -c "$COOKIE_JAR" -b "$COOKIE_JAR" + ) + [ -n "$body" ] && args+=(-d "$body") + + curl "${args[@]}" "${ENDPOINT}${path}" +} + +# Project calls: API key, the way a server SDK talks to Appwrite. +api() { + local method="$1" path="$2" body="${3:-}" + local args=( + --silent --show-error --fail-with-body --max-time 60 + -X "$method" + -H 'content-type: application/json' + -H "x-appwrite-project: ${PROJECT_ID:?PROJECT_ID not set}" + -H "x-appwrite-key: ${API_KEY:?API_KEY not set}" + ) + [ -n "$body" ] && args+=(-d "$body") + + curl "${args[@]}" "${ENDPOINT}${path}" +} + +# Columns are processed asynchronously, so rows cannot be written until every +# column on the table reports available. +wait_for_columns() { + local database="$1" table="$2" attempts="${3:-60}" i statuses + + for ((i = 1; i <= attempts; i++)); do + statuses=$(api GET "/tablesdb/${database}/tables/${table}/columns" \ + | jq -r '.columns[].status' | sort -u | tr '\n' ' ') + + case "$statuses" in + 'available ') return 0 ;; + *failed*) fail "Column processing failed on ${table}: ${statuses}" ;; + esac + sleep 1 + done + + fail "Columns on ${table} did not become available within ${attempts}s (last: ${statuses:-none})." +} + +manifest_get() { + jq -r "$1" "$MANIFEST" +} + +# Asserts on values read back through the API. Prints every check so a failed +# upgrade job reads as a list of what survived and what did not. +assert_equals() { + local what="$1" expected="$2" actual="$3" + + if [ "$expected" = "$actual" ]; then + echo " ok ${what}" + return 0 + fi + + echo " FAIL ${what}" + echo " expected: ${expected}" + echo " actual: ${actual}" + FAILURES=$((${FAILURES:-0} + 1)) +} diff --git a/.github/scripts/upgrade-seed.sh b/.github/scripts/upgrade-seed.sh new file mode 100755 index 0000000000..c538cb456c --- /dev/null +++ b/.github/scripts/upgrade-seed.sh @@ -0,0 +1,159 @@ +#!/usr/bin/env bash +# Seeds an Appwrite install with the data shapes the upgrade gate cares about: +# empty values, long strings, unicode, relationships and mixed permissions. +# Run against the PREVIOUS stable, before switching images. +# +# ENDPOINT=http://localhost:8080/v1 MANIFEST=upgrade-seed.json ./upgrade-seed.sh +# +# Writes every created ID to MANIFEST so upgrade-verify.sh can read the same +# resources back after the migration. + +set -euo pipefail + +# shellcheck source=.github/scripts/upgrade-lib.sh +source "$(dirname "${BASH_SOURCE[0]}")/upgrade-lib.sh" + +ROOT_EMAIL="upgrade-gate@example.com" +ROOT_PASSWORD="upgrade-gate-password-1" +DATABASE_ID="gate" +AUTHORS_TABLE="authors" +POSTS_TABLE="posts" +BUCKET_ID="gate" +FILE_ID="gate-file" + +# The shapes a migration is most likely to mangle. +EMPTY_STRING="" +LONG_STRING=$(printf 'l%.0s' $(seq 1 4000)) +UNICODE_STRING="ünïcødé — 日本語 — 🚂 — — O'Brien \"quoted\"" + +wait_for_appwrite +echo "Seeding $(server_version) at ${ENDPOINT}" + +# --- console: root account, organization, project, API key ------------------- + +echo "==> Console account and project" +rm -f "$COOKIE_JAR" + +# On a fresh install the first account becomes root. +console POST /account "$(jq -nc \ + --arg email "$ROOT_EMAIL" \ + --arg password "$ROOT_PASSWORD" \ + '{userId: "unique()", email: $email, password: $password, name: "Upgrade Gate"}')" \ + >/dev/null + +console POST /account/sessions/email "$(jq -nc \ + --arg email "$ROOT_EMAIL" \ + --arg password "$ROOT_PASSWORD" \ + '{email: $email, password: $password}')" >/dev/null + +TEAM_ID=$(console POST /teams '{"teamId":"unique()","name":"Upgrade Gate"}' | jq -r '.["$id"]') +[ -n "$TEAM_ID" ] && [ "$TEAM_ID" != "null" ] || fail "Could not create the organization." + +PROJECT_ID=$(console POST /projects "$(jq -nc \ + --arg teamId "$TEAM_ID" \ + '{projectId: "upgrade-gate", name: "Upgrade Gate", teamId: $teamId}')" | jq -r '.["$id"]') +[ -n "$PROJECT_ID" ] && [ "$PROJECT_ID" != "null" ] || fail "Could not create the project." +export PROJECT_ID + +API_KEY=$(console POST "/projects/${PROJECT_ID}/keys" "$(jq -nc \ + '{name: "upgrade-gate", scopes: ["users.read","users.write","tables.read","tables.write","databases.read","databases.write","rows.read","rows.write","documents.read","documents.write","collections.read","collections.write","files.read","files.write","buckets.read","buckets.write"]}')" \ + | jq -r '.secret') +[ -n "$API_KEY" ] && [ "$API_KEY" != "null" ] || fail "Could not create an API key." +export API_KEY + +# --- tables, columns, relationship ------------------------------------------- + +echo "==> Database, tables and columns" +api POST /tablesdb "$(jq -nc --arg id "$DATABASE_ID" '{databaseId: $id, name: "Gate"}')" >/dev/null + +api POST "/tablesdb/${DATABASE_ID}/tables" "$(jq -nc \ + --arg id "$AUTHORS_TABLE" \ + '{tableId: $id, name: "Authors", permissions: ["read(\"any\")"], rowSecurity: false}')" >/dev/null + +api POST "/tablesdb/${DATABASE_ID}/tables" "$(jq -nc \ + --arg id "$POSTS_TABLE" \ + '{tableId: $id, name: "Posts", permissions: ["read(\"any\")","create(\"users\")"], rowSecurity: true}')" >/dev/null + +api POST "/tablesdb/${DATABASE_ID}/tables/${AUTHORS_TABLE}/columns/string" \ + '{"key":"name","size":255,"required":true}' >/dev/null +wait_for_columns "$DATABASE_ID" "$AUTHORS_TABLE" + +api POST "/tablesdb/${DATABASE_ID}/tables/${POSTS_TABLE}/columns/string" \ + '{"key":"title","size":255,"required":true}' >/dev/null +api POST "/tablesdb/${DATABASE_ID}/tables/${POSTS_TABLE}/columns/string" \ + '{"key":"body","size":5000,"required":false}' >/dev/null +api POST "/tablesdb/${DATABASE_ID}/tables/${POSTS_TABLE}/columns/string" \ + '{"key":"tag","size":64,"required":false,"default":"untagged"}' >/dev/null +api POST "/tablesdb/${DATABASE_ID}/tables/${POSTS_TABLE}/columns/relationship" "$(jq -nc \ + --arg related "$AUTHORS_TABLE" \ + '{relatedTableId: $related, type: "manyToOne", twoWay: false, key: "author", onDelete: "setNull"}')" >/dev/null +wait_for_columns "$DATABASE_ID" "$POSTS_TABLE" + +# --- rows -------------------------------------------------------------------- + +echo "==> Rows" +AUTHOR_ID=$(api POST "/tablesdb/${DATABASE_ID}/tables/${AUTHORS_TABLE}/rows" \ + '{"rowId":"author-1","data":{"name":"Ada Lovelace"}}' | jq -r '.["$id"]') + +api POST "/tablesdb/${DATABASE_ID}/tables/${POSTS_TABLE}/rows" "$(jq -nc \ + --arg author "$AUTHOR_ID" \ + --arg body "$EMPTY_STRING" \ + '{rowId: "post-empty", data: {title: "Empty body", body: $body, author: $author}}')" >/dev/null + +api POST "/tablesdb/${DATABASE_ID}/tables/${POSTS_TABLE}/rows" "$(jq -nc \ + --arg body "$LONG_STRING" \ + '{rowId: "post-long", data: {title: "Long body", body: $body}}')" >/dev/null + +api POST "/tablesdb/${DATABASE_ID}/tables/${POSTS_TABLE}/rows" "$(jq -nc \ + --arg body "$UNICODE_STRING" \ + '{rowId: "post-unicode", data: {title: "Unicode body", body: $body}}')" >/dev/null + +# Row-level permissions, deliberately different from the table's. +api POST "/tablesdb/${DATABASE_ID}/tables/${POSTS_TABLE}/rows" "$(jq -nc \ + '{rowId: "post-perms", data: {title: "Mixed permissions", body: "kept"}, permissions: ["read(\"any\")","update(\"users\")"]}')" >/dev/null + +# --- users, storage ---------------------------------------------------------- + +echo "==> User and file" +USER_ID=$(api POST /users "$(jq -nc \ + --arg name "$UNICODE_STRING" \ + '{userId: "gate-user", email: "gate-user@example.com", password: "gate-user-password-1", name: $name}')" \ + | jq -r '.["$id"]') + +api POST /storage/buckets "$(jq -nc \ + --arg id "$BUCKET_ID" \ + '{bucketId: $id, name: "Gate", permissions: ["read(\"any\")"], fileSecurity: false}')" >/dev/null + +FILE_CONTENT=$(dirname "$MANIFEST")/gate-file.txt +printf '%s\n' "$UNICODE_STRING" > "$FILE_CONTENT" + +curl --silent --show-error --fail-with-body --max-time 60 \ + -X POST \ + -H "x-appwrite-project: ${PROJECT_ID}" \ + -H "x-appwrite-key: ${API_KEY}" \ + -F "fileId=${FILE_ID}" \ + -F "file=@${FILE_CONTENT}" \ + "${ENDPOINT}/storage/buckets/${BUCKET_ID}/files" >/dev/null + +# --- manifest ---------------------------------------------------------------- + +jq -nc \ + --arg version "$(server_version)" \ + --arg projectId "$PROJECT_ID" \ + --arg apiKey "$API_KEY" \ + --arg teamId "$TEAM_ID" \ + --arg databaseId "$DATABASE_ID" \ + --arg authorsTable "$AUTHORS_TABLE" \ + --arg postsTable "$POSTS_TABLE" \ + --arg authorId "$AUTHOR_ID" \ + --arg userId "$USER_ID" \ + --arg bucketId "$BUCKET_ID" \ + --arg fileId "$FILE_ID" \ + --arg longString "$LONG_STRING" \ + --arg unicodeString "$UNICODE_STRING" \ + '{seededOn: $version, projectId: $projectId, apiKey: $apiKey, teamId: $teamId, + databaseId: $databaseId, authorsTable: $authorsTable, postsTable: $postsTable, + authorId: $authorId, userId: $userId, bucketId: $bucketId, fileId: $fileId, + longString: $longString, unicodeString: $unicodeString}' > "$MANIFEST" + +echo "Seeded from $(server_version); manifest at ${MANIFEST}" diff --git a/.github/scripts/upgrade-verify.sh b/.github/scripts/upgrade-verify.sh new file mode 100755 index 0000000000..7a1a713fa8 --- /dev/null +++ b/.github/scripts/upgrade-verify.sh @@ -0,0 +1,120 @@ +#!/usr/bin/env bash +# Reads back everything upgrade-seed.sh created and asserts it survived the +# migration unchanged, then writes new data to prove the upgraded install is +# still usable. Run against the TARGET version, after migrate. +# +# ENDPOINT=http://localhost:8080/v1 MANIFEST=upgrade-seed.json ./upgrade-verify.sh + +set -euo pipefail + +# shellcheck source=.github/scripts/upgrade-lib.sh +source "$(dirname "${BASH_SOURCE[0]}")/upgrade-lib.sh" + +[ -f "$MANIFEST" ] || fail "No manifest at ${MANIFEST}; did upgrade-seed.sh run?" + +FAILURES=0 + +PROJECT_ID=$(manifest_get '.projectId'); export PROJECT_ID +API_KEY=$(manifest_get '.apiKey'); export API_KEY +DATABASE_ID=$(manifest_get '.databaseId') +AUTHORS_TABLE=$(manifest_get '.authorsTable') +POSTS_TABLE=$(manifest_get '.postsTable') +USER_ID=$(manifest_get '.userId') +BUCKET_ID=$(manifest_get '.bucketId') +FILE_ID=$(manifest_get '.fileId') +LONG_STRING=$(manifest_get '.longString') +UNICODE_STRING=$(manifest_get '.unicodeString') +SEEDED_ON=$(manifest_get '.seededOn') + +wait_for_appwrite +UPGRADED_TO=$(server_version) + +echo "Verifying upgrade ${SEEDED_ON} -> ${UPGRADED_TO}" + +if [ "$SEEDED_ON" = "$UPGRADED_TO" ]; then + fail "Still reporting ${UPGRADED_TO}; the image did not change, so this run proves nothing." +fi + +# --- the API key still authenticates ----------------------------------------- + +# The project, its API key and its scopes all have to survive the migration +# before any of the assertions below mean anything. +echo "==> Project access" +if ! api GET /users >/dev/null 2>&1; then + fail "The seeded API key no longer authenticates against project ${PROJECT_ID}." +fi +echo " ok seeded API key still authenticates" + +# --- seeded rows survived ---------------------------------------------------- + +echo "==> Rows" +row() { api GET "/tablesdb/${DATABASE_ID}/tables/${POSTS_TABLE}/rows/$1"; } + +empty=$(row post-empty) +assert_equals "post-empty title" "Empty body" "$(jq -r '.title' <<<"$empty")" +assert_equals "post-empty kept its empty string" "" "$(jq -r '.body' <<<"$empty")" +assert_equals "post-empty kept its column default" "untagged" "$(jq -r '.tag' <<<"$empty")" +assert_equals "post-empty kept its relationship" "$(manifest_get '.authorId')" \ + "$(jq -r 'if (.author | type) == "object" then .author["$id"] else .author end' <<<"$empty")" + +long=$(row post-long) +assert_equals "post-long body length" "${#LONG_STRING}" "$(jq -r '.body | length' <<<"$long")" +assert_equals "post-long body content" "$LONG_STRING" "$(jq -r '.body' <<<"$long")" + +unicode=$(row post-unicode) +assert_equals "post-unicode body survived byte-for-byte" "$UNICODE_STRING" "$(jq -r '.body' <<<"$unicode")" + +perms=$(row post-perms) +assert_equals "post-perms kept its row permissions" \ + 'read("any"),update("users")' \ + "$(jq -r '.["$permissions"] | sort | join(",")' <<<"$perms")" + +assert_equals "posts table kept its permissions" \ + 'create("users"),read("any")' \ + "$(api GET "/tablesdb/${DATABASE_ID}/tables/${POSTS_TABLE}" | jq -r '.["$permissions"] | sort | join(",")')" + +assert_equals "row count on posts" "4" \ + "$(api GET "/tablesdb/${DATABASE_ID}/tables/${POSTS_TABLE}/rows" | jq -r '.total')" + +# --- user and file survived -------------------------------------------------- + +echo "==> User and file" +user=$(api GET "/users/${USER_ID}") +assert_equals "user email" "gate-user@example.com" "$(jq -r '.email' <<<"$user")" +assert_equals "user name with unicode" "$UNICODE_STRING" "$(jq -r '.name' <<<"$user")" + +file=$(api GET "/storage/buckets/${BUCKET_ID}/files/${FILE_ID}") +assert_equals "file still listed" "$FILE_ID" "$(jq -r '.["$id"]' <<<"$file")" + +downloaded=$(curl --silent --show-error --fail-with-body --max-time 60 \ + -H "x-appwrite-project: ${PROJECT_ID}" \ + -H "x-appwrite-key: ${API_KEY}" \ + "${ENDPOINT}/storage/buckets/${BUCKET_ID}/files/${FILE_ID}/download") +assert_equals "file contents" "$UNICODE_STRING" "$(printf '%s' "$downloaded" | head -1)" + +# --- the upgraded install still takes writes --------------------------------- + +echo "==> Writes after upgrade" +written=$(api POST "/tablesdb/${DATABASE_ID}/tables/${POSTS_TABLE}/rows" "$(jq -nc \ + --arg body "$UNICODE_STRING" \ + '{rowId: "post-after-upgrade", data: {title: "Written after upgrade", body: $body}}')") +assert_equals "new row readable" "Written after upgrade" "$(jq -r '.title' <<<"$written")" + +api PATCH "/users/${USER_ID}/name" '{"name":"Renamed after upgrade"}' >/dev/null +assert_equals "user update applied" "Renamed after upgrade" \ + "$(api GET "/users/${USER_ID}" | jq -r '.name')" + +new_author=$(api POST "/tablesdb/${DATABASE_ID}/tables/${AUTHORS_TABLE}/rows" \ + '{"rowId":"author-after-upgrade","data":{"name":"Grace Hopper"}}' | jq -r '.["$id"]') +linked=$(api PATCH "/tablesdb/${DATABASE_ID}/tables/${POSTS_TABLE}/rows/post-after-upgrade" "$(jq -nc \ + --arg author "$new_author" '{data: {author: $author}}')") +assert_equals "relationship writable after upgrade" "$new_author" \ + "$(jq -r 'if (.author | type) == "object" then .author["$id"] else .author end' <<<"$linked")" + +# --- result ------------------------------------------------------------------ + +if [ "$FAILURES" -gt 0 ]; then + fail "${FAILURES} assertion(s) failed upgrading ${SEEDED_ON} -> ${UPGRADED_TO}." +fi + +echo "All assertions passed upgrading ${SEEDED_ON} -> ${UPGRADED_TO}." diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ffd484f36d..ffb778c2b0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,7 +15,9 @@ on: push: branches: - main - - 1.9.x + # Every release branch, so the nightly channel always has green commits to + # resolve on the active line without anyone remembering to add it here. + - '*.x' workflow_dispatch: inputs: response_format: @@ -923,3 +925,209 @@ jobs: - name: Fail benchmark if: always() && steps.benchmark_after.outcome != 'success' run: exit 1 + + # The two release gates AGENTS.md requires. They run on release branches and on + # demand, not on every PR to main: a fresh install plus an upgrade from the + # previous stable is what a release is blocked on, and it is the work we used to + # pay a person to repeat by hand before every patch. + install: + name: Release / Fresh install + if: >- + github.event_name == 'workflow_dispatch' || + (github.event_name == 'push' && github.ref != 'refs/heads/main') || + (github.event_name == 'pull_request' && github.base_ref != 'main') + runs-on: runs-on=${{ github.run_id }}/runner=4cpu-linux-x64/volume=60g/spot=price-capacity-optimized/retry=when-interrupted/extras=otel+ecr-cache + timeout-minutes: 25 + needs: build + permissions: + contents: read + packages: read + env: + COMPOSE_FILE: docker-compose.yml + COMPOSE_PROJECT_NAME: install-gate + COMPOSE_PROFILES: '' + _APP_IMAGE: appwrite-dev + _APP_VERSION: latest + _APP_HTTP_PORT: '8080' + _APP_HTTPS_PORT: '8443' + ENDPOINT: http://localhost:8080/v1 + steps: + - name: Checkout repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - name: Login to Docker Hub + if: ${{ env.DOCKERHUB_AUTH_AVAILABLE == 'true' }} + uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0 + with: + username: ${{ vars.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Pull the candidate image + run: | + docker pull ${{ env.RUNS_ON_ECR_CACHE }}:${{ env.IMAGE }}-${{ github.sha }} + docker tag ${{ env.RUNS_ON_ECR_CACHE }}:${{ env.IMAGE }}-${{ github.sha }} appwrite-dev:latest + + - name: Start from clean + timeout-minutes: 10 + run: docker compose up -d --quiet-pull --wait + + - name: Report version + run: | + set -euo pipefail + curl --silent --fail --retry 10 --retry-all-errors --retry-delay 3 \ + "${ENDPOINT}/health/version" | tee /dev/stderr | jq -e '.version' + + # A stack that answers /health/version can still be unusable, so drive the + # console and project APIs a self-hoster actually starts with. + - name: Exercise the public API + timeout-minutes: 10 + env: + MANIFEST: ${{ runner.temp }}/install-gate.json + run: .github/scripts/upgrade-seed.sh + + - name: Check for crash loops + if: always() + run: | + docker compose ps + # compose emits NDJSON on some versions and a JSON array on others; + # flatten handles both. + docker compose ps --format json \ + | jq -e -s 'flatten | map(select(.State != "running" and .State != "exited")) | length == 0' \ + || { echo "::error::Some containers are not running."; docker compose logs --tail=200; exit 1; } + + - name: Stop + if: always() + run: docker compose down -v || true + + upgrade: + name: Release / Upgrade from previous stable + if: >- + github.event_name == 'workflow_dispatch' || + (github.event_name == 'push' && github.ref != 'refs/heads/main') || + (github.event_name == 'pull_request' && github.base_ref != 'main') + runs-on: runs-on=${{ github.run_id }}/runner=4cpu-linux-x64/volume=60g/spot=price-capacity-optimized/retry=when-interrupted/extras=otel+ecr-cache + timeout-minutes: 45 + needs: build + permissions: + contents: read + packages: read + env: + COMPOSE_PROJECT_NAME: upgrade-gate + COMPOSE_PROFILES: '' + _APP_HTTP_PORT: '8080' + _APP_HTTPS_PORT: '8443' + ENDPOINT: http://localhost:8080/v1 + MANIFEST: ${{ github.workspace }}/upgrade-seed.json + steps: + - name: Checkout repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + fetch-depth: 0 + + - name: Login to Docker Hub + if: ${{ env.DOCKERHUB_AUTH_AVAILABLE == 'true' }} + uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0 + with: + username: ${{ vars.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + # Previous baseline = highest published stable strictly below this branch's + # APP_VERSION_STABLE. Docker Hub rather than git tags, because the image has + # to exist for the test to run at all. + - name: Resolve the previous stable + id: previous + run: | + set -euo pipefail + target=$(sed -n "s/^const APP_VERSION_STABLE = '\([^']*\)';.*/\1/p" app/init/constants.php) + previous=$(.github/scripts/previous-stable.py "${target}") + + echo "Upgrading ${previous} -> ${target}" + echo "previous=${previous}" >> "$GITHUB_OUTPUT" + echo "target=${target}" >> "$GITHUB_OUTPUT" + + # An upgrade starts from the compose file and env the old version shipped, + # not from this checkout's -- that is what the installation being upgraded + # was actually started from. + - name: Materialise the previous release's compose + env: + PREVIOUS: ${{ steps.previous.outputs.previous }} + run: | + set -euo pipefail + git fetch --no-tags --depth 1 origin "refs/tags/${PREVIOUS}:refs/tags/${PREVIOUS}" + git show "${PREVIOUS}:docker-compose.yml" > docker-compose.previous.yml + git show "${PREVIOUS}:.env" > .env.previous + + - name: Start the previous stable + timeout-minutes: 10 + env: + COMPOSE_FILE: docker-compose.previous.yml + _APP_IMAGE: appwrite/appwrite + _APP_VERSION: ${{ steps.previous.outputs.previous }} + run: docker compose --env-file .env.previous up -d --quiet-pull --wait + + - name: Seed through the public API + timeout-minutes: 10 + run: .github/scripts/upgrade-seed.sh + + - name: Stop the previous stable, keeping volumes + env: + COMPOSE_FILE: docker-compose.previous.yml + _APP_IMAGE: appwrite/appwrite + _APP_VERSION: ${{ steps.previous.outputs.previous }} + run: docker compose --env-file .env.previous down + + - name: Pull the candidate image + run: | + docker pull ${{ env.RUNS_ON_ECR_CACHE }}:${{ env.IMAGE }}-${{ github.sha }} + docker tag ${{ env.RUNS_ON_ECR_CACHE }}:${{ env.IMAGE }}-${{ github.sha }} appwrite-dev:latest + + - name: Start the candidate on the same volumes + timeout-minutes: 10 + env: + COMPOSE_FILE: docker-compose.yml + _APP_IMAGE: appwrite-dev + _APP_VERSION: latest + run: docker compose up -d --quiet-pull --wait + + - name: Migrate + timeout-minutes: 20 + env: + COMPOSE_FILE: docker-compose.yml + _APP_IMAGE: appwrite-dev + _APP_VERSION: latest + run: docker compose exec -T appwrite migrate + + - name: Verify the seeded data survived + timeout-minutes: 10 + run: .github/scripts/upgrade-verify.sh + + # A migration that is not idempotent breaks every retried or resumed + # upgrade, so run it again and re-assert. + - name: Migrate again and re-verify + timeout-minutes: 25 + env: + COMPOSE_FILE: docker-compose.yml + _APP_IMAGE: appwrite-dev + _APP_VERSION: latest + run: | + set -euo pipefail + docker compose exec -T appwrite migrate + .github/scripts/upgrade-verify.sh + + - name: Collect logs on failure + if: failure() + env: + COMPOSE_FILE: docker-compose.yml + _APP_IMAGE: appwrite-dev + _APP_VERSION: latest + run: | + docker compose ps || true + docker compose logs --tail=400 appwrite || true + + - name: Stop + if: always() + env: + COMPOSE_FILE: docker-compose.yml + _APP_IMAGE: appwrite-dev + _APP_VERSION: latest + run: docker compose down -v || true From ef56024ad9286f2b4a70af3f8b6ed93c5e7d603c Mon Sep 17 00:00:00 2001 From: Levi van Noort <73097785+levivannoort@users.noreply.github.com> Date: Mon, 14 Sep 2026 11:29:44 +0200 Subject: [PATCH 04/37] feat(releases): write release metadata with one task A patch bump touches APP_VERSION_STABLE, four install snippets in README.md, three in README-CN.md, and Migration::$versions, and nothing checked that they agree. `release X.Y.Z` writes all of them from one version number, and tests/unit/Release/MetadataTest.php fails when they drift apart again. The task also refuses to map a patch to a new migration class: a fix needing a schema change is a minor, which is what keeps a nightly user able to roll back to yesterday's build. --- bin/release | 3 + src/Appwrite/Platform/Services/Tasks.php | 2 + src/Appwrite/Platform/Tasks/Release.php | 233 +++++++++++++++++++++++ tests/unit/Release/MetadataTest.php | 128 +++++++++++++ 4 files changed, 366 insertions(+) create mode 100755 bin/release create mode 100644 src/Appwrite/Platform/Tasks/Release.php create mode 100644 tests/unit/Release/MetadataTest.php diff --git a/bin/release b/bin/release new file mode 100755 index 0000000000..02b369262e --- /dev/null +++ b/bin/release @@ -0,0 +1,3 @@ +#!/bin/sh + +exec php /usr/src/code/app/cli.php release "$@" \ No newline at end of file diff --git a/src/Appwrite/Platform/Services/Tasks.php b/src/Appwrite/Platform/Services/Tasks.php index 8481b1e2ef..2193fe8faa 100644 --- a/src/Appwrite/Platform/Services/Tasks.php +++ b/src/Appwrite/Platform/Services/Tasks.php @@ -8,6 +8,7 @@ use Appwrite\Platform\Tasks\Interval; use Appwrite\Platform\Tasks\Maintenance; use Appwrite\Platform\Tasks\Migrate; use Appwrite\Platform\Tasks\QueueRetry; +use Appwrite\Platform\Tasks\Release; use Appwrite\Platform\Tasks\Schedule; use Appwrite\Platform\Tasks\ScheduleExecutions; use Appwrite\Platform\Tasks\ScheduleFunctions; @@ -34,6 +35,7 @@ class Tasks extends Service ->addAction(Maintenance::getName(), new Maintenance()) ->addAction(Migrate::getName(), new Migrate()) ->addAction(QueueRetry::getName(), new QueueRetry()) + ->addAction(Release::getName(), new Release()) ->addAction(SDKs::getName(), new SDKs()) ->addAction(SSL::getName(), new SSL()) ->addAction(Screenshot::getName(), new Screenshot()) diff --git a/src/Appwrite/Platform/Tasks/Release.php b/src/Appwrite/Platform/Tasks/Release.php new file mode 100644 index 0000000000..24072085a8 --- /dev/null +++ b/src/Appwrite/Platform/Tasks/Release.php @@ -0,0 +1,233 @@ +root = (string) \realpath(__DIR__ . '/../../../..'); + + $this + ->desc('Write the release metadata for a version') + ->param('version', '', new Text(32), 'Version being released, e.g. 2.0.2') + ->param('migration', '', new Text(length: 8, min: 0), 'Migration class for this version. Defaults to the previous version\'s class, which is what a patch must use.', true) + ->param('console', '', new Text(length: 32, min: 0), 'Console image tag to pin (appwrite/new). Left alone when omitted.', true) + ->param('dry-run', 'N', new WhiteList(['Y', 'N']), 'Print the changes without writing them', true) + ->callback($this->action(...)); + } + + public function action(string $version, string $migration, string $console, string $dryRun): void + { + if (\preg_match('/^\d+\.\d+\.\d+(-rc\.\d+)?$/', $version) !== 1) { + Console::error("'{$version}' is not an X.Y.Z or X.Y.Z-rc.N version."); + + return; + } + + $current = APP_VERSION_STABLE; + + if (\version_compare($version, $current, '<=')) { + Console::error("{$version} is not newer than the current {$current}."); + + return; + } + + $previous = \array_key_last(Migration::$versions); + $migration = $migration !== '' ? $migration : (string) Migration::$versions[$previous]; + + if ($this->isPatch($current, $version) && $migration !== (string) Migration::$versions[$previous]) { + Console::error( + "{$version} is a patch of {$current}, so it must reuse {$previous}'s migration " + . "(" . Migration::$versions[$previous] . "), not {$migration}. A fix that needs a " + . "schema change is a minor -- see the Releases section of AGENTS.md." + ); + + return; + } + + if (!\class_exists('Appwrite\\Migration\\Version\\' . $migration)) { + Console::error("Migration class {$migration} does not exist."); + + return; + } + + $changes = [ + ...$this->stableVersion($current, $version), + ...$this->readmePins($current, $version), + ...$this->migrationVersion($version, $migration), + ...($console !== '' ? $this->consolePin($console) : []), + ]; + + if ($changes === []) { + Console::success("Everything already reads {$version}; nothing to write."); + + return; + } + + Console::log("Releasing {$current} -> {$version} (migration {$migration})\n"); + + foreach ($changes as $file => $change) { + Console::log(" {$file}"); + Console::log(" {$change['from']}"); + Console::log(" {$change['to']}\n"); + + if ($dryRun === 'N') { + \file_put_contents($this->path($file), $change['contents']); + } + } + + if ($dryRun === 'Y') { + Console::info('Dry run; nothing written.'); + + return; + } + + Console::success(\count($changes) . ' file(s) written.'); + Console::info('Still by hand: release notes on the changelog, specs if the API changed, and request/response filters for public breaks.'); + } + + private function isPatch(string $current, string $version): bool + { + [$currentMajor, $currentMinor] = \explode('.', $current); + [$major, $minor] = \explode('.', $version); + + return $currentMajor === $major && $currentMinor === $minor; + } + + private function path(string $file): string + { + return $this->root . '/' . $file; + } + + private function read(string $file): string + { + $contents = @\file_get_contents($this->path($file)); + + if ($contents === false) { + throw new \RuntimeException("Cannot read {$file} from {$this->root}."); + } + + return $contents; + } + + /** + * @return array + */ + private function stableVersion(string $current, string $version): array + { + $contents = $this->read(self::CONSTANTS); + $from = "const APP_VERSION_STABLE = '{$current}';"; + $to = "const APP_VERSION_STABLE = '{$version}';"; + + if (!\str_contains($contents, $from)) { + throw new \RuntimeException(self::CONSTANTS . " does not declare APP_VERSION_STABLE as '{$current}'."); + } + + return [self::CONSTANTS => [ + 'from' => "- {$from}", + 'to' => "+ {$to}", + 'contents' => \str_replace($from, $to, $contents), + ]]; + } + + /** + * @return array + */ + private function readmePins(string $current, string $version): array + { + $changes = []; + + foreach (self::READMES as $readme) { + $contents = $this->read($readme); + $count = 0; + $updated = \str_replace("appwrite/appwrite:{$current}", "appwrite/appwrite:{$version}", $contents, $count); + + if ($count === 0) { + throw new \RuntimeException("{$readme} does not pin appwrite/appwrite:{$current}."); + } + + $changes[$readme] = [ + 'from' => "- appwrite/appwrite:{$current} ({$count}x)", + 'to' => "+ appwrite/appwrite:{$version} ({$count}x)", + 'contents' => $updated, + ]; + } + + return $changes; + } + + /** + * @return array + */ + private function migrationVersion(string $version, string $migration): array + { + $contents = $this->read(self::MIGRATION); + + if (\preg_match("/^\s*'" . \preg_quote($version, '/') . "'\s*=>/m", $contents) === 1) { + return []; + } + + $previous = \array_key_last(Migration::$versions); + $anchor = "'{$previous}' => '" . Migration::$versions[$previous] . "',"; + + if (!\str_contains($contents, $anchor)) { + throw new \RuntimeException(self::MIGRATION . " does not end its \$versions map with {$anchor}."); + } + + $entry = "'{$version}' => '{$migration}',"; + + return [self::MIGRATION => [ + 'from' => " {$anchor}", + 'to' => "+ {$entry}", + 'contents' => \str_replace($anchor, $anchor . "\n " . $entry, $contents), + ]]; + } + + /** + * @return array + */ + private function consolePin(string $console): array + { + $contents = $this->read(self::COMPOSE); + + if (\preg_match('/image: appwrite\/new:(\S+)/', $contents, $matches) !== 1) { + throw new \RuntimeException(self::COMPOSE . ' does not pin an appwrite/new image.'); + } + + if ($matches[1] === $console) { + return []; + } + + return [self::COMPOSE => [ + 'from' => "- image: appwrite/new:{$matches[1]}", + 'to' => "+ image: appwrite/new:{$console}", + 'contents' => \str_replace($matches[0], "image: appwrite/new:{$console}", $contents), + ]]; + } +} diff --git a/tests/unit/Release/MetadataTest.php b/tests/unit/Release/MetadataTest.php new file mode 100644 index 0000000000..a6a9566b8c --- /dev/null +++ b/tests/unit/Release/MetadataTest.php @@ -0,0 +1,128 @@ +assertMatchesRegularExpression( + '/^\d+\.\d+\.\d+(-(rc\.\d+|RC\d+))?$/', + APP_VERSION_STABLE, + 'APP_VERSION_STABLE must be an X.Y.Z version, optionally a release candidate.' + ); + } + + /** + * @return array + */ + public static function readmes(): array + { + return [ + 'README.md' => ['README.md'], + 'README-CN.md' => ['README-CN.md'], + ]; + } + + /** + * Every install snippet pins the released version. A README that still points + * at the previous version sends new self-hosters to an older image. + * + * @dataProvider readmes + */ + public function testReadmePinsStableVersion(string $readme): void + { + $contents = \file_get_contents(self::ROOT . '/' . $readme); + $this->assertNotFalse($contents, "{$readme} is unreadable."); + + \preg_match_all('#appwrite/appwrite:(\S+)#', $contents, $matches); + + $this->assertNotEmpty($matches[1], "{$readme} has no appwrite/appwrite install snippet to check."); + + foreach (\array_unique($matches[1]) as $pinned) { + $this->assertSame( + APP_VERSION_STABLE, + $pinned, + "{$readme} pins appwrite/appwrite:{$pinned} but APP_VERSION_STABLE is " . APP_VERSION_STABLE . '.' + ); + } + } + + /** + * A version with no migration mapping cannot be upgraded to, so the release + * would install and then fail for everyone already running Appwrite. + */ + public function testStableVersionHasAMigration(): void + { + if (\str_contains(APP_VERSION_STABLE, 'RC') || \str_contains(APP_VERSION_STABLE, '-rc.')) { + $this->markTestSkipped('Release candidates are mapped when the final version is cut.'); + } + + $this->assertArrayHasKey( + APP_VERSION_STABLE, + Migration::$versions, + 'APP_VERSION_STABLE has no entry in Migration::$versions.' + ); + } + + /** + * The last mapped version is the one being released. An entry newer than + * APP_VERSION_STABLE means a version was mapped but never marked stable. + */ + public function testNoMigrationMappingIsAheadOfStable(): void + { + if (\str_contains(APP_VERSION_STABLE, 'RC') || \str_contains(APP_VERSION_STABLE, '-rc.')) { + $this->markTestSkipped('Release candidates are mapped when the final version is cut.'); + } + + foreach (\array_keys(Migration::$versions) as $mapped) { + $this->assertLessThanOrEqual( + 0, + \version_compare((string) $mapped, APP_VERSION_STABLE), + "Migration::\$versions maps {$mapped}, which is newer than APP_VERSION_STABLE " . APP_VERSION_STABLE . '.' + ); + } + } + + /** + * The console ships as its own image, so self-hosted installs only get a + * console fix when this pin moves. An unpinned or floating tag would make an + * install unreproducible. + */ + public function testConsoleImageIsPinnedToAVersion(): void + { + $compose = \file_get_contents(self::ROOT . '/docker-compose.yml'); + $this->assertNotFalse($compose, 'docker-compose.yml is unreadable.'); + + $this->assertSame( + 1, + \preg_match('#image: appwrite/new:(\S+)#', $compose, $matches), + 'docker-compose.yml does not pin an appwrite/new console image.' + ); + + $this->assertMatchesRegularExpression( + '/^\d+\.\d+\.\d+$/', + $matches[1], + "The console is pinned to '{$matches[1]}', which is not an exact X.Y.Z version." + ); + } +} From 8f43d96407aa9a5c1aa473aa916498b58df5ef06 Mon Sep 17 00:00:00 2001 From: Levi van Noort <73097785+levivannoort@users.noreply.github.com> Date: Mon, 14 Sep 2026 11:29:54 +0200 Subject: [PATCH 05/37] feat(install): add --channel to install and upgrade Opting into nightly already worked through _APP_VERSION, since every service resolves its own image tag, but it left self-hosters to work out that the tag has to stay rolling. --channel=nightly derives X.Y-nightly from the stable version so the install tracks the minor line instead of freezing on one build. --- src/Appwrite/Platform/Tasks/Install.php | 30 ++++++++++++++++++++++++- src/Appwrite/Platform/Tasks/Upgrade.php | 4 +++- 2 files changed, 32 insertions(+), 2 deletions(-) diff --git a/src/Appwrite/Platform/Tasks/Install.php b/src/Appwrite/Platform/Tasks/Install.php index 96a4224b4c..663b9b0713 100644 --- a/src/Appwrite/Platform/Tasks/Install.php +++ b/src/Appwrite/Platform/Tasks/Install.php @@ -36,6 +36,9 @@ class Install extends Action private const string PATTERN_DB_PASSWORD_VAR = '/^_APP_DB_.*_PASS$/'; private const string PATTERN_SESSION_COOKIE = '/a_session_console=([^;]+)/'; + public const string CHANNEL_STABLE = 'stable'; + public const string CHANNEL_NIGHTLY = 'nightly'; + private const string APPWRITE_API_URL = 'http://appwrite'; private const string GROWTH_API_URL = 'https://growth.appwrite.io/v1'; @@ -64,6 +67,7 @@ class Install extends Action ->param('no-start', false, new Boolean(true), 'Run an interactive session', true) ->param('database', 'postgresql', new WhiteList(['postgresql', 'mariadb', 'mongodb']), 'Database to use (postgresql|mariadb|mongodb)', true) ->param('topology', 'combined', new WhiteList(['combined', 'separate']), 'Worker and scheduler topology (combined|separate)', true) + ->param('channel', self::CHANNEL_STABLE, new WhiteList([self::CHANNEL_STABLE, self::CHANNEL_NIGHTLY]), 'Release channel to track (stable|nightly). Nightly is unsupported and moves daily.', true) ->callback($this->action(...)); } @@ -75,7 +79,8 @@ class Install extends Action string $interactive, bool $noStart, string $database, - string $topology + string $topology, + string $channel = self::CHANNEL_STABLE ): void { $isUpgrade = $this->isUpgrade; $defaultHttpPort = '80'; @@ -562,6 +567,14 @@ class Install extends Action $database = $input['_APP_DB_ADAPTER'] ?? 'postgresql'; $version = \getenv('_APP_VERSION') ?: (\defined('APP_VERSION_STABLE') ? APP_VERSION_STABLE : 'latest'); + + // The nightly channel tracks the minor line rather than one release, so the + // tag has to stay rolling -- pinning X.Y.Z would freeze the install on a + // single build. See the Releases section of AGENTS.md. + if ($channel === self::CHANNEL_NIGHTLY) { + $version = $this->nightlyTag($version); + } + if ($isLocalInstall) { $version = 'local'; } @@ -823,6 +836,21 @@ class Install extends Action } } + /** + * The rolling nightly tag for the minor line a stable version belongs to, + * e.g. 2.0.1 -> 2.0-nightly. + */ + private function nightlyTag(string $version): string + { + if (\preg_match('/^(\d+)\.(\d+)\./', $version, $matches) !== 1) { + Console::warning("Cannot derive a nightly tag from '{$version}'; using the bare nightly tag."); + + return 'nightly'; + } + + return "{$matches[1]}.{$matches[2]}-nightly"; + } + private function createInitialAdminAccount(array $account, ?callable $progress, string $apiUrl, string $domain): void { $name = $account['name'] ?? 'Admin'; diff --git a/src/Appwrite/Platform/Tasks/Upgrade.php b/src/Appwrite/Platform/Tasks/Upgrade.php index e161f62beb..55b310f9fd 100644 --- a/src/Appwrite/Platform/Tasks/Upgrade.php +++ b/src/Appwrite/Platform/Tasks/Upgrade.php @@ -33,6 +33,7 @@ class Upgrade extends Install ->param('database', '', new Text(length: 0, min: 0), 'Ignored: an upgrade always keeps the database the installation already uses', true) ->param('topology', 'combined', new WhiteList(['combined', 'separate']), 'Worker and scheduler topology (combined|separate)', true) ->param('migrate', false, new Boolean(true), 'Run database migration after upgrade', true) + ->param('channel', Install::CHANNEL_STABLE, new WhiteList([Install::CHANNEL_STABLE, Install::CHANNEL_NIGHTLY]), 'Release channel to track (stable|nightly). Nightly is unsupported and moves daily.', true) ->callback($this->action(...)); } @@ -46,6 +47,7 @@ class Upgrade extends Install string $database, string $topology = 'combined', bool $migrate = false, + string $channel = Install::CHANNEL_STABLE, ): void { $this->isUpgrade = true; $this->migrate = $migrate; @@ -101,7 +103,7 @@ class Upgrade extends Install $this->lockedDatabase = $database; - parent::action($httpPort, $httpsPort, $organization, $image, $interactive, $noStart, $database, $topology); + parent::action($httpPort, $httpsPort, $organization, $image, $interactive, $noStart, $database, $topology, $channel); } protected function startWebServer( From 15bf52bdebb622a99bff7eeedc5f669aa6f38709 Mon Sep 17 00:00:00 2001 From: Levi van Noort <73097785+levivannoort@users.noreply.github.com> Date: Mon, 14 Sep 2026 11:29:54 +0200 Subject: [PATCH 06/37] docs(releases): document the stable and nightly channels Records the tag contract for both channels, the four rules the nightly channel rests on, the weekly patch train, and how a self-hoster opts in. Also points the release checklist at the new task and CI gates, and drops the reference to app/views/install/compose.phtml, which no longer exists. --- AGENTS.md | 39 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 38 insertions(+), 1 deletion(-) diff --git a/AGENTS.md b/AGENTS.md index da9c7beadc..473065f4a9 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -298,6 +298,39 @@ Preview builds set the flag on **both** the `specs` and `sdks` steps in `.github ## Releases +Self-hosted ships on two channels. Cloud is separate: it builds from `cl-*` tags to `appwrite/ce` via [`publish.yml`](.github/workflows/publish.yml) and nothing below applies to it. + +| Channel | Source | Trigger | Tags on `appwrite/appwrite` | +|---------|--------|---------|------------------------------| +| stable | active release branch (`2.0.x`) | GitHub Release, published by a human | `X.Y.Z` (immutable), `X.Y`, `X`, `latest` | +| nightly | same branch, newest CI-green commit | daily at 00:00 UTC, or `workflow_dispatch` | `X.Y.Z-nightly.` (immutable), `X.Y-nightly`, `nightly` | + +[`nightly.yml`](.github/workflows/nightly.yml) builds the channel; [`security-scan.yml`](.github/workflows/security-scan.yml) is the Trivy scan that used to own that file name. `X.Y.Z` in a nightly tag is the **next** patch on the line — the nightly is a prerelease of the version it is heading for, so `2.0.2-nightly.20260915` sorts below `2.0.2` — and `` is `YYYYMMDD`. + +Four rules keep the channel safe. They are requirements, not preferences: + +1. **A patch never adds a migration.** Map a new patch to the previous version's class in `Migration::$versions`. A fix that needs a schema change is a minor. This is what lets a nightly user roll back to yesterday's build, and `1.9.6` (which introduced V25) is the exception not to repeat. +2. **Nightly publishes only a CI-green commit**, walking back from the tip rather than shipping it, so a red branch delays the channel instead of breaking it. +3. **The release branch stays releasable.** Backports land as complete cherry-picks, behind a flag when the fix is not finished. +4. **Nightly publishes to `appwrite/appwrite` only.** It must never be able to push to `appwrite/ce`. + +Self-hosters opt in through `_APP_VERSION`, which every service in `docker-compose.yml` already resolves (`${_APP_IMAGE:-appwrite/appwrite}:${_APP_VERSION:-latest}`), or with `--channel` on [`install`](src/Appwrite/Platform/Tasks/Install.php) and [`upgrade`](src/Appwrite/Platform/Tasks/Upgrade.php): + +``` +_APP_VERSION=2.0-nightly # then: docker compose pull && docker compose up -d && docker compose exec appwrite migrate +``` + +Nightly is unsupported, has no SLA, and only ever moves within one patch line. `X.Y-nightly` is the form to recommend; the bare `nightly` tag follows the newest line and will jump minors. + +### Patch train + +Patches leave on a schedule so that "is this worth a release?" stops being a per-fix decision: + +- **Weekly**, on the active release branch, if at least one user-visible fix landed since the last tag. Nothing landed, no release — silently. +- **Security and data-loss fixes ship out of band**, same day. +- **Backports are labelled, not remembered.** Fixes land on `main`; a `backport X.Y.x` label opens the cherry-pick PR. +- **A console publish is a release trigger.** The console is pinned (`appwrite/new:X.Y.Z`) in `docker-compose.yml`, so a console-only fix reaches self-hosters only through an `appwrite/appwrite` patch. Bump the pin, let CI pass, and let it ride the next train. + ### Patch version When bumping a patch (e.g. `1.9.0` → `1.9.1`): @@ -307,6 +340,8 @@ When bumping a patch (e.g. `1.9.0` → `1.9.1`): - [`README.md`](README.md) and [`README-CN.md`](README-CN.md) — `appwrite/appwrite:X.Y.Z` in all three install blocks each - [`src/Appwrite/Migration/Migration.php`](src/Appwrite/Migration/Migration.php) — add the version to `$versions`, mapping to a new migration class or the same class as the previous version +`docker compose exec appwrite release X.Y.Z` writes all four in one pass; `tests/unit/Release/MetadataTest.php` fails when they disagree. Edit them by hand only when the task cannot express the change. + Ask the user to review, publish notes on the [Appwrite changelog](https://appwrite.io/changelog), generate specs if the API changed, and add request/response filters if needed. `APP_CACHE_BUSTER` is not a version number and does not track releases. It salts the response cache key in [`Request::cacheIdentifier()`](src/Appwrite/Utopia/Request.php) for the routes labelled `cache` (file preview, avatars). Bump it only when cached output would now be wrong — a changed image pipeline or new bundled avatar assets — since every bump orphans every entry and regenerates them. @@ -315,10 +350,12 @@ Ask the user to review, publish notes on the [Appwrite changelog](https://appwri A release is not ready until a **fresh install** and an **upgrade from the previous stable** both work with realistic data. Previous baseline = highest stable semver tag lower than the target (ignore RC/beta/alpha; prefer `git ls-remote --tags origin`). +Both gates run in CI on release branches — the `install` and `upgrade` jobs in [`ci.yml`](.github/workflows/ci.yml), seeding through the public API with [`upgrade-seed.sh`](.github/scripts/upgrade-seed.sh) and asserting with [`upgrade-verify.sh`](.github/scripts/upgrade-verify.sh). Read those jobs before doing the work by hand; do it by hand when the release changes something they do not cover (a new service, a data shape they do not seed). + **Fresh install:** `docker compose down -v` then `up -d --force-recreate --build --wait`. Check `docker compose ps` / logs for crash loops, missing env, failed workers. Hit `/v1/health/version` on the public port. Run unit tests, `tests/e2e/General`, and service e2e. Exercise console users, projects, databases/rows, storage, and (when in scope) functions/sites through public APIs — not empty-stack health checks alone. **Upgrade:** install the previous stable image, seed broad data (empty values, long strings, relationships, mixed permissions), keep volumes, switch to the target image, run migrate. Migration must complete, be idempotent, and preserve seeded data through public API reads/writes. -**Metadata:** `APP_VERSION_STABLE`; Appwrite and console tags in `docker-compose.yml` and [`app/views/install/compose.phtml`](app/views/install/compose.phtml); README install snippets; `Migration.php` `$versions`; [changelog](https://appwrite.io/changelog). For public API breaks: request filters in `src/Appwrite/Utopia/Request/Filters/V*.php`, response filters in `src/Appwrite/Utopia/Response/Filters/V*.php`, registered in [`app/controllers/general.php`](app/controllers/general.php) for `x-appwrite-response-format`. Unit-test filters under `tests/unit/Utopia/{Request,Response}/Filters`; add e2e with that header when routing, auth, or persistence is involved. +**Metadata:** `APP_VERSION_STABLE`; Appwrite and console tags in `docker-compose.yml`; README install snippets; `Migration.php` `$versions`; [changelog](https://appwrite.io/changelog). For public API breaks: request filters in `src/Appwrite/Utopia/Request/Filters/V*.php`, response filters in `src/Appwrite/Utopia/Response/Filters/V*.php`, registered in [`app/controllers/general.php`](app/controllers/general.php) for `x-appwrite-response-format`. Unit-test filters under `tests/unit/Utopia/{Request,Response}/Filters`; add e2e with that header when routing, auth, or persistence is involved. Do not approve an RC/final until both gates pass, metadata matches the target, and unintended public breaks have filters (or the owner documents the break on the changelog). From 471e0870986ec3a5279024a390cc76f58c254c99 Mon Sep 17 00:00:00 2001 From: Levi van Noort <73097785+levivannoort@users.noreply.github.com> Date: Mon, 14 Sep 2026 12:09:16 +0200 Subject: [PATCH 07/37] refactor(ci): simplify the upgrade gate and drop python Verification was 20-odd hand-written assertions behind an assert helper and a failure counter. It is now one projected snapshot, taken before the upgrade and diffed after it -- shorter, and the failure output is a diff of what changed rather than a list of lines. The projection keeps only the fields the seed put in, so an upgrade is still free to add fields to a model. Writes are behind --write so the read-only comparison can run after every migrate; previously the second run diffed against rows the first run had written. The previous-stable resolver was a 89-line python script and is now four lines of shell: sort the published tags together with the target and take what lands before it. Same answers on live Docker Hub data (2.0.0 -> 1.9.6, 2.1.0 -> 2.0.0), and no python in CI. Also moves the candidate's compose env to the job level instead of repeating it on seven steps, and renames USER so the scripts stop shadowing the shell's own variable. 315 lines down from 477. --- .github/scripts/previous-stable.py | 89 ------------- .github/scripts/upgrade-lib.sh | 135 +++++++++++--------- .github/scripts/upgrade-seed.sh | 196 +++++++++++------------------ .github/scripts/upgrade-verify.sh | 130 ++++++------------- .github/workflows/ci.yml | 69 +++++----- 5 files changed, 229 insertions(+), 390 deletions(-) delete mode 100755 .github/scripts/previous-stable.py diff --git a/.github/scripts/previous-stable.py b/.github/scripts/previous-stable.py deleted file mode 100755 index 19221f8c27..0000000000 --- a/.github/scripts/previous-stable.py +++ /dev/null @@ -1,89 +0,0 @@ -#!/usr/bin/env python3 -"""Print the published stable version directly below a target version. - -The upgrade gate needs a baseline to upgrade *from*. Docker Hub is the source of -truth rather than git tags, because the image has to exist for the gate to run at -all -- a tag with no published image is not an upgrade anyone can perform. - - ./previous-stable.py 2.0.0 -> 1.9.6 - -Release candidates and channel tags are ignored: only X.Y.Z counts as a stable a -self-hoster could be upgrading from. -""" - -from __future__ import annotations - -import json -import sys -import urllib.error -import urllib.request - -REPOSITORY = "appwrite/appwrite" -TAGS_URL = f"https://hub.docker.com/v2/repositories/{REPOSITORY}/tags/" -PAGES = 3 -PAGE_SIZE = 100 - - -def parse(version: str) -> tuple[int, int, int] | None: - parts = version.split(".") - - if len(parts) != 3 or not all(part.isdigit() for part in parts): - return None - - major, minor, patch = (int(part) for part in parts) - - return major, minor, patch - - -def published() -> list[tuple[int, int, int]]: - versions: list[tuple[int, int, int]] = [] - url: str | None = f"{TAGS_URL}?page_size={PAGE_SIZE}&ordering=last_updated" - - for _ in range(PAGES): - if url is None: - break - - request = urllib.request.Request(url, headers={"User-Agent": "appwrite-upgrade-gate"}) - - with urllib.request.urlopen(request, timeout=60) as response: - payload = json.load(response) - - versions.extend( - version - for version in (parse(tag["name"]) for tag in payload.get("results", [])) - if version is not None - ) - - url = payload.get("next") - - return versions - - -def main() -> int: - if len(sys.argv) != 2: - print(f"usage: {sys.argv[0]} ", file=sys.stderr) - return 2 - - target = parse(sys.argv[1]) - - if target is None: - print(f"'{sys.argv[1]}' is not an X.Y.Z version.", file=sys.stderr) - return 2 - - try: - candidates = sorted(version for version in published() if version < target) - except (urllib.error.URLError, TimeoutError, json.JSONDecodeError) as error: - print(f"Could not read tags from Docker Hub: {error}", file=sys.stderr) - return 1 - - if not candidates: - print(f"No published stable below {sys.argv[1]}.", file=sys.stderr) - return 1 - - print(".".join(str(part) for part in candidates[-1])) - - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/.github/scripts/upgrade-lib.sh b/.github/scripts/upgrade-lib.sh index 055cff6627..9f03cd151c 100644 --- a/.github/scripts/upgrade-lib.sh +++ b/.github/scripts/upgrade-lib.sh @@ -1,81 +1,94 @@ #!/usr/bin/env bash -# Shared helpers for the upgrade gate scripts. Sourced, not executed. +# Shared ground for the upgrade gate. Sourced, not executed. # -# The upgrade gate talks to Appwrite the way a self-hoster does: over the public -# HTTP API, with no access to the container internals. That is the point -- it -# proves the data survives a version change through the surface users depend on. +# The gate talks to Appwrite the way a self-hoster does -- over the public HTTP +# API, with no access to container internals -- because that is the surface the +# data has to survive a version change on. + +# The resource names and value shapes below are read by the scripts that source +# this file, so shellcheck cannot see their use when linting it on its own. +# shellcheck disable=SC2034 set -euo pipefail : "${ENDPOINT:?ENDPOINT must be set, e.g. http://localhost:8080/v1}" MANIFEST="${MANIFEST:-upgrade-seed.json}" -COOKIE_JAR="${COOKIE_JAR:-$(dirname "$MANIFEST")/upgrade-cookies.txt}" +STATE="$(dirname "$MANIFEST")" +COOKIE_JAR="${STATE}/upgrade-cookies.txt" +SNAPSHOT="${STATE}/upgrade-snapshot.json" + +# Seeded resources. Fixed IDs so both scripts address the same rows without +# passing them around. +DATABASE="gate" +AUTHORS="authors" +POSTS="posts" +BUCKET="gate" +FILE_ID="gate-file" +USER_ID="gate-user" +AUTHOR="author-1" + +# The value shapes a migration is most likely to mangle. +LONG_VALUE="$(printf 'l%.0s' {1..4000})" +UNICODE_VALUE="ünïcødé — 日本語 — 🚂 — — O'Brien \"quoted\"" fail() { echo "::error::$*" >&2 exit 1 } -# Waits for the stack to answer on the public port. A fresh install spends a -# while on migrations and worker boot, so this is generous. +# A fresh install spends a while on migrations and worker boot. wait_for_appwrite() { - local attempts="${1:-90}" i + local i - for ((i = 1; i <= attempts; i++)); do + for ((i = 1; i <= 120; i++)); do if curl --silent --fail --max-time 5 "${ENDPOINT}/health/version" >/dev/null 2>&1; then - echo "Appwrite answering at ${ENDPOINT} after ${i}s." return 0 fi sleep 1 done - fail "Appwrite did not answer at ${ENDPOINT}/health/version within ${attempts}s." + fail "Appwrite did not answer at ${ENDPOINT}/health/version within 120s." } -server_version() { +version() { curl --silent --fail --max-time 10 "${ENDPOINT}/health/version" | jq -r '.version' } -# Console calls: cookie session, browser origin. Used to create the org and -# project a self-hoster starts with. +# Console session, browser origin: how the organization and project get made. console() { local method="$1" path="$2" body="${3:-}" - local args=( - --silent --show-error --fail-with-body --max-time 30 - -X "$method" - -H 'content-type: application/json' - -H 'origin: http://localhost' - -H 'x-appwrite-project: console' - -c "$COOKIE_JAR" -b "$COOKIE_JAR" - ) - [ -n "$body" ] && args+=(-d "$body") - curl "${args[@]}" "${ENDPOINT}${path}" + curl --silent --show-error --fail-with-body --max-time 30 \ + -X "$method" \ + -H 'content-type: application/json' \ + -H 'origin: http://localhost' \ + -H 'x-appwrite-project: console' \ + -c "$COOKIE_JAR" -b "$COOKIE_JAR" \ + ${body:+-d "$body"} \ + "${ENDPOINT}${path}" } -# Project calls: API key, the way a server SDK talks to Appwrite. +# API key: how a server SDK talks to a project. api() { local method="$1" path="$2" body="${3:-}" - local args=( - --silent --show-error --fail-with-body --max-time 60 - -X "$method" - -H 'content-type: application/json' - -H "x-appwrite-project: ${PROJECT_ID:?PROJECT_ID not set}" - -H "x-appwrite-key: ${API_KEY:?API_KEY not set}" - ) - [ -n "$body" ] && args+=(-d "$body") - curl "${args[@]}" "${ENDPOINT}${path}" + curl --silent --show-error --fail-with-body --max-time 60 \ + -X "$method" \ + -H 'content-type: application/json' \ + -H "x-appwrite-project: ${PROJECT:?PROJECT not set}" \ + -H "x-appwrite-key: ${KEY:?KEY not set}" \ + ${body:+-d "$body"} \ + "${ENDPOINT}${path}" } -# Columns are processed asynchronously, so rows cannot be written until every -# column on the table reports available. +# Columns are processed asynchronously; rows cannot be written until they are all +# available. wait_for_columns() { - local database="$1" table="$2" attempts="${3:-60}" i statuses + local table="$1" i statuses - for ((i = 1; i <= attempts; i++)); do - statuses=$(api GET "/tablesdb/${database}/tables/${table}/columns" \ + for ((i = 1; i <= 60; i++)); do + statuses=$(api GET "/tablesdb/${DATABASE}/tables/${table}/columns" \ | jq -r '.columns[].status' | sort -u | tr '\n' ' ') case "$statuses" in @@ -85,25 +98,31 @@ wait_for_columns() { sleep 1 done - fail "Columns on ${table} did not become available within ${attempts}s (last: ${statuses:-none})." + fail "Columns on ${table} did not become available within 60s (last: ${statuses:-none})." } -manifest_get() { - jq -r "$1" "$MANIFEST" -} - -# Asserts on values read back through the API. Prints every check so a failed -# upgrade job reads as a list of what survived and what did not. -assert_equals() { - local what="$1" expected="$2" actual="$3" - - if [ "$expected" = "$actual" ]; then - echo " ok ${what}" - return 0 - fi - - echo " FAIL ${what}" - echo " expected: ${expected}" - echo " actual: ${actual}" - FAILURES=$((${FAILURES:-0} + 1)) +# Everything the gate asserts on, projected to just the fields we seeded and +# sorted for a stable comparison. Projecting rather than dumping whole responses +# is deliberate: an upgrade is allowed to add fields to a model, and only the +# values a user put in have to come back unchanged. +snapshot() { + jq -nS \ + --argjson rows "$(api GET "/tablesdb/${DATABASE}/tables/${POSTS}/rows")" \ + --argjson table "$(api GET "/tablesdb/${DATABASE}/tables/${POSTS}")" \ + --argjson user "$(api GET "/users/${USER_ID}")" \ + --argjson file "$(api GET "/storage/buckets/${BUCKET}/files/${FILE_ID}")" \ + '{ + rows: [$rows.rows[] | { + id: .["$id"], + title: .title, + body: .body, + tag: .tag, + author: (if (.author | type) == "object" then .author["$id"] else .author end), + permissions: (.["$permissions"] | sort) + }] | sort_by(.id), + total: $rows.total, + table: { permissions: ($table["$permissions"] | sort), rowSecurity: $table.rowSecurity }, + user: { email: $user.email, name: $user.name }, + file: { id: $file["$id"], size: $file.sizeOriginal } + }' } diff --git a/.github/scripts/upgrade-seed.sh b/.github/scripts/upgrade-seed.sh index c538cb456c..2c4f4dad50 100755 --- a/.github/scripts/upgrade-seed.sh +++ b/.github/scripts/upgrade-seed.sh @@ -1,159 +1,115 @@ #!/usr/bin/env bash -# Seeds an Appwrite install with the data shapes the upgrade gate cares about: -# empty values, long strings, unicode, relationships and mixed permissions. -# Run against the PREVIOUS stable, before switching images. +# Seeds an install with the data shapes the upgrade gate cares about, then +# records a snapshot of them. Run against the PREVIOUS stable, before switching +# images. # -# ENDPOINT=http://localhost:8080/v1 MANIFEST=upgrade-seed.json ./upgrade-seed.sh -# -# Writes every created ID to MANIFEST so upgrade-verify.sh can read the same -# resources back after the migration. +# ENDPOINT=http://localhost:8080/v1 ./upgrade-seed.sh set -euo pipefail # shellcheck source=.github/scripts/upgrade-lib.sh source "$(dirname "${BASH_SOURCE[0]}")/upgrade-lib.sh" -ROOT_EMAIL="upgrade-gate@example.com" -ROOT_PASSWORD="upgrade-gate-password-1" -DATABASE_ID="gate" -AUTHORS_TABLE="authors" -POSTS_TABLE="posts" -BUCKET_ID="gate" -FILE_ID="gate-file" - -# The shapes a migration is most likely to mangle. -EMPTY_STRING="" -LONG_STRING=$(printf 'l%.0s' $(seq 1 4000)) -UNICODE_STRING="ünïcødé — 日本語 — 🚂 — — O'Brien \"quoted\"" - wait_for_appwrite -echo "Seeding $(server_version) at ${ENDPOINT}" +echo "Seeding $(version) at ${ENDPOINT}" -# --- console: root account, organization, project, API key ------------------- +# --- console: root account, organization, project, key ----------------------- -echo "==> Console account and project" rm -f "$COOKIE_JAR" -# On a fresh install the first account becomes root. -console POST /account "$(jq -nc \ - --arg email "$ROOT_EMAIL" \ - --arg password "$ROOT_PASSWORD" \ - '{userId: "unique()", email: $email, password: $password, name: "Upgrade Gate"}')" \ - >/dev/null +# The first account on a fresh install becomes root. +console POST /account \ + '{"userId":"unique()","email":"gate@example.com","password":"upgrade-gate-1","name":"Gate"}' >/dev/null +console POST /account/sessions/email \ + '{"email":"gate@example.com","password":"upgrade-gate-1"}' >/dev/null -console POST /account/sessions/email "$(jq -nc \ - --arg email "$ROOT_EMAIL" \ - --arg password "$ROOT_PASSWORD" \ - '{email: $email, password: $password}')" >/dev/null +TEAM=$(console POST /teams '{"teamId":"unique()","name":"Gate"}' | jq -r '.["$id"]') +PROJECT=$(console POST /projects "{\"projectId\":\"gate\",\"name\":\"Gate\",\"teamId\":\"${TEAM}\"}" | jq -r '.["$id"]') +export PROJECT -TEAM_ID=$(console POST /teams '{"teamId":"unique()","name":"Upgrade Gate"}' | jq -r '.["$id"]') -[ -n "$TEAM_ID" ] && [ "$TEAM_ID" != "null" ] || fail "Could not create the organization." - -PROJECT_ID=$(console POST /projects "$(jq -nc \ - --arg teamId "$TEAM_ID" \ - '{projectId: "upgrade-gate", name: "Upgrade Gate", teamId: $teamId}')" | jq -r '.["$id"]') -[ -n "$PROJECT_ID" ] && [ "$PROJECT_ID" != "null" ] || fail "Could not create the project." -export PROJECT_ID - -API_KEY=$(console POST "/projects/${PROJECT_ID}/keys" "$(jq -nc \ - '{name: "upgrade-gate", scopes: ["users.read","users.write","tables.read","tables.write","databases.read","databases.write","rows.read","rows.write","documents.read","documents.write","collections.read","collections.write","files.read","files.write","buckets.read","buckets.write"]}')" \ +KEY=$(console POST "/projects/${PROJECT}/keys" \ + '{"name":"gate","scopes":["users.read","users.write","tables.read","tables.write","rows.read","rows.write","files.read","files.write","buckets.read","buckets.write"]}' \ | jq -r '.secret') -[ -n "$API_KEY" ] && [ "$API_KEY" != "null" ] || fail "Could not create an API key." -export API_KEY +export KEY -# --- tables, columns, relationship ------------------------------------------- +[ "$PROJECT" = 'gate' ] && [ -n "$KEY" ] && [ "$KEY" != 'null' ] || fail 'Could not create a project and key.' -echo "==> Database, tables and columns" -api POST /tablesdb "$(jq -nc --arg id "$DATABASE_ID" '{databaseId: $id, name: "Gate"}')" >/dev/null +# --- tables ------------------------------------------------------------------ -api POST "/tablesdb/${DATABASE_ID}/tables" "$(jq -nc \ - --arg id "$AUTHORS_TABLE" \ - '{tableId: $id, name: "Authors", permissions: ["read(\"any\")"], rowSecurity: false}')" >/dev/null +echo '==> Tables' +api POST /tablesdb "{\"databaseId\":\"${DATABASE}\",\"name\":\"Gate\"}" >/dev/null -api POST "/tablesdb/${DATABASE_ID}/tables" "$(jq -nc \ - --arg id "$POSTS_TABLE" \ - '{tableId: $id, name: "Posts", permissions: ["read(\"any\")","create(\"users\")"], rowSecurity: true}')" >/dev/null +api POST "/tablesdb/${DATABASE}/tables" "$(jq -nc --arg id "$AUTHORS" '{ + tableId: $id, name: "Authors", permissions: ["read(\"any\")"], rowSecurity: false, + columns: [{key: "name", type: "string", size: 255, required: true}] +}')" >/dev/null -api POST "/tablesdb/${DATABASE_ID}/tables/${AUTHORS_TABLE}/columns/string" \ - '{"key":"name","size":255,"required":true}' >/dev/null -wait_for_columns "$DATABASE_ID" "$AUTHORS_TABLE" +api POST "/tablesdb/${DATABASE}/tables" "$(jq -nc --arg id "$POSTS" '{ + tableId: $id, name: "Posts", permissions: ["read(\"any\")", "create(\"users\")"], rowSecurity: true, + columns: [ + {key: "title", type: "string", size: 255, required: true}, + {key: "body", type: "string", size: 5000, required: false}, + {key: "tag", type: "string", size: 64, required: false, default: "untagged"} + ] +}')" >/dev/null -api POST "/tablesdb/${DATABASE_ID}/tables/${POSTS_TABLE}/columns/string" \ - '{"key":"title","size":255,"required":true}' >/dev/null -api POST "/tablesdb/${DATABASE_ID}/tables/${POSTS_TABLE}/columns/string" \ - '{"key":"body","size":5000,"required":false}' >/dev/null -api POST "/tablesdb/${DATABASE_ID}/tables/${POSTS_TABLE}/columns/string" \ - '{"key":"tag","size":64,"required":false,"default":"untagged"}' >/dev/null -api POST "/tablesdb/${DATABASE_ID}/tables/${POSTS_TABLE}/columns/relationship" "$(jq -nc \ - --arg related "$AUTHORS_TABLE" \ - '{relatedTableId: $related, type: "manyToOne", twoWay: false, key: "author", onDelete: "setNull"}')" >/dev/null -wait_for_columns "$DATABASE_ID" "$POSTS_TABLE" +# Relationships are not expressible in the inline columns array. +api POST "/tablesdb/${DATABASE}/tables/${POSTS}/columns/relationship" "$(jq -nc --arg related "$AUTHORS" '{ + relatedTableId: $related, type: "manyToOne", twoWay: false, key: "author", onDelete: "setNull" +}')" >/dev/null + +wait_for_columns "$AUTHORS" +wait_for_columns "$POSTS" # --- rows -------------------------------------------------------------------- -echo "==> Rows" -AUTHOR_ID=$(api POST "/tablesdb/${DATABASE_ID}/tables/${AUTHORS_TABLE}/rows" \ - '{"rowId":"author-1","data":{"name":"Ada Lovelace"}}' | jq -r '.["$id"]') +echo '==> Rows' +api POST "/tablesdb/${DATABASE}/tables/${AUTHORS}/rows" \ + "{\"rowId\":\"${AUTHOR}\",\"data\":{\"name\":\"Ada Lovelace\"}}" >/dev/null -api POST "/tablesdb/${DATABASE_ID}/tables/${POSTS_TABLE}/rows" "$(jq -nc \ - --arg author "$AUTHOR_ID" \ - --arg body "$EMPTY_STRING" \ - '{rowId: "post-empty", data: {title: "Empty body", body: $body, author: $author}}')" >/dev/null +# An empty string, a relationship, and a column default left to apply. +api POST "/tablesdb/${DATABASE}/tables/${POSTS}/rows" "$(jq -nc --arg author "$AUTHOR" '{ + rowId: "post-empty", data: {title: "Empty body", body: "", author: $author} +}')" >/dev/null -api POST "/tablesdb/${DATABASE_ID}/tables/${POSTS_TABLE}/rows" "$(jq -nc \ - --arg body "$LONG_STRING" \ - '{rowId: "post-long", data: {title: "Long body", body: $body}}')" >/dev/null +api POST "/tablesdb/${DATABASE}/tables/${POSTS}/rows" "$(jq -nc --arg body "$LONG_VALUE" '{ + rowId: "post-long", data: {title: "Long body", body: $body} +}')" >/dev/null -api POST "/tablesdb/${DATABASE_ID}/tables/${POSTS_TABLE}/rows" "$(jq -nc \ - --arg body "$UNICODE_STRING" \ - '{rowId: "post-unicode", data: {title: "Unicode body", body: $body}}')" >/dev/null +api POST "/tablesdb/${DATABASE}/tables/${POSTS}/rows" "$(jq -nc --arg body "$UNICODE_VALUE" '{ + rowId: "post-unicode", data: {title: "Unicode body", body: $body} +}')" >/dev/null -# Row-level permissions, deliberately different from the table's. -api POST "/tablesdb/${DATABASE_ID}/tables/${POSTS_TABLE}/rows" "$(jq -nc \ - '{rowId: "post-perms", data: {title: "Mixed permissions", body: "kept"}, permissions: ["read(\"any\")","update(\"users\")"]}')" >/dev/null +# Row permissions deliberately different from the table's. +api POST "/tablesdb/${DATABASE}/tables/${POSTS}/rows" '{ + "rowId": "post-perms", "data": {"title": "Mixed permissions", "body": "kept"}, + "permissions": ["read(\"any\")", "update(\"users\")"] +}' >/dev/null -# --- users, storage ---------------------------------------------------------- +# --- user and file ----------------------------------------------------------- -echo "==> User and file" -USER_ID=$(api POST /users "$(jq -nc \ - --arg name "$UNICODE_STRING" \ - '{userId: "gate-user", email: "gate-user@example.com", password: "gate-user-password-1", name: $name}')" \ - | jq -r '.["$id"]') +echo '==> User and file' +api POST /users "$(jq -nc --arg id "$USER_ID" --arg name "$UNICODE_VALUE" '{ + userId: $id, email: "gate-user@example.com", password: "gate-user-1", name: $name +}')" >/dev/null -api POST /storage/buckets "$(jq -nc \ - --arg id "$BUCKET_ID" \ - '{bucketId: $id, name: "Gate", permissions: ["read(\"any\")"], fileSecurity: false}')" >/dev/null - -FILE_CONTENT=$(dirname "$MANIFEST")/gate-file.txt -printf '%s\n' "$UNICODE_STRING" > "$FILE_CONTENT" +api POST /storage/buckets "$(jq -nc --arg id "$BUCKET" '{ + bucketId: $id, name: "Gate", permissions: ["read(\"any\")"], fileSecurity: false +}')" >/dev/null +printf '%s\n' "$UNICODE_VALUE" > "${STATE}/gate-file.txt" curl --silent --show-error --fail-with-body --max-time 60 \ -X POST \ - -H "x-appwrite-project: ${PROJECT_ID}" \ - -H "x-appwrite-key: ${API_KEY}" \ + -H "x-appwrite-project: ${PROJECT}" \ + -H "x-appwrite-key: ${KEY}" \ -F "fileId=${FILE_ID}" \ - -F "file=@${FILE_CONTENT}" \ - "${ENDPOINT}/storage/buckets/${BUCKET_ID}/files" >/dev/null + -F "file=@${STATE}/gate-file.txt" \ + "${ENDPOINT}/storage/buckets/${BUCKET}/files" >/dev/null -# --- manifest ---------------------------------------------------------------- +# --- record ------------------------------------------------------------------ -jq -nc \ - --arg version "$(server_version)" \ - --arg projectId "$PROJECT_ID" \ - --arg apiKey "$API_KEY" \ - --arg teamId "$TEAM_ID" \ - --arg databaseId "$DATABASE_ID" \ - --arg authorsTable "$AUTHORS_TABLE" \ - --arg postsTable "$POSTS_TABLE" \ - --arg authorId "$AUTHOR_ID" \ - --arg userId "$USER_ID" \ - --arg bucketId "$BUCKET_ID" \ - --arg fileId "$FILE_ID" \ - --arg longString "$LONG_STRING" \ - --arg unicodeString "$UNICODE_STRING" \ - '{seededOn: $version, projectId: $projectId, apiKey: $apiKey, teamId: $teamId, - databaseId: $databaseId, authorsTable: $authorsTable, postsTable: $postsTable, - authorId: $authorId, userId: $userId, bucketId: $bucketId, fileId: $fileId, - longString: $longString, unicodeString: $unicodeString}' > "$MANIFEST" +jq -nc --arg version "$(version)" --arg project "$PROJECT" --arg key "$KEY" \ + '{seededOn: $version, project: $project, key: $key}' > "$MANIFEST" +snapshot > "$SNAPSHOT" -echo "Seeded from $(server_version); manifest at ${MANIFEST}" +echo "Seeded from $(version); snapshot at ${SNAPSHOT}" diff --git a/.github/scripts/upgrade-verify.sh b/.github/scripts/upgrade-verify.sh index 7a1a713fa8..39a5a84387 100755 --- a/.github/scripts/upgrade-verify.sh +++ b/.github/scripts/upgrade-verify.sh @@ -1,120 +1,72 @@ #!/usr/bin/env bash -# Reads back everything upgrade-seed.sh created and asserts it survived the -# migration unchanged, then writes new data to prove the upgraded install is -# still usable. Run against the TARGET version, after migrate. +# Asserts the seeded data came through the migration unchanged. Run against the +# TARGET version, after migrate. # -# ENDPOINT=http://localhost:8080/v1 MANIFEST=upgrade-seed.json ./upgrade-verify.sh +# ENDPOINT=http://localhost:8080/v1 ./upgrade-verify.sh +# ENDPOINT=http://localhost:8080/v1 ./upgrade-verify.sh --write +# +# The default is read-only, so it can run after every migrate. --write also +# proves the upgraded install still takes writes; pass it once, on the last run, +# since the rows it creates would show up in the next comparison. set -euo pipefail # shellcheck source=.github/scripts/upgrade-lib.sh source "$(dirname "${BASH_SOURCE[0]}")/upgrade-lib.sh" -[ -f "$MANIFEST" ] || fail "No manifest at ${MANIFEST}; did upgrade-seed.sh run?" +[ -f "$SNAPSHOT" ] || fail "No snapshot at ${SNAPSHOT}; did upgrade-seed.sh run?" -FAILURES=0 - -PROJECT_ID=$(manifest_get '.projectId'); export PROJECT_ID -API_KEY=$(manifest_get '.apiKey'); export API_KEY -DATABASE_ID=$(manifest_get '.databaseId') -AUTHORS_TABLE=$(manifest_get '.authorsTable') -POSTS_TABLE=$(manifest_get '.postsTable') -USER_ID=$(manifest_get '.userId') -BUCKET_ID=$(manifest_get '.bucketId') -FILE_ID=$(manifest_get '.fileId') -LONG_STRING=$(manifest_get '.longString') -UNICODE_STRING=$(manifest_get '.unicodeString') -SEEDED_ON=$(manifest_get '.seededOn') +PROJECT=$(jq -r '.project' "$MANIFEST"); export PROJECT +KEY=$(jq -r '.key' "$MANIFEST"); export KEY +SEEDED_ON=$(jq -r '.seededOn' "$MANIFEST") wait_for_appwrite -UPGRADED_TO=$(server_version) +UPGRADED_TO=$(version) -echo "Verifying upgrade ${SEEDED_ON} -> ${UPGRADED_TO}" +echo "Verifying ${SEEDED_ON} -> ${UPGRADED_TO}" if [ "$SEEDED_ON" = "$UPGRADED_TO" ]; then fail "Still reporting ${UPGRADED_TO}; the image did not change, so this run proves nothing." fi -# --- the API key still authenticates ----------------------------------------- +# The project, its key and its scopes have to survive before anything else can +# be checked. +api GET /users >/dev/null 2>&1 || fail "The seeded API key no longer authenticates against project ${PROJECT}." -# The project, its API key and its scopes all have to survive the migration -# before any of the assertions below mean anything. -echo "==> Project access" -if ! api GET /users >/dev/null 2>&1; then - fail "The seeded API key no longer authenticates against project ${PROJECT_ID}." +# --- nothing the user put in changed ----------------------------------------- + +if ! diff --unified=1 --label "seeded on ${SEEDED_ON}" --label "read back on ${UPGRADED_TO}" \ + "$SNAPSHOT" <(snapshot); then + fail 'Seeded data changed across the upgrade; see the diff above.' fi -echo " ok seeded API key still authenticates" -# --- seeded rows survived ---------------------------------------------------- +echo 'ok: rows, permissions, relationship, user and file all read back unchanged' -echo "==> Rows" -row() { api GET "/tablesdb/${DATABASE_ID}/tables/${POSTS_TABLE}/rows/$1"; } - -empty=$(row post-empty) -assert_equals "post-empty title" "Empty body" "$(jq -r '.title' <<<"$empty")" -assert_equals "post-empty kept its empty string" "" "$(jq -r '.body' <<<"$empty")" -assert_equals "post-empty kept its column default" "untagged" "$(jq -r '.tag' <<<"$empty")" -assert_equals "post-empty kept its relationship" "$(manifest_get '.authorId')" \ - "$(jq -r 'if (.author | type) == "object" then .author["$id"] else .author end' <<<"$empty")" - -long=$(row post-long) -assert_equals "post-long body length" "${#LONG_STRING}" "$(jq -r '.body | length' <<<"$long")" -assert_equals "post-long body content" "$LONG_STRING" "$(jq -r '.body' <<<"$long")" - -unicode=$(row post-unicode) -assert_equals "post-unicode body survived byte-for-byte" "$UNICODE_STRING" "$(jq -r '.body' <<<"$unicode")" - -perms=$(row post-perms) -assert_equals "post-perms kept its row permissions" \ - 'read("any"),update("users")' \ - "$(jq -r '.["$permissions"] | sort | join(",")' <<<"$perms")" - -assert_equals "posts table kept its permissions" \ - 'create("users"),read("any")' \ - "$(api GET "/tablesdb/${DATABASE_ID}/tables/${POSTS_TABLE}" | jq -r '.["$permissions"] | sort | join(",")')" - -assert_equals "row count on posts" "4" \ - "$(api GET "/tablesdb/${DATABASE_ID}/tables/${POSTS_TABLE}/rows" | jq -r '.total')" - -# --- user and file survived -------------------------------------------------- - -echo "==> User and file" -user=$(api GET "/users/${USER_ID}") -assert_equals "user email" "gate-user@example.com" "$(jq -r '.email' <<<"$user")" -assert_equals "user name with unicode" "$UNICODE_STRING" "$(jq -r '.name' <<<"$user")" - -file=$(api GET "/storage/buckets/${BUCKET_ID}/files/${FILE_ID}") -assert_equals "file still listed" "$FILE_ID" "$(jq -r '.["$id"]' <<<"$file")" - -downloaded=$(curl --silent --show-error --fail-with-body --max-time 60 \ - -H "x-appwrite-project: ${PROJECT_ID}" \ - -H "x-appwrite-key: ${API_KEY}" \ - "${ENDPOINT}/storage/buckets/${BUCKET_ID}/files/${FILE_ID}/download") -assert_equals "file contents" "$UNICODE_STRING" "$(printf '%s' "$downloaded" | head -1)" +if [ "${1:-}" != '--write' ]; then + exit 0 +fi # --- the upgraded install still takes writes --------------------------------- -echo "==> Writes after upgrade" -written=$(api POST "/tablesdb/${DATABASE_ID}/tables/${POSTS_TABLE}/rows" "$(jq -nc \ - --arg body "$UNICODE_STRING" \ - '{rowId: "post-after-upgrade", data: {title: "Written after upgrade", body: $body}}')") -assert_equals "new row readable" "Written after upgrade" "$(jq -r '.title' <<<"$written")" +# A read-only check passes against a half-migrated schema, so write too. +api POST "/tablesdb/${DATABASE}/tables/${POSTS}/rows" "$(jq -nc --arg body "$UNICODE_VALUE" '{ + rowId: "post-after-upgrade", data: {title: "Written after upgrade", body: $body} +}')" >/dev/null api PATCH "/users/${USER_ID}/name" '{"name":"Renamed after upgrade"}' >/dev/null -assert_equals "user update applied" "Renamed after upgrade" \ - "$(api GET "/users/${USER_ID}" | jq -r '.name')" -new_author=$(api POST "/tablesdb/${DATABASE_ID}/tables/${AUTHORS_TABLE}/rows" \ +author=$(api POST "/tablesdb/${DATABASE}/tables/${AUTHORS}/rows" \ '{"rowId":"author-after-upgrade","data":{"name":"Grace Hopper"}}' | jq -r '.["$id"]') -linked=$(api PATCH "/tablesdb/${DATABASE_ID}/tables/${POSTS_TABLE}/rows/post-after-upgrade" "$(jq -nc \ - --arg author "$new_author" '{data: {author: $author}}')") -assert_equals "relationship writable after upgrade" "$new_author" \ - "$(jq -r 'if (.author | type) == "object" then .author["$id"] else .author end' <<<"$linked")" +api PATCH "/tablesdb/${DATABASE}/tables/${POSTS}/rows/post-after-upgrade" \ + "{\"data\":{\"author\":\"${author}\"}}" >/dev/null -# --- result ------------------------------------------------------------------ +written=$(snapshot) +total=$(jq -r '.total' <<<"$written") -if [ "$FAILURES" -gt 0 ]; then - fail "${FAILURES} assertion(s) failed upgrading ${SEEDED_ON} -> ${UPGRADED_TO}." -fi +[ "$total" = '5' ] || fail "Expected 5 rows after writing one; found ${total}." +[ "$(jq -r '.user.name' <<<"$written")" = 'Renamed after upgrade' ] || fail 'The user update did not apply.' +[ "$(jq -r '.rows[] | select(.id == "post-after-upgrade") | .author' <<<"$written")" = "$author" ] \ + || fail 'The relationship written after the upgrade did not stick.' -echo "All assertions passed upgrading ${SEEDED_ON} -> ${UPGRADED_TO}." +echo 'ok: rows, users and relationships are still writable' +echo "All checks passed upgrading ${SEEDED_ON} -> ${UPGRADED_TO}." diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ffb778c2b0..b3ec7bc4c9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1011,9 +1011,14 @@ jobs: permissions: contents: read packages: read + # Defaults describe the candidate. The two steps that run the previous + # release override them; everything else is the version being tested. env: + COMPOSE_FILE: docker-compose.yml COMPOSE_PROJECT_NAME: upgrade-gate COMPOSE_PROFILES: '' + _APP_IMAGE: appwrite-dev + _APP_VERSION: latest _APP_HTTP_PORT: '8080' _APP_HTTPS_PORT: '8443' ENDPOINT: http://localhost:8080/v1 @@ -1031,23 +1036,38 @@ jobs: username: ${{ vars.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - # Previous baseline = highest published stable strictly below this branch's - # APP_VERSION_STABLE. Docker Hub rather than git tags, because the image has - # to exist for the test to run at all. + # Baseline = highest published stable below this branch's version. Docker + # Hub rather than git tags, because an image nobody can pull is not an + # upgrade anyone can perform. - name: Resolve the previous stable id: previous run: | set -euo pipefail target=$(sed -n "s/^const APP_VERSION_STABLE = '\([^']*\)';.*/\1/p" app/init/constants.php) - previous=$(.github/scripts/previous-stable.py "${target}") + + # Sort the published X.Y.Z tags together with the target, then take + # whatever lands immediately before it. + previous=$( + curl --silent --fail --retry 3 --retry-all-errors --max-time 60 \ + 'https://hub.docker.com/v2/repositories/appwrite/appwrite/tags/?page_size=100&ordering=last_updated' \ + | jq -r '.results[].name' \ + | grep -E '^[0-9]+\.[0-9]+\.[0-9]+$' \ + | { cat; echo "${target}"; } \ + | sort -u -V \ + | grep -B1 -x -- "${target}" \ + | head -1 + ) + + if [ -z "$previous" ] || [ "$previous" = "$target" ]; then + echo "::error::No published stable below ${target} in the 100 most recent Docker Hub tags." + exit 1 + fi echo "Upgrading ${previous} -> ${target}" echo "previous=${previous}" >> "$GITHUB_OUTPUT" - echo "target=${target}" >> "$GITHUB_OUTPUT" # An upgrade starts from the compose file and env the old version shipped, - # not from this checkout's -- that is what the installation being upgraded - # was actually started from. + # which is what the installation being upgraded was actually started from. - name: Materialise the previous release's compose env: PREVIOUS: ${{ steps.previous.outputs.previous }} @@ -1083,18 +1103,10 @@ jobs: - name: Start the candidate on the same volumes timeout-minutes: 10 - env: - COMPOSE_FILE: docker-compose.yml - _APP_IMAGE: appwrite-dev - _APP_VERSION: latest run: docker compose up -d --quiet-pull --wait - name: Migrate timeout-minutes: 20 - env: - COMPOSE_FILE: docker-compose.yml - _APP_IMAGE: appwrite-dev - _APP_VERSION: latest run: docker compose exec -T appwrite migrate - name: Verify the seeded data survived @@ -1102,32 +1114,21 @@ jobs: run: .github/scripts/upgrade-verify.sh # A migration that is not idempotent breaks every retried or resumed - # upgrade, so run it again and re-assert. - - name: Migrate again and re-verify - timeout-minutes: 25 - env: - COMPOSE_FILE: docker-compose.yml - _APP_IMAGE: appwrite-dev - _APP_VERSION: latest - run: | - set -euo pipefail - docker compose exec -T appwrite migrate - .github/scripts/upgrade-verify.sh + # upgrade. + - name: Migrate again + timeout-minutes: 20 + run: docker compose exec -T appwrite migrate + + - name: Verify again, and that writes still work + timeout-minutes: 10 + run: .github/scripts/upgrade-verify.sh --write - name: Collect logs on failure if: failure() - env: - COMPOSE_FILE: docker-compose.yml - _APP_IMAGE: appwrite-dev - _APP_VERSION: latest run: | docker compose ps || true docker compose logs --tail=400 appwrite || true - name: Stop if: always() - env: - COMPOSE_FILE: docker-compose.yml - _APP_IMAGE: appwrite-dev - _APP_VERSION: latest run: docker compose down -v || true From 341ad9f58e63e53a8fadc6f2da6e9b09f45275f0 Mon Sep 17 00:00:00 2001 From: Levi van Noort <73097785+levivannoort@users.noreply.github.com> Date: Mon, 14 Sep 2026 13:24:44 +0200 Subject: [PATCH 08/37] fix(install): keep Upgrade::action compatible with Install::action Adding $channel as the ninth parameter of Install::action() put a string where Upgrade::action() already had bool $migrate, and an override may only add optional parameters, not change the type of an existing position. Every process that boots the platform died on the incompatible declaration, which is why the whole pipeline went red rather than one job. $channel also has to reach performInstallation(), a different method, so it now travels on $this the way $topology already does instead of being read out of scope. Also yields the README data provider, per Rector's YieldDataProviderRector. --- src/Appwrite/Platform/Tasks/Install.php | 4 +++- src/Appwrite/Platform/Tasks/Upgrade.php | 4 ++-- tests/unit/Release/MetadataTest.php | 15 +++++++-------- 3 files changed, 12 insertions(+), 11 deletions(-) diff --git a/src/Appwrite/Platform/Tasks/Install.php b/src/Appwrite/Platform/Tasks/Install.php index 663b9b0713..8ca0daaf90 100644 --- a/src/Appwrite/Platform/Tasks/Install.php +++ b/src/Appwrite/Platform/Tasks/Install.php @@ -49,6 +49,7 @@ class Install extends Action protected ?array $installerConfig = null; protected string $path = '/usr/src/code/appwrite'; protected string $topology = 'combined'; + protected string $channel = self::CHANNEL_STABLE; public static function getName(): string { @@ -82,6 +83,7 @@ class Install extends Action string $topology, string $channel = self::CHANNEL_STABLE ): void { + $this->channel = $channel; $isUpgrade = $this->isUpgrade; $defaultHttpPort = '80'; $defaultHttpsPort = '443'; @@ -571,7 +573,7 @@ class Install extends Action // The nightly channel tracks the minor line rather than one release, so the // tag has to stay rolling -- pinning X.Y.Z would freeze the install on a // single build. See the Releases section of AGENTS.md. - if ($channel === self::CHANNEL_NIGHTLY) { + if ($this->channel === self::CHANNEL_NIGHTLY) { $version = $this->nightlyTag($version); } diff --git a/src/Appwrite/Platform/Tasks/Upgrade.php b/src/Appwrite/Platform/Tasks/Upgrade.php index 55b310f9fd..dc01ff09d0 100644 --- a/src/Appwrite/Platform/Tasks/Upgrade.php +++ b/src/Appwrite/Platform/Tasks/Upgrade.php @@ -32,8 +32,8 @@ class Upgrade extends Install ->param('no-start', false, new Boolean(true), 'Run an interactive session', true) ->param('database', '', new Text(length: 0, min: 0), 'Ignored: an upgrade always keeps the database the installation already uses', true) ->param('topology', 'combined', new WhiteList(['combined', 'separate']), 'Worker and scheduler topology (combined|separate)', true) - ->param('migrate', false, new Boolean(true), 'Run database migration after upgrade', true) ->param('channel', Install::CHANNEL_STABLE, new WhiteList([Install::CHANNEL_STABLE, Install::CHANNEL_NIGHTLY]), 'Release channel to track (stable|nightly). Nightly is unsupported and moves daily.', true) + ->param('migrate', false, new Boolean(true), 'Run database migration after upgrade', true) ->callback($this->action(...)); } @@ -46,8 +46,8 @@ class Upgrade extends Install bool $noStart, string $database, string $topology = 'combined', - bool $migrate = false, string $channel = Install::CHANNEL_STABLE, + bool $migrate = false, ): void { $this->isUpgrade = true; $this->migrate = $migrate; diff --git a/tests/unit/Release/MetadataTest.php b/tests/unit/Release/MetadataTest.php index a6a9566b8c..c293a03040 100644 --- a/tests/unit/Release/MetadataTest.php +++ b/tests/unit/Release/MetadataTest.php @@ -33,14 +33,12 @@ final class MetadataTest extends TestCase } /** - * @return array + * @return \Iterator */ - public static function readmes(): array + public static function readmes(): \Iterator { - return [ - 'README.md' => ['README.md'], - 'README-CN.md' => ['README-CN.md'], - ]; + yield 'README.md' => ['README.md']; + yield 'README-CN.md' => ['README-CN.md']; } /** @@ -119,10 +117,11 @@ final class MetadataTest extends TestCase 'docker-compose.yml does not pin an appwrite/new console image.' ); + // A build suffix is fine (1.1.78-self-hosted); a floating tag is not. $this->assertMatchesRegularExpression( - '/^\d+\.\d+\.\d+$/', + '/^\d+\.\d+\.\d+(-[0-9A-Za-z.-]+)?$/', $matches[1], - "The console is pinned to '{$matches[1]}', which is not an exact X.Y.Z version." + "The console is pinned to '{$matches[1]}', which is not an exact version." ); } } From 134820422bb6e57ffd84bb389995fe2492c4cb0b Mon Sep 17 00:00:00 2001 From: Levi van Noort <73097785+levivannoort@users.noreply.github.com> Date: Mon, 14 Sep 2026 13:33:36 +0200 Subject: [PATCH 09/37] fix(ci): check release metadata where the files exist tests/unit/Release/MetadataTest.php could not do its job: unit tests run inside the appwrite container, whose /usr/src/code holds only app, dev, public, src and vendor. The READMEs are neither copied into the image nor mounted by the override, so reading them returned false and the test failed for a reason that had nothing to do with the metadata. The file-level pins now live in a Check release metadata step in the Checks job, which runs on the runner where the files are. The one assertion that is genuinely about code -- no version mapped ahead of APP_VERSION_STABLE -- moves to MigrationVersionsTest, next to the existing assertion about that same map. Also loosens the console pin check: main pins 1.1.78-self-hosted, so a build suffix is valid where a floating tag is not. --- .github/workflows/ci.yml | 37 +++++ AGENTS.md | 2 +- src/Appwrite/Platform/Tasks/Release.php | 4 +- .../unit/Migration/MigrationVersionsTest.php | 22 +++ tests/unit/Release/MetadataTest.php | 127 ------------------ 5 files changed, 62 insertions(+), 130 deletions(-) delete mode 100644 tests/unit/Release/MetadataTest.php diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ab2e185b0d..331e5b49e2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -163,6 +163,43 @@ jobs: if: ${{ !cancelled() && steps.install.outcome == 'success' }} run: _APP_STORAGE_LIMIT=5368709120 php app/cli.php specs --version=latest --git=no + # The release metadata has to agree across files, and a partial bump is + # how a published version ends up disagreeing with the repo. Checked + # here rather than in tests/unit because the READMEs are not in the + # image the unit suite runs inside. + - name: Check release metadata + if: ${{ !cancelled() }} + run: | + set -euo pipefail + version=$(sed -n "s/^const APP_VERSION_STABLE = '\([^']*\)';.*/\1/p" app/init/constants.php) + echo "APP_VERSION_STABLE is ${version}" + + for readme in README.md README-CN.md; do + pinned=$(grep -oE 'appwrite/appwrite:[^ `"'"'"')]+' "$readme" | cut -d: -f2 | sort -u) + + if [ -z "$pinned" ]; then + echo "::error file=${readme}::No appwrite/appwrite install snippet found." + exit 1 + fi + + if [ "$pinned" != "$version" ]; then + echo "::error file=${readme}::Pins $(echo "$pinned" | tr '\n' ' ')but APP_VERSION_STABLE is ${version}." + exit 1 + fi + + echo " ok ${readme}" + done + + # A floating console tag would make an install unreproducible. + console=$(grep -oE 'image: appwrite/new:\S+' docker-compose.yml | cut -d: -f3) + + if ! echo "$console" | grep -qE '^[0-9]+\.[0-9]+\.[0-9]+(-[0-9A-Za-z.-]+)?$'; then + echo "::error file=docker-compose.yml::The console is pinned to '${console}', which is not an exact version." + exit 1 + fi + + echo " ok docker-compose.yml console pin (${console})" + locale: name: Checks / Locale runs-on: ubuntu-latest diff --git a/AGENTS.md b/AGENTS.md index ae065b8050..e4738287f7 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -341,7 +341,7 @@ When bumping a patch (e.g. `1.9.0` → `1.9.1`): - [`README.md`](README.md) and [`README-CN.md`](README-CN.md) — `appwrite/appwrite:X.Y.Z` in all three install blocks each - [`src/Appwrite/Migration/Migration.php`](src/Appwrite/Migration/Migration.php) — add the version to `$versions`, mapping to a new migration class or the same class as the previous version -`docker compose exec appwrite release X.Y.Z` writes all four in one pass; `tests/unit/Release/MetadataTest.php` fails when they disagree. Edit them by hand only when the task cannot express the change. +`docker compose exec appwrite release X.Y.Z` writes all four in one pass, and the `Check release metadata` step in [`ci.yml`](.github/workflows/ci.yml) fails when they disagree. Edit them by hand only when the task cannot express the change. Ask the user to review, publish notes on the [Appwrite changelog](https://appwrite.io/changelog), generate specs if the API changed, and add request/response filters if needed. diff --git a/src/Appwrite/Platform/Tasks/Release.php b/src/Appwrite/Platform/Tasks/Release.php index 24072085a8..414e7e5576 100644 --- a/src/Appwrite/Platform/Tasks/Release.php +++ b/src/Appwrite/Platform/Tasks/Release.php @@ -13,8 +13,8 @@ class Release extends Action /** * Release metadata lives in four files that have to agree. Editing them by * hand is how a published version ends up disagreeing with the repo, so this - * writes all four from one version number and tests/unit/Release/MetadataTest.php - * fails when they drift apart again. + * writes all four from one version number, and the Check release metadata + * step in ci.yml fails when they drift apart again. */ private const string CONSTANTS = 'app/init/constants.php'; private const string MIGRATION = 'src/Appwrite/Migration/Migration.php'; diff --git a/tests/unit/Migration/MigrationVersionsTest.php b/tests/unit/Migration/MigrationVersionsTest.php index 09bc7a3588..1e604d9902 100644 --- a/tests/unit/Migration/MigrationVersionsTest.php +++ b/tests/unit/Migration/MigrationVersionsTest.php @@ -35,6 +35,28 @@ final class MigrationVersionsTest extends TestCase } } + /** + * A mapping newer than APP_VERSION_STABLE means a version was given a + * migration but never marked stable, which is how 1.9.1 through 1.9.4 came + * to exist in this map with no release behind them. + */ + public function testNoVersionIsMappedAheadOfStable(): void + { + require_once __DIR__ . '/../../../app/init.php'; + + if (\str_contains(APP_VERSION_STABLE, 'RC') || \str_contains(APP_VERSION_STABLE, '-rc.')) { + $this->markTestSkipped('Release candidates are mapped when the final version is cut.'); + } + + foreach (\array_keys(Migration::$versions) as $mapped) { + $this->assertLessThanOrEqual( + 0, + \version_compare((string) $mapped, APP_VERSION_STABLE), + "Migration::\$versions maps {$mapped}, which is newer than APP_VERSION_STABLE " . APP_VERSION_STABLE . '.' + ); + } + } + public function testV24CreatesAlertsCollectionForConsoleProject(): void { require_once __DIR__ . '/../../../app/init.php'; diff --git a/tests/unit/Release/MetadataTest.php b/tests/unit/Release/MetadataTest.php deleted file mode 100644 index c293a03040..0000000000 --- a/tests/unit/Release/MetadataTest.php +++ /dev/null @@ -1,127 +0,0 @@ -assertMatchesRegularExpression( - '/^\d+\.\d+\.\d+(-(rc\.\d+|RC\d+))?$/', - APP_VERSION_STABLE, - 'APP_VERSION_STABLE must be an X.Y.Z version, optionally a release candidate.' - ); - } - - /** - * @return \Iterator - */ - public static function readmes(): \Iterator - { - yield 'README.md' => ['README.md']; - yield 'README-CN.md' => ['README-CN.md']; - } - - /** - * Every install snippet pins the released version. A README that still points - * at the previous version sends new self-hosters to an older image. - * - * @dataProvider readmes - */ - public function testReadmePinsStableVersion(string $readme): void - { - $contents = \file_get_contents(self::ROOT . '/' . $readme); - $this->assertNotFalse($contents, "{$readme} is unreadable."); - - \preg_match_all('#appwrite/appwrite:(\S+)#', $contents, $matches); - - $this->assertNotEmpty($matches[1], "{$readme} has no appwrite/appwrite install snippet to check."); - - foreach (\array_unique($matches[1]) as $pinned) { - $this->assertSame( - APP_VERSION_STABLE, - $pinned, - "{$readme} pins appwrite/appwrite:{$pinned} but APP_VERSION_STABLE is " . APP_VERSION_STABLE . '.' - ); - } - } - - /** - * A version with no migration mapping cannot be upgraded to, so the release - * would install and then fail for everyone already running Appwrite. - */ - public function testStableVersionHasAMigration(): void - { - if (\str_contains(APP_VERSION_STABLE, 'RC') || \str_contains(APP_VERSION_STABLE, '-rc.')) { - $this->markTestSkipped('Release candidates are mapped when the final version is cut.'); - } - - $this->assertArrayHasKey( - APP_VERSION_STABLE, - Migration::$versions, - 'APP_VERSION_STABLE has no entry in Migration::$versions.' - ); - } - - /** - * The last mapped version is the one being released. An entry newer than - * APP_VERSION_STABLE means a version was mapped but never marked stable. - */ - public function testNoMigrationMappingIsAheadOfStable(): void - { - if (\str_contains(APP_VERSION_STABLE, 'RC') || \str_contains(APP_VERSION_STABLE, '-rc.')) { - $this->markTestSkipped('Release candidates are mapped when the final version is cut.'); - } - - foreach (\array_keys(Migration::$versions) as $mapped) { - $this->assertLessThanOrEqual( - 0, - \version_compare((string) $mapped, APP_VERSION_STABLE), - "Migration::\$versions maps {$mapped}, which is newer than APP_VERSION_STABLE " . APP_VERSION_STABLE . '.' - ); - } - } - - /** - * The console ships as its own image, so self-hosted installs only get a - * console fix when this pin moves. An unpinned or floating tag would make an - * install unreproducible. - */ - public function testConsoleImageIsPinnedToAVersion(): void - { - $compose = \file_get_contents(self::ROOT . '/docker-compose.yml'); - $this->assertNotFalse($compose, 'docker-compose.yml is unreadable.'); - - $this->assertSame( - 1, - \preg_match('#image: appwrite/new:(\S+)#', $compose, $matches), - 'docker-compose.yml does not pin an appwrite/new console image.' - ); - - // A build suffix is fine (1.1.78-self-hosted); a floating tag is not. - $this->assertMatchesRegularExpression( - '/^\d+\.\d+\.\d+(-[0-9A-Za-z.-]+)?$/', - $matches[1], - "The console is pinned to '{$matches[1]}', which is not an exact version." - ); - } -} From f49f8cd53ba779cd4cab993e84ff6a79063b4812 Mon Sep 17 00:00:00 2001 From: Levi van Noort <73097785+levivannoort@users.noreply.github.com> Date: Tue, 15 Sep 2026 07:44:51 +0200 Subject: [PATCH 10/37] refactor(ci): resolve nightly builds from the marker tag alone The workflow asked "has this already shipped?" twice: the nightly- marker tag answered it for the commit, and a Docker Hub tag lookup answered it for the day. Keep the marker, drop the registry call and its status-code handling -- a same-day rebuild now reuses the day's tag instead of taking a run-number suffix, so that tag is one a day rather than immutable. Also drops guards that guard nothing: the walk-back loop over green runs (gh already returns the newest green run, and release branches are not force pushed), the branch-name mismatch warning, and the step refusing to publish anywhere but appwrite/appwrite -- the tags are built from the hardcoded IMAGE env six lines above it. --- .github/workflows/nightly.yml | 92 ++++++----------------------------- AGENTS.md | 2 +- 2 files changed, 16 insertions(+), 78 deletions(-) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 44b2c6e6cb..b3d754243d 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -5,9 +5,7 @@ name: "Nightly" # Cloud builds from cl-* tags through publish.yml and is not part of this channel. # # Tags pushed (see the Releases section of AGENTS.md): -# X.Y.Z-nightly. immutable, the rollback anchor. X.Y.Z is the NEXT patch -# on the line, so the build sorts below the release it is -# heading for. +# X.Y.Z-nightly. the rollback anchor, one per day # X.Y-nightly rolling, per minor line. The form to recommend. # nightly rolling, newest line only. @@ -78,27 +76,18 @@ jobs: echo "branch=${branch}" >> "$GITHUB_OUTPUT" # Invariant 2: build the newest CI-green commit, not the tip, so a red - # branch delays the channel instead of shipping through it. gh returns - # newest first; a run's commit can have been force-pushed away, so the - # candidate must still be on the branch. - sha="" - while read -r candidate; do - [ -n "$candidate" ] || continue - if git merge-base --is-ancestor "$candidate" "origin/${branch}" 2>/dev/null; then - sha="$candidate" - break - fi - done < <(gh run list \ + # branch delays the channel instead of shipping through it. + sha=$(gh run list \ --workflow=ci.yml \ --branch="${branch}" \ --event=push \ --status=success \ - --limit=30 \ + --limit=1 \ --json headSha \ - --jq '.[].headSha') + --jq '.[0].headSha // empty') if [ -z "$sha" ]; then - echo "::warning::No green CI run found on ${branch} in the last 30 pushes; nothing to publish." + echo "::warning::No green CI run found on ${branch}; nothing to publish." echo "skip=true" >> "$GITHUB_OUTPUT" exit 0 fi @@ -107,13 +96,12 @@ jobs: # Nothing new since the last nightly. The marker tag is moved by the # build job below and is the only state this workflow keeps. - if git rev-parse --verify "refs/tags/nightly-${branch}" >/dev/null 2>&1; then - previous=$(git rev-parse "refs/tags/nightly-${branch}^{commit}") - if [ "$previous" = "$sha" ]; then - echo "::notice::${branch} has not moved since the last nightly (${sha}); skipping." - echo "skip=true" >> "$GITHUB_OUTPUT" - exit 0 - fi + previous=$(git rev-parse -q --verify "refs/tags/nightly-${branch}^{commit}" || true) + + if [ "$previous" = "$sha" ]; then + echo "::notice::${branch} has not moved since the last nightly (${sha}); skipping." + echo "skip=true" >> "$GITHUB_OUTPUT" + exit 0 fi # A nightly is a prerelease of the next patch on the line, so read the @@ -130,36 +118,11 @@ jobs: rest=${stable#*.} minor=${rest%%.*} patch=${rest#*.} - next="${major}.${minor}.$((patch + 1))" - if [ "${branch}" != "${major}.${minor}.x" ]; then - echo "::warning::${branch} carries APP_VERSION_STABLE=${stable}; tagging as ${next} from the constant rather than the branch name." - fi - - # Published nightlies are immutable, so a second green commit on the - # same day takes its own suffix rather than overwriting the day's tag. - build="$(date -u +%Y%m%d)" - status=$(curl --silent --output /dev/null --write-out '%{http_code}' \ - --retry 3 --retry-all-errors --max-time 30 \ - "https://hub.docker.com/v2/repositories/${IMAGE}/tags/${next}-nightly.${build}" || true) - - case "$status" in - 404) ;; - 200) - build="${build}.${GITHUB_RUN_NUMBER}" - echo "::notice::${next}-nightly.$(date -u +%Y%m%d) is already published; using ${build}." - ;; - *) - echo "::error::Unexpected response ${status} from Docker Hub while checking for an existing nightly tag. Refusing to publish." - exit 1 - ;; - esac - - version="${next}-nightly.${build}" + version="${major}.${minor}.$((patch + 1))-nightly.$(date -u +%Y%m%d)" echo "version=${version}" >> "$GITHUB_OUTPUT" - tags="${IMAGE}:${version}" - tags="${tags},${IMAGE}:${major}.${minor}-nightly" + tags="${IMAGE}:${version},${IMAGE}:${major}.${minor}-nightly" # The bare tag follows the newest line only, so a dispatched build of an # older branch cannot drag it backwards. @@ -170,16 +133,7 @@ jobs: echo "tags=${tags}" >> "$GITHUB_OUTPUT" echo "skip=false" >> "$GITHUB_OUTPUT" - { - echo "### Nightly" - echo - echo "| | |" - echo "|---|---|" - echo "| Branch | \`${branch}\` |" - echo "| Commit | \`${sha}\` |" - echo "| Version | \`${version}\` |" - echo "| Tags | \`${tags//,/\`, \`}\` |" - } >> "$GITHUB_STEP_SUMMARY" + echo "Publishing ${tags//,/, } from ${branch} (${sha})." >> "$GITHUB_STEP_SUMMARY" build: name: Build and publish @@ -196,22 +150,6 @@ jobs: ref: ${{ needs.resolve.outputs.sha }} submodules: recursive - - name: Refuse to publish anywhere but appwrite/appwrite - env: - TAGS: ${{ needs.resolve.outputs.tags }} - run: | - set -euo pipefail - echo "$TAGS" | tr ',' '\n' | while read -r tag; do - [ -n "$tag" ] || continue - case "$tag" in - appwrite/appwrite:*) ;; - *) - echo "::error::Refusing to publish ${tag}: the nightly channel is appwrite/appwrite only." - exit 1 - ;; - esac - done - - name: Set up QEMU uses: docker/setup-qemu-action@ce360397dd3f832beb865e1373c09c0e9f86d70a # v4.0.0 diff --git a/AGENTS.md b/AGENTS.md index e4738287f7..dc0f23ec13 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -304,7 +304,7 @@ Self-hosted ships on two channels. Cloud is separate: it builds from `cl-*` tags | Channel | Source | Trigger | Tags on `appwrite/appwrite` | |---------|--------|---------|------------------------------| | stable | active release branch (`2.0.x`) | GitHub Release, published by a human | `X.Y.Z` (immutable), `X.Y`, `X`, `latest` | -| nightly | same branch, newest CI-green commit | daily at 00:00 UTC, or `workflow_dispatch` | `X.Y.Z-nightly.` (immutable), `X.Y-nightly`, `nightly` | +| nightly | same branch, newest CI-green commit | daily at 00:00 UTC, or `workflow_dispatch` | `X.Y.Z-nightly.` (one a day), `X.Y-nightly`, `nightly` | [`nightly.yml`](.github/workflows/nightly.yml) builds the channel; [`security-scan.yml`](.github/workflows/security-scan.yml) is the Trivy scan that used to own that file name. `X.Y.Z` in a nightly tag is the **next** patch on the line — the nightly is a prerelease of the version it is heading for, so `2.0.2-nightly.20260915` sorts below `2.0.2` — and `` is `YYYYMMDD`. From 3efe083a2f50632fa724dba43f4d1e17a420fc7f Mon Sep 17 00:00:00 2001 From: Levi van Noort <73097785+levivannoort@users.noreply.github.com> Date: Tue, 15 Sep 2026 07:44:51 +0200 Subject: [PATCH 11/37] refactor(releases): drop the regexes from the release and install tasks AGENTS.md rules out preg_* in new code. Version parsing, the migration map lookup and the console pin all read fine with explode, ctype_digit, str_contains and strcspn. --- src/Appwrite/Platform/Tasks/Install.php | 6 ++-- src/Appwrite/Platform/Tasks/Release.php | 46 +++++++++++++++++++++---- 2 files changed, 43 insertions(+), 9 deletions(-) diff --git a/src/Appwrite/Platform/Tasks/Install.php b/src/Appwrite/Platform/Tasks/Install.php index 8ca0daaf90..2c9b5c0439 100644 --- a/src/Appwrite/Platform/Tasks/Install.php +++ b/src/Appwrite/Platform/Tasks/Install.php @@ -844,13 +844,15 @@ class Install extends Action */ private function nightlyTag(string $version): string { - if (\preg_match('/^(\d+)\.(\d+)\./', $version, $matches) !== 1) { + [$major, $minor] = \array_pad(\explode('.', $version), 2, ''); + + if (!\ctype_digit($major) || !\ctype_digit($minor)) { Console::warning("Cannot derive a nightly tag from '{$version}'; using the bare nightly tag."); return 'nightly'; } - return "{$matches[1]}.{$matches[2]}-nightly"; + return "{$major}.{$minor}-nightly"; } private function createInitialAdminAccount(array $account, ?callable $progress, string $apiUrl, string $domain): void diff --git a/src/Appwrite/Platform/Tasks/Release.php b/src/Appwrite/Platform/Tasks/Release.php index 414e7e5576..d6848af98c 100644 --- a/src/Appwrite/Platform/Tasks/Release.php +++ b/src/Appwrite/Platform/Tasks/Release.php @@ -44,7 +44,7 @@ class Release extends Action public function action(string $version, string $migration, string $console, string $dryRun): void { - if (\preg_match('/^\d+\.\d+\.\d+(-rc\.\d+)?$/', $version) !== 1) { + if (!$this->isVersion($version)) { Console::error("'{$version}' is not an X.Y.Z or X.Y.Z-rc.N version."); return; @@ -112,6 +112,32 @@ class Release extends Action Console::info('Still by hand: release notes on the changelog, specs if the API changed, and request/response filters for public breaks.'); } + /** + * X.Y.Z, optionally an -rc.N candidate of it. + */ + private function isVersion(string $version): bool + { + $parts = \explode('-rc.', $version, 2); + + if (\count($parts) === 2 && !\ctype_digit($parts[1])) { + return false; + } + + $numbers = \explode('.', $parts[0]); + + if (\count($numbers) !== 3) { + return false; + } + + foreach ($numbers as $number) { + if (!\ctype_digit($number)) { + return false; + } + } + + return true; + } + private function isPatch(string $current, string $version): bool { [$currentMajor, $currentMinor] = \explode('.', $current); @@ -189,7 +215,7 @@ class Release extends Action { $contents = $this->read(self::MIGRATION); - if (\preg_match("/^\s*'" . \preg_quote($version, '/') . "'\s*=>/m", $contents) === 1) { + if (\str_contains($contents, "'{$version}' =>")) { return []; } @@ -216,18 +242,24 @@ class Release extends Action { $contents = $this->read(self::COMPOSE); - if (\preg_match('/image: appwrite\/new:(\S+)/', $contents, $matches) !== 1) { + $pin = 'image: appwrite/new:'; + $start = \strpos($contents, $pin); + + if ($start === false) { throw new \RuntimeException(self::COMPOSE . ' does not pin an appwrite/new image.'); } - if ($matches[1] === $console) { + $start += \strlen($pin); + $current = \substr($contents, $start, \strcspn($contents, " \t\r\n", $start)); + + if ($current === $console) { return []; } return [self::COMPOSE => [ - 'from' => "- image: appwrite/new:{$matches[1]}", - 'to' => "+ image: appwrite/new:{$console}", - 'contents' => \str_replace($matches[0], "image: appwrite/new:{$console}", $contents), + 'from' => "- {$pin}{$current}", + 'to' => "+ {$pin}{$console}", + 'contents' => \str_replace($pin . $current, $pin . $console, $contents), ]]; } } From e578dde102b035dfb4ee0d0c5e765a1c9702c197 Mon Sep 17 00:00:00 2001 From: Levi van Noort <73097785+levivannoort@users.noreply.github.com> Date: Tue, 15 Sep 2026 07:54:25 +0200 Subject: [PATCH 12/37] fix(ci): resolve the upgrade baseline from git tags The upgrade gate read the previous stable out of Docker Hub's 100 most recent tags. The nightly channel now pushes a tag a day to that same repository, so within weeks the stables fall out of that window and the gate fails to find a baseline it can upgrade from. Git tags carry every release, sort by semver and are already here from fetch-depth: 0 -- which also makes the explicit tag fetch redundant. Also stops the crash-loop check from accepting exited containers. Every service in the default profile is long-running, so an exited one is a service that stopped serving. --- .github/workflows/ci.yml | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 331e5b49e2..9c8312eec4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1042,7 +1042,7 @@ jobs: # compose emits NDJSON on some versions and a JSON array on others; # flatten handles both. docker compose ps --format json \ - | jq -e -s 'flatten | map(select(.State != "running" and .State != "exited")) | length == 0' \ + | jq -e -s 'flatten | map(select(.State != "running")) | length == 0' \ || { echo "::error::Some containers are not running."; docker compose logs --tail=200; exit 1; } - name: Stop @@ -1095,12 +1095,12 @@ jobs: set -euo pipefail target=$(sed -n "s/^const APP_VERSION_STABLE = '\([^']*\)';.*/\1/p" app/init/constants.php) - # Sort the published X.Y.Z tags together with the target, then take - # whatever lands immediately before it. + # Sort the released X.Y.Z tags together with the target, then take + # whatever lands immediately before it. Git rather than Docker Hub's + # tag listing: the nightly channel pushes a tag a day to the same + # repository, so the stables drop out of any recent-tags window. previous=$( - curl --silent --fail --retry 3 --retry-all-errors --max-time 60 \ - 'https://hub.docker.com/v2/repositories/appwrite/appwrite/tags/?page_size=100&ordering=last_updated' \ - | jq -r '.results[].name' \ + git tag --list \ | grep -E '^[0-9]+\.[0-9]+\.[0-9]+$' \ | { cat; echo "${target}"; } \ | sort -u -V \ @@ -1109,7 +1109,7 @@ jobs: ) if [ -z "$previous" ] || [ "$previous" = "$target" ]; then - echo "::error::No published stable below ${target} in the 100 most recent Docker Hub tags." + echo "::error::No released stable below ${target}." exit 1 fi @@ -1123,7 +1123,6 @@ jobs: PREVIOUS: ${{ steps.previous.outputs.previous }} run: | set -euo pipefail - git fetch --no-tags --depth 1 origin "refs/tags/${PREVIOUS}:refs/tags/${PREVIOUS}" git show "${PREVIOUS}:docker-compose.yml" > docker-compose.previous.yml git show "${PREVIOUS}:.env" > .env.previous From b5a864d6c5c58b66a4daa0889da9275fcba4152a Mon Sep 17 00:00:00 2001 From: Levi van Noort <73097785+levivannoort@users.noreply.github.com> Date: Tue, 15 Sep 2026 07:59:01 +0200 Subject: [PATCH 13/37] refactor(ci): fold the gate scripts into one upgrade-lib.sh existed only to be sourced by the two scripts either side of it, and both run in the same job. One upgrade.sh with seed and verify subcommands says the same thing without the indirection, the shellcheck source directives or the unused-variable suppression the split forced. Also waits on the signal that means ready. /health/version answers before the platform schema is initialized -- the e2e jobs already poll /account for a 401 for exactly this reason -- so the gate could start seeding against a database that was not there yet. That also makes the install gate's separate version step redundant. --- .github/scripts/upgrade-lib.sh | 128 ------------- .github/scripts/upgrade-seed.sh | 115 ----------- .github/scripts/upgrade-verify.sh | 72 ------- .github/scripts/upgrade.sh | 309 ++++++++++++++++++++++++++++++ .github/workflows/ci.yml | 19 +- AGENTS.md | 2 +- 6 files changed, 317 insertions(+), 328 deletions(-) delete mode 100644 .github/scripts/upgrade-lib.sh delete mode 100755 .github/scripts/upgrade-seed.sh delete mode 100755 .github/scripts/upgrade-verify.sh create mode 100755 .github/scripts/upgrade.sh diff --git a/.github/scripts/upgrade-lib.sh b/.github/scripts/upgrade-lib.sh deleted file mode 100644 index 9f03cd151c..0000000000 --- a/.github/scripts/upgrade-lib.sh +++ /dev/null @@ -1,128 +0,0 @@ -#!/usr/bin/env bash -# Shared ground for the upgrade gate. Sourced, not executed. -# -# The gate talks to Appwrite the way a self-hoster does -- over the public HTTP -# API, with no access to container internals -- because that is the surface the -# data has to survive a version change on. - -# The resource names and value shapes below are read by the scripts that source -# this file, so shellcheck cannot see their use when linting it on its own. -# shellcheck disable=SC2034 - -set -euo pipefail - -: "${ENDPOINT:?ENDPOINT must be set, e.g. http://localhost:8080/v1}" - -MANIFEST="${MANIFEST:-upgrade-seed.json}" -STATE="$(dirname "$MANIFEST")" -COOKIE_JAR="${STATE}/upgrade-cookies.txt" -SNAPSHOT="${STATE}/upgrade-snapshot.json" - -# Seeded resources. Fixed IDs so both scripts address the same rows without -# passing them around. -DATABASE="gate" -AUTHORS="authors" -POSTS="posts" -BUCKET="gate" -FILE_ID="gate-file" -USER_ID="gate-user" -AUTHOR="author-1" - -# The value shapes a migration is most likely to mangle. -LONG_VALUE="$(printf 'l%.0s' {1..4000})" -UNICODE_VALUE="ünïcødé — 日本語 — 🚂 — — O'Brien \"quoted\"" - -fail() { - echo "::error::$*" >&2 - exit 1 -} - -# A fresh install spends a while on migrations and worker boot. -wait_for_appwrite() { - local i - - for ((i = 1; i <= 120; i++)); do - if curl --silent --fail --max-time 5 "${ENDPOINT}/health/version" >/dev/null 2>&1; then - return 0 - fi - sleep 1 - done - - fail "Appwrite did not answer at ${ENDPOINT}/health/version within 120s." -} - -version() { - curl --silent --fail --max-time 10 "${ENDPOINT}/health/version" | jq -r '.version' -} - -# Console session, browser origin: how the organization and project get made. -console() { - local method="$1" path="$2" body="${3:-}" - - curl --silent --show-error --fail-with-body --max-time 30 \ - -X "$method" \ - -H 'content-type: application/json' \ - -H 'origin: http://localhost' \ - -H 'x-appwrite-project: console' \ - -c "$COOKIE_JAR" -b "$COOKIE_JAR" \ - ${body:+-d "$body"} \ - "${ENDPOINT}${path}" -} - -# API key: how a server SDK talks to a project. -api() { - local method="$1" path="$2" body="${3:-}" - - curl --silent --show-error --fail-with-body --max-time 60 \ - -X "$method" \ - -H 'content-type: application/json' \ - -H "x-appwrite-project: ${PROJECT:?PROJECT not set}" \ - -H "x-appwrite-key: ${KEY:?KEY not set}" \ - ${body:+-d "$body"} \ - "${ENDPOINT}${path}" -} - -# Columns are processed asynchronously; rows cannot be written until they are all -# available. -wait_for_columns() { - local table="$1" i statuses - - for ((i = 1; i <= 60; i++)); do - statuses=$(api GET "/tablesdb/${DATABASE}/tables/${table}/columns" \ - | jq -r '.columns[].status' | sort -u | tr '\n' ' ') - - case "$statuses" in - 'available ') return 0 ;; - *failed*) fail "Column processing failed on ${table}: ${statuses}" ;; - esac - sleep 1 - done - - fail "Columns on ${table} did not become available within 60s (last: ${statuses:-none})." -} - -# Everything the gate asserts on, projected to just the fields we seeded and -# sorted for a stable comparison. Projecting rather than dumping whole responses -# is deliberate: an upgrade is allowed to add fields to a model, and only the -# values a user put in have to come back unchanged. -snapshot() { - jq -nS \ - --argjson rows "$(api GET "/tablesdb/${DATABASE}/tables/${POSTS}/rows")" \ - --argjson table "$(api GET "/tablesdb/${DATABASE}/tables/${POSTS}")" \ - --argjson user "$(api GET "/users/${USER_ID}")" \ - --argjson file "$(api GET "/storage/buckets/${BUCKET}/files/${FILE_ID}")" \ - '{ - rows: [$rows.rows[] | { - id: .["$id"], - title: .title, - body: .body, - tag: .tag, - author: (if (.author | type) == "object" then .author["$id"] else .author end), - permissions: (.["$permissions"] | sort) - }] | sort_by(.id), - total: $rows.total, - table: { permissions: ($table["$permissions"] | sort), rowSecurity: $table.rowSecurity }, - user: { email: $user.email, name: $user.name }, - file: { id: $file["$id"], size: $file.sizeOriginal } - }' -} diff --git a/.github/scripts/upgrade-seed.sh b/.github/scripts/upgrade-seed.sh deleted file mode 100755 index 2c4f4dad50..0000000000 --- a/.github/scripts/upgrade-seed.sh +++ /dev/null @@ -1,115 +0,0 @@ -#!/usr/bin/env bash -# Seeds an install with the data shapes the upgrade gate cares about, then -# records a snapshot of them. Run against the PREVIOUS stable, before switching -# images. -# -# ENDPOINT=http://localhost:8080/v1 ./upgrade-seed.sh - -set -euo pipefail - -# shellcheck source=.github/scripts/upgrade-lib.sh -source "$(dirname "${BASH_SOURCE[0]}")/upgrade-lib.sh" - -wait_for_appwrite -echo "Seeding $(version) at ${ENDPOINT}" - -# --- console: root account, organization, project, key ----------------------- - -rm -f "$COOKIE_JAR" - -# The first account on a fresh install becomes root. -console POST /account \ - '{"userId":"unique()","email":"gate@example.com","password":"upgrade-gate-1","name":"Gate"}' >/dev/null -console POST /account/sessions/email \ - '{"email":"gate@example.com","password":"upgrade-gate-1"}' >/dev/null - -TEAM=$(console POST /teams '{"teamId":"unique()","name":"Gate"}' | jq -r '.["$id"]') -PROJECT=$(console POST /projects "{\"projectId\":\"gate\",\"name\":\"Gate\",\"teamId\":\"${TEAM}\"}" | jq -r '.["$id"]') -export PROJECT - -KEY=$(console POST "/projects/${PROJECT}/keys" \ - '{"name":"gate","scopes":["users.read","users.write","tables.read","tables.write","rows.read","rows.write","files.read","files.write","buckets.read","buckets.write"]}' \ - | jq -r '.secret') -export KEY - -[ "$PROJECT" = 'gate' ] && [ -n "$KEY" ] && [ "$KEY" != 'null' ] || fail 'Could not create a project and key.' - -# --- tables ------------------------------------------------------------------ - -echo '==> Tables' -api POST /tablesdb "{\"databaseId\":\"${DATABASE}\",\"name\":\"Gate\"}" >/dev/null - -api POST "/tablesdb/${DATABASE}/tables" "$(jq -nc --arg id "$AUTHORS" '{ - tableId: $id, name: "Authors", permissions: ["read(\"any\")"], rowSecurity: false, - columns: [{key: "name", type: "string", size: 255, required: true}] -}')" >/dev/null - -api POST "/tablesdb/${DATABASE}/tables" "$(jq -nc --arg id "$POSTS" '{ - tableId: $id, name: "Posts", permissions: ["read(\"any\")", "create(\"users\")"], rowSecurity: true, - columns: [ - {key: "title", type: "string", size: 255, required: true}, - {key: "body", type: "string", size: 5000, required: false}, - {key: "tag", type: "string", size: 64, required: false, default: "untagged"} - ] -}')" >/dev/null - -# Relationships are not expressible in the inline columns array. -api POST "/tablesdb/${DATABASE}/tables/${POSTS}/columns/relationship" "$(jq -nc --arg related "$AUTHORS" '{ - relatedTableId: $related, type: "manyToOne", twoWay: false, key: "author", onDelete: "setNull" -}')" >/dev/null - -wait_for_columns "$AUTHORS" -wait_for_columns "$POSTS" - -# --- rows -------------------------------------------------------------------- - -echo '==> Rows' -api POST "/tablesdb/${DATABASE}/tables/${AUTHORS}/rows" \ - "{\"rowId\":\"${AUTHOR}\",\"data\":{\"name\":\"Ada Lovelace\"}}" >/dev/null - -# An empty string, a relationship, and a column default left to apply. -api POST "/tablesdb/${DATABASE}/tables/${POSTS}/rows" "$(jq -nc --arg author "$AUTHOR" '{ - rowId: "post-empty", data: {title: "Empty body", body: "", author: $author} -}')" >/dev/null - -api POST "/tablesdb/${DATABASE}/tables/${POSTS}/rows" "$(jq -nc --arg body "$LONG_VALUE" '{ - rowId: "post-long", data: {title: "Long body", body: $body} -}')" >/dev/null - -api POST "/tablesdb/${DATABASE}/tables/${POSTS}/rows" "$(jq -nc --arg body "$UNICODE_VALUE" '{ - rowId: "post-unicode", data: {title: "Unicode body", body: $body} -}')" >/dev/null - -# Row permissions deliberately different from the table's. -api POST "/tablesdb/${DATABASE}/tables/${POSTS}/rows" '{ - "rowId": "post-perms", "data": {"title": "Mixed permissions", "body": "kept"}, - "permissions": ["read(\"any\")", "update(\"users\")"] -}' >/dev/null - -# --- user and file ----------------------------------------------------------- - -echo '==> User and file' -api POST /users "$(jq -nc --arg id "$USER_ID" --arg name "$UNICODE_VALUE" '{ - userId: $id, email: "gate-user@example.com", password: "gate-user-1", name: $name -}')" >/dev/null - -api POST /storage/buckets "$(jq -nc --arg id "$BUCKET" '{ - bucketId: $id, name: "Gate", permissions: ["read(\"any\")"], fileSecurity: false -}')" >/dev/null - -printf '%s\n' "$UNICODE_VALUE" > "${STATE}/gate-file.txt" -curl --silent --show-error --fail-with-body --max-time 60 \ - -X POST \ - -H "x-appwrite-project: ${PROJECT}" \ - -H "x-appwrite-key: ${KEY}" \ - -F "fileId=${FILE_ID}" \ - -F "file=@${STATE}/gate-file.txt" \ - "${ENDPOINT}/storage/buckets/${BUCKET}/files" >/dev/null - -# --- record ------------------------------------------------------------------ - -jq -nc --arg version "$(version)" --arg project "$PROJECT" --arg key "$KEY" \ - '{seededOn: $version, project: $project, key: $key}' > "$MANIFEST" -snapshot > "$SNAPSHOT" - -echo "Seeded from $(version); snapshot at ${SNAPSHOT}" diff --git a/.github/scripts/upgrade-verify.sh b/.github/scripts/upgrade-verify.sh deleted file mode 100755 index 39a5a84387..0000000000 --- a/.github/scripts/upgrade-verify.sh +++ /dev/null @@ -1,72 +0,0 @@ -#!/usr/bin/env bash -# Asserts the seeded data came through the migration unchanged. Run against the -# TARGET version, after migrate. -# -# ENDPOINT=http://localhost:8080/v1 ./upgrade-verify.sh -# ENDPOINT=http://localhost:8080/v1 ./upgrade-verify.sh --write -# -# The default is read-only, so it can run after every migrate. --write also -# proves the upgraded install still takes writes; pass it once, on the last run, -# since the rows it creates would show up in the next comparison. - -set -euo pipefail - -# shellcheck source=.github/scripts/upgrade-lib.sh -source "$(dirname "${BASH_SOURCE[0]}")/upgrade-lib.sh" - -[ -f "$SNAPSHOT" ] || fail "No snapshot at ${SNAPSHOT}; did upgrade-seed.sh run?" - -PROJECT=$(jq -r '.project' "$MANIFEST"); export PROJECT -KEY=$(jq -r '.key' "$MANIFEST"); export KEY -SEEDED_ON=$(jq -r '.seededOn' "$MANIFEST") - -wait_for_appwrite -UPGRADED_TO=$(version) - -echo "Verifying ${SEEDED_ON} -> ${UPGRADED_TO}" - -if [ "$SEEDED_ON" = "$UPGRADED_TO" ]; then - fail "Still reporting ${UPGRADED_TO}; the image did not change, so this run proves nothing." -fi - -# The project, its key and its scopes have to survive before anything else can -# be checked. -api GET /users >/dev/null 2>&1 || fail "The seeded API key no longer authenticates against project ${PROJECT}." - -# --- nothing the user put in changed ----------------------------------------- - -if ! diff --unified=1 --label "seeded on ${SEEDED_ON}" --label "read back on ${UPGRADED_TO}" \ - "$SNAPSHOT" <(snapshot); then - fail 'Seeded data changed across the upgrade; see the diff above.' -fi - -echo 'ok: rows, permissions, relationship, user and file all read back unchanged' - -if [ "${1:-}" != '--write' ]; then - exit 0 -fi - -# --- the upgraded install still takes writes --------------------------------- - -# A read-only check passes against a half-migrated schema, so write too. -api POST "/tablesdb/${DATABASE}/tables/${POSTS}/rows" "$(jq -nc --arg body "$UNICODE_VALUE" '{ - rowId: "post-after-upgrade", data: {title: "Written after upgrade", body: $body} -}')" >/dev/null - -api PATCH "/users/${USER_ID}/name" '{"name":"Renamed after upgrade"}' >/dev/null - -author=$(api POST "/tablesdb/${DATABASE}/tables/${AUTHORS}/rows" \ - '{"rowId":"author-after-upgrade","data":{"name":"Grace Hopper"}}' | jq -r '.["$id"]') -api PATCH "/tablesdb/${DATABASE}/tables/${POSTS}/rows/post-after-upgrade" \ - "{\"data\":{\"author\":\"${author}\"}}" >/dev/null - -written=$(snapshot) -total=$(jq -r '.total' <<<"$written") - -[ "$total" = '5' ] || fail "Expected 5 rows after writing one; found ${total}." -[ "$(jq -r '.user.name' <<<"$written")" = 'Renamed after upgrade' ] || fail 'The user update did not apply.' -[ "$(jq -r '.rows[] | select(.id == "post-after-upgrade") | .author' <<<"$written")" = "$author" ] \ - || fail 'The relationship written after the upgrade did not stick.' - -echo 'ok: rows, users and relationships are still writable' -echo "All checks passed upgrading ${SEEDED_ON} -> ${UPGRADED_TO}." diff --git a/.github/scripts/upgrade.sh b/.github/scripts/upgrade.sh new file mode 100755 index 0000000000..ca4d0977f9 --- /dev/null +++ b/.github/scripts/upgrade.sh @@ -0,0 +1,309 @@ +#!/usr/bin/env bash +# The payload of the release gates: seed an install with the data shapes a +# migration is most likely to mangle, then assert they came back unchanged. +# +# ENDPOINT=http://localhost:8080/v1 upgrade.sh seed # on the previous stable +# ENDPOINT=http://localhost:8080/v1 upgrade.sh verify # on the target, after migrate +# ENDPOINT=http://localhost:8080/v1 upgrade.sh verify --write # ... and that writes still work +# +# verify is read-only by default so it can run after every migrate. --write also +# proves the upgraded install takes writes; pass it once, on the last run, since +# the rows it creates would show up in the next comparison. +# +# It talks to Appwrite the way a self-hoster does -- over the public HTTP API, +# with no access to container internals -- because that is the surface the data +# has to survive a version change on. + +set -euo pipefail + +: "${ENDPOINT:?ENDPOINT must be set, e.g. http://localhost:8080/v1}" + +MANIFEST="${MANIFEST:-upgrade-seed.json}" +STATE="$(dirname "$MANIFEST")" +COOKIE_JAR="${STATE}/upgrade-cookies.txt" +SNAPSHOT="${STATE}/upgrade-snapshot.json" + +# Seeded resources. Fixed IDs so seed and verify address the same rows without +# passing them around. +DATABASE="gate" +AUTHORS="authors" +POSTS="posts" +BUCKET="gate" +FILE_ID="gate-file" +USER_ID="gate-user" +AUTHOR="author-1" + +# The value shapes a migration is most likely to mangle. +LONG_VALUE="$(printf 'l%.0s' {1..4000})" +UNICODE_VALUE="ünïcødé — 日本語 — 🚂 — — O'Brien \"quoted\"" + +fail() { + echo "::error::$*" >&2 + exit 1 +} + +# /health/version answers before the platform schema is initialized, so wait for +# the console API to reject an anonymous read instead. +wait_for_appwrite() { + local i status + + for ((i = 1; i <= 120; i++)); do + status=$(curl --silent --output /dev/null --write-out '%{http_code}' --max-time 5 \ + -H 'x-appwrite-project: console' "${ENDPOINT}/account" || true) + + if [ "$status" = '401' ]; then + return 0 + fi + sleep 1 + done + + fail "Appwrite did not serve ${ENDPOINT}/account within 120s (last status: ${status:-none})." +} + +version() { + curl --silent --fail --max-time 10 "${ENDPOINT}/health/version" | jq -r '.version' +} + +# Console session, browser origin: how the organization and project get made. +console() { + local method="$1" path="$2" body="${3:-}" + + curl --silent --show-error --fail-with-body --max-time 30 \ + -X "$method" \ + -H 'content-type: application/json' \ + -H 'origin: http://localhost' \ + -H 'x-appwrite-project: console' \ + -c "$COOKIE_JAR" -b "$COOKIE_JAR" \ + ${body:+-d "$body"} \ + "${ENDPOINT}${path}" +} + +# API key: how a server SDK talks to a project. +api() { + local method="$1" path="$2" body="${3:-}" + + curl --silent --show-error --fail-with-body --max-time 60 \ + -X "$method" \ + -H 'content-type: application/json' \ + -H "x-appwrite-project: ${PROJECT:?PROJECT not set}" \ + -H "x-appwrite-key: ${KEY:?KEY not set}" \ + ${body:+-d "$body"} \ + "${ENDPOINT}${path}" +} + +# Columns are processed asynchronously; rows cannot be written until they are all +# available. +wait_for_columns() { + local table="$1" i statuses + + for ((i = 1; i <= 60; i++)); do + statuses=$(api GET "/tablesdb/${DATABASE}/tables/${table}/columns" \ + | jq -r '.columns[].status' | sort -u | tr '\n' ' ') + + case "$statuses" in + 'available ') return 0 ;; + *failed*) fail "Column processing failed on ${table}: ${statuses}" ;; + esac + sleep 1 + done + + fail "Columns on ${table} did not become available within 60s (last: ${statuses:-none})." +} + +# Everything the gate asserts on, projected to just the fields we seeded and +# sorted for a stable comparison. Projecting rather than dumping whole responses +# is deliberate: an upgrade is allowed to add fields to a model, and only the +# values a user put in have to come back unchanged. +snapshot() { + jq -nS \ + --argjson rows "$(api GET "/tablesdb/${DATABASE}/tables/${POSTS}/rows")" \ + --argjson table "$(api GET "/tablesdb/${DATABASE}/tables/${POSTS}")" \ + --argjson user "$(api GET "/users/${USER_ID}")" \ + --argjson file "$(api GET "/storage/buckets/${BUCKET}/files/${FILE_ID}")" \ + '{ + rows: [$rows.rows[] | { + id: .["$id"], + title: .title, + body: .body, + tag: .tag, + author: (if (.author | type) == "object" then .author["$id"] else .author end), + permissions: (.["$permissions"] | sort) + }] | sort_by(.id), + total: $rows.total, + table: { permissions: ($table["$permissions"] | sort), rowSecurity: $table.rowSecurity }, + user: { email: $user.email, name: $user.name }, + file: { id: $file["$id"], size: $file.sizeOriginal } + }' +} + +seed() { + wait_for_appwrite + echo "Seeding $(version) at ${ENDPOINT}" + + # --- console: root account, organization, project, key ------------------- + + rm -f "$COOKIE_JAR" + + # The first account on a fresh install becomes root. + console POST /account \ + '{"userId":"unique()","email":"gate@example.com","password":"upgrade-gate-1","name":"Gate"}' >/dev/null + console POST /account/sessions/email \ + '{"email":"gate@example.com","password":"upgrade-gate-1"}' >/dev/null + + TEAM=$(console POST /teams '{"teamId":"unique()","name":"Gate"}' | jq -r '.["$id"]') + PROJECT=$(console POST /projects "{\"projectId\":\"gate\",\"name\":\"Gate\",\"teamId\":\"${TEAM}\"}" | jq -r '.["$id"]') + export PROJECT + + KEY=$(console POST "/projects/${PROJECT}/keys" \ + '{"name":"gate","scopes":["users.read","users.write","tables.read","tables.write","rows.read","rows.write","files.read","files.write","buckets.read","buckets.write"]}' \ + | jq -r '.secret') + export KEY + + [ "$PROJECT" = 'gate' ] && [ -n "$KEY" ] && [ "$KEY" != 'null' ] || fail 'Could not create a project and key.' + + # --- tables -------------------------------------------------------------- + + echo '==> Tables' + api POST /tablesdb "{\"databaseId\":\"${DATABASE}\",\"name\":\"Gate\"}" >/dev/null + + api POST "/tablesdb/${DATABASE}/tables" "$(jq -nc --arg id "$AUTHORS" '{ + tableId: $id, name: "Authors", permissions: ["read(\"any\")"], rowSecurity: false, + columns: [{key: "name", type: "string", size: 255, required: true}] + }')" >/dev/null + + api POST "/tablesdb/${DATABASE}/tables" "$(jq -nc --arg id "$POSTS" '{ + tableId: $id, name: "Posts", permissions: ["read(\"any\")", "create(\"users\")"], rowSecurity: true, + columns: [ + {key: "title", type: "string", size: 255, required: true}, + {key: "body", type: "string", size: 5000, required: false}, + {key: "tag", type: "string", size: 64, required: false, default: "untagged"} + ] + }')" >/dev/null + + # Relationships are not expressible in the inline columns array. + api POST "/tablesdb/${DATABASE}/tables/${POSTS}/columns/relationship" "$(jq -nc --arg related "$AUTHORS" '{ + relatedTableId: $related, type: "manyToOne", twoWay: false, key: "author", onDelete: "setNull" + }')" >/dev/null + + wait_for_columns "$AUTHORS" + wait_for_columns "$POSTS" + + # --- rows ---------------------------------------------------------------- + + echo '==> Rows' + api POST "/tablesdb/${DATABASE}/tables/${AUTHORS}/rows" \ + "{\"rowId\":\"${AUTHOR}\",\"data\":{\"name\":\"Ada Lovelace\"}}" >/dev/null + + # An empty string, a relationship, and a column default left to apply. + api POST "/tablesdb/${DATABASE}/tables/${POSTS}/rows" "$(jq -nc --arg author "$AUTHOR" '{ + rowId: "post-empty", data: {title: "Empty body", body: "", author: $author} + }')" >/dev/null + + api POST "/tablesdb/${DATABASE}/tables/${POSTS}/rows" "$(jq -nc --arg body "$LONG_VALUE" '{ + rowId: "post-long", data: {title: "Long body", body: $body} + }')" >/dev/null + + api POST "/tablesdb/${DATABASE}/tables/${POSTS}/rows" "$(jq -nc --arg body "$UNICODE_VALUE" '{ + rowId: "post-unicode", data: {title: "Unicode body", body: $body} + }')" >/dev/null + + # Row permissions deliberately different from the table's. + api POST "/tablesdb/${DATABASE}/tables/${POSTS}/rows" '{ + "rowId": "post-perms", "data": {"title": "Mixed permissions", "body": "kept"}, + "permissions": ["read(\"any\")", "update(\"users\")"] + }' >/dev/null + + # --- user and file ------------------------------------------------------- + + echo '==> User and file' + api POST /users "$(jq -nc --arg id "$USER_ID" --arg name "$UNICODE_VALUE" '{ + userId: $id, email: "gate-user@example.com", password: "gate-user-1", name: $name + }')" >/dev/null + + api POST /storage/buckets "$(jq -nc --arg id "$BUCKET" '{ + bucketId: $id, name: "Gate", permissions: ["read(\"any\")"], fileSecurity: false + }')" >/dev/null + + printf '%s\n' "$UNICODE_VALUE" > "${STATE}/gate-file.txt" + curl --silent --show-error --fail-with-body --max-time 60 \ + -X POST \ + -H "x-appwrite-project: ${PROJECT}" \ + -H "x-appwrite-key: ${KEY}" \ + -F "fileId=${FILE_ID}" \ + -F "file=@${STATE}/gate-file.txt" \ + "${ENDPOINT}/storage/buckets/${BUCKET}/files" >/dev/null + + # --- record -------------------------------------------------------------- + + jq -nc --arg version "$(version)" --arg project "$PROJECT" --arg key "$KEY" \ + '{seededOn: $version, project: $project, key: $key}' > "$MANIFEST" + snapshot > "$SNAPSHOT" + + echo "Seeded from $(version); snapshot at ${SNAPSHOT}" +} + +verify() { + [ -f "$SNAPSHOT" ] || fail "No snapshot at ${SNAPSHOT}; did the seed run?" + + PROJECT=$(jq -r '.project' "$MANIFEST"); export PROJECT + KEY=$(jq -r '.key' "$MANIFEST"); export KEY + local seeded upgraded author written total + seeded=$(jq -r '.seededOn' "$MANIFEST") + + wait_for_appwrite + upgraded=$(version) + + echo "Verifying ${seeded} -> ${upgraded}" + + if [ "$seeded" = "$upgraded" ]; then + fail "Still reporting ${upgraded}; the image did not change, so this run proves nothing." + fi + + # The project, its key and its scopes have to survive before anything else + # can be checked. + api GET /users >/dev/null 2>&1 || fail "The seeded API key no longer authenticates against project ${PROJECT}." + + # --- nothing the user put in changed ------------------------------------- + + if ! diff --unified=1 --label "seeded on ${seeded}" --label "read back on ${upgraded}" \ + "$SNAPSHOT" <(snapshot); then + fail 'Seeded data changed across the upgrade; see the diff above.' + fi + + echo 'ok: rows, permissions, relationship, user and file all read back unchanged' + + if [ "${1:-}" != '--write' ]; then + return 0 + fi + + # --- the upgraded install still takes writes ----------------------------- + + # A read-only check passes against a half-migrated schema, so write too. + api POST "/tablesdb/${DATABASE}/tables/${POSTS}/rows" "$(jq -nc --arg body "$UNICODE_VALUE" '{ + rowId: "post-after-upgrade", data: {title: "Written after upgrade", body: $body} + }')" >/dev/null + + api PATCH "/users/${USER_ID}/name" '{"name":"Renamed after upgrade"}' >/dev/null + + author=$(api POST "/tablesdb/${DATABASE}/tables/${AUTHORS}/rows" \ + '{"rowId":"author-after-upgrade","data":{"name":"Grace Hopper"}}' | jq -r '.["$id"]') + api PATCH "/tablesdb/${DATABASE}/tables/${POSTS}/rows/post-after-upgrade" \ + "{\"data\":{\"author\":\"${author}\"}}" >/dev/null + + written=$(snapshot) + total=$(jq -r '.total' <<<"$written") + + [ "$total" = '5' ] || fail "Expected 5 rows after writing one; found ${total}." + [ "$(jq -r '.user.name' <<<"$written")" = 'Renamed after upgrade' ] || fail 'The user update did not apply.' + [ "$(jq -r '.rows[] | select(.id == "post-after-upgrade") | .author' <<<"$written")" = "$author" ] \ + || fail 'The relationship written after the upgrade did not stick.' + + echo 'ok: rows, users and relationships are still writable' + echo "All checks passed upgrading ${seeded} -> ${upgraded}." +} + +case "${1:-}" in + seed) seed ;; + verify) shift; verify "$@" ;; + *) fail "Usage: $(basename "$0") seed|verify [--write]" ;; +esac diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9c8312eec4..cb5e634309 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1021,19 +1021,14 @@ jobs: timeout-minutes: 10 run: docker compose up -d --quiet-pull --wait - - name: Report version - run: | - set -euo pipefail - curl --silent --fail --retry 10 --retry-all-errors --retry-delay 3 \ - "${ENDPOINT}/health/version" | tee /dev/stderr | jq -e '.version' - - # A stack that answers /health/version can still be unusable, so drive the - # console and project APIs a self-hoster actually starts with. + # A stack that boots can still be unusable, so drive the console and + # project APIs a self-hoster actually starts with. The script reports the + # version it reached before it seeds anything. - name: Exercise the public API timeout-minutes: 10 env: MANIFEST: ${{ runner.temp }}/install-gate.json - run: .github/scripts/upgrade-seed.sh + run: .github/scripts/upgrade.sh seed - name: Check for crash loops if: always() @@ -1136,7 +1131,7 @@ jobs: - name: Seed through the public API timeout-minutes: 10 - run: .github/scripts/upgrade-seed.sh + run: .github/scripts/upgrade.sh seed - name: Stop the previous stable, keeping volumes env: @@ -1160,7 +1155,7 @@ jobs: - name: Verify the seeded data survived timeout-minutes: 10 - run: .github/scripts/upgrade-verify.sh + run: .github/scripts/upgrade.sh verify # A migration that is not idempotent breaks every retried or resumed # upgrade. @@ -1170,7 +1165,7 @@ jobs: - name: Verify again, and that writes still work timeout-minutes: 10 - run: .github/scripts/upgrade-verify.sh --write + run: .github/scripts/upgrade.sh verify --write - name: Collect logs on failure if: failure() diff --git a/AGENTS.md b/AGENTS.md index dc0f23ec13..17954f9f62 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -351,7 +351,7 @@ Ask the user to review, publish notes on the [Appwrite changelog](https://appwri A release is not ready until a **fresh install** and an **upgrade from the previous stable** both work with realistic data. Previous baseline = highest stable semver tag lower than the target (ignore RC/beta/alpha; prefer `git ls-remote --tags origin`). -Both gates run in CI on release branches — the `install` and `upgrade` jobs in [`ci.yml`](.github/workflows/ci.yml), seeding through the public API with [`upgrade-seed.sh`](.github/scripts/upgrade-seed.sh) and asserting with [`upgrade-verify.sh`](.github/scripts/upgrade-verify.sh). Read those jobs before doing the work by hand; do it by hand when the release changes something they do not cover (a new service, a data shape they do not seed). +Both gates run in CI on release branches — the `install` and `upgrade` jobs in [`ci.yml`](.github/workflows/ci.yml), seeding and asserting through the public API with [`upgrade.sh`](.github/scripts/upgrade.sh). Read those jobs before doing the work by hand; do it by hand when the release changes something they do not cover (a new service, a data shape they do not seed). **Fresh install:** `docker compose down -v` then `up -d --force-recreate --build --wait`. Check `docker compose ps` / logs for crash loops, missing env, failed workers. Hit `/v1/health/version` on the public port. Run unit tests, `tests/e2e/General`, and service e2e. Exercise console users, projects, databases/rows, storage, and (when in scope) functions/sites through public APIs — not empty-stack health checks alone. From 196bb009536afb216b8204b9767892bb8ea792ec Mon Sep 17 00:00:00 2001 From: Levi van Noort <73097785+levivannoort@users.noreply.github.com> Date: Tue, 15 Sep 2026 08:10:57 +0200 Subject: [PATCH 14/37] refactor(ci): take the nightly line from the branch name The workflow read APP_VERSION_STABLE out of the resolved commit, validated its shape and bumped the patch, all to name a tag. Nothing reads that name: migrate keys off the constant compiled into the image and the VERSION build arg only sets _APP_VERSION, so 2.0.x can simply publish 2.0-nightly.. That also drops the failure mode where a release candidate on the branch stopped the channel. Drops the nightly- marker tag with it, and the write access and tag push it needed. Skipping an unchanged branch was the wrong instinct: a daily rebuild is how the channel picks up base image security fixes, which is most of what a nightly is for. nightly.yml is now 148 lines, from 270 when the channel was added. --- .github/workflows/nightly.yml | 75 +++++++++-------------------------- AGENTS.md | 6 +-- 2 files changed, 21 insertions(+), 60 deletions(-) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index b3d754243d..63aa98229f 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -5,9 +5,13 @@ name: "Nightly" # Cloud builds from cl-* tags through publish.yml and is not part of this channel. # # Tags pushed (see the Releases section of AGENTS.md): -# X.Y.Z-nightly. the rollback anchor, one per day -# X.Y-nightly rolling, per minor line. The form to recommend. -# nightly rolling, newest line only. +# X.Y-nightly. the day's build, to pin or roll back to +# X.Y-nightly rolling, per minor line. The form to recommend. +# nightly rolling, newest line only. +# +# The line comes from the branch name, not from APP_VERSION_STABLE: migrate keys +# off the constant compiled into the image, and VERSION only sets _APP_VERSION, +# so a nightly tag is a label rather than something the product reads. on: schedule: @@ -36,15 +40,12 @@ jobs: runs-on: ubuntu-latest outputs: skip: ${{ steps.resolve.outputs.skip }} - branch: ${{ steps.resolve.outputs.branch }} sha: ${{ steps.resolve.outputs.sha }} version: ${{ steps.resolve.outputs.version }} tags: ${{ steps.resolve.outputs.tags }} steps: - name: Checkout repository uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - fetch-depth: 0 - name: Resolve branch, commit and version id: resolve @@ -55,25 +56,19 @@ jobs: set -euo pipefail # The active release branch, unless workflow_dispatch named one. - newest=$(git branch -r --format='%(refname:short)' \ - | sed -n 's#^origin/##p' \ + branches=$(git ls-remote --heads origin \ + | sed -n 's#.*refs/heads/##p' \ | grep -E '^[0-9]+\.[0-9]+\.x$' \ - | sort -V \ - | tail -1) - - if [ -z "$newest" ]; then - echo "::error::No X.Y.x release branch found on origin." - exit 1 - fi - + | sort -V) + newest=$(echo "$branches" | tail -1) branch="${REQUESTED_BRANCH:-$newest}" - if ! git rev-parse --verify "origin/${branch}" >/dev/null 2>&1; then - echo "::error::Branch ${branch} does not exist on origin." + if ! echo "$branches" | grep -qx -- "$branch"; then + echo "::error::'${branch}' is not an X.Y.x release branch on origin." exit 1 fi - echo "branch=${branch}" >> "$GITHUB_OUTPUT" + line="${branch%.x}" # Invariant 2: build the newest CI-green commit, not the tip, so a red # branch delays the channel instead of shipping through it. @@ -94,35 +89,12 @@ jobs: echo "sha=${sha}" >> "$GITHUB_OUTPUT" - # Nothing new since the last nightly. The marker tag is moved by the - # build job below and is the only state this workflow keeps. - previous=$(git rev-parse -q --verify "refs/tags/nightly-${branch}^{commit}" || true) - - if [ "$previous" = "$sha" ]; then - echo "::notice::${branch} has not moved since the last nightly (${sha}); skipping." - echo "skip=true" >> "$GITHUB_OUTPUT" - exit 0 - fi - - # A nightly is a prerelease of the next patch on the line, so read the - # branch's own stable version and bump the patch. - stable=$(git show "${sha}:app/init/constants.php" \ - | sed -n "s/^const APP_VERSION_STABLE = '\([^']*\)';.*/\1/p") - - if ! printf '%s' "$stable" | grep -qE '^[0-9]+\.[0-9]+\.[0-9]+$'; then - echo "::error::APP_VERSION_STABLE on ${branch} is '${stable}', which is not a released X.Y.Z. Refusing to guess a nightly version." - exit 1 - fi - - major=${stable%%.*} - rest=${stable#*.} - minor=${rest%%.*} - patch=${rest#*.} - - version="${major}.${minor}.$((patch + 1))-nightly.$(date -u +%Y%m%d)" + # An unchanged branch still gets a build: it picks up base image + # security fixes, which is most of what a nightly is good for. + version="${line}-nightly.$(date -u +%Y%m%d)" echo "version=${version}" >> "$GITHUB_OUTPUT" - tags="${IMAGE}:${version},${IMAGE}:${major}.${minor}-nightly" + tags="${IMAGE}:${version},${IMAGE}:${line}-nightly" # The bare tag follows the newest line only, so a dispatched build of an # older branch cannot drag it backwards. @@ -141,8 +113,6 @@ jobs: if: needs.resolve.outputs.skip == 'false' runs-on: ubuntu-latest timeout-minutes: 120 - permissions: - contents: write # moves the nightly- marker tag steps: - name: Checkout the resolved commit uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 @@ -176,12 +146,3 @@ jobs: org.opencontainers.image.revision=${{ needs.resolve.outputs.sha }} org.opencontainers.image.version=${{ needs.resolve.outputs.version }} io.appwrite.channel=nightly - - - name: Move the marker tag - env: - BRANCH: ${{ needs.resolve.outputs.branch }} - SHA: ${{ needs.resolve.outputs.sha }} - run: | - set -euo pipefail - git tag --force "nightly-${BRANCH}" "${SHA}" - git push --force origin "nightly-${BRANCH}" diff --git a/AGENTS.md b/AGENTS.md index 17954f9f62..2310b03e59 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -304,14 +304,14 @@ Self-hosted ships on two channels. Cloud is separate: it builds from `cl-*` tags | Channel | Source | Trigger | Tags on `appwrite/appwrite` | |---------|--------|---------|------------------------------| | stable | active release branch (`2.0.x`) | GitHub Release, published by a human | `X.Y.Z` (immutable), `X.Y`, `X`, `latest` | -| nightly | same branch, newest CI-green commit | daily at 00:00 UTC, or `workflow_dispatch` | `X.Y.Z-nightly.` (one a day), `X.Y-nightly`, `nightly` | +| nightly | same branch, newest CI-green commit | daily at 00:00 UTC, or `workflow_dispatch` | `X.Y-nightly.` (one a day), `X.Y-nightly`, `nightly` | -[`nightly.yml`](.github/workflows/nightly.yml) builds the channel; [`security-scan.yml`](.github/workflows/security-scan.yml) is the Trivy scan that used to own that file name. `X.Y.Z` in a nightly tag is the **next** patch on the line — the nightly is a prerelease of the version it is heading for, so `2.0.2-nightly.20260915` sorts below `2.0.2` — and `` is `YYYYMMDD`. +[`nightly.yml`](.github/workflows/nightly.yml) builds the channel; [`security-scan.yml`](.github/workflows/security-scan.yml) is the Trivy scan that used to own that file name. `X.Y` comes from the branch name and `` is `YYYYMMDD`, so `2.0.x` publishes `2.0-nightly.20260915`. The tag is a label, not a version the product reads: `migrate` keys off `APP_VERSION_STABLE` compiled into the image, and the `VERSION` build arg only sets `_APP_VERSION`. The branch is rebuilt daily whether or not it moved, so the channel carries base image security fixes. Four rules keep the channel safe. They are requirements, not preferences: 1. **A patch never adds a migration.** Map a new patch to the previous version's class in `Migration::$versions`. A fix that needs a schema change is a minor. This is what lets a nightly user roll back to yesterday's build, and `1.9.6` (which introduced V25) is the exception not to repeat. -2. **Nightly publishes only a CI-green commit**, walking back from the tip rather than shipping it, so a red branch delays the channel instead of breaking it. +2. **Nightly publishes only a CI-green commit** rather than the tip, so a red branch delays the channel instead of breaking it. 3. **The release branch stays releasable.** Backports land as complete cherry-picks, behind a flag when the fix is not finished. 4. **Nightly publishes to `appwrite/appwrite` only.** It must never be able to push to `appwrite/ce`. From 879f85083b56e429ac7ec216e9aa07752fd742af Mon Sep 17 00:00:00 2001 From: harsh mahajan Date: Tue, 15 Sep 2026 12:24:43 +0530 Subject: [PATCH 15/37] fix(console): register /console web redirects only for the legacy URL scheme With _APP_CONSOLE_URL_SCHEME=root the console serves /, /auth/*, /card/*, /invite, /login, /mfa, /recover and /register/* at the root, so redirecting them to /console sends users to legacy paths. --- .../Modules/Console/Services/Http.php | 21 +++++++++++-------- tests/e2e/General/HTTPTest.php | 4 +++- 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/src/Appwrite/Platform/Modules/Console/Services/Http.php b/src/Appwrite/Platform/Modules/Console/Services/Http.php index 78b2835402..b0a41ca89d 100644 --- a/src/Appwrite/Platform/Modules/Console/Services/Http.php +++ b/src/Appwrite/Platform/Modules/Console/Services/Http.php @@ -20,6 +20,7 @@ use Appwrite\Platform\Modules\Console\Http\Scopes\Project\XList as ListKeyScopes use Appwrite\Platform\Modules\Console\Http\Templates\Email\Get as GetEmailTemplate; use Appwrite\Platform\Modules\Console\Http\Variables\Get as GetVariables; use Utopia\Platform\Service; +use Utopia\System\System; class Http extends Service { @@ -39,14 +40,16 @@ class Http extends Service $this->addAction(CreateAssistantQuery::getName(), new CreateAssistantQuery()); $this->addAction(GetResourceAvailability::getName(), new GetResourceAvailability()); - // web redirects to /console - $this->addAction(RedirectRoot::getName(), new RedirectRoot()); - $this->addAction(RedirectAuth::getName(), new RedirectAuth()); - $this->addAction(RedirectInvite::getName(), new RedirectInvite()); - $this->addAction(RedirectLogin::getName(), new RedirectLogin()); - $this->addAction(RedirectMFA::getName(), new RedirectMFA()); - $this->addAction(RedirectCard::getName(), new RedirectCard()); - $this->addAction(RedirectRecover::getName(), new RedirectRecover()); - $this->addAction(RedirectRegister::getName(), new RedirectRegister()); + // web redirects to /console; root-scheme consoles serve these paths themselves + if (System::getEnv('_APP_CONSOLE_URL_SCHEME', 'legacy') !== 'root') { + $this->addAction(RedirectRoot::getName(), new RedirectRoot()); + $this->addAction(RedirectAuth::getName(), new RedirectAuth()); + $this->addAction(RedirectInvite::getName(), new RedirectInvite()); + $this->addAction(RedirectLogin::getName(), new RedirectLogin()); + $this->addAction(RedirectMFA::getName(), new RedirectMFA()); + $this->addAction(RedirectCard::getName(), new RedirectCard()); + $this->addAction(RedirectRecover::getName(), new RedirectRecover()); + $this->addAction(RedirectRegister::getName(), new RedirectRegister()); + } } } diff --git a/tests/e2e/General/HTTPTest.php b/tests/e2e/General/HTTPTest.php index 42d49bbabc..ecbd88f822 100644 --- a/tests/e2e/General/HTTPTest.php +++ b/tests/e2e/General/HTTPTest.php @@ -9,6 +9,7 @@ use Tests\E2E\Scopes\ProjectNone; use Tests\E2E\Scopes\Scope; use Tests\E2E\Scopes\SideNone; use Utopia\Config\Config; +use Utopia\System\System; final class HTTPTest extends Scope { @@ -241,7 +242,8 @@ final class HTTPTest extends Scope $response = $this->client->call(Client::METHOD_GET, $endpoint, [], [], true, false); - $this->assertEquals('/console' . $endpoint, $response['headers']['location']); + $location = System::getEnv('_APP_CONSOLE_URL_SCHEME', 'legacy') !== 'root' ? '/console' . $endpoint : null; + $this->assertEquals($location, $response['headers']['location'] ?? null); } public function testConsoleServed() From 891ba49e09af391700ef862113ee8e652afb6f09 Mon Sep 17 00:00:00 2001 From: harsh mahajan Date: Tue, 15 Sep 2026 12:25:09 +0530 Subject: [PATCH 16/37] fix(router): link function build errors to function deployments The build failed, canceled and not-ready pages always linked to the sites console, so a function domain whose deployment was not ready sent users to a site that does not exist. Function links now use the functions shape: legacy .../functions/function-{id}[/deployment-{deploymentId}] and root /projects/{projectId}/functions/{id}[/deployments/{deploymentId}]. --- app/controllers/general.php | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/app/controllers/general.php b/app/controllers/general.php index 40a0751174..b8580aa742 100644 --- a/app/controllers/general.php +++ b/app/controllers/general.php @@ -357,12 +357,15 @@ function router(Http $utopia, Database $dbForPlatform, callable $getProjectDB, S $status = $deployment->getAttribute('status'); $region = $project->getAttribute('region', 'default'); $legacyConsolePaths = System::getEnv('_APP_CONSOLE_URL_SCHEME', 'legacy') !== 'root'; - $siteUrl = $legacyConsolePaths - ? "/console/project-{$region}-{$project->getId()}/sites/site-{$resource->getId()}" - : "/projects/{$project->getId()}/sites/{$resource->getId()}"; + $collection = $type === 'function' ? 'functions' : 'sites'; + $resourceUrl = $legacyConsolePaths + ? "/console/project-{$region}-{$project->getId()}/{$collection}/{$type}-{$resource->getId()}" + : "/projects/{$project->getId()}/{$collection}/{$resource->getId()}"; + // Function deployments are listed on the function page itself + $deploymentsUrl = $type === 'function' ? $resourceUrl : "{$resourceUrl}/deployments"; $deploymentUrl = $legacyConsolePaths - ? "{$siteUrl}/deployments/deployment-{$deployment->getId()}" - : "{$siteUrl}/deployments/{$deployment->getId()}"; + ? "{$deploymentsUrl}/deployment-{$deployment->getId()}" + : "{$resourceUrl}/deployments/{$deployment->getId()}"; switch ($status) { case 'failed': @@ -371,7 +374,7 @@ function router(Http $utopia, Database $dbForPlatform, callable $getProjectDB, S break; case 'canceled': $exception = new AppwriteException(AppwriteException::BUILD_CANCELED, view: $errorView); - $exception->addCTA('View deployments', $url . $siteUrl . '/deployments'); + $exception->addCTA('View deployments', $url . $deploymentsUrl); break; default: $exception = new AppwriteException(AppwriteException::BUILD_NOT_READY, view: $errorView); From d16e8868afaa4138122a10d1ae1654459d1ef591 Mon Sep 17 00:00:00 2001 From: harsh mahajan Date: Tue, 15 Sep 2026 12:28:21 +0530 Subject: [PATCH 17/37] chore(console): describe when the /console web redirects are registered The vibes console has no /invite, /login, /recover or /register routes, so the registration comment no longer claims root-scheme consoles serve those paths. --- src/Appwrite/Platform/Modules/Console/Services/Http.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Appwrite/Platform/Modules/Console/Services/Http.php b/src/Appwrite/Platform/Modules/Console/Services/Http.php index b0a41ca89d..00b981c495 100644 --- a/src/Appwrite/Platform/Modules/Console/Services/Http.php +++ b/src/Appwrite/Platform/Modules/Console/Services/Http.php @@ -40,7 +40,7 @@ class Http extends Service $this->addAction(CreateAssistantQuery::getName(), new CreateAssistantQuery()); $this->addAction(GetResourceAvailability::getName(), new GetResourceAvailability()); - // web redirects to /console; root-scheme consoles serve these paths themselves + // web redirects to /console, only where the console is served under /console if (System::getEnv('_APP_CONSOLE_URL_SCHEME', 'legacy') !== 'root') { $this->addAction(RedirectRoot::getName(), new RedirectRoot()); $this->addAction(RedirectAuth::getName(), new RedirectAuth()); From a132f646bf1d6ef623d00b627a39dc17cf96df0d Mon Sep 17 00:00:00 2001 From: Levi van Noort <73097785+levivannoort@users.noreply.github.com> Date: Tue, 15 Sep 2026 09:03:41 +0200 Subject: [PATCH 18/37] refactor(ci): cut the nightly workflow to one job Everything that was not the build itself is gone: the resolve job and the outputs plumbing between it and the build, the skip path (a release branch with no green CI run is an error worth seeing, not a quiet no-op), the workflow_dispatch branch input and the branch validation it needed, and the IMAGE indirection -- the one place a tag is written now names the repository it publishes to. Dropping the branch input means only the active release branch is built, so an X.Y-nightly tag stops moving when a newer line opens. That is what the channel is for. nightly.yml is 89 lines, from 270 when the channel was added. --- .github/workflows/nightly.yml | 149 ++++++++++------------------------ AGENTS.md | 4 +- 2 files changed, 47 insertions(+), 106 deletions(-) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 63aa98229f..f49161e3ac 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -1,123 +1,68 @@ name: "Nightly" -# Publishes the nightly channel for self-hosting: one build a day from the newest -# CI-green commit on the active release branch. Never publishes to appwrite/ce -- -# Cloud builds from cl-* tags through publish.yml and is not part of this channel. +# One build a day of the active release branch, for self-hosters who want to +# track it. Unsupported, and never publishes to appwrite/ce -- Cloud builds from +# cl-* tags through publish.yml. # -# Tags pushed (see the Releases section of AGENTS.md): -# X.Y-nightly. the day's build, to pin or roll back to -# X.Y-nightly rolling, per minor line. The form to recommend. -# nightly rolling, newest line only. -# -# The line comes from the branch name, not from APP_VERSION_STABLE: migrate keys -# off the constant compiled into the image, and VERSION only sets _APP_VERSION, -# so a nightly tag is a label rather than something the product reads. +# Tags on appwrite/appwrite: X.Y-nightly. to pin or roll back to, +# X.Y-nightly to follow the line, nightly to follow whatever line is active. on: schedule: - cron: '0 0 * * *' # 12am UTC daily workflow_dispatch: - inputs: - branch: - description: 'Release branch to build (default: newest X.Y.x branch)' - required: false - type: string concurrency: group: nightly cancel-in-progress: false -env: - # Invariant 4 in AGENTS.md: the nightly channel publishes here and nowhere else. - IMAGE: appwrite/appwrite - permissions: contents: read jobs: - resolve: - name: Resolve build - runs-on: ubuntu-latest - outputs: - skip: ${{ steps.resolve.outputs.skip }} - sha: ${{ steps.resolve.outputs.sha }} - version: ${{ steps.resolve.outputs.version }} - tags: ${{ steps.resolve.outputs.tags }} - steps: - - name: Checkout repository - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - - name: Resolve branch, commit and version - id: resolve - env: - GH_TOKEN: ${{ github.token }} - REQUESTED_BRANCH: ${{ inputs.branch }} - run: | - set -euo pipefail - - # The active release branch, unless workflow_dispatch named one. - branches=$(git ls-remote --heads origin \ - | sed -n 's#.*refs/heads/##p' \ - | grep -E '^[0-9]+\.[0-9]+\.x$' \ - | sort -V) - newest=$(echo "$branches" | tail -1) - branch="${REQUESTED_BRANCH:-$newest}" - - if ! echo "$branches" | grep -qx -- "$branch"; then - echo "::error::'${branch}' is not an X.Y.x release branch on origin." - exit 1 - fi - - line="${branch%.x}" - - # Invariant 2: build the newest CI-green commit, not the tip, so a red - # branch delays the channel instead of shipping through it. - sha=$(gh run list \ - --workflow=ci.yml \ - --branch="${branch}" \ - --event=push \ - --status=success \ - --limit=1 \ - --json headSha \ - --jq '.[0].headSha // empty') - - if [ -z "$sha" ]; then - echo "::warning::No green CI run found on ${branch}; nothing to publish." - echo "skip=true" >> "$GITHUB_OUTPUT" - exit 0 - fi - - echo "sha=${sha}" >> "$GITHUB_OUTPUT" - - # An unchanged branch still gets a build: it picks up base image - # security fixes, which is most of what a nightly is good for. - version="${line}-nightly.$(date -u +%Y%m%d)" - echo "version=${version}" >> "$GITHUB_OUTPUT" - - tags="${IMAGE}:${version},${IMAGE}:${line}-nightly" - - # The bare tag follows the newest line only, so a dispatched build of an - # older branch cannot drag it backwards. - if [ "${branch}" = "${newest}" ]; then - tags="${tags},${IMAGE}:nightly" - fi - - echo "tags=${tags}" >> "$GITHUB_OUTPUT" - echo "skip=false" >> "$GITHUB_OUTPUT" - - echo "Publishing ${tags//,/, } from ${branch} (${sha})." >> "$GITHUB_STEP_SUMMARY" - - build: + nightly: name: Build and publish - needs: resolve - if: needs.resolve.outputs.skip == 'false' runs-on: ubuntu-latest timeout-minutes: 120 steps: - - name: Checkout the resolved commit + - name: Resolve the build + env: + GH_TOKEN: ${{ github.token }} + GH_REPO: ${{ github.repository }} + REMOTE: ${{ github.server_url }}/${{ github.repository }} + run: | + set -euo pipefail + + branch=$(git ls-remote --heads "$REMOTE" \ + | sed -n 's#.*refs/heads/##p' \ + | grep -E '^[0-9]+\.[0-9]+\.x$' \ + | sort -V \ + | tail -1) + + [ -n "$branch" ] || { echo '::error::No X.Y.x release branch on origin.'; exit 1; } + + # The newest CI-green commit rather than the tip, so a red branch + # delays the channel instead of shipping through it. + sha=$(gh run list --workflow=ci.yml --branch="$branch" --event=push \ + --status=success --limit=1 --json headSha --jq '.[0].headSha // empty') + + [ -n "$sha" ] || { echo "::error::No green CI run on ${branch}."; exit 1; } + + line="${branch%.x}" + version="${line}-nightly.$(date -u +%Y%m%d)" + + { + echo "SHA=${sha}" + echo "VERSION=${version}" + echo "TAGS=appwrite/appwrite:${version},appwrite/appwrite:${line}-nightly,appwrite/appwrite:nightly" + } >> "$GITHUB_ENV" + + echo "Publishing ${version} from ${branch} (${sha})." >> "$GITHUB_STEP_SUMMARY" + + - name: Check out the resolved commit uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: - ref: ${{ needs.resolve.outputs.sha }} + ref: ${{ env.SHA }} submodules: recursive - name: Set up QEMU @@ -138,11 +83,7 @@ jobs: context: . platforms: linux/amd64,linux/arm64 target: production - build-args: | - VERSION=${{ needs.resolve.outputs.version }} + build-args: VERSION=${{ env.VERSION }} push: true - tags: ${{ needs.resolve.outputs.tags }} - labels: | - org.opencontainers.image.revision=${{ needs.resolve.outputs.sha }} - org.opencontainers.image.version=${{ needs.resolve.outputs.version }} - io.appwrite.channel=nightly + tags: ${{ env.TAGS }} + labels: org.opencontainers.image.revision=${{ env.SHA }} diff --git a/AGENTS.md b/AGENTS.md index 2310b03e59..5789573bc9 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -304,9 +304,9 @@ Self-hosted ships on two channels. Cloud is separate: it builds from `cl-*` tags | Channel | Source | Trigger | Tags on `appwrite/appwrite` | |---------|--------|---------|------------------------------| | stable | active release branch (`2.0.x`) | GitHub Release, published by a human | `X.Y.Z` (immutable), `X.Y`, `X`, `latest` | -| nightly | same branch, newest CI-green commit | daily at 00:00 UTC, or `workflow_dispatch` | `X.Y-nightly.` (one a day), `X.Y-nightly`, `nightly` | +| nightly | newest CI-green commit on that branch | daily at 00:00 UTC, or `workflow_dispatch` | `X.Y-nightly.` (one a day), `X.Y-nightly`, `nightly` | -[`nightly.yml`](.github/workflows/nightly.yml) builds the channel; [`security-scan.yml`](.github/workflows/security-scan.yml) is the Trivy scan that used to own that file name. `X.Y` comes from the branch name and `` is `YYYYMMDD`, so `2.0.x` publishes `2.0-nightly.20260915`. The tag is a label, not a version the product reads: `migrate` keys off `APP_VERSION_STABLE` compiled into the image, and the `VERSION` build arg only sets `_APP_VERSION`. The branch is rebuilt daily whether or not it moved, so the channel carries base image security fixes. +[`nightly.yml`](.github/workflows/nightly.yml) builds the channel; [`security-scan.yml`](.github/workflows/security-scan.yml) is the Trivy scan that used to own that file name. `X.Y` comes from the branch name and `` is `YYYYMMDD`, so `2.0.x` publishes `2.0-nightly.20260915`. The tag is a label, not a version the product reads: `migrate` keys off `APP_VERSION_STABLE` compiled into the image, and the `VERSION` build arg only sets `_APP_VERSION`. The branch is rebuilt daily whether or not it moved, so the channel carries base image security fixes. Only the newest release branch is built, so an `X.Y-nightly` tag stops moving once a newer line opens. Four rules keep the channel safe. They are requirements, not preferences: From 7d1121d83d91a98a5e3daa7d4951fab195da0ae4 Mon Sep 17 00:00:00 2001 From: harsh mahajan Date: Tue, 15 Sep 2026 13:04:26 +0530 Subject: [PATCH 19/37] feat(console): build console links from _APP_CONSOLE_URL Drops _APP_CONSOLE_URL_SCHEME and the legacy /console/project-{region}-{id} link shapes. OAuth2 and magic URL defaults, router error CTAs, preview authorization, VCS commit statuses and PR comments, and the paused-webhook email now link to the console root routes on _APP_CONSOLE_URL, which defaults to the console domain over the configured protocol. The webhooks and jobs workers receive the variables the default is derived from. --- .env | 1 + app/config/platform.php | 2 + app/config/variables.php | 8 +-- app/controllers/api/account.php | 36 +++++------- app/controllers/general.php | 30 +++------- docker-compose.yml | 15 +++-- src/Appwrite/Deployment/GitAction.php | 7 +-- .../Modules/VCS/Http/GitHub/Deployment.php | 11 +--- src/Appwrite/Platform/Workers/Webhooks.php | 9 +-- src/Appwrite/Vcs/Comment.php | 8 +-- .../Account/AccountCustomClientTest.php | 4 +- tests/unit/Platform/Workers/WebhooksTest.php | 55 +------------------ 12 files changed, 52 insertions(+), 134 deletions(-) diff --git a/.env b/.env index a09c1da12d..86d469ccfc 100644 --- a/.env +++ b/.env @@ -28,6 +28,7 @@ _APP_OPTIONS_ROUTER_PROTECTION=disabled _APP_OPTIONS_ABUSE=disabled _APP_OPTIONS_ABUSE_INCREASED_LIMIT_PROJECTS=increased-limit-project _APP_CONSOLE_DOMAIN=localhost +_APP_CONSOLE_URL= _APP_CONSOLE_HOSTNAMES=localhost,appwrite.io,*.appwrite.io _APP_CONSOLE_SCHEMA=appwriteio _APP_CONSOLE_SESSION_ALERTS=enabled diff --git a/app/config/platform.php b/app/config/platform.php index 21aedaad98..7e4f99fdf2 100644 --- a/app/config/platform.php +++ b/app/config/platform.php @@ -19,6 +19,8 @@ if (\str_contains($functionsDomain, ',')) { return [ 'apiHostname' => System::getEnv('_APP_DOMAIN', 'localhost'), 'consoleHostname' => System::getEnv('_APP_CONSOLE_DOMAIN', System::getEnv('_APP_DOMAIN', 'localhost')), + // Console web app origin for links; it can live apart from the API host above + 'consoleUrl' => \rtrim(System::getEnv('_APP_CONSOLE_URL', (System::getEnv('_APP_OPTIONS_FORCE_HTTPS') === 'disabled' ? 'http' : 'https') . '://' . System::getEnv('_APP_CONSOLE_DOMAIN', System::getEnv('_APP_DOMAIN', 'localhost'))), '/'), 'hostnames' => array_filter(array_unique([ System::getEnv('_APP_DOMAIN', 'localhost'), System::getEnv('_APP_CONSOLE_DOMAIN', 'localhost'), diff --git a/app/config/variables.php b/app/config/variables.php index 297a129fd7..d67f07d406 100644 --- a/app/config/variables.php +++ b/app/config/variables.php @@ -242,10 +242,10 @@ return [ 'filter' => '' ], [ - 'name' => '_APP_CONSOLE_URL_SCHEME', - 'description' => 'Console URL scheme used when the backend generates links to the console (OAuth callbacks, emails, error page CTAs, VCS comments). Set to \'root\' for the new console served at the root path (appwrite/new), or \'legacy\' for the older console served under the /console path prefix. The default value is \'legacy\'.', - 'introduction' => '2.0.0', - 'default' => 'legacy', + 'name' => '_APP_CONSOLE_URL', + 'description' => 'Origin of the Appwrite console web app, such as https://console.example.com, used when the backend generates links to the console (OAuth callbacks, emails, error page CTAs, VCS comments and commit statuses). Set it when the console is served on a different host than the API. When empty, links use _APP_CONSOLE_DOMAIN (or _APP_DOMAIN) over https, or over http when _APP_OPTIONS_FORCE_HTTPS is disabled.', + 'introduction' => '2.1.1', + 'default' => '', 'required' => false, 'question' => '', 'filter' => '' diff --git a/app/controllers/api/account.php b/app/controllers/api/account.php index 4bad06ca4e..5c8d7ff94e 100644 --- a/app/controllers/api/account.php +++ b/app/controllers/api/account.php @@ -77,10 +77,8 @@ use Utopia\Validator\Range; use Utopia\Validator\Text; use Utopia\Validator\WhiteList; -$legacyConsolePaths = System::getEnv('_APP_CONSOLE_URL_SCHEME', 'legacy') !== 'root'; - -$oauthDefaultSuccess = $legacyConsolePaths ? '/console/auth/oauth2/success' : '/auth/oauth2/success'; -$oauthDefaultFailure = $legacyConsolePaths ? '/console/auth/oauth2/failure' : '/auth/oauth2/failure'; +$oauthDefaultSuccess = '/auth/oauth2/success'; +$oauthDefaultFailure = '/auth/oauth2/failure'; $createSession = function (string $userId, string $secret, Request $request, Response $response, User $user, Database $dbForProject, Document $project, array $platform, Locale $locale, Geo $geo, Event $queueForEvents, Bus $bus, Store $store, ProofsToken $proofForToken, ProofsCode $proofForCode, bool $domainVerification, ?string $cookieDomain, Authorization $authorization) { @@ -1378,11 +1376,9 @@ Http::get('/v1/account/sessions/oauth2/:provider') throw new Exception(Exception::PROJECT_PROVIDER_UNSUPPORTED); } - $host = $platform['consoleHostname'] ?? ''; - $redirectBase = $protocol . '://' . $host; - if ($protocol === 'https' && $port !== '443') { - $redirectBase .= ':' . $port; - } elseif ($protocol === 'http' && $port !== '80') { + $redirectBase = $platform['consoleUrl'] ?? ''; + // Only the derived console URL follows the request port; an explicit one carries its own + if (empty(System::getEnv('_APP_CONSOLE_URL')) && $port !== ($protocol === 'https' ? '443' : '80')) { $redirectBase .= ':' . $port; } @@ -2285,13 +2281,11 @@ Http::get('/v1/account/tokens/oauth2/:provider') throw new Exception(Exception::PROJECT_PROVIDER_UNSUPPORTED); } - $host = $platform['consoleHostname'] ?? ''; $protocol = System::getEnv('_APP_OPTIONS_FORCE_HTTPS') === 'disabled' ? 'http' : 'https'; $port = $request->getPort(); - $redirectBase = $protocol . '://' . $host; - if ($protocol === 'https' && $port !== '443') { - $redirectBase .= ':' . $port; - } elseif ($protocol === 'http' && $port !== '80') { + $redirectBase = $platform['consoleUrl'] ?? ''; + // Only the derived console URL follows the request port; an explicit one carries its own + if (empty(System::getEnv('_APP_CONSOLE_URL')) && $port !== ($protocol === 'https' ? '443' : '80')) { $redirectBase .= ':' . $port; } @@ -2358,7 +2352,7 @@ Http::post('/v1/account/tokens/magic-url') ->inject('proofForPassword') ->inject('platform') ->inject('authorization') - ->action(function (string $userId, string $email, string $url, bool $phrase, Request $request, Response $response, Document $user, Document $project, Database $dbForProject, Locale $locale, Event $queueForEvents, MailPublisher $publisherForMails, array $plan, ProofsPassword $proofForPassword, array $platform, Authorization $authorization) use ($legacyConsolePaths) { + ->action(function (string $userId, string $email, string $url, bool $phrase, Request $request, Response $response, Document $user, Document $project, Database $dbForProject, Locale $locale, Event $queueForEvents, MailPublisher $publisherForMails, array $plan, ProofsPassword $proofForPassword, array $platform, Authorization $authorization) { if (empty(System::getEnv('_APP_SMTP_HOST'))) { throw new Exception(Exception::GENERAL_SMTP_DISABLED, 'SMTP disabled'); } @@ -2496,17 +2490,13 @@ Http::post('/v1/account/tokens/magic-url') if (empty($url)) { $protocol = System::getEnv('_APP_OPTIONS_FORCE_HTTPS') === 'disabled' ? 'http' : 'https'; - $host = $platform['consoleHostname'] ?? ''; $port = $request->getPort(); - $callbackBase = $protocol . '://' . $host; - if ($protocol === 'https' && $port !== '443') { - $callbackBase .= ':' . $port; - } elseif ($protocol === 'http' && $port !== '80') { + $callbackBase = $platform['consoleUrl'] ?? ''; + // Only the derived console URL follows the request port; an explicit one carries its own + if (empty(System::getEnv('_APP_CONSOLE_URL')) && $port !== ($protocol === 'https' ? '443' : '80')) { $callbackBase .= ':' . $port; } - $url = $legacyConsolePaths - ? "{$callbackBase}/console/auth/magic-url" - : "{$callbackBase}/auth/magic-url"; + $url = "{$callbackBase}/auth/magic-url"; } $url = Template::parseURL($url); diff --git a/app/controllers/general.php b/app/controllers/general.php index b8580aa742..7d7b6204f2 100644 --- a/app/controllers/general.php +++ b/app/controllers/general.php @@ -98,8 +98,7 @@ function router(Http $utopia, Database $dbForPlatform, callable $getProjectDB, S } $errorView = __DIR__ . '/../views/general/error.phtml'; - $protocol = System::getEnv('_APP_OPTIONS_FORCE_HTTPS') == 'disabled' ? 'http' : 'https'; - $url = $protocol . '://' . $platform['consoleHostname']; + $url = $platform['consoleUrl'] ?? ''; $platformHostnames = $platform['hostnames'] ?? []; if ($rule->isEmpty()) { @@ -126,7 +125,7 @@ function router(Http $utopia, Database $dbForPlatform, callable $getProjectDB, S if (\str_ends_with($host, $denyDomain)) { $exception = new AppwriteException(AppwriteException::RULE_NOT_FOUND, 'This domain is not connected to any Appwrite resources. Visit domains tab under function/site settings to configure it.', view: $errorView); - $exception->addCTA('Start with this domain', System::getEnv('_APP_CONSOLE_URL_SCHEME', 'legacy') !== 'root' ? "{$url}/console" : $url); + $exception->addCTA('Start with this domain', $url); throw $exception; } } @@ -205,11 +204,7 @@ function router(Http $utopia, Database $dbForPlatform, callable $getProjectDB, S $resourceId = $rule->getAttribute('deploymentResourceId', ''); $type = ($resourceType === 'site') ? 'sites' : 'functions'; $exception = new AppwriteException(AppwriteException::DEPLOYMENT_NOT_FOUND, view: $errorView); - $region = $project->getAttribute('region', 'default'); - $ctaUrl = System::getEnv('_APP_CONSOLE_URL_SCHEME', 'legacy') !== 'root' - ? "/console/project-{$region}-{$projectId}/{$type}/{$resourceType}-{$resourceId}" - : "/projects/{$projectId}/{$type}/{$resourceId}"; - $exception->addCTA('View deployments', $url . $ctaUrl); + $exception->addCTA('View deployments', $url . "/projects/{$projectId}/{$type}/{$resourceId}"); throw $exception; } @@ -294,11 +289,10 @@ function router(Http $utopia, Database $dbForPlatform, callable $getProjectDB, S } if (!$authorized) { - $url = $protocol . "://" . $platform['consoleHostname']; $response ->addHeader('Cache-Control', 'no-store, no-cache, must-revalidate, max-age=0') ->addHeader('Pragma', 'no-cache') - ->redirect($url . (System::getEnv('_APP_CONSOLE_URL_SCHEME', 'legacy') !== 'root' ? '/console/auth/preview?' : '/auth/preview?') + ->redirect($url . '/auth/preview?' . \http_build_query([ 'projectId' => $projectId, 'origin' => $protocol . '://' . $host, @@ -355,17 +349,11 @@ function router(Http $utopia, Database $dbForPlatform, callable $getProjectDB, S $allowAnyStatus = !\is_null($apiKey) && $apiKey->isDeploymentStatusIgnored(); if (!$allowAnyStatus && $deployment->getAttribute('status') !== 'ready') { $status = $deployment->getAttribute('status'); - $region = $project->getAttribute('region', 'default'); - $legacyConsolePaths = System::getEnv('_APP_CONSOLE_URL_SCHEME', 'legacy') !== 'root'; $collection = $type === 'function' ? 'functions' : 'sites'; - $resourceUrl = $legacyConsolePaths - ? "/console/project-{$region}-{$project->getId()}/{$collection}/{$type}-{$resource->getId()}" - : "/projects/{$project->getId()}/{$collection}/{$resource->getId()}"; + $resourceUrl = "/projects/{$project->getId()}/{$collection}/{$resource->getId()}"; // Function deployments are listed on the function page itself $deploymentsUrl = $type === 'function' ? $resourceUrl : "{$resourceUrl}/deployments"; - $deploymentUrl = $legacyConsolePaths - ? "{$deploymentsUrl}/deployment-{$deployment->getId()}" - : "{$resourceUrl}/deployments/{$deployment->getId()}"; + $deploymentUrl = "{$resourceUrl}/deployments/{$deployment->getId()}"; switch ($status) { case 'failed': @@ -389,11 +377,7 @@ function router(Http $utopia, Database $dbForPlatform, callable $getProjectDB, S $permissions = $resource->getAttribute('execute'); if (!(\in_array('any', $permissions)) && !(\in_array('guests', $permissions))) { $exception = new AppwriteException(AppwriteException::FUNCTION_EXECUTE_PERMISSION_MISSING, view: $errorView); - $region = $project->getAttribute('region', 'default'); - $ctaUrl = System::getEnv('_APP_CONSOLE_URL_SCHEME', 'legacy') !== 'root' - ? "/console/project-{$region}-{$project->getId()}/functions/function-{$resource->getId()}/settings" - : "/projects/{$project->getId()}/functions/{$resource->getId()}/settings"; - $exception->addCTA('View settings', $url . $ctaUrl); + $exception->addCTA('View settings', $url . "/projects/{$project->getId()}/functions/{$resource->getId()}/settings"); throw $exception; } } diff --git a/docker-compose.yml b/docker-compose.yml index 95793ba74f..8c6d1e1bf5 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -124,7 +124,7 @@ services: - _APP_NOTIFICATIONS_TRACKING_SECRET - _APP_DOMAIN - _APP_CONSOLE_DOMAIN - - _APP_CONSOLE_URL_SCHEME=root + - _APP_CONSOLE_URL - _APP_CONSOLE_TRUSTED_PROJECTS - _APP_DOMAIN_TARGET_CNAME - _APP_DOMAIN_TARGET_AAAA @@ -427,8 +427,8 @@ services: - _APP_LOGGING_FORMAT - _APP_QUEUE_NAME - _APP_DATABASE_SHARED_TABLES - - _APP_CONSOLE_URL_SCHEME=root - _APP_CONSOLE_DOMAIN + - _APP_CONSOLE_URL - _APP_CONSOLE_TRUSTED_PROJECTS - _APP_DOMAIN - _APP_DOMAIN_TARGET_CNAME @@ -634,7 +634,10 @@ services: - _APP_LOGGING_FORMAT - _APP_WEBHOOK_MAX_FAILED_ATTEMPTS - _APP_DATABASE_SHARED_TABLES - - _APP_CONSOLE_URL_SCHEME=root + - _APP_OPTIONS_FORCE_HTTPS + - _APP_DOMAIN + - _APP_CONSOLE_DOMAIN + - _APP_CONSOLE_URL appwrite-worker-deletes: profiles: - separate @@ -847,7 +850,7 @@ services: - _APP_OPTIONS_ROUTER_FORCE_HTTPS - _APP_DOMAIN - _APP_CONSOLE_DOMAIN - - _APP_CONSOLE_URL_SCHEME=root + - _APP_CONSOLE_URL - _APP_CONSOLE_TRUSTED_PROJECTS - _APP_STORAGE_DEVICE - _APP_STORAGE_S3_ACCESS_KEY @@ -942,6 +945,10 @@ services: - _APP_LOGGING_CONFIG - _APP_LOGGING_FORMAT - _APP_DATABASE_SHARED_TABLES + - _APP_OPTIONS_FORCE_HTTPS + - _APP_DOMAIN + - _APP_CONSOLE_DOMAIN + - _APP_CONSOLE_URL extra_hosts: - host.docker.internal:host-gateway diff --git a/src/Appwrite/Deployment/GitAction.php b/src/Appwrite/Deployment/GitAction.php index 0d5d4103d6..c668029e49 100644 --- a/src/Appwrite/Deployment/GitAction.php +++ b/src/Appwrite/Deployment/GitAction.php @@ -50,13 +50,8 @@ final class GitAction default => $status }; - $hostname = $platform['consoleHostname'] ?? ''; - $region = $project->getAttribute('region', 'default'); $collection = $isSite ? 'sites' : 'functions'; - $type = $isSite ? 'site' : 'function'; - $targetUrl = System::getEnv('_APP_CONSOLE_URL_SCHEME', 'legacy') !== 'root' - ? "{$protocol}://{$hostname}/console/project-{$region}-{$project->getId()}/{$collection}/{$type}-{$resource->getId()}" - : "{$protocol}://{$hostname}/projects/{$project->getId()}/{$collection}/{$resource->getId()}"; + $targetUrl = ($platform['consoleUrl'] ?? '') . "/projects/{$project->getId()}/{$collection}/{$resource->getId()}"; $name = $resource->getAttribute('name') . ' (' . $project->getAttribute('name') . ')'; $vcs->updateCommitStatus($repositoryName, $commitHash, $owner, $state, $message, $targetUrl, $name); diff --git a/src/Appwrite/Platform/Modules/VCS/Http/GitHub/Deployment.php b/src/Appwrite/Platform/Modules/VCS/Http/GitHub/Deployment.php index d5ea712b6c..1f26df9b1b 100644 --- a/src/Appwrite/Platform/Modules/VCS/Http/GitHub/Deployment.php +++ b/src/Appwrite/Platform/Modules/VCS/Http/GitHub/Deployment.php @@ -164,11 +164,9 @@ trait Deployment Span::add("{$logBase}.authorized", $isAuthorized); $protocol = System::getEnv('_APP_OPTIONS_FORCE_HTTPS') === 'disabled' ? 'http' : 'https'; - $hostname = $platform['consoleHostname'] ?? ''; + $consoleUrl = $platform['consoleUrl'] ?? ''; - $authorizeUrl = System::getEnv('_APP_CONSOLE_URL_SCHEME', 'legacy') !== 'root' - ? $protocol . '://' . $hostname . "/console/git/authorize-contributor?projectId={$projectId}&installationId={$installationId}&repositoryId={$repositoryId}&providerPullRequestId={$providerPullRequestId}" - : $protocol . '://' . $hostname . "/git/authorize-contributor?projectId={$projectId}&installationId={$installationId}&repositoryId={$repositoryId}&providerPullRequestId={$providerPullRequestId}"; + $authorizeUrl = $consoleUrl . "/git/authorize-contributor?projectId={$projectId}&installationId={$installationId}&repositoryId={$repositoryId}&providerPullRequestId={$providerPullRequestId}"; $action = $isAuthorized ? ['type' => 'logs'] : ['type' => 'authorize', 'url' => $authorizeUrl]; @@ -557,7 +555,6 @@ trait Deployment if (!empty($providerCommitHash) && $resource->getAttribute('providerSilentMode', false) === false) { $resourceName = $resource->getAttribute('name'); $projectName = $project->getAttribute('name'); - $region = $project->getAttribute('region', 'default'); $name = "{$resourceName} ({$projectName})"; $message = 'Starting...'; @@ -569,9 +566,7 @@ trait Deployment } $owner = $vcs->getOwnerName($providerInstallationId, (int) $providerRepositoryId); - $providerTargetUrl = System::getEnv('_APP_CONSOLE_URL_SCHEME', 'legacy') !== 'root' - ? $protocol . '://' . $hostname . "/console/project-$region-$projectId/$resourceCollection/$resourceType-$resourceId" - : $protocol . '://' . $hostname . "/projects/$projectId/$resourceCollection/$resourceId"; + $providerTargetUrl = $consoleUrl . "/projects/$projectId/$resourceCollection/$resourceId"; $vcs->updateCommitStatus($repositoryName, $providerCommitHash, $owner, 'pending', $message, $providerTargetUrl, $name); } diff --git a/src/Appwrite/Platform/Workers/Webhooks.php b/src/Appwrite/Platform/Workers/Webhooks.php index 76667cc350..f4c58cac17 100644 --- a/src/Appwrite/Platform/Workers/Webhooks.php +++ b/src/Appwrite/Platform/Workers/Webhooks.php @@ -276,11 +276,10 @@ class Webhooks extends Action $projectId = $project->getId(); $projectInternalId = $project->getSequence(); - $region = $project->getAttribute('region', 'default'); - $webhookId = $webhook->getId(); $protocol = System::getEnv('_APP_OPTIONS_FORCE_HTTPS', 'disabled') === 'disabled' ? 'http' : 'https'; $consoleHostname = System::getEnv('_APP_CONSOLE_DOMAIN', System::getEnv('_APP_DOMAIN', 'localhost')); + $consoleUrl = \rtrim(System::getEnv('_APP_CONSOLE_URL', $protocol . '://' . $consoleHostname), '/'); $subject = 'Webhook deliveries have been paused'; $preview = 'Webhook "' . $webhook->getAttribute('name') . '" has been paused after ' . $attempts . ' failed delivery attempts.'; @@ -320,10 +319,8 @@ class Webhooks extends Action $template->setParam('{{project}}', $project->getAttribute('name')); $template->setParam('{{url}}', $webhook->getAttribute('url')); $template->setParam('{{error}}', 'The server returned ' . $statusCode . ' status code'); - $template->setParam('{{host}}', $protocol . '://' . $consoleHostname); - $template->setParam('{{path}}', System::getEnv('_APP_CONSOLE_URL_SCHEME', 'legacy') !== 'root' - ? "/console/project-{$region}-{$projectId}/settings/webhooks/{$webhookId}" - : "/projects/{$projectId}/settings/webhooks"); + $template->setParam('{{host}}', $consoleUrl); + $template->setParam('{{path}}', "/projects/{$projectId}/settings/webhooks"); $template->setParam('{{attempts}}', $attempts); $publisherForNotifications->enqueue(new NotificationMessage( diff --git a/src/Appwrite/Vcs/Comment.php b/src/Appwrite/Vcs/Comment.php index 328dcaa19a..be36c92299 100644 --- a/src/Appwrite/Vcs/Comment.php +++ b/src/Appwrite/Vcs/Comment.php @@ -162,9 +162,7 @@ class Comment }; if ($site['action']['type'] === 'logs') { - $logsUrl = System::getEnv('_APP_CONSOLE_URL_SCHEME', 'legacy') !== 'root' - ? "{$protocol}://{$hostname}/console/project-{$site['region']}-{$projectId}/sites/site-{$siteId}/deployments/deployment-{$site['deploymentId']}" - : "{$protocol}://{$hostname}/projects/{$projectId}/sites/{$siteId}/deployments/{$site['deploymentId']}"; + $logsUrl = ($this->platform['consoleUrl'] ?? '') . "/projects/{$projectId}/sites/{$siteId}/deployments/{$site['deploymentId']}"; $action = "[View Logs]({$logsUrl})"; } else { $action = '[Authorize](' . $site['action']['url'] . ')'; @@ -215,9 +213,7 @@ class Comment }; if ($function['action']['type'] === 'logs') { - $logsUrl = System::getEnv('_APP_CONSOLE_URL_SCHEME', 'legacy') !== 'root' - ? "{$protocol}://{$hostname}/console/project-{$function['region']}-{$projectId}/functions/function-{$functionId}/deployment-{$function['deploymentId']}" - : "{$protocol}://{$hostname}/projects/{$projectId}/functions/{$functionId}/deployments/{$function['deploymentId']}"; + $logsUrl = ($this->platform['consoleUrl'] ?? '') . "/projects/{$projectId}/functions/{$functionId}/deployments/{$function['deploymentId']}"; $action = "[View Logs]({$logsUrl})"; } else { $action = '[Authorize](' . $function['action']['url'] . ')'; diff --git a/tests/e2e/Services/Account/AccountCustomClientTest.php b/tests/e2e/Services/Account/AccountCustomClientTest.php index 4d8768470d..3d460b8c21 100644 --- a/tests/e2e/Services/Account/AccountCustomClientTest.php +++ b/tests/e2e/Services/Account/AccountCustomClientTest.php @@ -3440,7 +3440,7 @@ final class AccountCustomClientTest extends Scope $this->assertEquals(200, $response['headers']['status-code']); - // Omit failure so Appwrite uses the default relay URL (/auth/oauth2/failure or legacy /console/...) + // Omit failure so Appwrite uses the default relay URL (/auth/oauth2/failure) $response = $this->client->call(Client::METHOD_GET, '/account/sessions/oauth2/' . $provider, [ 'origin' => 'http://localhost', 'content-type' => 'application/json', @@ -3481,7 +3481,7 @@ final class AccountCustomClientTest extends Scope $query = []; \parse_str((string) \parse_url($location, PHP_URL_QUERY), $query); - $this->assertContains($path, ['/auth/oauth2/failure', '/console/auth/oauth2/failure']); + $this->assertEquals('/auth/oauth2/failure', $path); $this->assertEquals($projectId, $query['project'] ?? null); $this->assertNotEmpty($query['error'] ?? null); diff --git a/tests/unit/Platform/Workers/WebhooksTest.php b/tests/unit/Platform/Workers/WebhooksTest.php index e846531633..f514e5404f 100644 --- a/tests/unit/Platform/Workers/WebhooksTest.php +++ b/tests/unit/Platform/Workers/WebhooksTest.php @@ -31,7 +31,7 @@ final class WebhooksTest extends TestCase $publisherForNotifications = new NotificationPublisher($publisher, new Queue('v1-notifications')); $worker = new Webhooks(); - $this->withConsoleUrlScheme('legacy', fn () => $worker->sendAlert( + $worker->sendAlert( attempts: 10, statusCode: 500, webhook: new Document([ @@ -50,7 +50,7 @@ final class WebhooksTest extends TestCase dbForPlatform: $database, publisherForNotifications: $publisherForNotifications, plan: [] - )); + ); $events = $publisher->getEvents('v1-notifications'); @@ -68,7 +68,7 @@ final class WebhooksTest extends TestCase ); $this->assertStringContainsString('Payments', (string) $payload['body']); $this->assertStringContainsString('Ada Lovelace', (string) $payload['body']); - $this->assertStringContainsString('/console/project-fra-project-1/settings/webhooks/webhook-1', (string) $payload['body']); + $this->assertStringContainsString('/projects/project-1/settings/webhooks', (string) $payload['body']); $this->assertStringNotContainsString('{{', (string) $payload['body']); $this->assertSame(APP_NAME, $payload['variables']['platform']); $this->assertSame(APP_EMAIL_LOGO_URL, $payload['variables']['logoUrl']); @@ -95,43 +95,6 @@ final class WebhooksTest extends TestCase ], $payload['recipients'][1]); } - public function testSendAlertUsesRootConsolePathsWhenConfigured(): void - { - $database = $this->createPlatformDatabase(); - $this->seedOwnerUser($database); - - $publisher = new MockPublisher(); - $publisherForNotifications = new NotificationPublisher($publisher, new Queue('v1-notifications')); - $worker = new Webhooks(); - - $this->withConsoleUrlScheme('root', fn () => $worker->sendAlert( - attempts: 10, - statusCode: 500, - webhook: new Document([ - '$id' => 'webhook-1', - '$updatedAt' => '2026-01-01T00:00:00.000+00:00', - 'name' => 'Payments', - 'url' => 'https://example.test/webhook', - ]), - project: new Document([ - '$id' => 'project-1', - '$sequence' => 'project-internal-1', - 'name' => 'Production', - 'teamInternalId' => 'team-internal-1', - 'region' => 'fra', - ]), - dbForPlatform: $database, - publisherForNotifications: $publisherForNotifications, - plan: [] - )); - - $events = $publisher->getEvents('v1-notifications'); - - $this->assertCount(1, $events); - $this->assertStringContainsString('/projects/project-1/settings/webhooks', (string) $events[0]['body']); - $this->assertStringNotContainsString('/console/', (string) $events[0]['body']); - } - public function testSendAlertPersonalizesBodyPerOwner(): void { $database = $this->createPlatformDatabase(); @@ -234,18 +197,6 @@ final class WebhooksTest extends TestCase $this->assertSame('webhook:webhook-1:paused:2026-01-02T00:00:00.000+00:00', $events[1]['deduplicationKey']); } - private function withConsoleUrlScheme(string $scheme, callable $callback): void - { - $original = \getenv('_APP_CONSOLE_URL_SCHEME'); - \putenv("_APP_CONSOLE_URL_SCHEME={$scheme}"); - - try { - $callback(); - } finally { - \putenv($original === false ? '_APP_CONSOLE_URL_SCHEME' : "_APP_CONSOLE_URL_SCHEME={$original}"); - } - } - private function createPlatformDatabase(): Database { $authorization = new Authorization(); From 514c9e4d255eed9a17dbe1f39fe0fdb01e1a4d61 Mon Sep 17 00:00:00 2001 From: harsh mahajan Date: Tue, 15 Sep 2026 13:04:26 +0530 Subject: [PATCH 20/37] fix(vcs): send redirect-less installation callbacks to project settings The console has no git-installations page; Git configuration lives on the project settings page. --- .../Modules/VCS/Http/Callback/Base.php | 5 ++- .../Modules/VCS/Http/GitHub/Callback/Get.php | 8 ++--- .../Modules/VCS/Http/Origin/Callback/Get.php | 8 ++--- .../VCSGitea/VCSGiteaConsoleClientTest.php | 33 ++++++++----------- 4 files changed, 19 insertions(+), 35 deletions(-) diff --git a/src/Appwrite/Platform/Modules/VCS/Http/Callback/Base.php b/src/Appwrite/Platform/Modules/VCS/Http/Callback/Base.php index 55f9ee1d2f..1b5d697c1d 100644 --- a/src/Appwrite/Platform/Modules/VCS/Http/Callback/Base.php +++ b/src/Appwrite/Platform/Modules/VCS/Http/Callback/Base.php @@ -101,13 +101,12 @@ abstract class Base extends Action return; } - $region = $project->getAttribute('region', 'default'); $protocol = System::getEnv('_APP_OPTIONS_FORCE_HTTPS') === 'disabled' ? 'http' : 'https'; $hostname = $platform['consoleHostname'] ?? ''; $defaultState = [ - 'success' => $protocol . '://' . $hostname . "/console/project-$region-$projectId/settings/git-installations", - 'failure' => $protocol . '://' . $hostname . "/console/project-$region-$projectId/settings/git-installations", + 'success' => ($platform['consoleUrl'] ?? '') . "/projects/$projectId/settings", + 'failure' => ($platform['consoleUrl'] ?? '') . "/projects/$projectId/settings", ]; $redirectSuccess = empty($state['success']) ? $defaultState['success'] : $state['success']; diff --git a/src/Appwrite/Platform/Modules/VCS/Http/GitHub/Callback/Get.php b/src/Appwrite/Platform/Modules/VCS/Http/GitHub/Callback/Get.php index eb53da9857..2b17642924 100644 --- a/src/Appwrite/Platform/Modules/VCS/Http/GitHub/Callback/Get.php +++ b/src/Appwrite/Platform/Modules/VCS/Http/GitHub/Callback/Get.php @@ -98,13 +98,9 @@ class Get extends Action throw new Exception(Exception::PROJECT_NOT_FOUND, $error); } - $region = $project->getAttribute('region', 'default'); - $protocol = System::getEnv('_APP_OPTIONS_FORCE_HTTPS') === 'disabled' ? 'http' : 'https'; - $hostname = $platform['consoleHostname'] ?? ''; - $defaultState = [ - 'success' => $protocol . '://' . $hostname . "/console/project-$region-$projectId/settings/git-installations", - 'failure' => $protocol . '://' . $hostname . "/console/project-$region-$projectId/settings/git-installations", + 'success' => ($platform['consoleUrl'] ?? '') . "/projects/$projectId/settings", + 'failure' => ($platform['consoleUrl'] ?? '') . "/projects/$projectId/settings", ]; $redirectSuccess = empty($state['success']) ? $defaultState['success'] : $state['success']; diff --git a/src/Appwrite/Platform/Modules/VCS/Http/Origin/Callback/Get.php b/src/Appwrite/Platform/Modules/VCS/Http/Origin/Callback/Get.php index 9488a34343..5ed3364133 100644 --- a/src/Appwrite/Platform/Modules/VCS/Http/Origin/Callback/Get.php +++ b/src/Appwrite/Platform/Modules/VCS/Http/Origin/Callback/Get.php @@ -104,13 +104,9 @@ class Get extends Action return; } - $region = $project->getAttribute('region', 'default'); - $protocol = System::getEnv('_APP_OPTIONS_FORCE_HTTPS') === 'disabled' ? 'http' : 'https'; - $hostname = $platform['consoleHostname'] ?? ''; - $defaultState = [ - 'success' => $protocol . '://' . $hostname . "/console/project-$region-$projectId/settings/git-installations", - 'failure' => $protocol . '://' . $hostname . "/console/project-$region-$projectId/settings/git-installations", + 'success' => ($platform['consoleUrl'] ?? '') . "/projects/$projectId/settings", + 'failure' => ($platform['consoleUrl'] ?? '') . "/projects/$projectId/settings", ]; $state = \array_merge($defaultState, \array_filter($state)); diff --git a/tests/e2e/Services/VCSGitea/VCSGiteaConsoleClientTest.php b/tests/e2e/Services/VCSGitea/VCSGiteaConsoleClientTest.php index baa2805529..8db0756915 100644 --- a/tests/e2e/Services/VCSGitea/VCSGiteaConsoleClientTest.php +++ b/tests/e2e/Services/VCSGitea/VCSGiteaConsoleClientTest.php @@ -238,7 +238,7 @@ final class VCSGiteaConsoleClientTest extends Scope /** @var array $cookies */ $cookies = []; $this->giteaCookies = $cookies; - $consoleUrl = $this->gitInstallationsUrl($projectId); + $consoleUrl = $this->settingsUrl($projectId); $authorize = $this->client->call(Client::METHOD_GET, '/vcs/gitea/authorize', \array_merge([ 'x-appwrite-project' => $projectId, @@ -490,7 +490,7 @@ final class VCSGiteaConsoleClientTest extends Scope public function testCreateInstallationWithTamperedState(): void { $projectId = $this->getProject()['$id']; - $consoleUrl = $this->gitInstallationsUrl($projectId); + $consoleUrl = $this->settingsUrl($projectId); $state = \json_decode($this->buildGiteaState($projectId, $consoleUrl, $consoleUrl), true); $state['projectId'] = 'victim-project'; @@ -506,7 +506,7 @@ final class VCSGiteaConsoleClientTest extends Scope public function testCreateInstallationWithoutCode(): void { $projectId = $this->getProject()['$id']; - $consoleUrl = $this->gitInstallationsUrl($projectId); + $consoleUrl = $this->settingsUrl($projectId); // Signed state, no code: the failure redirect carries the error as a query string $response = $this->callGiteaCallbackHelper([ @@ -529,7 +529,7 @@ final class VCSGiteaConsoleClientTest extends Scope ]); $this->assertEquals(301, $response['headers']['status-code']); - $this->assertStringStartsWith($this->gitInstallationsUrl($projectId) . '?error=', (string) $response['headers']['location']); + $this->assertStringStartsWith($this->settingsUrl($projectId) . '?error=', (string) $response['headers']['location']); } public function testCreateInstallationWithInvalidState(): void @@ -541,7 +541,7 @@ final class VCSGiteaConsoleClientTest extends Scope public function testCreateInstallationWithUnknownProject(): void { - $consoleUrl = $this->gitInstallationsUrl('missing'); + $consoleUrl = $this->settingsUrl('missing'); $response = $this->callGiteaCallbackHelper([ 'code' => 'unused', @@ -555,7 +555,7 @@ final class VCSGiteaConsoleClientTest extends Scope public function testCreateInstallationWithLongState(): void { $projectId = $this->getProject()['$id']; - $consoleUrl = $this->gitInstallationsUrl($projectId); + $consoleUrl = $this->settingsUrl($projectId); // Past the old 2048 cap: redirect URLs are not length-limited, so the // authorize endpoint can produce a state this size itself. @@ -579,16 +579,9 @@ final class VCSGiteaConsoleClientTest extends Scope $this->assertNotEmpty($installation['$id']); } - /** - * The callback builds its fallback redirect from the project's region, so the - * expected URL follows the region the scope's project was created in: Cloud - * CI creates projects in a region other than default. - */ - private function gitInstallationsUrl(string $projectId): string + private function settingsUrl(string $projectId): string { - $region = $this->getProject()['region']; - - return "http://localhost/console/project-{$region}-{$projectId}/settings/git-installations"; + return "http://localhost/projects/{$projectId}/settings"; } /** @@ -834,7 +827,7 @@ final class VCSGiteaConsoleClientTest extends Scope public function testCreateInstallationWithUnsignedState(): void { $projectId = $this->getProject()['$id']; - $consoleUrl = 'http://localhost/console/project-default-' . $projectId . '/settings/git-installations'; + $consoleUrl = 'http://localhost/projects/' . $projectId . '/settings'; $state = \json_decode($this->buildGiteaState($projectId, $consoleUrl, $consoleUrl), true); unset($state['signature']); @@ -847,7 +840,7 @@ final class VCSGiteaConsoleClientTest extends Scope public function testCreateInstallationWithTamperedRedirects(): void { $projectId = $this->getProject()['$id']; - $consoleUrl = 'http://localhost/console/project-default-' . $projectId . '/settings/git-installations'; + $consoleUrl = 'http://localhost/projects/' . $projectId . '/settings'; foreach (['success', 'failure'] as $field) { $state = \json_decode($this->buildGiteaState($projectId, $consoleUrl, $consoleUrl), true); @@ -862,7 +855,7 @@ final class VCSGiteaConsoleClientTest extends Scope public function testCreateInstallationWithReplayedSignature(): void { $projectId = $this->getProject()['$id']; - $consoleUrl = 'http://localhost/console/project-default-' . $projectId . '/settings/git-installations'; + $consoleUrl = 'http://localhost/projects/' . $projectId . '/settings'; $state = \json_decode($this->buildGiteaState($projectId, $consoleUrl, $consoleUrl), true); $state['signature'] = \json_decode($this->buildGiteaState('victim-project', $consoleUrl, $consoleUrl), true)['signature']; @@ -875,7 +868,7 @@ final class VCSGiteaConsoleClientTest extends Scope public function testCreateInstallationWithNonStringSignature(): void { $projectId = $this->getProject()['$id']; - $consoleUrl = 'http://localhost/console/project-default-' . $projectId . '/settings/git-installations'; + $consoleUrl = 'http://localhost/projects/' . $projectId . '/settings'; $state = \json_decode($this->buildGiteaState($projectId, $consoleUrl, $consoleUrl), true); $state['signature'] = 1234; @@ -890,7 +883,7 @@ final class VCSGiteaConsoleClientTest extends Scope public function testCreateInstallationWithOversizedRedirects(): void { $projectId = $this->getProject()['$id']; - $consoleUrl = 'http://localhost/console/project-default-' . $projectId . '/settings/git-installations'; + $consoleUrl = 'http://localhost/projects/' . $projectId . '/settings'; // Authorize must refuse rather than mint a state its own callback would reject. $authorize = $this->client->call(Client::METHOD_GET, '/vcs/gitea/authorize', \array_merge([ From 9fd34b73a7b76f8d1574f9ea69d48f4c09bf1029 Mon Sep 17 00:00:00 2001 From: harsh mahajan Date: Tue, 15 Sep 2026 13:04:26 +0530 Subject: [PATCH 21/37] fix(console): redirect legacy web paths to console routes /login, /register, /recover and /invite land on /sign-in, /sign-up, /reset and /join on the console URL, while /, /auth/* and /mfa keep their path. The /card redirect is removed since the console has no card page. Requests already on the console host are not redirected, so they cannot loop back through the proxy. --- .../Modules/Console/Http/Redirects/Base.php | 21 +++++++++++++++++-- .../Console/Http/Redirects/Card/Get.php | 18 ---------------- .../Console/Http/Redirects/Invite/Get.php | 5 +++++ .../Console/Http/Redirects/Login/Get.php | 5 +++++ .../Console/Http/Redirects/Recover/Get.php | 5 +++++ .../Console/Http/Redirects/Register/Get.php | 5 +++++ .../Modules/Console/Services/Http.php | 21 +++++++------------ tests/e2e/General/HTTPTest.php | 11 +++++----- 8 files changed, 52 insertions(+), 39 deletions(-) delete mode 100644 src/Appwrite/Platform/Modules/Console/Http/Redirects/Card/Get.php diff --git a/src/Appwrite/Platform/Modules/Console/Http/Redirects/Base.php b/src/Appwrite/Platform/Modules/Console/Http/Redirects/Base.php index 4b36319255..7c426731d4 100644 --- a/src/Appwrite/Platform/Modules/Console/Http/Redirects/Base.php +++ b/src/Appwrite/Platform/Modules/Console/Http/Redirects/Base.php @@ -2,6 +2,7 @@ namespace Appwrite\Platform\Modules\Console\Http\Redirects; +use Appwrite\Extend\Exception; use Appwrite\Utopia\Request; use Appwrite\Utopia\Response; use Utopia\Platform\Action; @@ -21,6 +22,14 @@ abstract class Base extends Action */ abstract protected function getPath(): string; + /** + * Console route to land on, the request path unless the console names it differently. + */ + protected function getTarget(string $path): string + { + return $path; + } + public function __construct() { $this @@ -31,13 +40,21 @@ abstract class Base extends Action ->label('scope', 'home') ->inject('request') ->inject('response') + ->inject('platform') ->callback($this->action(...)); } - public function action(Request $request, Response $response): void + public function action(Request $request, Response $response, array $platform): void { + $consoleUrl = $platform['consoleUrl'] ?? ''; + + // On the console's own host the proxy serves these paths, so redirecting would loop + if ($request->getHostname() === \parse_url($consoleUrl, PHP_URL_HOST)) { + throw new Exception(Exception::GENERAL_ROUTE_NOT_FOUND); + } + $url = parse_url($request->getURI()); - $target = "/console" . ($url['path'] ?? ''); + $target = $consoleUrl . $this->getTarget($url['path'] ?? ''); $params = $request->getParams(); if (!empty($params)) { $target .= "?" . \http_build_query($params); diff --git a/src/Appwrite/Platform/Modules/Console/Http/Redirects/Card/Get.php b/src/Appwrite/Platform/Modules/Console/Http/Redirects/Card/Get.php deleted file mode 100644 index c98c125f4e..0000000000 --- a/src/Appwrite/Platform/Modules/Console/Http/Redirects/Card/Get.php +++ /dev/null @@ -1,18 +0,0 @@ -addAction(CreateAssistantQuery::getName(), new CreateAssistantQuery()); $this->addAction(GetResourceAvailability::getName(), new GetResourceAvailability()); - // web redirects to /console, only where the console is served under /console - if (System::getEnv('_APP_CONSOLE_URL_SCHEME', 'legacy') !== 'root') { - $this->addAction(RedirectRoot::getName(), new RedirectRoot()); - $this->addAction(RedirectAuth::getName(), new RedirectAuth()); - $this->addAction(RedirectInvite::getName(), new RedirectInvite()); - $this->addAction(RedirectLogin::getName(), new RedirectLogin()); - $this->addAction(RedirectMFA::getName(), new RedirectMFA()); - $this->addAction(RedirectCard::getName(), new RedirectCard()); - $this->addAction(RedirectRecover::getName(), new RedirectRecover()); - $this->addAction(RedirectRegister::getName(), new RedirectRegister()); - } + // web redirects to the console + $this->addAction(RedirectRoot::getName(), new RedirectRoot()); + $this->addAction(RedirectAuth::getName(), new RedirectAuth()); + $this->addAction(RedirectInvite::getName(), new RedirectInvite()); + $this->addAction(RedirectLogin::getName(), new RedirectLogin()); + $this->addAction(RedirectMFA::getName(), new RedirectMFA()); + $this->addAction(RedirectRecover::getName(), new RedirectRecover()); + $this->addAction(RedirectRegister::getName(), new RedirectRegister()); } } diff --git a/tests/e2e/General/HTTPTest.php b/tests/e2e/General/HTTPTest.php index ecbd88f822..d9e9d2c9e5 100644 --- a/tests/e2e/General/HTTPTest.php +++ b/tests/e2e/General/HTTPTest.php @@ -9,7 +9,6 @@ use Tests\E2E\Scopes\ProjectNone; use Tests\E2E\Scopes\Scope; use Tests\E2E\Scopes\SideNone; use Utopia\Config\Config; -use Utopia\System\System; final class HTTPTest extends Scope { @@ -121,11 +120,11 @@ final class HTTPTest extends Scope public function testDefaultOAuth2() { - $response = $this->client->call(Client::METHOD_GET, '/console/auth/oauth2/success', $this->getHeaders()); + $response = $this->client->call(Client::METHOD_GET, '/auth/oauth2/success', $this->getHeaders()); $this->assertEquals(200, $response['headers']['status-code']); - $response = $this->client->call(Client::METHOD_GET, '/console/auth/oauth2/failure', $this->getHeaders()); + $response = $this->client->call(Client::METHOD_GET, '/auth/oauth2/failure', $this->getHeaders()); $this->assertEquals(200, $response['headers']['status-code']); } @@ -240,10 +239,10 @@ final class HTTPTest extends Scope $endpoint = '/invite?membershipId=123&userId=asdf'; - $response = $this->client->call(Client::METHOD_GET, $endpoint, [], [], true, false); + // Requests on the console's own host are left to the proxy, so arrive on the API host + $response = $this->client->call(Client::METHOD_GET, $endpoint, ['host' => 'appwrite.test'], [], true, false); - $location = System::getEnv('_APP_CONSOLE_URL_SCHEME', 'legacy') !== 'root' ? '/console' . $endpoint : null; - $this->assertEquals($location, $response['headers']['location'] ?? null); + $this->assertEquals('http://localhost/join?membershipId=123&userId=asdf', $response['headers']['location']); } public function testConsoleServed() From 87e0f9c6a282c903104aa1d0b69e2ea953c03d7d Mon Sep 17 00:00:00 2001 From: harsh mahajan Date: Tue, 15 Sep 2026 13:12:03 +0530 Subject: [PATCH 22/37] fix(console): allow redirects to the console host when _APP_CONSOLE_URL is set The default OAuth2 success and failure URLs are built on the console URL, and the callback validates them against the allowed hostnames. A console served on its own host was missing from that list, so the default redirects were rejected. --- app/init/resources/request.php | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/app/init/resources/request.php b/app/init/resources/request.php index 7f4d949e77..3453c7e5a6 100644 --- a/app/init/resources/request.php +++ b/app/init/resources/request.php @@ -251,6 +251,12 @@ return function (Container $context): void { $context->set('allowedHostnames', function (array $platform, Document $project, Document $rule, Document $devKey, Request $request) { $allowed = [...($platform['hostnames'] ?? [])]; + /* Add the console host, the default OAuth2 redirects land on it even when _APP_CONSOLE_URL points elsewhere */ + $consoleHostname = \parse_url($platform['consoleUrl'] ?? '', PHP_URL_HOST); + if (! empty($consoleHostname)) { + $allowed[] = $consoleHostname; + } + /* Add platform configured hostnames */ if (! $project->isEmpty() && $project->getId() !== 'console') { $platforms = $project->getAttribute('platforms', []); From 2c5aa827c2c5e51ad4be638cd6f1d9ffafd75b1f Mon Sep 17 00:00:00 2001 From: Levi van Noort <73097785+levivannoort@users.noreply.github.com> Date: Tue, 15 Sep 2026 09:47:48 +0200 Subject: [PATCH 23/37] refactor: move the release gates and metadata task to their own branch The install and upgrade gates, upgrade.sh and the release task are release process, not the nightly channel: nothing here imports them, and they were 80% of a diff meant to answer "should we publish nightly builds?". They are unchanged in #13692. What stays is the channel itself -- nightly.yml, --channel on install and upgrade, and the docs that describe them -- plus the *.x push trigger, which nightly needs so there are green CI runs on release branches to resolve. --- .github/scripts/upgrade.sh | 309 ------------------ .github/workflows/ci.yml | 238 -------------- AGENTS.md | 4 - bin/release | 3 - src/Appwrite/Platform/Services/Tasks.php | 2 - src/Appwrite/Platform/Tasks/Release.php | 265 --------------- .../unit/Migration/MigrationVersionsTest.php | 22 -- 7 files changed, 843 deletions(-) delete mode 100755 .github/scripts/upgrade.sh delete mode 100755 bin/release delete mode 100644 src/Appwrite/Platform/Tasks/Release.php diff --git a/.github/scripts/upgrade.sh b/.github/scripts/upgrade.sh deleted file mode 100755 index ca4d0977f9..0000000000 --- a/.github/scripts/upgrade.sh +++ /dev/null @@ -1,309 +0,0 @@ -#!/usr/bin/env bash -# The payload of the release gates: seed an install with the data shapes a -# migration is most likely to mangle, then assert they came back unchanged. -# -# ENDPOINT=http://localhost:8080/v1 upgrade.sh seed # on the previous stable -# ENDPOINT=http://localhost:8080/v1 upgrade.sh verify # on the target, after migrate -# ENDPOINT=http://localhost:8080/v1 upgrade.sh verify --write # ... and that writes still work -# -# verify is read-only by default so it can run after every migrate. --write also -# proves the upgraded install takes writes; pass it once, on the last run, since -# the rows it creates would show up in the next comparison. -# -# It talks to Appwrite the way a self-hoster does -- over the public HTTP API, -# with no access to container internals -- because that is the surface the data -# has to survive a version change on. - -set -euo pipefail - -: "${ENDPOINT:?ENDPOINT must be set, e.g. http://localhost:8080/v1}" - -MANIFEST="${MANIFEST:-upgrade-seed.json}" -STATE="$(dirname "$MANIFEST")" -COOKIE_JAR="${STATE}/upgrade-cookies.txt" -SNAPSHOT="${STATE}/upgrade-snapshot.json" - -# Seeded resources. Fixed IDs so seed and verify address the same rows without -# passing them around. -DATABASE="gate" -AUTHORS="authors" -POSTS="posts" -BUCKET="gate" -FILE_ID="gate-file" -USER_ID="gate-user" -AUTHOR="author-1" - -# The value shapes a migration is most likely to mangle. -LONG_VALUE="$(printf 'l%.0s' {1..4000})" -UNICODE_VALUE="ünïcødé — 日本語 — 🚂 — — O'Brien \"quoted\"" - -fail() { - echo "::error::$*" >&2 - exit 1 -} - -# /health/version answers before the platform schema is initialized, so wait for -# the console API to reject an anonymous read instead. -wait_for_appwrite() { - local i status - - for ((i = 1; i <= 120; i++)); do - status=$(curl --silent --output /dev/null --write-out '%{http_code}' --max-time 5 \ - -H 'x-appwrite-project: console' "${ENDPOINT}/account" || true) - - if [ "$status" = '401' ]; then - return 0 - fi - sleep 1 - done - - fail "Appwrite did not serve ${ENDPOINT}/account within 120s (last status: ${status:-none})." -} - -version() { - curl --silent --fail --max-time 10 "${ENDPOINT}/health/version" | jq -r '.version' -} - -# Console session, browser origin: how the organization and project get made. -console() { - local method="$1" path="$2" body="${3:-}" - - curl --silent --show-error --fail-with-body --max-time 30 \ - -X "$method" \ - -H 'content-type: application/json' \ - -H 'origin: http://localhost' \ - -H 'x-appwrite-project: console' \ - -c "$COOKIE_JAR" -b "$COOKIE_JAR" \ - ${body:+-d "$body"} \ - "${ENDPOINT}${path}" -} - -# API key: how a server SDK talks to a project. -api() { - local method="$1" path="$2" body="${3:-}" - - curl --silent --show-error --fail-with-body --max-time 60 \ - -X "$method" \ - -H 'content-type: application/json' \ - -H "x-appwrite-project: ${PROJECT:?PROJECT not set}" \ - -H "x-appwrite-key: ${KEY:?KEY not set}" \ - ${body:+-d "$body"} \ - "${ENDPOINT}${path}" -} - -# Columns are processed asynchronously; rows cannot be written until they are all -# available. -wait_for_columns() { - local table="$1" i statuses - - for ((i = 1; i <= 60; i++)); do - statuses=$(api GET "/tablesdb/${DATABASE}/tables/${table}/columns" \ - | jq -r '.columns[].status' | sort -u | tr '\n' ' ') - - case "$statuses" in - 'available ') return 0 ;; - *failed*) fail "Column processing failed on ${table}: ${statuses}" ;; - esac - sleep 1 - done - - fail "Columns on ${table} did not become available within 60s (last: ${statuses:-none})." -} - -# Everything the gate asserts on, projected to just the fields we seeded and -# sorted for a stable comparison. Projecting rather than dumping whole responses -# is deliberate: an upgrade is allowed to add fields to a model, and only the -# values a user put in have to come back unchanged. -snapshot() { - jq -nS \ - --argjson rows "$(api GET "/tablesdb/${DATABASE}/tables/${POSTS}/rows")" \ - --argjson table "$(api GET "/tablesdb/${DATABASE}/tables/${POSTS}")" \ - --argjson user "$(api GET "/users/${USER_ID}")" \ - --argjson file "$(api GET "/storage/buckets/${BUCKET}/files/${FILE_ID}")" \ - '{ - rows: [$rows.rows[] | { - id: .["$id"], - title: .title, - body: .body, - tag: .tag, - author: (if (.author | type) == "object" then .author["$id"] else .author end), - permissions: (.["$permissions"] | sort) - }] | sort_by(.id), - total: $rows.total, - table: { permissions: ($table["$permissions"] | sort), rowSecurity: $table.rowSecurity }, - user: { email: $user.email, name: $user.name }, - file: { id: $file["$id"], size: $file.sizeOriginal } - }' -} - -seed() { - wait_for_appwrite - echo "Seeding $(version) at ${ENDPOINT}" - - # --- console: root account, organization, project, key ------------------- - - rm -f "$COOKIE_JAR" - - # The first account on a fresh install becomes root. - console POST /account \ - '{"userId":"unique()","email":"gate@example.com","password":"upgrade-gate-1","name":"Gate"}' >/dev/null - console POST /account/sessions/email \ - '{"email":"gate@example.com","password":"upgrade-gate-1"}' >/dev/null - - TEAM=$(console POST /teams '{"teamId":"unique()","name":"Gate"}' | jq -r '.["$id"]') - PROJECT=$(console POST /projects "{\"projectId\":\"gate\",\"name\":\"Gate\",\"teamId\":\"${TEAM}\"}" | jq -r '.["$id"]') - export PROJECT - - KEY=$(console POST "/projects/${PROJECT}/keys" \ - '{"name":"gate","scopes":["users.read","users.write","tables.read","tables.write","rows.read","rows.write","files.read","files.write","buckets.read","buckets.write"]}' \ - | jq -r '.secret') - export KEY - - [ "$PROJECT" = 'gate' ] && [ -n "$KEY" ] && [ "$KEY" != 'null' ] || fail 'Could not create a project and key.' - - # --- tables -------------------------------------------------------------- - - echo '==> Tables' - api POST /tablesdb "{\"databaseId\":\"${DATABASE}\",\"name\":\"Gate\"}" >/dev/null - - api POST "/tablesdb/${DATABASE}/tables" "$(jq -nc --arg id "$AUTHORS" '{ - tableId: $id, name: "Authors", permissions: ["read(\"any\")"], rowSecurity: false, - columns: [{key: "name", type: "string", size: 255, required: true}] - }')" >/dev/null - - api POST "/tablesdb/${DATABASE}/tables" "$(jq -nc --arg id "$POSTS" '{ - tableId: $id, name: "Posts", permissions: ["read(\"any\")", "create(\"users\")"], rowSecurity: true, - columns: [ - {key: "title", type: "string", size: 255, required: true}, - {key: "body", type: "string", size: 5000, required: false}, - {key: "tag", type: "string", size: 64, required: false, default: "untagged"} - ] - }')" >/dev/null - - # Relationships are not expressible in the inline columns array. - api POST "/tablesdb/${DATABASE}/tables/${POSTS}/columns/relationship" "$(jq -nc --arg related "$AUTHORS" '{ - relatedTableId: $related, type: "manyToOne", twoWay: false, key: "author", onDelete: "setNull" - }')" >/dev/null - - wait_for_columns "$AUTHORS" - wait_for_columns "$POSTS" - - # --- rows ---------------------------------------------------------------- - - echo '==> Rows' - api POST "/tablesdb/${DATABASE}/tables/${AUTHORS}/rows" \ - "{\"rowId\":\"${AUTHOR}\",\"data\":{\"name\":\"Ada Lovelace\"}}" >/dev/null - - # An empty string, a relationship, and a column default left to apply. - api POST "/tablesdb/${DATABASE}/tables/${POSTS}/rows" "$(jq -nc --arg author "$AUTHOR" '{ - rowId: "post-empty", data: {title: "Empty body", body: "", author: $author} - }')" >/dev/null - - api POST "/tablesdb/${DATABASE}/tables/${POSTS}/rows" "$(jq -nc --arg body "$LONG_VALUE" '{ - rowId: "post-long", data: {title: "Long body", body: $body} - }')" >/dev/null - - api POST "/tablesdb/${DATABASE}/tables/${POSTS}/rows" "$(jq -nc --arg body "$UNICODE_VALUE" '{ - rowId: "post-unicode", data: {title: "Unicode body", body: $body} - }')" >/dev/null - - # Row permissions deliberately different from the table's. - api POST "/tablesdb/${DATABASE}/tables/${POSTS}/rows" '{ - "rowId": "post-perms", "data": {"title": "Mixed permissions", "body": "kept"}, - "permissions": ["read(\"any\")", "update(\"users\")"] - }' >/dev/null - - # --- user and file ------------------------------------------------------- - - echo '==> User and file' - api POST /users "$(jq -nc --arg id "$USER_ID" --arg name "$UNICODE_VALUE" '{ - userId: $id, email: "gate-user@example.com", password: "gate-user-1", name: $name - }')" >/dev/null - - api POST /storage/buckets "$(jq -nc --arg id "$BUCKET" '{ - bucketId: $id, name: "Gate", permissions: ["read(\"any\")"], fileSecurity: false - }')" >/dev/null - - printf '%s\n' "$UNICODE_VALUE" > "${STATE}/gate-file.txt" - curl --silent --show-error --fail-with-body --max-time 60 \ - -X POST \ - -H "x-appwrite-project: ${PROJECT}" \ - -H "x-appwrite-key: ${KEY}" \ - -F "fileId=${FILE_ID}" \ - -F "file=@${STATE}/gate-file.txt" \ - "${ENDPOINT}/storage/buckets/${BUCKET}/files" >/dev/null - - # --- record -------------------------------------------------------------- - - jq -nc --arg version "$(version)" --arg project "$PROJECT" --arg key "$KEY" \ - '{seededOn: $version, project: $project, key: $key}' > "$MANIFEST" - snapshot > "$SNAPSHOT" - - echo "Seeded from $(version); snapshot at ${SNAPSHOT}" -} - -verify() { - [ -f "$SNAPSHOT" ] || fail "No snapshot at ${SNAPSHOT}; did the seed run?" - - PROJECT=$(jq -r '.project' "$MANIFEST"); export PROJECT - KEY=$(jq -r '.key' "$MANIFEST"); export KEY - local seeded upgraded author written total - seeded=$(jq -r '.seededOn' "$MANIFEST") - - wait_for_appwrite - upgraded=$(version) - - echo "Verifying ${seeded} -> ${upgraded}" - - if [ "$seeded" = "$upgraded" ]; then - fail "Still reporting ${upgraded}; the image did not change, so this run proves nothing." - fi - - # The project, its key and its scopes have to survive before anything else - # can be checked. - api GET /users >/dev/null 2>&1 || fail "The seeded API key no longer authenticates against project ${PROJECT}." - - # --- nothing the user put in changed ------------------------------------- - - if ! diff --unified=1 --label "seeded on ${seeded}" --label "read back on ${upgraded}" \ - "$SNAPSHOT" <(snapshot); then - fail 'Seeded data changed across the upgrade; see the diff above.' - fi - - echo 'ok: rows, permissions, relationship, user and file all read back unchanged' - - if [ "${1:-}" != '--write' ]; then - return 0 - fi - - # --- the upgraded install still takes writes ----------------------------- - - # A read-only check passes against a half-migrated schema, so write too. - api POST "/tablesdb/${DATABASE}/tables/${POSTS}/rows" "$(jq -nc --arg body "$UNICODE_VALUE" '{ - rowId: "post-after-upgrade", data: {title: "Written after upgrade", body: $body} - }')" >/dev/null - - api PATCH "/users/${USER_ID}/name" '{"name":"Renamed after upgrade"}' >/dev/null - - author=$(api POST "/tablesdb/${DATABASE}/tables/${AUTHORS}/rows" \ - '{"rowId":"author-after-upgrade","data":{"name":"Grace Hopper"}}' | jq -r '.["$id"]') - api PATCH "/tablesdb/${DATABASE}/tables/${POSTS}/rows/post-after-upgrade" \ - "{\"data\":{\"author\":\"${author}\"}}" >/dev/null - - written=$(snapshot) - total=$(jq -r '.total' <<<"$written") - - [ "$total" = '5' ] || fail "Expected 5 rows after writing one; found ${total}." - [ "$(jq -r '.user.name' <<<"$written")" = 'Renamed after upgrade' ] || fail 'The user update did not apply.' - [ "$(jq -r '.rows[] | select(.id == "post-after-upgrade") | .author' <<<"$written")" = "$author" ] \ - || fail 'The relationship written after the upgrade did not stick.' - - echo 'ok: rows, users and relationships are still writable' - echo "All checks passed upgrading ${seeded} -> ${upgraded}." -} - -case "${1:-}" in - seed) seed ;; - verify) shift; verify "$@" ;; - *) fail "Usage: $(basename "$0") seed|verify [--write]" ;; -esac diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cb5e634309..a62847242b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -163,43 +163,6 @@ jobs: if: ${{ !cancelled() && steps.install.outcome == 'success' }} run: _APP_STORAGE_LIMIT=5368709120 php app/cli.php specs --version=latest --git=no - # The release metadata has to agree across files, and a partial bump is - # how a published version ends up disagreeing with the repo. Checked - # here rather than in tests/unit because the READMEs are not in the - # image the unit suite runs inside. - - name: Check release metadata - if: ${{ !cancelled() }} - run: | - set -euo pipefail - version=$(sed -n "s/^const APP_VERSION_STABLE = '\([^']*\)';.*/\1/p" app/init/constants.php) - echo "APP_VERSION_STABLE is ${version}" - - for readme in README.md README-CN.md; do - pinned=$(grep -oE 'appwrite/appwrite:[^ `"'"'"')]+' "$readme" | cut -d: -f2 | sort -u) - - if [ -z "$pinned" ]; then - echo "::error file=${readme}::No appwrite/appwrite install snippet found." - exit 1 - fi - - if [ "$pinned" != "$version" ]; then - echo "::error file=${readme}::Pins $(echo "$pinned" | tr '\n' ' ')but APP_VERSION_STABLE is ${version}." - exit 1 - fi - - echo " ok ${readme}" - done - - # A floating console tag would make an install unreproducible. - console=$(grep -oE 'image: appwrite/new:\S+' docker-compose.yml | cut -d: -f3) - - if ! echo "$console" | grep -qE '^[0-9]+\.[0-9]+\.[0-9]+(-[0-9A-Za-z.-]+)?$'; then - echo "::error file=docker-compose.yml::The console is pinned to '${console}', which is not an exact version." - exit 1 - fi - - echo " ok docker-compose.yml console pin (${console})" - locale: name: Checks / Locale runs-on: ubuntu-latest @@ -975,204 +938,3 @@ jobs: - name: Fail benchmark if: always() && steps.benchmark_after.outcome != 'success' run: exit 1 - - # The two release gates AGENTS.md requires. They run on release branches and on - # demand, not on every PR to main: a fresh install plus an upgrade from the - # previous stable is what a release is blocked on, and it is the work we used to - # pay a person to repeat by hand before every patch. - install: - name: Release / Fresh install - if: >- - github.event_name == 'workflow_dispatch' || - (github.event_name == 'push' && github.ref != 'refs/heads/main') || - (github.event_name == 'pull_request' && github.base_ref != 'main') - runs-on: runs-on=${{ github.run_id }}/runner=4cpu-linux-x64/volume=60g/spot=price-capacity-optimized/retry=when-interrupted/extras=otel+ecr-cache - timeout-minutes: 25 - needs: build - permissions: - contents: read - packages: read - env: - COMPOSE_FILE: docker-compose.yml - COMPOSE_PROJECT_NAME: install-gate - COMPOSE_PROFILES: '' - _APP_IMAGE: appwrite-dev - _APP_VERSION: latest - _APP_HTTP_PORT: '8080' - _APP_HTTPS_PORT: '8443' - ENDPOINT: http://localhost:8080/v1 - steps: - - name: Checkout repository - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - - name: Login to Docker Hub - if: ${{ env.DOCKERHUB_AUTH_AVAILABLE == 'true' }} - uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0 - with: - username: ${{ vars.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - - name: Pull the candidate image - run: | - docker pull ${{ env.RUNS_ON_ECR_CACHE }}:${{ env.IMAGE }}-${{ github.sha }} - docker tag ${{ env.RUNS_ON_ECR_CACHE }}:${{ env.IMAGE }}-${{ github.sha }} appwrite-dev:latest - - - name: Start from clean - timeout-minutes: 10 - run: docker compose up -d --quiet-pull --wait - - # A stack that boots can still be unusable, so drive the console and - # project APIs a self-hoster actually starts with. The script reports the - # version it reached before it seeds anything. - - name: Exercise the public API - timeout-minutes: 10 - env: - MANIFEST: ${{ runner.temp }}/install-gate.json - run: .github/scripts/upgrade.sh seed - - - name: Check for crash loops - if: always() - run: | - docker compose ps - # compose emits NDJSON on some versions and a JSON array on others; - # flatten handles both. - docker compose ps --format json \ - | jq -e -s 'flatten | map(select(.State != "running")) | length == 0' \ - || { echo "::error::Some containers are not running."; docker compose logs --tail=200; exit 1; } - - - name: Stop - if: always() - run: docker compose down -v || true - - upgrade: - name: Release / Upgrade from previous stable - if: >- - github.event_name == 'workflow_dispatch' || - (github.event_name == 'push' && github.ref != 'refs/heads/main') || - (github.event_name == 'pull_request' && github.base_ref != 'main') - runs-on: runs-on=${{ github.run_id }}/runner=4cpu-linux-x64/volume=60g/spot=price-capacity-optimized/retry=when-interrupted/extras=otel+ecr-cache - timeout-minutes: 45 - needs: build - permissions: - contents: read - packages: read - # Defaults describe the candidate. The two steps that run the previous - # release override them; everything else is the version being tested. - env: - COMPOSE_FILE: docker-compose.yml - COMPOSE_PROJECT_NAME: upgrade-gate - COMPOSE_PROFILES: '' - _APP_IMAGE: appwrite-dev - _APP_VERSION: latest - _APP_HTTP_PORT: '8080' - _APP_HTTPS_PORT: '8443' - ENDPOINT: http://localhost:8080/v1 - MANIFEST: ${{ github.workspace }}/upgrade-seed.json - steps: - - name: Checkout repository - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - fetch-depth: 0 - - - name: Login to Docker Hub - if: ${{ env.DOCKERHUB_AUTH_AVAILABLE == 'true' }} - uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0 - with: - username: ${{ vars.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - # Baseline = highest published stable below this branch's version. Docker - # Hub rather than git tags, because an image nobody can pull is not an - # upgrade anyone can perform. - - name: Resolve the previous stable - id: previous - run: | - set -euo pipefail - target=$(sed -n "s/^const APP_VERSION_STABLE = '\([^']*\)';.*/\1/p" app/init/constants.php) - - # Sort the released X.Y.Z tags together with the target, then take - # whatever lands immediately before it. Git rather than Docker Hub's - # tag listing: the nightly channel pushes a tag a day to the same - # repository, so the stables drop out of any recent-tags window. - previous=$( - git tag --list \ - | grep -E '^[0-9]+\.[0-9]+\.[0-9]+$' \ - | { cat; echo "${target}"; } \ - | sort -u -V \ - | grep -B1 -x -- "${target}" \ - | head -1 - ) - - if [ -z "$previous" ] || [ "$previous" = "$target" ]; then - echo "::error::No released stable below ${target}." - exit 1 - fi - - echo "Upgrading ${previous} -> ${target}" - echo "previous=${previous}" >> "$GITHUB_OUTPUT" - - # An upgrade starts from the compose file and env the old version shipped, - # which is what the installation being upgraded was actually started from. - - name: Materialise the previous release's compose - env: - PREVIOUS: ${{ steps.previous.outputs.previous }} - run: | - set -euo pipefail - git show "${PREVIOUS}:docker-compose.yml" > docker-compose.previous.yml - git show "${PREVIOUS}:.env" > .env.previous - - - name: Start the previous stable - timeout-minutes: 10 - env: - COMPOSE_FILE: docker-compose.previous.yml - _APP_IMAGE: appwrite/appwrite - _APP_VERSION: ${{ steps.previous.outputs.previous }} - run: docker compose --env-file .env.previous up -d --quiet-pull --wait - - - name: Seed through the public API - timeout-minutes: 10 - run: .github/scripts/upgrade.sh seed - - - name: Stop the previous stable, keeping volumes - env: - COMPOSE_FILE: docker-compose.previous.yml - _APP_IMAGE: appwrite/appwrite - _APP_VERSION: ${{ steps.previous.outputs.previous }} - run: docker compose --env-file .env.previous down - - - name: Pull the candidate image - run: | - docker pull ${{ env.RUNS_ON_ECR_CACHE }}:${{ env.IMAGE }}-${{ github.sha }} - docker tag ${{ env.RUNS_ON_ECR_CACHE }}:${{ env.IMAGE }}-${{ github.sha }} appwrite-dev:latest - - - name: Start the candidate on the same volumes - timeout-minutes: 10 - run: docker compose up -d --quiet-pull --wait - - - name: Migrate - timeout-minutes: 20 - run: docker compose exec -T appwrite migrate - - - name: Verify the seeded data survived - timeout-minutes: 10 - run: .github/scripts/upgrade.sh verify - - # A migration that is not idempotent breaks every retried or resumed - # upgrade. - - name: Migrate again - timeout-minutes: 20 - run: docker compose exec -T appwrite migrate - - - name: Verify again, and that writes still work - timeout-minutes: 10 - run: .github/scripts/upgrade.sh verify --write - - - name: Collect logs on failure - if: failure() - run: | - docker compose ps || true - docker compose logs --tail=400 appwrite || true - - - name: Stop - if: always() - run: docker compose down -v || true diff --git a/AGENTS.md b/AGENTS.md index 5789573bc9..0703ef9770 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -341,8 +341,6 @@ When bumping a patch (e.g. `1.9.0` → `1.9.1`): - [`README.md`](README.md) and [`README-CN.md`](README-CN.md) — `appwrite/appwrite:X.Y.Z` in all three install blocks each - [`src/Appwrite/Migration/Migration.php`](src/Appwrite/Migration/Migration.php) — add the version to `$versions`, mapping to a new migration class or the same class as the previous version -`docker compose exec appwrite release X.Y.Z` writes all four in one pass, and the `Check release metadata` step in [`ci.yml`](.github/workflows/ci.yml) fails when they disagree. Edit them by hand only when the task cannot express the change. - Ask the user to review, publish notes on the [Appwrite changelog](https://appwrite.io/changelog), generate specs if the API changed, and add request/response filters if needed. `APP_CACHE_BUSTER` is not a version number and does not track releases. It salts the response cache key in [`Request::cacheIdentifier()`](src/Appwrite/Utopia/Request.php) for the routes labelled `cache` (file preview, avatars). Bump it only when cached output would now be wrong — a changed image pipeline or new bundled avatar assets — since every bump orphans every entry and regenerates them. @@ -351,8 +349,6 @@ Ask the user to review, publish notes on the [Appwrite changelog](https://appwri A release is not ready until a **fresh install** and an **upgrade from the previous stable** both work with realistic data. Previous baseline = highest stable semver tag lower than the target (ignore RC/beta/alpha; prefer `git ls-remote --tags origin`). -Both gates run in CI on release branches — the `install` and `upgrade` jobs in [`ci.yml`](.github/workflows/ci.yml), seeding and asserting through the public API with [`upgrade.sh`](.github/scripts/upgrade.sh). Read those jobs before doing the work by hand; do it by hand when the release changes something they do not cover (a new service, a data shape they do not seed). - **Fresh install:** `docker compose down -v` then `up -d --force-recreate --build --wait`. Check `docker compose ps` / logs for crash loops, missing env, failed workers. Hit `/v1/health/version` on the public port. Run unit tests, `tests/e2e/General`, and service e2e. Exercise console users, projects, databases/rows, storage, and (when in scope) functions/sites through public APIs — not empty-stack health checks alone. **Upgrade:** install the previous stable image, seed broad data (empty values, long strings, relationships, mixed permissions), keep volumes, switch to the target image, run migrate. Migration must complete, be idempotent, and preserve seeded data through public API reads/writes. diff --git a/bin/release b/bin/release deleted file mode 100755 index 02b369262e..0000000000 --- a/bin/release +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh - -exec php /usr/src/code/app/cli.php release "$@" \ No newline at end of file diff --git a/src/Appwrite/Platform/Services/Tasks.php b/src/Appwrite/Platform/Services/Tasks.php index 2193fe8faa..8481b1e2ef 100644 --- a/src/Appwrite/Platform/Services/Tasks.php +++ b/src/Appwrite/Platform/Services/Tasks.php @@ -8,7 +8,6 @@ use Appwrite\Platform\Tasks\Interval; use Appwrite\Platform\Tasks\Maintenance; use Appwrite\Platform\Tasks\Migrate; use Appwrite\Platform\Tasks\QueueRetry; -use Appwrite\Platform\Tasks\Release; use Appwrite\Platform\Tasks\Schedule; use Appwrite\Platform\Tasks\ScheduleExecutions; use Appwrite\Platform\Tasks\ScheduleFunctions; @@ -35,7 +34,6 @@ class Tasks extends Service ->addAction(Maintenance::getName(), new Maintenance()) ->addAction(Migrate::getName(), new Migrate()) ->addAction(QueueRetry::getName(), new QueueRetry()) - ->addAction(Release::getName(), new Release()) ->addAction(SDKs::getName(), new SDKs()) ->addAction(SSL::getName(), new SSL()) ->addAction(Screenshot::getName(), new Screenshot()) diff --git a/src/Appwrite/Platform/Tasks/Release.php b/src/Appwrite/Platform/Tasks/Release.php deleted file mode 100644 index d6848af98c..0000000000 --- a/src/Appwrite/Platform/Tasks/Release.php +++ /dev/null @@ -1,265 +0,0 @@ -root = (string) \realpath(__DIR__ . '/../../../..'); - - $this - ->desc('Write the release metadata for a version') - ->param('version', '', new Text(32), 'Version being released, e.g. 2.0.2') - ->param('migration', '', new Text(length: 8, min: 0), 'Migration class for this version. Defaults to the previous version\'s class, which is what a patch must use.', true) - ->param('console', '', new Text(length: 32, min: 0), 'Console image tag to pin (appwrite/new). Left alone when omitted.', true) - ->param('dry-run', 'N', new WhiteList(['Y', 'N']), 'Print the changes without writing them', true) - ->callback($this->action(...)); - } - - public function action(string $version, string $migration, string $console, string $dryRun): void - { - if (!$this->isVersion($version)) { - Console::error("'{$version}' is not an X.Y.Z or X.Y.Z-rc.N version."); - - return; - } - - $current = APP_VERSION_STABLE; - - if (\version_compare($version, $current, '<=')) { - Console::error("{$version} is not newer than the current {$current}."); - - return; - } - - $previous = \array_key_last(Migration::$versions); - $migration = $migration !== '' ? $migration : (string) Migration::$versions[$previous]; - - if ($this->isPatch($current, $version) && $migration !== (string) Migration::$versions[$previous]) { - Console::error( - "{$version} is a patch of {$current}, so it must reuse {$previous}'s migration " - . "(" . Migration::$versions[$previous] . "), not {$migration}. A fix that needs a " - . "schema change is a minor -- see the Releases section of AGENTS.md." - ); - - return; - } - - if (!\class_exists('Appwrite\\Migration\\Version\\' . $migration)) { - Console::error("Migration class {$migration} does not exist."); - - return; - } - - $changes = [ - ...$this->stableVersion($current, $version), - ...$this->readmePins($current, $version), - ...$this->migrationVersion($version, $migration), - ...($console !== '' ? $this->consolePin($console) : []), - ]; - - if ($changes === []) { - Console::success("Everything already reads {$version}; nothing to write."); - - return; - } - - Console::log("Releasing {$current} -> {$version} (migration {$migration})\n"); - - foreach ($changes as $file => $change) { - Console::log(" {$file}"); - Console::log(" {$change['from']}"); - Console::log(" {$change['to']}\n"); - - if ($dryRun === 'N') { - \file_put_contents($this->path($file), $change['contents']); - } - } - - if ($dryRun === 'Y') { - Console::info('Dry run; nothing written.'); - - return; - } - - Console::success(\count($changes) . ' file(s) written.'); - Console::info('Still by hand: release notes on the changelog, specs if the API changed, and request/response filters for public breaks.'); - } - - /** - * X.Y.Z, optionally an -rc.N candidate of it. - */ - private function isVersion(string $version): bool - { - $parts = \explode('-rc.', $version, 2); - - if (\count($parts) === 2 && !\ctype_digit($parts[1])) { - return false; - } - - $numbers = \explode('.', $parts[0]); - - if (\count($numbers) !== 3) { - return false; - } - - foreach ($numbers as $number) { - if (!\ctype_digit($number)) { - return false; - } - } - - return true; - } - - private function isPatch(string $current, string $version): bool - { - [$currentMajor, $currentMinor] = \explode('.', $current); - [$major, $minor] = \explode('.', $version); - - return $currentMajor === $major && $currentMinor === $minor; - } - - private function path(string $file): string - { - return $this->root . '/' . $file; - } - - private function read(string $file): string - { - $contents = @\file_get_contents($this->path($file)); - - if ($contents === false) { - throw new \RuntimeException("Cannot read {$file} from {$this->root}."); - } - - return $contents; - } - - /** - * @return array - */ - private function stableVersion(string $current, string $version): array - { - $contents = $this->read(self::CONSTANTS); - $from = "const APP_VERSION_STABLE = '{$current}';"; - $to = "const APP_VERSION_STABLE = '{$version}';"; - - if (!\str_contains($contents, $from)) { - throw new \RuntimeException(self::CONSTANTS . " does not declare APP_VERSION_STABLE as '{$current}'."); - } - - return [self::CONSTANTS => [ - 'from' => "- {$from}", - 'to' => "+ {$to}", - 'contents' => \str_replace($from, $to, $contents), - ]]; - } - - /** - * @return array - */ - private function readmePins(string $current, string $version): array - { - $changes = []; - - foreach (self::READMES as $readme) { - $contents = $this->read($readme); - $count = 0; - $updated = \str_replace("appwrite/appwrite:{$current}", "appwrite/appwrite:{$version}", $contents, $count); - - if ($count === 0) { - throw new \RuntimeException("{$readme} does not pin appwrite/appwrite:{$current}."); - } - - $changes[$readme] = [ - 'from' => "- appwrite/appwrite:{$current} ({$count}x)", - 'to' => "+ appwrite/appwrite:{$version} ({$count}x)", - 'contents' => $updated, - ]; - } - - return $changes; - } - - /** - * @return array - */ - private function migrationVersion(string $version, string $migration): array - { - $contents = $this->read(self::MIGRATION); - - if (\str_contains($contents, "'{$version}' =>")) { - return []; - } - - $previous = \array_key_last(Migration::$versions); - $anchor = "'{$previous}' => '" . Migration::$versions[$previous] . "',"; - - if (!\str_contains($contents, $anchor)) { - throw new \RuntimeException(self::MIGRATION . " does not end its \$versions map with {$anchor}."); - } - - $entry = "'{$version}' => '{$migration}',"; - - return [self::MIGRATION => [ - 'from' => " {$anchor}", - 'to' => "+ {$entry}", - 'contents' => \str_replace($anchor, $anchor . "\n " . $entry, $contents), - ]]; - } - - /** - * @return array - */ - private function consolePin(string $console): array - { - $contents = $this->read(self::COMPOSE); - - $pin = 'image: appwrite/new:'; - $start = \strpos($contents, $pin); - - if ($start === false) { - throw new \RuntimeException(self::COMPOSE . ' does not pin an appwrite/new image.'); - } - - $start += \strlen($pin); - $current = \substr($contents, $start, \strcspn($contents, " \t\r\n", $start)); - - if ($current === $console) { - return []; - } - - return [self::COMPOSE => [ - 'from' => "- {$pin}{$current}", - 'to' => "+ {$pin}{$console}", - 'contents' => \str_replace($pin . $current, $pin . $console, $contents), - ]]; - } -} diff --git a/tests/unit/Migration/MigrationVersionsTest.php b/tests/unit/Migration/MigrationVersionsTest.php index 1e604d9902..09bc7a3588 100644 --- a/tests/unit/Migration/MigrationVersionsTest.php +++ b/tests/unit/Migration/MigrationVersionsTest.php @@ -35,28 +35,6 @@ final class MigrationVersionsTest extends TestCase } } - /** - * A mapping newer than APP_VERSION_STABLE means a version was given a - * migration but never marked stable, which is how 1.9.1 through 1.9.4 came - * to exist in this map with no release behind them. - */ - public function testNoVersionIsMappedAheadOfStable(): void - { - require_once __DIR__ . '/../../../app/init.php'; - - if (\str_contains(APP_VERSION_STABLE, 'RC') || \str_contains(APP_VERSION_STABLE, '-rc.')) { - $this->markTestSkipped('Release candidates are mapped when the final version is cut.'); - } - - foreach (\array_keys(Migration::$versions) as $mapped) { - $this->assertLessThanOrEqual( - 0, - \version_compare((string) $mapped, APP_VERSION_STABLE), - "Migration::\$versions maps {$mapped}, which is newer than APP_VERSION_STABLE " . APP_VERSION_STABLE . '.' - ); - } - } - public function testV24CreatesAlertsCollectionForConsoleProject(): void { require_once __DIR__ . '/../../../app/init.php'; From bf1b0bf4cd8bf2c0c191d77014a6f6f0f445b65c Mon Sep 17 00:00:00 2001 From: Levi van Noort <73097785+levivannoort@users.noreply.github.com> Date: Tue, 15 Sep 2026 10:34:22 +0200 Subject: [PATCH 24/37] fix(install): resolve the channel tag from the release, not the pulled tag A nightly image reports _APP_VERSION=X.Y-nightly., the tag it was pulled under. Both channels were derived from that value, so an install or upgrade run from inside a nightly image got the wrong tag either way: --channel nightly could not read a minor line out of it and fell back to the bare nightly tag, which jumps minors, and --channel stable wrote the dated nightly tag straight back, leaving the install on nightly. Take the release compiled into the image instead whenever the tag names a channel. Both channels already derive from it, so nightly pins to its own line and stable returns to the release the image was built from. --- src/Appwrite/Platform/Tasks/Install.php | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/Appwrite/Platform/Tasks/Install.php b/src/Appwrite/Platform/Tasks/Install.php index 2c9b5c0439..31bf9b50b1 100644 --- a/src/Appwrite/Platform/Tasks/Install.php +++ b/src/Appwrite/Platform/Tasks/Install.php @@ -568,7 +568,17 @@ class Install extends Action $database = $input['_APP_DB_ADAPTER'] ?? 'postgresql'; - $version = \getenv('_APP_VERSION') ?: (\defined('APP_VERSION_STABLE') ? APP_VERSION_STABLE : 'latest'); + $stableVersion = \defined('APP_VERSION_STABLE') ? APP_VERSION_STABLE : 'latest'; + $version = \getenv('_APP_VERSION') ?: $stableVersion; + + // A nightly image reports the tag it was pulled under -- 2.0-nightly. -- + // which names a channel rather than a release, and so cannot be carried forward: + // there is no X.Y to re-derive a nightly tag from, and writing it back would keep + // an install asking for the stable channel on nightly. The release the image was + // built from is compiled in, and both channels take their tag from it. + if (\str_contains($version, 'nightly')) { + $version = $stableVersion; + } // The nightly channel tracks the minor line rather than one release, so the // tag has to stay rolling -- pinning X.Y.Z would freeze the install on a From 7cc7063e4053e5c2cc2d4e5708de292f1521558c Mon Sep 17 00:00:00 2001 From: harsh mahajan Date: Tue, 15 Sep 2026 14:35:31 +0530 Subject: [PATCH 25/37] fix(console): build worker console links from platform consoleUrl The notifications and webhooks workers now take the console origin from the injected platform config instead of rebuilding it from _APP_CONSOLE_DOMAIN, so email links follow _APP_CONSOLE_URL and per-platform overrides. Rename the VCS comment QR variable since it points at the API host. --- app/config/platform.php | 1 + .../Platform/Workers/Notifications.php | 14 ++--- src/Appwrite/Platform/Workers/Webhooks.php | 20 +++--- src/Appwrite/Vcs/Comment.php | 6 +- .../Platform/Workers/NotificationsTest.php | 61 ++++++++++--------- tests/unit/Platform/Workers/WebhooksTest.php | 5 +- 6 files changed, 57 insertions(+), 50 deletions(-) diff --git a/app/config/platform.php b/app/config/platform.php index 7e4f99fdf2..d815cc2ad0 100644 --- a/app/config/platform.php +++ b/app/config/platform.php @@ -18,6 +18,7 @@ if (\str_contains($functionsDomain, ',')) { */ return [ 'apiHostname' => System::getEnv('_APP_DOMAIN', 'localhost'), + // Also serves the API (VCS OAuth callbacks, avatars, images); never point it at the console web origin 'consoleHostname' => System::getEnv('_APP_CONSOLE_DOMAIN', System::getEnv('_APP_DOMAIN', 'localhost')), // Console web app origin for links; it can live apart from the API host above 'consoleUrl' => \rtrim(System::getEnv('_APP_CONSOLE_URL', (System::getEnv('_APP_OPTIONS_FORCE_HTTPS') === 'disabled' ? 'http' : 'https') . '://' . System::getEnv('_APP_CONSOLE_DOMAIN', System::getEnv('_APP_DOMAIN', 'localhost'))), '/'), diff --git a/src/Appwrite/Platform/Workers/Notifications.php b/src/Appwrite/Platform/Workers/Notifications.php index caca7bb060..a5de91224b 100644 --- a/src/Appwrite/Platform/Workers/Notifications.php +++ b/src/Appwrite/Platform/Workers/Notifications.php @@ -52,10 +52,11 @@ class Notifications extends Action ->inject('project') ->inject('register') ->inject('dbForPlatform') + ->inject('platform') ->callback($this->action(...)); } - public function action(Message $message, Document $project, Registry $register, Database $dbForPlatform): void + public function action(Message $message, Document $project, Registry $register, Database $dbForPlatform, array $platform): void { $payload = $message->getPayload(); @@ -90,7 +91,7 @@ class Notifications extends Action } try { - $alertId = $this->dispatch($recipient, $messageId, $payload, $project, $register, $dbForPlatform); + $alertId = $this->dispatch($recipient, $messageId, $payload, $project, $register, $dbForPlatform, $platform); if ($messageId !== '' && $channel === NOTIFICATION_TYPE_WEBHOOK && $alertId === null) { $this->persistAlert($dbForPlatform, $messageId, $recipient, $payload, $project); } @@ -173,11 +174,12 @@ class Notifications extends Action Document $project, Registry $register, Database $dbForPlatform, + array $platform, ): ?string { $channel = $recipient['channel']; return match ($channel) { - NOTIFICATION_TYPE_EMAIL => $this->dispatchEmail($recipient, $messageId, $payload, $project, $register, $dbForPlatform), + NOTIFICATION_TYPE_EMAIL => $this->dispatchEmail($recipient, $messageId, $payload, $project, $register, $dbForPlatform, $platform), NOTIFICATION_TYPE_CONSOLE => $this->dispatchConsole($recipient, $messageId, $payload, $project, $dbForPlatform), NOTIFICATION_TYPE_WEBHOOK => $this->dispatchWebhook($recipient, $payload), default => throw new Exception('Unsupported notification channel: ' . $channel), @@ -194,6 +196,7 @@ class Notifications extends Action Document $project, Registry $register, Database $dbForPlatform, + array $platform, ): ?string { $address = $recipient['address']; $smtp = $this->resolveSmtpConfig($project, $payload); @@ -206,13 +209,10 @@ class Notifications extends Action $type = empty($smtp) ? 'cloud' : 'smtp'; Span::add('type', $type); - $protocol = System::getEnv('_APP_OPTIONS_FORCE_HTTPS', 'disabled') === 'disabled' ? 'http' : 'https'; - $consoleHostname = System::getEnv('_APP_CONSOLE_DOMAIN', System::getEnv('_APP_DOMAIN', 'localhost')); - $subject = $payload['subject'] ?? ''; $variables = $payload['variables'] ?? []; $variables = \array_merge($variables, $payload['templateParams'] ?? []); - $variables['host'] = $protocol . '://' . $consoleHostname; + $variables['host'] = $platform['consoleUrl'] ?? ''; $name = $payload['name'] ?? ''; $body = $payload['body'] ?? ''; $preview = $payload['preview'] ?? ''; diff --git a/src/Appwrite/Platform/Workers/Webhooks.php b/src/Appwrite/Platform/Workers/Webhooks.php index f4c58cac17..297529630c 100644 --- a/src/Appwrite/Platform/Workers/Webhooks.php +++ b/src/Appwrite/Platform/Workers/Webhooks.php @@ -45,6 +45,7 @@ class Webhooks extends Action ->inject('dbForPlatform') ->inject('publisherForNotifications') ->inject('publisherForUsage') + ->inject('platform') ->inject('plan') ->callback($this->action(...)); } @@ -55,11 +56,12 @@ class Webhooks extends Action * @param Database $dbForPlatform * @param NotificationPublisher $publisherForNotifications * @param UsagePublisher $publisherForUsage + * @param array $platform * @param array $plan * @return void * @throws Exception */ - public function action(Message $message, Document $project, Database $dbForPlatform, NotificationPublisher $publisherForNotifications, UsagePublisher $publisherForUsage, array $plan): void + public function action(Message $message, Document $project, Database $dbForPlatform, NotificationPublisher $publisherForNotifications, UsagePublisher $publisherForUsage, array $platform, array $plan): void { $payload = $message->getPayload(); @@ -76,7 +78,7 @@ class Webhooks extends Action $errors = []; foreach ($project->getAttribute('webhooks', []) as $webhook) { if (array_intersect($webhook->getAttribute('events', []), $events)) { - $error = $this->execute($events, $webhookPayload, $webhook, $user, $project, $dbForPlatform, $publisherForNotifications, $publisherForUsage, $plan); + $error = $this->execute($events, $webhookPayload, $webhook, $user, $project, $dbForPlatform, $publisherForNotifications, $publisherForUsage, $platform, $plan); if ($error !== null) { $errors[] = $error; } @@ -97,10 +99,11 @@ class Webhooks extends Action * @param Database $dbForPlatform * @param NotificationPublisher $publisherForNotifications * @param UsagePublisher $publisherForUsage + * @param array $platform * @param array $plan * @return string|null The error log if the delivery failed, otherwise null */ - private function execute(array $events, string $payload, Document $webhook, Document $user, Document $project, Database $dbForPlatform, NotificationPublisher $publisherForNotifications, UsagePublisher $publisherForUsage, array $plan): ?string + private function execute(array $events, string $payload, Document $webhook, Document $user, Document $project, Database $dbForPlatform, NotificationPublisher $publisherForNotifications, UsagePublisher $publisherForUsage, array $platform, array $plan): ?string { if ($webhook->getAttribute('enabled') !== true) { return null; @@ -196,7 +199,7 @@ class Webhooks extends Action if ($attempts >= \intval(System::getEnv('_APP_WEBHOOK_MAX_FAILED_ATTEMPTS', '10'))) { $webhook->setAttribute('enabled', false); $updatePayload['enabled'] = false; - $this->sendAlert($attempts, $statusCode, $webhook, $project, $dbForPlatform, $publisherForNotifications, $plan); + $this->sendAlert($attempts, $statusCode, $webhook, $project, $dbForPlatform, $publisherForNotifications, $platform, $plan); } $dbForPlatform->updateDocument('webhooks', $webhook->getId(), new Document($updatePayload)); @@ -237,10 +240,11 @@ class Webhooks extends Action * @param Document $project * @param Database $dbForPlatform * @param NotificationPublisher $publisherForNotifications + * @param array $platform * @param array $plan * @return void */ - public function sendAlert(int $attempts, mixed $statusCode, Document $webhook, Document $project, Database $dbForPlatform, NotificationPublisher $publisherForNotifications, array $plan): void + public function sendAlert(int $attempts, mixed $statusCode, Document $webhook, Document $project, Database $dbForPlatform, NotificationPublisher $publisherForNotifications, array $platform, array $plan): void { $memberships = $dbForPlatform->find('memberships', [ Query::equal('teamInternalId', [$project->getAttribute('teamInternalId')]), @@ -277,10 +281,6 @@ class Webhooks extends Action $projectId = $project->getId(); $projectInternalId = $project->getSequence(); - $protocol = System::getEnv('_APP_OPTIONS_FORCE_HTTPS', 'disabled') === 'disabled' ? 'http' : 'https'; - $consoleHostname = System::getEnv('_APP_CONSOLE_DOMAIN', System::getEnv('_APP_DOMAIN', 'localhost')); - $consoleUrl = \rtrim(System::getEnv('_APP_CONSOLE_URL', $protocol . '://' . $consoleHostname), '/'); - $subject = 'Webhook deliveries have been paused'; $preview = 'Webhook "' . $webhook->getAttribute('name') . '" has been paused after ' . $attempts . ' failed delivery attempts.'; @@ -319,7 +319,7 @@ class Webhooks extends Action $template->setParam('{{project}}', $project->getAttribute('name')); $template->setParam('{{url}}', $webhook->getAttribute('url')); $template->setParam('{{error}}', 'The server returned ' . $statusCode . ' status code'); - $template->setParam('{{host}}', $consoleUrl); + $template->setParam('{{host}}', $platform['consoleUrl'] ?? ''); $template->setParam('{{path}}', "/projects/{$projectId}/settings/webhooks"); $template->setParam('{{attempts}}', $attempts); diff --git a/src/Appwrite/Vcs/Comment.php b/src/Appwrite/Vcs/Comment.php index be36c92299..700f37b5e1 100644 --- a/src/Appwrite/Vcs/Comment.php +++ b/src/Appwrite/Vcs/Comment.php @@ -171,10 +171,10 @@ class Comment $qrImagePathLight = '/images/vcs/qr-light.svg'; $qrImagePathDark = '/images/vcs/qr-dark.svg'; - $consoleUrl = $protocol . '://' . $hostname . '/v1/avatars/qr?text=' . \urlencode($site['previewUrl']); + $qrUrl = $protocol . '://' . $hostname . '/v1/avatars/qr?text=' . \urlencode($site['previewUrl']); $qr = $this->withImages - ? '[' . $this->generatImage($qrImagePathLight, $qrImagePathDark, 'QR Code', 28) . '](' . $consoleUrl . ')' - : '[QR Code](' . $consoleUrl . ')'; + ? '[' . $this->generatImage($qrImagePathLight, $qrImagePathDark, 'QR Code', 28) . '](' . $qrUrl . ')' + : '[QR Code](' . $qrUrl . ')'; $preview = '[Preview URL](' . $site['previewUrl'] . ')'; diff --git a/tests/unit/Platform/Workers/NotificationsTest.php b/tests/unit/Platform/Workers/NotificationsTest.php index dc04a34bcd..93be857969 100644 --- a/tests/unit/Platform/Workers/NotificationsTest.php +++ b/tests/unit/Platform/Workers/NotificationsTest.php @@ -45,6 +45,7 @@ final class SpyNotifications extends Notifications Document $project, Registry $register, Database $dbForPlatform, + array $platform, ): ?string { $channel = $recipient['channel']; $this->dispatched[] = [ @@ -157,6 +158,7 @@ final class NotificationsTest extends TestCase private Authorization $authorization; private Registry $registry; private Document $project; + private array $platform = ['consoleUrl' => 'https://console.example.test']; private Span $span; protected function setUp(): void @@ -291,7 +293,7 @@ final class NotificationsTest extends TestCase 'deduplicationKey' => 'event-1', ]; - $worker->action($this->buildMessage($payload), $this->project, $this->registry, $this->database); + $worker->action($this->buildMessage($payload), $this->project, $this->registry, $this->database, $this->platform); $this->assertCount(3, $worker->dispatched); $channels = \array_map(static fn ($d) => $d['channel'], $worker->dispatched); @@ -313,7 +315,7 @@ final class NotificationsTest extends TestCase 'permissions' => [Permission::read(Role::any())], ]; - $worker->action($this->buildMessage($payload), $this->project, $this->registry, $this->database); + $worker->action($this->buildMessage($payload), $this->project, $this->registry, $this->database, $this->platform); $rows = $this->database->find('notifications'); $this->assertCount(2, $rows); @@ -345,11 +347,11 @@ final class NotificationsTest extends TestCase 'deduplicationKey' => 'dup-key', ]; - $worker->action($this->buildMessage($payload), $this->project, $this->registry, $this->database); + $worker->action($this->buildMessage($payload), $this->project, $this->registry, $this->database, $this->platform); $this->assertCount(1, $worker->dispatched); $worker->dispatched = []; - $worker->action($this->buildMessage($payload), $this->project, $this->registry, $this->database); + $worker->action($this->buildMessage($payload), $this->project, $this->registry, $this->database, $this->platform); $this->assertCount(0, $worker->dispatched, 'second invocation must short-circuit on dedup hit'); } @@ -363,7 +365,8 @@ final class NotificationsTest extends TestCase $this->buildMessage(['project' => ['$id' => 'project-x'], 'subject' => '', 'body' => '']), $this->project, $this->registry, - $this->database + $this->database, + $this->platform ); } @@ -377,7 +380,7 @@ final class NotificationsTest extends TestCase 'body' => 'Y', ]; - $worker->action($this->buildMessage($payload), $this->project, $this->registry, $this->database); + $worker->action($this->buildMessage($payload), $this->project, $this->registry, $this->database, $this->platform); $this->assertCount(1, $worker->dispatched); $this->assertSame('legacy@example.test', $worker->dispatched[0]['address']); @@ -409,7 +412,7 @@ final class NotificationsTest extends TestCase ], ]; - $worker->action($this->buildMessage($payload), $this->project, $this->registry, $this->database); + $worker->action($this->buildMessage($payload), $this->project, $this->registry, $this->database, $this->platform); } finally { \putenv($previousSmtpHost === false ? '_APP_SMTP_HOST' : '_APP_SMTP_HOST=' . $previousSmtpHost); } @@ -443,7 +446,7 @@ final class NotificationsTest extends TestCase $this->expectException(\Exception::class); $this->expectExceptionMessage('Skipped mail processing. No SMTP configuration has been set.'); - $worker->action($this->buildMessage($payload), $this->project, $this->registry, $this->database); + $worker->action($this->buildMessage($payload), $this->project, $this->registry, $this->database, $this->platform); } finally { \putenv($previousSmtpHost === false ? '_APP_SMTP_HOST' : '_APP_SMTP_HOST=' . $previousSmtpHost); } @@ -469,7 +472,7 @@ final class NotificationsTest extends TestCase 'body' => 'b', ]; - $worker->action($this->buildMessage($payload), $this->project, $this->registry, $this->database); + $worker->action($this->buildMessage($payload), $this->project, $this->registry, $this->database, $this->platform); $this->assertCount(2, $worker->dispatched); $this->assertSame('tenant-secret', $worker->dispatched[0]['signatureKey']); @@ -490,7 +493,7 @@ final class NotificationsTest extends TestCase ]; try { - $worker->action($this->buildMessage($payload), $this->project, $this->registry, $this->database); + $worker->action($this->buildMessage($payload), $this->project, $this->registry, $this->database, $this->platform); $this->fail('expected exception to propagate'); } catch (\Throwable $error) { $this->assertSame('boom', $error->getMessage()); @@ -521,7 +524,7 @@ final class NotificationsTest extends TestCase ]; try { - $worker->action($this->buildMessage($payload), $this->project, $this->registry, $this->database); + $worker->action($this->buildMessage($payload), $this->project, $this->registry, $this->database, $this->platform); $this->fail('expected webhook failure to propagate'); } catch (\RuntimeException $error) { $this->assertSame('webhook down', $error->getMessage()); @@ -534,7 +537,7 @@ final class NotificationsTest extends TestCase $this->assertSame(NOTIFICATION_TYPE_CONSOLE, $rows[0]->getAttribute('channel')); $retry = new SpyNotifications(); - $retry->action($this->buildMessage($payload), $this->project, $this->registry, $this->database); + $retry->action($this->buildMessage($payload), $this->project, $this->registry, $this->database, $this->platform); $this->assertCount(1, $retry->dispatched, 'retry should dispatch only the previously undelivered webhook'); $this->assertSame(NOTIFICATION_TYPE_WEBHOOK, $retry->dispatched[0]['channel']); @@ -559,7 +562,7 @@ final class NotificationsTest extends TestCase 'deduplicationKey' => 'console-skip', ]; - $worker->action($this->buildMessage($payload), $this->project, $this->registry, $this->database); + $worker->action($this->buildMessage($payload), $this->project, $this->registry, $this->database, $this->platform); // The Console adapter wrote exactly one alert; the action loop // must NOT have called persistAlert (otherwise we'd see 2 rows or @@ -586,7 +589,7 @@ final class NotificationsTest extends TestCase ]; try { - $worker->action($this->buildMessage($payload), $this->project, $this->registry, $this->database); + $worker->action($this->buildMessage($payload), $this->project, $this->registry, $this->database, $this->platform); $this->fail('expected console zero-delivery to throw'); } catch (\Throwable $error) { $this->assertStringContainsString('Console alert delivery failed', $error->getMessage()); @@ -610,7 +613,7 @@ final class NotificationsTest extends TestCase 'deduplicationKey' => 'fanout', ]; - $worker->action($this->buildMessage($payload), $this->project, $this->registry, $this->database); + $worker->action($this->buildMessage($payload), $this->project, $this->registry, $this->database, $this->platform); $rows = $this->database->find('notifications'); $this->assertCount(2, $rows, 'two recipients must produce two distinct alert rows'); @@ -647,7 +650,7 @@ final class NotificationsTest extends TestCase 'deduplicationKey' => 'roundtrip', ]; - $worker->action($this->buildMessage($payload), $this->project, $this->registry, $this->database); + $worker->action($this->buildMessage($payload), $this->project, $this->registry, $this->database, $this->platform); $rows = $this->database->find('notifications'); $this->assertCount(1, $rows); @@ -688,7 +691,7 @@ final class NotificationsTest extends TestCase 'deduplicationKey' => 'logo-key', ]; - $worker->action($this->buildMessage($payload), $this->project, $this->registry, $this->database); + $worker->action($this->buildMessage($payload), $this->project, $this->registry, $this->database, $this->platform); } finally { \putenv($previousSmtpHost === false ? '_APP_SMTP_HOST' : '_APP_SMTP_HOST=' . $previousSmtpHost); \putenv($previousTrackingSecret === false ? '_APP_NOTIFICATIONS_TRACKING_SECRET' : '_APP_NOTIFICATIONS_TRACKING_SECRET=' . $previousTrackingSecret); @@ -760,7 +763,7 @@ final class NotificationsTest extends TestCase 'deduplicationKey' => 'logo-key-no-tracking-secret', ]; - $worker->action($this->buildMessage($payload), $this->project, $this->registry, $this->database); + $worker->action($this->buildMessage($payload), $this->project, $this->registry, $this->database, $this->platform); } finally { \putenv($previousSmtpHost === false ? '_APP_SMTP_HOST' : '_APP_SMTP_HOST=' . $previousSmtpHost); \putenv($previousTrackingSecret === false ? '_APP_NOTIFICATIONS_TRACKING_SECRET' : '_APP_NOTIFICATIONS_TRACKING_SECRET=' . $previousTrackingSecret); @@ -792,7 +795,7 @@ final class NotificationsTest extends TestCase 'deduplicationKey' => 'persist-email', ]; - $worker->action($this->buildMessage($payload), $this->project, $this->registry, $this->database); + $worker->action($this->buildMessage($payload), $this->project, $this->registry, $this->database, $this->platform); } finally { \putenv($previousSmtpHost === false ? '_APP_SMTP_HOST' : '_APP_SMTP_HOST=' . $previousSmtpHost); } @@ -845,7 +848,7 @@ final class NotificationsTest extends TestCase $threw = false; try { - $worker->action($this->buildMessage($payload), $this->project, $this->registry, $this->database); + $worker->action($this->buildMessage($payload), $this->project, $this->registry, $this->database, $this->platform); } catch (\Throwable $error) { $threw = true; $this->assertStringContainsString('SMTP unavailable', $error->getMessage()); @@ -866,7 +869,7 @@ final class NotificationsTest extends TestCase $this->registry->set('smtp', static fn () => $working); $retryWorker = new Notifications(); - $retryWorker->action($this->buildMessage($payload), $this->project, $this->registry, $this->database); + $retryWorker->action($this->buildMessage($payload), $this->project, $this->registry, $this->database, $this->platform); $this->assertSame(1, $working->sendCount, 'retry must invoke the working adapter'); @@ -908,7 +911,7 @@ final class NotificationsTest extends TestCase $threw = false; try { - $worker->action($this->buildMessage($payload), $this->project, $this->registry, $this->database); + $worker->action($this->buildMessage($payload), $this->project, $this->registry, $this->database, $this->platform); } catch (\Throwable $error) { $threw = true; $this->assertStringContainsString('SMTP unavailable', $error->getMessage()); @@ -933,7 +936,7 @@ final class NotificationsTest extends TestCase $this->registry->set('smtp', static fn () => $working); $retryWorker = new Notifications(); - $retryWorker->action($this->buildMessage($payload), $this->project, $this->registry, $this->database); + $retryWorker->action($this->buildMessage($payload), $this->project, $this->registry, $this->database, $this->platform); $this->assertSame(1, $working->sendCount, 'retry must still deliver the email recipient'); $rows = $this->database->find('notifications', [ @@ -976,7 +979,7 @@ final class NotificationsTest extends TestCase 'deduplicationKey' => 'happy-email', ]; - $worker->action($this->buildMessage($payload), $this->project, $this->registry, $this->database); + $worker->action($this->buildMessage($payload), $this->project, $this->registry, $this->database, $this->platform); } finally { \putenv($previousSmtpHost === false ? '_APP_SMTP_HOST' : '_APP_SMTP_HOST=' . $previousSmtpHost); \putenv($previousTrackingSecret === false ? '_APP_NOTIFICATIONS_TRACKING_SECRET' : '_APP_NOTIFICATIONS_TRACKING_SECRET=' . $previousTrackingSecret); @@ -1045,7 +1048,7 @@ final class NotificationsTest extends TestCase 'deduplicationKey' => 'happy-console', ]; - $worker->action($this->buildMessage($payload), $this->project, $this->registry, $this->database); + $worker->action($this->buildMessage($payload), $this->project, $this->registry, $this->database, $this->platform); $rows = $this->database->find('notifications', [ Query::equal('channel', ['console']), @@ -1088,7 +1091,7 @@ final class NotificationsTest extends TestCase 'deduplicationKey' => 'console-preview', ]; - $worker->action($this->buildMessage($payload), $this->project, $this->registry, $this->database); + $worker->action($this->buildMessage($payload), $this->project, $this->registry, $this->database, $this->platform); $rows = $this->database->find('notifications'); $this->assertCount(1, $rows); @@ -1116,7 +1119,7 @@ final class NotificationsTest extends TestCase ]; try { - $worker->action($this->buildMessage($payload), $this->project, $this->registry, $this->database); + $worker->action($this->buildMessage($payload), $this->project, $this->registry, $this->database, $this->platform); } catch (\Exception $error) { $this->assertStringContainsString('No SMTP configuration has been set', $error->getMessage()); $threw = true; @@ -1148,7 +1151,7 @@ final class NotificationsTest extends TestCase $this->expectException(\Exception::class); $this->expectExceptionMessage('Invalid console alert resourceId'); - $worker->action($this->buildMessage($payload), $this->project, $this->registry, $this->database); + $worker->action($this->buildMessage($payload), $this->project, $this->registry, $this->database, $this->platform); } /** @@ -1220,7 +1223,7 @@ final class NotificationsTest extends TestCase 'deduplicationKey' => 'happy-webhook', ]; - $worker->action($this->buildMessage($payload), $this->project, $this->registry, $this->database); + $worker->action($this->buildMessage($payload), $this->project, $this->registry, $this->database, $this->platform); $this->assertCount(1, $worker->captured, 'adapter must POST exactly once'); $request = $worker->captured[0]; diff --git a/tests/unit/Platform/Workers/WebhooksTest.php b/tests/unit/Platform/Workers/WebhooksTest.php index f514e5404f..47e898d9af 100644 --- a/tests/unit/Platform/Workers/WebhooksTest.php +++ b/tests/unit/Platform/Workers/WebhooksTest.php @@ -49,6 +49,7 @@ final class WebhooksTest extends TestCase ]), dbForPlatform: $database, publisherForNotifications: $publisherForNotifications, + platform: ['consoleUrl' => 'https://console.example.test'], plan: [] ); @@ -68,7 +69,7 @@ final class WebhooksTest extends TestCase ); $this->assertStringContainsString('Payments', (string) $payload['body']); $this->assertStringContainsString('Ada Lovelace', (string) $payload['body']); - $this->assertStringContainsString('/projects/project-1/settings/webhooks', (string) $payload['body']); + $this->assertStringContainsString('https://console.example.test/projects/project-1/settings/webhooks', (string) $payload['body']); $this->assertStringNotContainsString('{{', (string) $payload['body']); $this->assertSame(APP_NAME, $payload['variables']['platform']); $this->assertSame(APP_EMAIL_LOGO_URL, $payload['variables']['logoUrl']); @@ -134,6 +135,7 @@ final class WebhooksTest extends TestCase ]), dbForPlatform: $database, publisherForNotifications: $publisherForNotifications, + platform: ['consoleUrl' => 'https://console.example.test'], plan: [] ); @@ -186,6 +188,7 @@ final class WebhooksTest extends TestCase project: $project, dbForPlatform: $database, publisherForNotifications: $publisherForNotifications, + platform: ['consoleUrl' => 'https://console.example.test'], plan: [] ); } From 195dfd7bf9943fa4db7652eb3a2329a78607798d Mon Sep 17 00:00:00 2001 From: Levi van Noort <73097785+levivannoort@users.noreply.github.com> Date: Tue, 15 Sep 2026 11:09:24 +0200 Subject: [PATCH 26/37] fix(install): only treat this channel's tags as nightly str_contains($version, 'nightly') claimed any tag with the word in it, so a self-hoster running their own customer-nightly-fix image would have had it replaced with APP_VERSION_STABLE and persisted to .env, quietly pulling a different image than the one they asked for. Match what nightly.yml actually publishes instead: nightly, X.Y-nightly and X.Y-nightly.. --- src/Appwrite/Platform/Tasks/Install.php | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/src/Appwrite/Platform/Tasks/Install.php b/src/Appwrite/Platform/Tasks/Install.php index 31bf9b50b1..fc21563e92 100644 --- a/src/Appwrite/Platform/Tasks/Install.php +++ b/src/Appwrite/Platform/Tasks/Install.php @@ -576,7 +576,7 @@ class Install extends Action // there is no X.Y to re-derive a nightly tag from, and writing it back would keep // an install asking for the stable channel on nightly. The release the image was // built from is compiled in, and both channels take their tag from it. - if (\str_contains($version, 'nightly')) { + if ($this->isNightlyTag($version)) { $version = $stableVersion; } @@ -848,6 +848,18 @@ class Install extends Action } } + /** + * The tags nightly.yml publishes: `nightly`, `X.Y-nightly` and + * `X.Y-nightly.`. A self-hoster's own tag that happens to mention the + * word is theirs, not this channel's, and is carried forward untouched. + */ + private function isNightlyTag(string $version): bool + { + return $version === 'nightly' + || \str_ends_with($version, '-nightly') + || \str_contains($version, '-nightly.'); + } + /** * The rolling nightly tag for the minor line a stable version belongs to, * e.g. 2.0.1 -> 2.0-nightly. From 4ed17e66024d6f56ca29490bf70d18a0709791ca Mon Sep 17 00:00:00 2001 From: harsh mahajan Date: Tue, 15 Sep 2026 14:53:04 +0530 Subject: [PATCH 27/37] fix(console): send /recover without a reset token to the recovery form The console reset page requires userId and secret, so /recover only lands there when both are present and otherwise opens /recovery. --- .../Platform/Modules/Console/Http/Redirects/Base.php | 4 ++-- .../Platform/Modules/Console/Http/Redirects/Invite/Get.php | 2 +- .../Platform/Modules/Console/Http/Redirects/Login/Get.php | 2 +- .../Modules/Console/Http/Redirects/Recover/Get.php | 7 ++++++- .../Modules/Console/Http/Redirects/Register/Get.php | 2 +- 5 files changed, 11 insertions(+), 6 deletions(-) diff --git a/src/Appwrite/Platform/Modules/Console/Http/Redirects/Base.php b/src/Appwrite/Platform/Modules/Console/Http/Redirects/Base.php index 7c426731d4..888bf711f8 100644 --- a/src/Appwrite/Platform/Modules/Console/Http/Redirects/Base.php +++ b/src/Appwrite/Platform/Modules/Console/Http/Redirects/Base.php @@ -25,7 +25,7 @@ abstract class Base extends Action /** * Console route to land on, the request path unless the console names it differently. */ - protected function getTarget(string $path): string + protected function getTarget(string $path, array $params): string { return $path; } @@ -54,8 +54,8 @@ abstract class Base extends Action } $url = parse_url($request->getURI()); - $target = $consoleUrl . $this->getTarget($url['path'] ?? ''); $params = $request->getParams(); + $target = $consoleUrl . $this->getTarget($url['path'] ?? '', $params); if (!empty($params)) { $target .= "?" . \http_build_query($params); } diff --git a/src/Appwrite/Platform/Modules/Console/Http/Redirects/Invite/Get.php b/src/Appwrite/Platform/Modules/Console/Http/Redirects/Invite/Get.php index 12e686113e..ef70ba5618 100644 --- a/src/Appwrite/Platform/Modules/Console/Http/Redirects/Invite/Get.php +++ b/src/Appwrite/Platform/Modules/Console/Http/Redirects/Invite/Get.php @@ -16,7 +16,7 @@ class Get extends Base return '/invite'; } - protected function getTarget(string $path): string + protected function getTarget(string $path, array $params): string { return '/join'; } diff --git a/src/Appwrite/Platform/Modules/Console/Http/Redirects/Login/Get.php b/src/Appwrite/Platform/Modules/Console/Http/Redirects/Login/Get.php index cf44ef7010..4d1bdce0f9 100644 --- a/src/Appwrite/Platform/Modules/Console/Http/Redirects/Login/Get.php +++ b/src/Appwrite/Platform/Modules/Console/Http/Redirects/Login/Get.php @@ -16,7 +16,7 @@ class Get extends Base return '/login'; } - protected function getTarget(string $path): string + protected function getTarget(string $path, array $params): string { return '/sign-in'; } diff --git a/src/Appwrite/Platform/Modules/Console/Http/Redirects/Recover/Get.php b/src/Appwrite/Platform/Modules/Console/Http/Redirects/Recover/Get.php index 9c29db9d34..38f8aec7ea 100644 --- a/src/Appwrite/Platform/Modules/Console/Http/Redirects/Recover/Get.php +++ b/src/Appwrite/Platform/Modules/Console/Http/Redirects/Recover/Get.php @@ -16,8 +16,13 @@ class Get extends Base return '/recover'; } - protected function getTarget(string $path): string + protected function getTarget(string $path, array $params): string { + // The reset page requires the emailed userId and secret, without them only the request form is usable + if (empty($params['userId']) || empty($params['secret'])) { + return '/recovery'; + } + return '/reset'; } } diff --git a/src/Appwrite/Platform/Modules/Console/Http/Redirects/Register/Get.php b/src/Appwrite/Platform/Modules/Console/Http/Redirects/Register/Get.php index 8a39daedd8..a48afeb1e9 100644 --- a/src/Appwrite/Platform/Modules/Console/Http/Redirects/Register/Get.php +++ b/src/Appwrite/Platform/Modules/Console/Http/Redirects/Register/Get.php @@ -16,7 +16,7 @@ class Get extends Base return '/register/*'; } - protected function getTarget(string $path): string + protected function getTarget(string $path, array $params): string { return '/sign-up'; } From 8788b6b6da9c31f56bb9c28e70eae317850bbb28 Mon Sep 17 00:00:00 2001 From: harsh mahajan Date: Tue, 15 Sep 2026 14:53:04 +0530 Subject: [PATCH 28/37] fix(account): hand OAuth2 session params only to the console's own relay The default success and failure relays were matched by path alone, so any allowed host serving /auth/oauth2/success received the session key and secret. Match the console host from platform consoleUrl as well. --- app/controllers/api/account.php | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/app/controllers/api/account.php b/app/controllers/api/account.php index 5c8d7ff94e..5fede6a7fd 100644 --- a/app/controllers/api/account.php +++ b/app/controllers/api/account.php @@ -1508,7 +1508,8 @@ Http::get('/v1/account/sessions/oauth2/:provider/redirect') ->inject('domainVerification') ->inject('cookieDomain') ->inject('authorization') - ->action(function (string $provider, string $code, string $state, string $error, string $error_description, Request $request, Response $response, Document $project, Validator $redirectValidator, Document $devKey, User $user, Database $dbForProject, Geo $geo, Database $dbForPlatform, Event $queueForEvents, Store $store, ProofsPassword $proofForPassword, ProofsToken $proofForToken, array $plan, bool $domainVerification, ?string $cookieDomain, Authorization $authorization) use ($oauthDefaultSuccess, $oauthDefaultFailure) { + ->inject('platform') + ->action(function (string $provider, string $code, string $state, string $error, string $error_description, Request $request, Response $response, Document $project, Validator $redirectValidator, Document $devKey, User $user, Database $dbForProject, Geo $geo, Database $dbForPlatform, Event $queueForEvents, Store $store, ProofsPassword $proofForPassword, ProofsToken $proofForToken, array $plan, bool $domainVerification, ?string $cookieDomain, Authorization $authorization, array $platform) use ($oauthDefaultSuccess, $oauthDefaultFailure) { $protocol = System::getEnv('_APP_OPTIONS_FORCE_HTTPS') === 'disabled' ? 'http' : 'https'; $port = $request->getPort(); $callbackBase = $protocol . '://' . $request->getHostname(); @@ -1576,12 +1577,15 @@ Http::get('/v1/account/sessions/oauth2/:provider/redirect') if ($devKey->isEmpty() && !empty($state['failure']) && !$redirectValidator->isValid($state['failure'])) { throw new Exception(Exception::PROJECT_INVALID_FAILURE_URL); } + // The default relays live on the console host; the same path on any other allowed host is a customer page + $consoleHostname = \parse_url($platform['consoleUrl'] ?? '', PHP_URL_HOST); + $failure = []; if (!empty($state['failure'])) { $failure = URLParser::parse($state['failure']); } - $failureRedirect = (function (string $type, ?string $message = null, ?int $code = null, ?\Throwable $previous = null, array $params = []) use ($failure, $response, $project, $oauthDefaultFailure) { + $failureRedirect = (function (string $type, ?string $message = null, ?int $code = null, ?\Throwable $previous = null, array $params = []) use ($failure, $response, $project, $oauthDefaultFailure, $consoleHostname) { $exception = new Exception($type, $message, $code, $previous, params: $params); if (!empty($failure)) { $query = URLParser::parseQuery($failure['query']); @@ -1592,7 +1596,7 @@ Http::get('/v1/account/sessions/oauth2/:provider/redirect') ]); // Mirror success path: default OAuth failure relay needs project to deep-link // back into the native app via appwrite-callback-{project}:// - if (($failure['path'] ?? '') === $oauthDefaultFailure) { + if ($failure['host'] === $consoleHostname && $failure['path'] === $oauthDefaultFailure) { $query['project'] = $project->getId(); } $failure['query'] = URLParser::unparseQuery($query); @@ -2175,7 +2179,7 @@ Http::get('/v1/account/sessions/oauth2/:provider/redirect') ; // TODO: Remove this deprecated workaround - support only token - if ($state['success']['path'] == $oauthDefaultSuccess) { + if ($state['success']['host'] === $consoleHostname && $state['success']['path'] === $oauthDefaultSuccess) { $query['project'] = $project->getId(); $query['domain'] = $cookieDomain; $query['key'] = $store->getKey(); From 264f03038d18003527166819bcd5d006f815b6e9 Mon Sep 17 00:00:00 2001 From: harsh mahajan Date: Tue, 15 Sep 2026 14:53:04 +0530 Subject: [PATCH 29/37] chore(variables): introduce _APP_CONSOLE_URL in 2.2.1 --- app/config/variables.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/config/variables.php b/app/config/variables.php index d67f07d406..2ee7fc690c 100644 --- a/app/config/variables.php +++ b/app/config/variables.php @@ -244,7 +244,7 @@ return [ [ 'name' => '_APP_CONSOLE_URL', 'description' => 'Origin of the Appwrite console web app, such as https://console.example.com, used when the backend generates links to the console (OAuth callbacks, emails, error page CTAs, VCS comments and commit statuses). Set it when the console is served on a different host than the API. When empty, links use _APP_CONSOLE_DOMAIN (or _APP_DOMAIN) over https, or over http when _APP_OPTIONS_FORCE_HTTPS is disabled.', - 'introduction' => '2.1.1', + 'introduction' => '2.2.1', 'default' => '', 'required' => false, 'question' => '', From 2c784df641a4c3a2104093aa8402d94340f631c8 Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Mon, 14 Sep 2026 13:57:43 +0100 Subject: [PATCH 30/37] feat(stats-resources): spread project counts across the interval with the scheduler Replace the hourly Console::loop that enqueued every active project at once with a Utopia\Schedule\Scheduler over a Stats source. Each project takes a slot derived from its id inside the interval, as scheduled functions do, so a region's resource counts spread across the hour instead of bursting at the top of it. Concurrency sampling keeps its cadence as a row of the same source. Incremental syncs read the accessedAt index every minute and a full snapshot runs once per interval, so platform DB load stays where it was. Project sub-queries are skipped in the source instead of disabled globally. Co-Authored-By: Claude Fable 5.1 --- app/config/variables.php | 2 +- src/Appwrite/Platform/Tasks/Schedule.php | 2 +- .../Platform/Tasks/StatsResources.php | 118 ++++++++++-------- src/Appwrite/Schedule/Source/Stats.php | 95 ++++++++++++++ tests/unit/Schedule/Source/StatsTest.php | 68 ++++++++++ 5 files changed, 228 insertions(+), 57 deletions(-) create mode 100644 src/Appwrite/Schedule/Source/Stats.php create mode 100644 tests/unit/Schedule/Source/StatsTest.php diff --git a/app/config/variables.php b/app/config/variables.php index 297a129fd7..4421d51ba0 100644 --- a/app/config/variables.php +++ b/app/config/variables.php @@ -1866,7 +1866,7 @@ return [ ], [ 'name' => '_APP_STATS_RESOURCES_INTERVAL', - 'description' => 'Interval in seconds between full resource-count snapshots.', + 'description' => 'Interval in seconds between resource-count snapshots. Each active project is counted once per interval, at a slot spread across it.', 'introduction' => '', 'default' => '3600', 'required' => false, diff --git a/src/Appwrite/Platform/Tasks/Schedule.php b/src/Appwrite/Platform/Tasks/Schedule.php index c3fc853f58..7ee431a6c3 100644 --- a/src/Appwrite/Platform/Tasks/Schedule.php +++ b/src/Appwrite/Platform/Tasks/Schedule.php @@ -95,7 +95,7 @@ class Schedule extends Action $this->loop(fn () => (new ScheduleMessages())->action($publisherForMessaging, $getIsResourceBlocked, $dbForPlatform, $getProjectDB, $telemetry, $locks)); if ($publisherForStatsResources !== null && $usageConnection !== null) { - $this->loop(fn () => (new StatsResources())->action($dbForPlatform, $publisherForStatsResources, $usageConnection)); + $this->loop(fn () => (new StatsResources())->action($dbForPlatform, $publisherForStatsResources, $usageConnection, $telemetry)); } while (true) { diff --git a/src/Appwrite/Platform/Tasks/StatsResources.php b/src/Appwrite/Platform/Tasks/StatsResources.php index 60315e862f..6c7947a080 100644 --- a/src/Appwrite/Platform/Tasks/StatsResources.php +++ b/src/Appwrite/Platform/Tasks/StatsResources.php @@ -4,15 +4,17 @@ namespace Appwrite\Platform\Tasks; use Appwrite\Event\Message\StatsResources as StatsResourcesMessage; use Appwrite\Event\Publisher\StatsResources as StatsResourcesPublisher; -use Appwrite\Platform\Action; +use Appwrite\Schedule\Source; use Appwrite\Usage\Concurrency; use Appwrite\Usage\Connection; use Utopia\Console; use Utopia\Database\Database; -use Utopia\Database\DateTime; -use Utopia\Database\Query; +use Utopia\Platform\Action; +use Utopia\Schedule\Occurrence; +use Utopia\Schedule\Scheduler; use Utopia\Span\Span; use Utopia\System\System; +use Utopia\Telemetry\Adapter as Telemetry; class StatsResources extends Action { @@ -32,77 +34,83 @@ class StatsResources extends Action ->inject('dbForPlatform') ->inject('publisherForStatsResources') ->inject('usageConnection') + ->inject('telemetry') ->callback($this->action(...)); } - public function action(Database $dbForPlatform, StatsResourcesPublisher $publisherForStatsResources, Connection $usageConnection): void + public function action(Database $dbForPlatform, StatsResourcesPublisher $publisherForStatsResources, Connection $usageConnection, Telemetry $telemetry): void { if (!$usageConnection->isEnabled()) { Console::info('Usage statistics are disabled'); return; } - $this->disableSubqueries(); - // Floor of 1 guards against a zero/negative interval spinning the loop; - // test stacks legitimately run short intervals (Cloud CI uses 2s). + // Floor of 1 guards against a zero/negative interval; test stacks + // legitimately run short intervals (Cloud CI uses 2s). $interval = max(1, (int) System::getEnv('_APP_STATS_RESOURCES_INTERVAL', 3600)); - Console::loop(function () use ($dbForPlatform, $publisherForStatsResources, $usageConnection): void { - Span::init('stats_resources_task'); - $projectsQueued = 0; - $projectsFailed = 0; + $source = new Source\Stats($dbForPlatform, $interval); + + $scheduler = new Scheduler( + source: $source, + syncSeconds: min($interval, 60), + snapshotSeconds: $interval, + telemetry: $telemetry, + onError: function (\Throwable $error): void { + Span::init('schedule.stats.reconcile'); + Span::current()?->finish(error: $error); + }, + ); + + $scheduler->run(fn (array $occurrences): null => $this->dispatch($occurrences, $publisherForStatsResources, $usageConnection)); + + Span::init('schedule.stats.stopped'); + Span::current()?->finish(error: new \RuntimeException('Scheduler loop returned')); + } + + /** + * Nothing here may throw: an exception ends the scheduler loop, and the + * process then stays alive and idle with nothing observing that + * scheduling has stopped. A failure costs one occurrence, not the loop. + * + * @param list $occurrences + */ + private function dispatch(array $occurrences, StatsResourcesPublisher $publisherForStatsResources, Connection $usageConnection): null + { + if (!$usageConnection->isReady()) { + Console::error('stats resources: usage schema is not ready, skipping ' . \count($occurrences) . ' occurrences'); + return null; + } + + $batch = \count($occurrences); + + foreach (\array_values($occurrences) as $index => $occurrence) { + Span::init('schedule.stats.enqueue'); + $error = null; - // Nothing here may end the loop. An exception escaping this closure - // ends Console::loop, and the process then stays alive and idle: it - // never exits, so restartPolicy never fires, and with no liveness - // probe nothing observes that scheduling has stopped. A single - // transient ClickHouse timeout on one tick silently ended gauge - // collection for a whole region, and the task went on reporting - // Ready for weeks while queueing nothing. try { - if (!$usageConnection->isReady()) { - Span::add('stats_resources_task.skipped', 'usage_schema_not_ready'); - Console::error('stats resources: usage schema is not ready, skipping cycle'); - return; - } + Span::add('occurrence.due', $occurrence->due->format('c')); + Span::add('occurrence.late', \round(\microtime(true) - (float) $occurrence->due->format('U.u'), 3)); + Span::add('occurrence.batch', $batch); + Span::add('occurrence.index', $index); - // Concurrency sampling reads the usage store; project scheduling - // reads the platform DB. They share no data, so a failure in the - // first must not cost the second -- that coupling is what turned - // one bad read into no scheduling at all. - try { + if ($occurrence->id === Source\Stats::CONCURRENCY) { $this->concurrency->sample($usageConnection->getUsage()); - } catch (\Throwable $th) { - Console::error('stats resources: concurrency sample failed, continuing: ' . $th->getMessage()); + continue; } - $last24Hours = (new \DateTime())->sub(new \DateInterval('P1D')); - $this->foreachDocument($dbForPlatform, 'projects', [ - Query::greaterThanEqual('accessedAt', DateTime::format($last24Hours)), - Query::equal('region', [System::getEnv('_APP_REGION', 'default')]), - Query::orderAsc('$sequence'), // accessedAt Can be updated during iteration - ], function ($project) use ($publisherForStatsResources, &$projectsQueued, &$projectsFailed): void { - // enqueue() swallows a publish failure, reporting it only to - // Console and returning false — as it does when stats are off. - if ($publisherForStatsResources->enqueue(new StatsResourcesMessage(project: $project)) === false) { - $projectsFailed++; - return; - } - - $projectsQueued++; - }); + Span::add('project.id', $occurrence->id); + if ($publisherForStatsResources->enqueue(new StatsResourcesMessage(project: $occurrence->payload)) === false) { + $error = new \RuntimeException('Failed to enqueue'); + } } catch (\Throwable $th) { - // Cost a cycle, not the process: the next tick retries in full. - Span::add('stats_resources_task.error', $th->getMessage()); - Console::error('stats resources: cycle failed, retrying next interval: ' . $th->getMessage()); + $error = $th; + Console::error('stats resources: ' . $occurrence->id . ' failed: ' . $th->getMessage()); } finally { - // An unfinished span exports nothing, including the early return. - Span::add('stats_resources_task.projects_queued', $projectsQueued); - Span::add('stats_resources_task.projects_failed', $projectsFailed); - Span::current()?->finish(); + Span::current()?->finish(error: $error); } - }, $interval, 0, function (\Throwable $th) { - Span::current()?->finish(error: $th); - }); + } + + return null; } } diff --git a/src/Appwrite/Schedule/Source/Stats.php b/src/Appwrite/Schedule/Source/Stats.php new file mode 100644 index 0000000000..a0befec9ec --- /dev/null +++ b/src/Appwrite/Schedule/Source/Stats.php @@ -0,0 +1,95 @@ +projects((new \DateTimeImmutable())->sub(new \DateInterval('P1D'))); + } + + #[\Override] + public function since(\DateTimeImmutable $moment): iterable + { + yield from $this->projects($moment); + } + + #[\Override] + public function make(Row $row): Entry + { + if ($row->id === self::CONCURRENCY) { + return new Entry(new Interval($this->interval)); + } + + // The worker reloads the project by id, so the payload carries no more. + return new Entry( + new Shifted(new Interval($this->interval), \abs(\crc32($row->id)) % $this->interval), + new Document(['$id' => $row->id]), + ); + } + + /** + * @return iterable + */ + private function projects(\DateTimeImmutable $accessedSince): iterable + { + $limit = 1000; + $sum = $limit; + $latest = null; + + while ($sum === $limit) { + $queries = [ + Query::limit($limit), + Query::equal('region', [System::getEnv('_APP_REGION', 'default')]), + Query::greaterThanEqual('accessedAt', DateTime::format(\DateTime::createFromImmutable($accessedSince))), + Query::orderAsc('$sequence'), // accessedAt can be updated during iteration + ]; + + if ($latest !== null) { + $queries[] = Query::cursorAfter($latest); + } + + $projects = $this->dbForPlatform->skipFilters( + fn () => $this->dbForPlatform->find('projects', $queries), + APP_PROJECTS_SUBQUERIES + ); + $sum = \count($projects); + + foreach ($projects as $project) { + yield new Row(id: $project->getId(), version: ''); + } + + $latest = \end($projects) ?: null; + } + } +} diff --git a/tests/unit/Schedule/Source/StatsTest.php b/tests/unit/Schedule/Source/StatsTest.php new file mode 100644 index 0000000000..8f18f706c8 --- /dev/null +++ b/tests/unit/Schedule/Source/StatsTest.php @@ -0,0 +1,68 @@ + $row->id, iterator_to_array($source->snapshot(), false)); + + $this->assertSame([Stats::CONCURRENCY, 'alpha', 'beta'], $ids); + } + + public function testSpreadsProjectsAcrossTheInterval(): void + { + $source = new Stats(new StatsDatabase([]), 3600); + $start = new \DateTimeImmutable('2026-09-14 10:00:00', new \DateTimeZone('UTC')); + $end = $start->modify('+3600 seconds'); + + $concurrency = $source->make(new Row(Stats::CONCURRENCY, '')); + $alpha = $source->make(new Row('alpha', '')); + $beta = $source->make(new Row('beta', '')); + + $this->assertNull($concurrency->payload); + $this->assertEquals([$start], $concurrency->trigger->occurrencesBetween($start, $end)); + + $this->assertInstanceOf(Document::class, $alpha->payload); + $this->assertSame('alpha', $alpha->payload->getId()); + $this->assertCount(1, $alpha->trigger->occurrencesBetween($start, $end)); + $this->assertCount(1, $beta->trigger->occurrencesBetween($start, $end)); + $this->assertNotEquals( + $alpha->trigger->occurrencesBetween($start, $end), + $beta->trigger->occurrencesBetween($start, $end), + ); + $this->assertEquals( + $alpha->trigger->occurrencesBetween($start, $end), + $source->make(new Row('alpha', ''))->trigger->occurrencesBetween($start, $end), + ); + } +} + +final class StatsDatabase extends Database +{ + /** @param list $ids */ + public function __construct(private readonly array $ids) + { + } + + public function skipFilters(callable $callback, ?array $filters = null): mixed + { + return $callback(); + } + + public function find(string $collection, array $queries = [], string $forPermission = Database::PERMISSION_READ): array + { + return array_map(fn (string $id): Document => new Document(['$id' => $id]), $this->ids); + } +} From f57ca361548984bac13431a13ee348263105b7e0 Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Mon, 14 Sep 2026 14:01:08 +0100 Subject: [PATCH 31/37] fix(stats-resources): skip every decode filter on the project listing Only the id is read, so no filter needs to run. Skipping them all rather than APP_PROJECTS_SUBQUERIES also covers subqueries Cloud registers on projects, which is the one thing Cloud overrode on this task. Co-Authored-By: Claude Fable 5.1 --- src/Appwrite/Schedule/Source/Stats.php | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Appwrite/Schedule/Source/Stats.php b/src/Appwrite/Schedule/Source/Stats.php index a0befec9ec..046149dc64 100644 --- a/src/Appwrite/Schedule/Source/Stats.php +++ b/src/Appwrite/Schedule/Source/Stats.php @@ -79,9 +79,10 @@ final class Stats implements Source, Changes $queries[] = Query::cursorAfter($latest); } + // Only the id is read, so no decode filter -- and no subquery, + // whichever edition registered it -- needs to run. $projects = $this->dbForPlatform->skipFilters( fn () => $this->dbForPlatform->find('projects', $queries), - APP_PROJECTS_SUBQUERIES ); $sum = \count($projects); From da5d3b82e393c1207696d4c818ba51168002df12 Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Mon, 14 Sep 2026 14:07:15 +0100 Subject: [PATCH 32/37] test(stats-resources): list projects through a query-aware database double The double now answers region, accessedAt, order, limit and cursor queries, so the tests prove stale and out-of-region projects are excluded, since() honours its cutoff, and paging covers every project. Co-Authored-By: Claude Fable 5.1 --- tests/unit/Schedule/Source/StatsTest.php | 102 +++++++++++++++++++++-- 1 file changed, 94 insertions(+), 8 deletions(-) diff --git a/tests/unit/Schedule/Source/StatsTest.php b/tests/unit/Schedule/Source/StatsTest.php index 8f18f706c8..4eb261c3fa 100644 --- a/tests/unit/Schedule/Source/StatsTest.php +++ b/tests/unit/Schedule/Source/StatsTest.php @@ -8,17 +8,51 @@ use Appwrite\Schedule\Source\Stats; use PHPUnit\Framework\TestCase; use Utopia\Database\Database; use Utopia\Database\Document; +use Utopia\Database\Query; use Utopia\Schedule\Source\Row; final class StatsTest extends TestCase { - public function testSnapshotListsConcurrencyAndProjects(): void + public function testSnapshotListsConcurrencyAndActiveProjectsOfTheRegion(): void { - $source = new Stats(new StatsDatabase(['alpha', 'beta']), 3600); + $database = new StatsDatabase([ + $this->project('recent', 'default', '-1 hour'), + $this->project('stale', 'default', '-2 days'), + $this->project('elsewhere', 'fra', '-1 hour'), + ]); - $ids = array_map(fn (Row $row): string => $row->id, iterator_to_array($source->snapshot(), false)); + $ids = $this->ids((new Stats($database, 3600))->snapshot()); - $this->assertSame([Stats::CONCURRENCY, 'alpha', 'beta'], $ids); + $this->assertContains(Stats::CONCURRENCY, $ids); + $this->assertContains('recent', $ids); + $this->assertNotContains('stale', $ids); + $this->assertNotContains('elsewhere', $ids); + } + + public function testSinceListsProjectsAccessedAfterTheMoment(): void + { + $database = new StatsDatabase([ + $this->project('before', 'default', '-10 minutes'), + $this->project('after', 'default', '-1 minute'), + ]); + + $ids = $this->ids((new Stats($database, 3600))->since(new \DateTimeImmutable('-5 minutes'))); + + $this->assertSame(['after'], $ids); + } + + public function testSnapshotPagesThroughEveryProject(): void + { + $projects = []; + for ($i = 0; $i < 2500; $i++) { + $projects[] = $this->project("project{$i}", 'default', '-1 hour'); + } + $database = new StatsDatabase($projects); + + $ids = $this->ids((new Stats($database, 3600))->snapshot()); + + $this->assertCount(2501, $ids); + $this->assertCount(2501, array_unique($ids)); } public function testSpreadsProjectsAcrossTheInterval(): void @@ -32,7 +66,7 @@ final class StatsTest extends TestCase $beta = $source->make(new Row('beta', '')); $this->assertNull($concurrency->payload); - $this->assertEquals([$start], $concurrency->trigger->occurrencesBetween($start, $end)); + $this->assertCount(1, $concurrency->trigger->occurrencesBetween($start, $end)); $this->assertInstanceOf(Document::class, $alpha->payload); $this->assertSame('alpha', $alpha->payload->getId()); @@ -47,13 +81,41 @@ final class StatsTest extends TestCase $source->make(new Row('alpha', ''))->trigger->occurrencesBetween($start, $end), ); } + + private function project(string $id, string $region, string $accessed): Document + { + return new Document([ + '$id' => $id, + 'region' => $region, + 'accessedAt' => (new \DateTime($accessed))->format('Y-m-d H:i:s.v'), + ]); + } + + /** + * @param iterable $rows + * @return list + */ + private function ids(iterable $rows): array + { + return array_map(fn (Row $row): string => $row->id, iterator_to_array($rows, false)); + } } +/** + * Answers project listings the way the platform database would: region and + * accessedAt filters, `$sequence` order, limit and cursor pagination. + */ final class StatsDatabase extends Database { - /** @param list $ids */ - public function __construct(private readonly array $ids) + /** @var list */ + private array $projects = []; + + /** @param list $projects */ + public function __construct(array $projects) { + foreach ($projects as $sequence => $project) { + $this->projects[] = new Document(array_merge($project->getArrayCopy(), ['$sequence' => (string) ($sequence + 1)])); + } } public function skipFilters(callable $callback, ?array $filters = null): mixed @@ -63,6 +125,30 @@ final class StatsDatabase extends Database public function find(string $collection, array $queries = [], string $forPermission = Database::PERMISSION_READ): array { - return array_map(fn (string $id): Document => new Document(['$id' => $id]), $this->ids); + $limit = PHP_INT_MAX; + $after = 0; + $projects = $this->projects; + + foreach ($queries as $query) { + switch ($query->getMethod()) { + case Query::TYPE_EQUAL: + $projects = array_filter($projects, fn (Document $p): bool => in_array($p->getAttribute($query->getAttribute()), $query->getValues(), true)); + break; + case Query::TYPE_GREATER_EQUAL: + $projects = array_filter($projects, fn (Document $p): bool => $p->getAttribute($query->getAttribute()) >= $query->getValue()); + break; + case Query::TYPE_LIMIT: + $limit = $query->getValue(); + break; + case Query::TYPE_CURSOR_AFTER: + $after = (int) $query->getValue()->getSequence(); + break; + } + } + + $projects = array_filter($projects, fn (Document $p): bool => (int) $p->getSequence() > $after); + usort($projects, fn (Document $a, Document $b): int => (int) $a->getSequence() <=> (int) $b->getSequence()); + + return array_slice($projects, 0, $limit); } } From 70dea1d780dfe2cc318be90fe90d91abddede4a2 Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Mon, 14 Sep 2026 14:10:31 +0100 Subject: [PATCH 33/37] fix(stats-resources): skip only the project subqueries, with a hook for more Disabling every filter also skipped the JSON decode of collection metadata, and the shared collection cache then handed HTTP a string where it expected attributes. Skip the named project subqueries as Source\Database does, and let an edition extend the list through a protected hook, as ScheduleFunctions does for spreadWindow. Co-Authored-By: Claude Fable 5.1 --- src/Appwrite/Platform/Tasks/StatsResources.php | 13 ++++++++++++- src/Appwrite/Schedule/Source/Stats.php | 7 +++++-- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/src/Appwrite/Platform/Tasks/StatsResources.php b/src/Appwrite/Platform/Tasks/StatsResources.php index 6c7947a080..8bc465cb7f 100644 --- a/src/Appwrite/Platform/Tasks/StatsResources.php +++ b/src/Appwrite/Platform/Tasks/StatsResources.php @@ -49,7 +49,7 @@ class StatsResources extends Action // legitimately run short intervals (Cloud CI uses 2s). $interval = max(1, (int) System::getEnv('_APP_STATS_RESOURCES_INTERVAL', 3600)); - $source = new Source\Stats($dbForPlatform, $interval); + $source = new Source\Stats($dbForPlatform, $interval, $this->subqueries()); $scheduler = new Scheduler( source: $source, @@ -68,6 +68,17 @@ class StatsResources extends Action Span::current()?->finish(error: new \RuntimeException('Scheduler loop returned')); } + /** + * Project decode filters the listing skips. Editions that register more + * subqueries on projects extend this. + * + * @return list + */ + protected function subqueries(): array + { + return APP_PROJECTS_SUBQUERIES; + } + /** * Nothing here may throw: an exception ends the scheduler loop, and the * process then stays alive and idle with nothing observing that diff --git a/src/Appwrite/Schedule/Source/Stats.php b/src/Appwrite/Schedule/Source/Stats.php index 046149dc64..d146d009ef 100644 --- a/src/Appwrite/Schedule/Source/Stats.php +++ b/src/Appwrite/Schedule/Source/Stats.php @@ -25,9 +25,13 @@ final class Stats implements Source, Changes { public const string CONCURRENCY = 'concurrency'; + /** + * @param list $subqueries project decode filters to skip; only the id is read + */ public function __construct( private readonly Database $dbForPlatform, private readonly int $interval, + private readonly array $subqueries = APP_PROJECTS_SUBQUERIES, ) { } @@ -79,10 +83,9 @@ final class Stats implements Source, Changes $queries[] = Query::cursorAfter($latest); } - // Only the id is read, so no decode filter -- and no subquery, - // whichever edition registered it -- needs to run. $projects = $this->dbForPlatform->skipFilters( fn () => $this->dbForPlatform->find('projects', $queries), + $this->subqueries ); $sum = \count($projects); From 24530f0e070022769b94ec3b19e0e009a85ccadf Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Mon, 14 Sep 2026 14:52:04 +0100 Subject: [PATCH 34/37] test(stats-resources): drop the source unit test Co-Authored-By: Claude Fable 5.1 --- tests/unit/Schedule/Source/StatsTest.php | 154 ----------------------- 1 file changed, 154 deletions(-) delete mode 100644 tests/unit/Schedule/Source/StatsTest.php diff --git a/tests/unit/Schedule/Source/StatsTest.php b/tests/unit/Schedule/Source/StatsTest.php deleted file mode 100644 index 4eb261c3fa..0000000000 --- a/tests/unit/Schedule/Source/StatsTest.php +++ /dev/null @@ -1,154 +0,0 @@ -project('recent', 'default', '-1 hour'), - $this->project('stale', 'default', '-2 days'), - $this->project('elsewhere', 'fra', '-1 hour'), - ]); - - $ids = $this->ids((new Stats($database, 3600))->snapshot()); - - $this->assertContains(Stats::CONCURRENCY, $ids); - $this->assertContains('recent', $ids); - $this->assertNotContains('stale', $ids); - $this->assertNotContains('elsewhere', $ids); - } - - public function testSinceListsProjectsAccessedAfterTheMoment(): void - { - $database = new StatsDatabase([ - $this->project('before', 'default', '-10 minutes'), - $this->project('after', 'default', '-1 minute'), - ]); - - $ids = $this->ids((new Stats($database, 3600))->since(new \DateTimeImmutable('-5 minutes'))); - - $this->assertSame(['after'], $ids); - } - - public function testSnapshotPagesThroughEveryProject(): void - { - $projects = []; - for ($i = 0; $i < 2500; $i++) { - $projects[] = $this->project("project{$i}", 'default', '-1 hour'); - } - $database = new StatsDatabase($projects); - - $ids = $this->ids((new Stats($database, 3600))->snapshot()); - - $this->assertCount(2501, $ids); - $this->assertCount(2501, array_unique($ids)); - } - - public function testSpreadsProjectsAcrossTheInterval(): void - { - $source = new Stats(new StatsDatabase([]), 3600); - $start = new \DateTimeImmutable('2026-09-14 10:00:00', new \DateTimeZone('UTC')); - $end = $start->modify('+3600 seconds'); - - $concurrency = $source->make(new Row(Stats::CONCURRENCY, '')); - $alpha = $source->make(new Row('alpha', '')); - $beta = $source->make(new Row('beta', '')); - - $this->assertNull($concurrency->payload); - $this->assertCount(1, $concurrency->trigger->occurrencesBetween($start, $end)); - - $this->assertInstanceOf(Document::class, $alpha->payload); - $this->assertSame('alpha', $alpha->payload->getId()); - $this->assertCount(1, $alpha->trigger->occurrencesBetween($start, $end)); - $this->assertCount(1, $beta->trigger->occurrencesBetween($start, $end)); - $this->assertNotEquals( - $alpha->trigger->occurrencesBetween($start, $end), - $beta->trigger->occurrencesBetween($start, $end), - ); - $this->assertEquals( - $alpha->trigger->occurrencesBetween($start, $end), - $source->make(new Row('alpha', ''))->trigger->occurrencesBetween($start, $end), - ); - } - - private function project(string $id, string $region, string $accessed): Document - { - return new Document([ - '$id' => $id, - 'region' => $region, - 'accessedAt' => (new \DateTime($accessed))->format('Y-m-d H:i:s.v'), - ]); - } - - /** - * @param iterable $rows - * @return list - */ - private function ids(iterable $rows): array - { - return array_map(fn (Row $row): string => $row->id, iterator_to_array($rows, false)); - } -} - -/** - * Answers project listings the way the platform database would: region and - * accessedAt filters, `$sequence` order, limit and cursor pagination. - */ -final class StatsDatabase extends Database -{ - /** @var list */ - private array $projects = []; - - /** @param list $projects */ - public function __construct(array $projects) - { - foreach ($projects as $sequence => $project) { - $this->projects[] = new Document(array_merge($project->getArrayCopy(), ['$sequence' => (string) ($sequence + 1)])); - } - } - - public function skipFilters(callable $callback, ?array $filters = null): mixed - { - return $callback(); - } - - public function find(string $collection, array $queries = [], string $forPermission = Database::PERMISSION_READ): array - { - $limit = PHP_INT_MAX; - $after = 0; - $projects = $this->projects; - - foreach ($queries as $query) { - switch ($query->getMethod()) { - case Query::TYPE_EQUAL: - $projects = array_filter($projects, fn (Document $p): bool => in_array($p->getAttribute($query->getAttribute()), $query->getValues(), true)); - break; - case Query::TYPE_GREATER_EQUAL: - $projects = array_filter($projects, fn (Document $p): bool => $p->getAttribute($query->getAttribute()) >= $query->getValue()); - break; - case Query::TYPE_LIMIT: - $limit = $query->getValue(); - break; - case Query::TYPE_CURSOR_AFTER: - $after = (int) $query->getValue()->getSequence(); - break; - } - } - - $projects = array_filter($projects, fn (Document $p): bool => (int) $p->getSequence() > $after); - usort($projects, fn (Document $a, Document $b): int => (int) $a->getSequence() <=> (int) $b->getSequence()); - - return array_slice($projects, 0, $limit); - } -} From 6ddb62badfaa675b5cbe9c36d0b7ad0277f9a049 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Tue, 15 Sep 2026 09:42:10 +0000 Subject: [PATCH 35/37] fix(executions): stop creating MySQL collection Remove the project-database executions schema now that ClickHouse is the only execution store. Keep execution and site-log query validation independent from the physical MySQL collection definition. Co-authored-by: Torsten Dittmann --- app/config/collections/projects.php | 271 ------------------ .../Database/Validator/Queries/Base.php | 22 +- .../Database/Validator/Queries/Executions.php | 14 +- .../Database/Validator/Queries/Logs.php | 2 +- tests/unit/General/CollectionsTest.php | 18 ++ 5 files changed, 44 insertions(+), 283 deletions(-) diff --git a/app/config/collections/projects.php b/app/config/collections/projects.php index 459772d10a..8d5c27060f 100644 --- a/app/config/collections/projects.php +++ b/app/config/collections/projects.php @@ -2012,277 +2012,6 @@ return [ ], ], - 'executions' => [ - '$collection' => ID::custom(Database::METADATA), - '$id' => ID::custom('executions'), - 'name' => 'Executions', - 'attributes' => [ - [ - '$id' => ID::custom('resourceInternalId'), - 'type' => Database::VAR_STRING, - 'format' => '', - 'size' => Database::LENGTH_KEY, - 'signed' => true, - 'required' => false, - 'default' => null, - 'array' => false, - 'filters' => [], - ], - [ - '$id' => ID::custom('resourceId'), - 'type' => Database::VAR_STRING, - 'format' => '', - 'size' => Database::LENGTH_KEY, - 'signed' => true, - 'required' => false, - 'default' => null, - 'array' => false, - 'filters' => [], - ], - [ - '$id' => ID::custom('resourceType'), - 'type' => Database::VAR_STRING, - 'format' => '', - 'size' => Database::LENGTH_KEY, - 'signed' => true, - 'required' => false, - 'default' => null, - 'array' => false, - 'filters' => [], - ], - [ - '$id' => ID::custom('deploymentInternalId'), - 'type' => Database::VAR_STRING, - 'format' => '', - 'size' => Database::LENGTH_KEY, - 'signed' => true, - 'required' => false, - 'default' => null, - 'array' => false, - 'filters' => [], - ], - [ - '$id' => ID::custom('deploymentId'), - 'type' => Database::VAR_STRING, - 'format' => '', - 'size' => Database::LENGTH_KEY, - 'signed' => true, - 'required' => false, - 'default' => null, - 'array' => false, - 'filters' => [], - ], - [ - 'array' => false, - '$id' => ID::custom('trigger'), - 'type' => Database::VAR_STRING, - 'format' => '', - 'size' => 128, - 'signed' => true, - 'required' => false, - 'default' => null, - 'filters' => [], - ], - [ - '$id' => ID::custom('status'), - 'type' => Database::VAR_STRING, - 'format' => '', - 'size' => 128, - 'signed' => true, - 'required' => false, - 'default' => null, - 'array' => false, - 'filters' => [], - ], - [ - '$id' => ID::custom('duration'), - 'type' => Database::VAR_FLOAT, - 'format' => '', - 'size' => 0, - 'signed' => true, - 'required' => false, - 'default' => null, - 'array' => false, - 'filters' => [], - ], - [ - '$id' => ID::custom('errors'), - 'type' => Database::VAR_STRING, - 'format' => '', - 'size' => APP_FUNCTION_ERROR_LENGTH_LIMIT, - 'signed' => true, - 'required' => false, - 'default' => null, - 'array' => false, - 'filters' => [], - ], - [ - '$id' => ID::custom('logs'), - 'type' => Database::VAR_STRING, - 'format' => '', - 'size' => APP_FUNCTION_LOG_LENGTH_LIMIT, - 'signed' => true, - 'required' => false, - 'default' => null, - 'array' => false, - 'filters' => [], - ], - [ - 'array' => false, - '$id' => ID::custom('requestMethod'), - 'type' => Database::VAR_STRING, - 'format' => '', - 'size' => 128, - 'signed' => true, - 'required' => false, - 'default' => null, - 'filters' => [], - ], - [ - 'array' => false, - '$id' => ID::custom('requestPath'), - 'type' => Database::VAR_STRING, - 'format' => '', - 'size' => 2048, - 'signed' => true, - 'required' => false, - 'default' => null, - 'filters' => [], - ], - [ - '$id' => ID::custom('requestHeaders'), - 'type' => Database::VAR_STRING, - 'format' => '', - 'size' => 16384, - 'signed' => true, - 'required' => false, - 'default' => null, - 'array' => false, - 'filters' => ['json'], - ], - [ - '$id' => ID::custom('responseStatusCode'), - 'type' => Database::VAR_INTEGER, - 'format' => '', - 'size' => 0, - 'signed' => true, - 'required' => false, - 'default' => null, - 'array' => false, - 'filters' => [], - ], - [ - '$id' => ID::custom('responseHeaders'), - 'type' => Database::VAR_STRING, - 'format' => '', - 'size' => 16384, - 'signed' => true, - 'required' => false, - 'default' => null, - 'array' => false, - 'filters' => ['json'], - ], - [ - '$id' => ID::custom('scheduledAt'), - 'type' => Database::VAR_DATETIME, - 'format' => '', - 'size' => 0, - 'signed' => false, - 'required' => false, - 'default' => null, - 'array' => false, - 'filters' => ['datetime'], - ], - [ - '$id' => ID::custom('scheduleInternalId'), - 'type' => Database::VAR_STRING, - 'format' => '', - 'size' => Database::LENGTH_KEY, - 'signed' => true, - 'required' => false, - 'default' => null, - 'array' => false, - 'filters' => [], - ], - [ - '$id' => ID::custom('scheduleId'), - 'type' => Database::VAR_STRING, - 'format' => '', - 'size' => Database::LENGTH_KEY, - 'signed' => true, - 'required' => false, - 'default' => null, - 'array' => false, - 'filters' => [], - ], - ], - 'indexes' => [ - [ - '$id' => ID::custom('_key_resource'), - 'type' => Database::INDEX_KEY, - 'attributes' => ['resourceInternalId', 'resourceType', 'resourceId'], - 'lengths' => [Database::LENGTH_KEY], - 'orders' => [Database::ORDER_ASC], - ], - [ - '$id' => ID::custom('_key_trigger'), - 'type' => Database::INDEX_KEY, - 'attributes' => ['trigger'], - 'lengths' => [32], - 'orders' => [Database::ORDER_ASC], - ], - [ - '$id' => ID::custom('_key_status'), - 'type' => Database::INDEX_KEY, - 'attributes' => ['status'], - 'lengths' => [32], - 'orders' => [Database::ORDER_ASC], - ], - [ - '$id' => ID::custom('_key_requestMethod'), - 'type' => Database::INDEX_KEY, - 'attributes' => ['requestMethod'], - 'lengths' => [128], - 'orders' => [Database::ORDER_ASC], - ], - [ - '$id' => ID::custom('_key_requestPath'), - 'type' => Database::INDEX_KEY, - 'attributes' => ['requestPath'], - 'lengths' => [Database::LENGTH_KEY], - 'orders' => [Database::ORDER_ASC], - ], - [ - '$id' => ID::custom('_key_deployment'), - 'type' => Database::INDEX_KEY, - 'attributes' => ['deploymentId'], - 'lengths' => [Database::LENGTH_KEY], - 'orders' => [Database::ORDER_ASC], - ], - [ - '$id' => ID::custom('_key_responseStatusCode'), - 'type' => Database::INDEX_KEY, - 'attributes' => ['responseStatusCode'], - 'lengths' => [], - 'orders' => [Database::ORDER_ASC], - ], - [ - '$id' => ID::custom('_key_duration'), - 'type' => Database::INDEX_KEY, - 'attributes' => ['duration'], - 'lengths' => [], - 'orders' => [Database::ORDER_ASC], - ], - [ - '$id' => ID::custom('_key_resourceType'), - 'type' => Database::INDEX_KEY, - 'attributes' => ['resourceType'], - 'lengths' => [], - 'orders' => [Database::ORDER_ASC], - ], - ], - ], - 'variables' => [ '$collection' => Database::METADATA, '$id' => 'variables', diff --git a/src/Appwrite/Utopia/Database/Validator/Queries/Base.php b/src/Appwrite/Utopia/Database/Validator/Queries/Base.php index 688a67a20d..056eb467b4 100644 --- a/src/Appwrite/Utopia/Database/Validator/Queries/Base.php +++ b/src/Appwrite/Utopia/Database/Validator/Queries/Base.php @@ -18,22 +18,24 @@ class Base extends Queries /** * Expression constructor * - * @param string $collection + * @param string|array $collection * @param string[] $allowedAttributes * @throws \Exception */ - public function __construct(string $collection, array $allowedAttributes) + public function __construct(string|array $collection, array $allowedAttributes) { - $config = Config::getParam('collections', []); + if (\is_string($collection)) { + $config = Config::getParam('collections', []); - $collections = \array_merge( - $config['projects'], - $config['buckets'], - $config['databases'], - $config['console'] - ); + $collections = \array_merge( + $config['projects'], + $config['buckets'], + $config['databases'], + $config['console'] + ); - $collection = $collections[$collection]; + $collection = $collections[$collection]; + } $allowedAttributesLookup = []; foreach ($allowedAttributes as $attribute) { diff --git a/src/Appwrite/Utopia/Database/Validator/Queries/Executions.php b/src/Appwrite/Utopia/Database/Validator/Queries/Executions.php index 4c0a38b46f..92b915daf8 100644 --- a/src/Appwrite/Utopia/Database/Validator/Queries/Executions.php +++ b/src/Appwrite/Utopia/Database/Validator/Queries/Executions.php @@ -2,8 +2,20 @@ namespace Appwrite\Utopia\Database\Validator\Queries; +use Utopia\Database\Database; + class Executions extends Base { + public const ATTRIBUTES = [ + ['$id' => 'trigger', 'type' => Database::VAR_STRING, 'array' => false], + ['$id' => 'status', 'type' => Database::VAR_STRING, 'array' => false], + ['$id' => 'responseStatusCode', 'type' => Database::VAR_INTEGER, 'array' => false], + ['$id' => 'duration', 'type' => Database::VAR_FLOAT, 'array' => false], + ['$id' => 'requestMethod', 'type' => Database::VAR_STRING, 'array' => false], + ['$id' => 'requestPath', 'type' => Database::VAR_STRING, 'array' => false], + ['$id' => 'deploymentId', 'type' => Database::VAR_STRING, 'array' => false], + ]; + public const ALLOWED_ATTRIBUTES = [ 'trigger', 'status', @@ -20,6 +32,6 @@ class Executions extends Base */ public function __construct() { - parent::__construct('executions', self::ALLOWED_ATTRIBUTES); + parent::__construct(['attributes' => self::ATTRIBUTES], self::ALLOWED_ATTRIBUTES); } } diff --git a/src/Appwrite/Utopia/Database/Validator/Queries/Logs.php b/src/Appwrite/Utopia/Database/Validator/Queries/Logs.php index d07d3dd3d2..ab1982c29f 100644 --- a/src/Appwrite/Utopia/Database/Validator/Queries/Logs.php +++ b/src/Appwrite/Utopia/Database/Validator/Queries/Logs.php @@ -19,6 +19,6 @@ class Logs extends Base */ public function __construct() { - parent::__construct('executions', self::ALLOWED_ATTRIBUTES); //TODO: Update this later + parent::__construct(['attributes' => Executions::ATTRIBUTES], self::ALLOWED_ATTRIBUTES); } } diff --git a/tests/unit/General/CollectionsTest.php b/tests/unit/General/CollectionsTest.php index d90905abda..1a125656f3 100644 --- a/tests/unit/General/CollectionsTest.php +++ b/tests/unit/General/CollectionsTest.php @@ -4,7 +4,10 @@ declare(strict_types=1); namespace Tests\Unit\General; +use Appwrite\Utopia\Database\Validator\Queries\Executions; +use Appwrite\Utopia\Database\Validator\Queries\Logs; use PHPUnit\Framework\TestCase; +use Utopia\Database\Query; final class CollectionsTest extends TestCase { @@ -33,4 +36,19 @@ final class CollectionsTest extends TestCase } } } + + public function testProjectsDoNotDefineExecutionsCollection(): void + { + $this->assertArrayNotHasKey('executions', $this->collections['projects']); + } + + public function testExecutionQueryValidatorsDoNotNeedCollectionSchema(): void + { + $this->assertTrue((new Executions())->isValid([ + Query::equal('status', ['completed']), + ])); + $this->assertTrue((new Logs())->isValid([ + Query::equal('status', ['completed']), + ])); + } } From 484b8b5aa28f4593e4a72bb8b47f2ec7545b1462 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Tue, 15 Sep 2026 09:48:36 +0000 Subject: [PATCH 36/37] test(executions): drop config-shape assertion Co-authored-by: Torsten Dittmann --- tests/unit/General/CollectionsTest.php | 5 ----- 1 file changed, 5 deletions(-) diff --git a/tests/unit/General/CollectionsTest.php b/tests/unit/General/CollectionsTest.php index 1a125656f3..ebc3586f0c 100644 --- a/tests/unit/General/CollectionsTest.php +++ b/tests/unit/General/CollectionsTest.php @@ -37,11 +37,6 @@ final class CollectionsTest extends TestCase } } - public function testProjectsDoNotDefineExecutionsCollection(): void - { - $this->assertArrayNotHasKey('executions', $this->collections['projects']); - } - public function testExecutionQueryValidatorsDoNotNeedCollectionSchema(): void { $this->assertTrue((new Executions())->isValid([ From 52a1d0bfb68a35e95b182319c12d79ec1f5a6cf9 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Tue, 15 Sep 2026 09:58:45 +0000 Subject: [PATCH 37/37] fix(executions): localize query attributes Co-authored-by: Torsten Dittmann --- .../Database/Validator/Queries/Base.php | 22 +++--- .../Database/Validator/Queries/Executions.php | 72 ++++++++++++++++--- .../Database/Validator/Queries/Logs.php | 4 +- 3 files changed, 73 insertions(+), 25 deletions(-) diff --git a/src/Appwrite/Utopia/Database/Validator/Queries/Base.php b/src/Appwrite/Utopia/Database/Validator/Queries/Base.php index 056eb467b4..688a67a20d 100644 --- a/src/Appwrite/Utopia/Database/Validator/Queries/Base.php +++ b/src/Appwrite/Utopia/Database/Validator/Queries/Base.php @@ -18,24 +18,22 @@ class Base extends Queries /** * Expression constructor * - * @param string|array $collection + * @param string $collection * @param string[] $allowedAttributes * @throws \Exception */ - public function __construct(string|array $collection, array $allowedAttributes) + public function __construct(string $collection, array $allowedAttributes) { - if (\is_string($collection)) { - $config = Config::getParam('collections', []); + $config = Config::getParam('collections', []); - $collections = \array_merge( - $config['projects'], - $config['buckets'], - $config['databases'], - $config['console'] - ); + $collections = \array_merge( + $config['projects'], + $config['buckets'], + $config['databases'], + $config['console'] + ); - $collection = $collections[$collection]; - } + $collection = $collections[$collection]; $allowedAttributesLookup = []; foreach ($allowedAttributes as $attribute) { diff --git a/src/Appwrite/Utopia/Database/Validator/Queries/Executions.php b/src/Appwrite/Utopia/Database/Validator/Queries/Executions.php index 92b915daf8..36fefa010b 100644 --- a/src/Appwrite/Utopia/Database/Validator/Queries/Executions.php +++ b/src/Appwrite/Utopia/Database/Validator/Queries/Executions.php @@ -3,17 +3,24 @@ namespace Appwrite\Utopia\Database\Validator\Queries; use Utopia\Database\Database; +use Utopia\Database\Document; +use Utopia\Database\Validator\Queries; +use Utopia\Database\Validator\Query\Cursor; +use Utopia\Database\Validator\Query\Filter; +use Utopia\Database\Validator\Query\Limit; +use Utopia\Database\Validator\Query\Offset; +use Utopia\Database\Validator\Query\Order; -class Executions extends Base +class Executions extends Queries { - public const ATTRIBUTES = [ - ['$id' => 'trigger', 'type' => Database::VAR_STRING, 'array' => false], - ['$id' => 'status', 'type' => Database::VAR_STRING, 'array' => false], - ['$id' => 'responseStatusCode', 'type' => Database::VAR_INTEGER, 'array' => false], - ['$id' => 'duration', 'type' => Database::VAR_FLOAT, 'array' => false], - ['$id' => 'requestMethod', 'type' => Database::VAR_STRING, 'array' => false], - ['$id' => 'requestPath', 'type' => Database::VAR_STRING, 'array' => false], - ['$id' => 'deploymentId', 'type' => Database::VAR_STRING, 'array' => false], + protected const ATTRIBUTE_TYPES = [ + 'trigger' => Database::VAR_STRING, + 'status' => Database::VAR_STRING, + 'responseStatusCode' => Database::VAR_INTEGER, + 'duration' => Database::VAR_FLOAT, + 'requestMethod' => Database::VAR_STRING, + 'requestPath' => Database::VAR_STRING, + 'deploymentId' => Database::VAR_STRING, ]; public const ALLOWED_ATTRIBUTES = [ @@ -30,8 +37,51 @@ class Executions extends Base * Expression constructor * */ - public function __construct() + public function __construct(array $allowedAttributes = self::ALLOWED_ATTRIBUTES) { - parent::__construct(['attributes' => self::ATTRIBUTES], self::ALLOWED_ATTRIBUTES); + $attributes = []; + foreach ($allowedAttributes as $attribute) { + $attributes[] = new Document([ + 'key' => $attribute, + 'type' => self::ATTRIBUTE_TYPES[$attribute], + 'array' => false, + ]); + } + + $attributes = [ + ...$attributes, + new Document([ + 'key' => '$id', + 'type' => Database::VAR_STRING, + 'array' => false, + ]), + new Document([ + 'key' => '$createdAt', + 'type' => Database::VAR_DATETIME, + 'array' => false, + ]), + new Document([ + 'key' => '$updatedAt', + 'type' => Database::VAR_DATETIME, + 'array' => false, + ]), + new Document([ + 'key' => '$sequence', + 'type' => Database::VAR_INTEGER, + 'array' => false, + ]), + ]; + + parent::__construct([ + new Limit(), + new Offset(), + new Cursor(), + new Filter( + attributes: $attributes, + idAttributeType: Database::VAR_INTEGER, + maxValuesCount: APP_DATABASE_QUERY_MAX_VALUES + ), + new Order($attributes), + ]); } } diff --git a/src/Appwrite/Utopia/Database/Validator/Queries/Logs.php b/src/Appwrite/Utopia/Database/Validator/Queries/Logs.php index ab1982c29f..302fe672a7 100644 --- a/src/Appwrite/Utopia/Database/Validator/Queries/Logs.php +++ b/src/Appwrite/Utopia/Database/Validator/Queries/Logs.php @@ -2,7 +2,7 @@ namespace Appwrite\Utopia\Database\Validator\Queries; -class Logs extends Base +class Logs extends Executions { public const ALLOWED_ATTRIBUTES = [ 'status', @@ -19,6 +19,6 @@ class Logs extends Base */ public function __construct() { - parent::__construct(['attributes' => Executions::ATTRIBUTES], self::ALLOWED_ATTRIBUTES); + parent::__construct(self::ALLOWED_ATTRIBUTES); } }