diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8eb89ceb31..7f20ef876f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -517,7 +517,7 @@ jobs: -e _APP_E2E_RESPONSE_FORMAT="${{ github.event.inputs.response_format }}" \ -e _TESTS_OAUTH2_GITHUB_CLIENT_ID="${{ secrets.TESTS_OAUTH2_GITHUB_CLIENT_ID }}" \ -e _TESTS_OAUTH2_GITHUB_CLIENT_SECRET="${{ secrets.TESTS_OAUTH2_GITHUB_CLIENT_SECRET }}" \ - appwrite vendor/bin/paratest --processes "$PARATEST_PROCESSES" $FUNCTIONAL_FLAG "$SERVICE_PATH" --exclude-group abuseEnabled --exclude-group screenshots --log-junit tests/e2e/Services/${{ matrix.service.name }}/junit.xml + appwrite vendor/bin/paratest --processes "$PARATEST_PROCESSES" $FUNCTIONAL_FLAG "$SERVICE_PATH" --exclude-group abuseEnabled --exclude-group screenshots --exclude-group antivirus --log-junit tests/e2e/Services/${{ matrix.service.name }}/junit.xml - name: Failure Logs if: failure() @@ -525,10 +525,10 @@ jobs: echo "=== Appwrite Logs ===" docker compose logs --timestamps - e2e_abuse: - name: Tests / E2E / Abuse (${{ matrix.mode }}) + e2e_grouped: + name: Tests / E2E / ${{ matrix.config.label }} (${{ matrix.mode }}) runs-on: runs-on=${{ github.run_id }}/runner=4cpu-linux-x64/volume=60g/spot=price-capacity-optimized/retry=when-interrupted/extras=otel+ecr-cache - timeout-minutes: 15 + timeout-minutes: 35 needs: [build, matrix] permissions: contents: read @@ -537,6 +537,30 @@ jobs: fail-fast: false matrix: mode: ${{ fromJSON(needs.matrix.outputs.modes) }} + config: + - label: Abuse + group: abuseEnabled + path: /usr/src/code/tests/e2e + options_abuse: enabled + storage_antivirus: disabled + compose_profiles: '' + wait: '' + - label: Screenshots + group: screenshots + path: /usr/src/code/tests/e2e/Services/Sites + options_abuse: disabled + storage_antivirus: disabled + compose_profiles: '' + wait: executor + - label: Antivirus + group: antivirus + path: /usr/src/code/tests/e2e/Services/Storage + options_abuse: disabled + storage_antivirus: enabled + compose_profiles: antivirus + wait: clamav + env: + COMPOSE_PROFILES: ${{ matrix.config.compose_profiles }} steps: - name: Checkout repository uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 @@ -574,82 +598,22 @@ jobs: done - name: Load and Start Appwrite - timeout-minutes: 5 + timeout-minutes: ${{ matrix.config.wait == 'clamav' && 15 || 5 }} env: - _APP_OPTIONS_ABUSE: enabled + _APP_OPTIONS_ABUSE: ${{ matrix.config.options_abuse }} + _APP_STORAGE_ANTIVIRUS: ${{ matrix.config.storage_antivirus }} _APP_DATABASE_SHARED_TABLES: ${{ matrix.mode != 'dedicated' && 'database_db_main' || '' }} _APP_DATABASE_DOCUMENTSDB_SHARED_TABLES: ${{ matrix.mode != 'dedicated' && 'documentsdb_db_main' || '' }} _APP_DATABASE_VECTORSDB_SHARED_TABLES: ${{ matrix.mode != 'dedicated' && 'vectorsdb_db_main' || '' }} - run: docker compose up -d --quiet-pull --wait - - - name: Run tests - timeout-minutes: 15 - run: >- - docker compose exec -T - -e _APP_E2E_RESPONSE_FORMAT="${{ github.event.inputs.response_format }}" - appwrite test /usr/src/code/tests/e2e --group=abuseEnabled - - - name: Failure Logs - if: failure() run: | - echo "=== Appwrite Logs ===" - docker compose logs --timestamps - - e2e_screenshots: - name: Tests / E2E / Screenshots (${{ matrix.mode }}) - runs-on: runs-on=${{ github.run_id }}/runner=4cpu-linux-x64/volume=60g/spot=price-capacity-optimized/retry=when-interrupted/extras=otel+ecr-cache - timeout-minutes: 15 - needs: [build, matrix] - permissions: - contents: read - packages: read - strategy: - fail-fast: false - matrix: - mode: ${{ fromJSON(needs.matrix.outputs.modes) }} - steps: - - name: Checkout repository - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - - name: Login to Docker Hub - if: ${{ env.DOCKERHUB_AUTH_AVAILABLE == 'true' }} - uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0 - with: - username: ${{ vars.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - - parallel: - - name: Pull Docker Image - run: | - docker pull ${{ env.RUNS_ON_ECR_CACHE }}:${{ env.IMAGE }}-${{ github.sha }} - docker tag ${{ env.RUNS_ON_ECR_CACHE }}:${{ env.IMAGE }}-${{ github.sha }} ${{ env.IMAGE }} - - - name: Pull Docker Compose images - timeout-minutes: 5 - run: | - # Docker Hub registry timeouts are a common transient CI failure - # (e.g. a traefik manifest fetch timing out). Retry the image pull; - # keep `up` single-shot so genuine container health failures still surface. - for attempt in 1 2 3; do - if docker compose pull --quiet --ignore-buildable; then - break - elif [ "$attempt" -eq 3 ]; then - echo "::error::docker compose pull failed after 3 attempts; aborting" - exit 1 - fi - echo "::warning::docker compose pull failed (attempt ${attempt}/3); retrying in 15s" - sleep 15 - done - - - name: Load and Start Appwrite - timeout-minutes: 5 - env: - _APP_DATABASE_SHARED_TABLES: ${{ matrix.mode != 'dedicated' && 'database_db_main' || '' }} - _APP_DATABASE_DOCUMENTSDB_SHARED_TABLES: ${{ matrix.mode != 'dedicated' && 'documentsdb_db_main' || '' }} - _APP_DATABASE_VECTORSDB_SHARED_TABLES: ${{ matrix.mode != 'dedicated' && 'vectorsdb_db_main' || '' }} - run: docker compose up -d --quiet-pull --wait + if [ "${{ matrix.config.wait }}" = "clamav" ]; then + docker compose up -d --quiet-pull --wait --wait-timeout 780 + else + docker compose up -d --quiet-pull --wait + fi - name: Wait for Open Runtimes + if: matrix.config.wait == 'executor' timeout-minutes: 3 run: | while ! docker logs exc1 | grep -q "Executor is ready."; do @@ -657,18 +621,44 @@ jobs: sleep 1 done + - name: Wait for ClamAV + if: matrix.config.wait == 'clamav' + timeout-minutes: 10 + run: | + for attempt in $(seq 1 120); do + if docker compose exec -T appwrite php -r ' + require "/usr/src/code/vendor/autoload.php"; + $host = getenv("_APP_STORAGE_ANTIVIRUS_HOST") ?: "clamav"; + $port = (int) (getenv("_APP_STORAGE_ANTIVIRUS_PORT") ?: 3310); + $av = new Appwrite\ClamAV\Network($host, $port); + exit(@$av->ping() ? 0 : 1); + '; then + echo "ClamAV is ready" + exit 0 + fi + echo "Waiting for ClamAV (attempt ${attempt}/120)" + sleep 5 + done + echo "::error::ClamAV did not become ready" + docker compose logs --timestamps clamav || true + exit 1 + - name: Run tests timeout-minutes: 15 run: >- docker compose exec -T -e _APP_E2E_RESPONSE_FORMAT="${{ github.event.inputs.response_format }}" - appwrite test /usr/src/code/tests/e2e/Services/Sites --group=screenshots + appwrite test ${{ matrix.config.path }} --group=${{ matrix.config.group }} - name: Failure Logs if: failure() run: | echo "=== Appwrite Logs ===" docker compose logs --timestamps + if [ "${{ matrix.config.wait }}" = "clamav" ]; then + echo "=== ClamAV Logs ===" + docker compose logs --timestamps clamav || true + fi benchmark: name: Benchmark diff --git a/app/controllers/api/account.php b/app/controllers/api/account.php index 2f24001acc..3dde47b268 100644 --- a/app/controllers/api/account.php +++ b/app/controllers/api/account.php @@ -1499,7 +1499,7 @@ Http::get('/v1/account/sessions/oauth2/:provider/redirect') ->inject('domainVerification') ->inject('cookieDomain') ->inject('authorization') - ->action(function (string $provider, string $code, string $state, string $error, string $error_description, Request $request, Response $response, Document $project, Validator $redirectValidator, Document $devKey, User $user, Database $dbForProject, GeoRecord $geoRecord, Database $dbForPlatform, Event $queueForEvents, Store $store, ProofsPassword $proofForPassword, ProofsToken $proofForToken, array $plan, bool $domainVerification, ?string $cookieDomain, Authorization $authorization) use ($oauthDefaultSuccess) { + ->action(function (string $provider, string $code, string $state, string $error, string $error_description, Request $request, Response $response, Document $project, Validator $redirectValidator, Document $devKey, User $user, Database $dbForProject, GeoRecord $geoRecord, Database $dbForPlatform, Event $queueForEvents, Store $store, ProofsPassword $proofForPassword, ProofsToken $proofForToken, array $plan, bool $domainVerification, ?string $cookieDomain, Authorization $authorization) use ($oauthDefaultSuccess, $oauthDefaultFailure) { $protocol = System::getEnv('_APP_OPTIONS_FORCE_HTTPS') === 'disabled' ? 'http' : 'https'; $port = $request->getPort(); $callbackBase = $protocol . '://' . $request->getHostname(); @@ -1572,7 +1572,7 @@ Http::get('/v1/account/sessions/oauth2/:provider/redirect') $failure = URLParser::parse($state['failure']); } - $failureRedirect = (function (string $type, ?string $message = null, ?int $code = null, ?\Throwable $previous = null, array $params = []) use ($failure, $response) { + $failureRedirect = (function (string $type, ?string $message = null, ?int $code = null, ?\Throwable $previous = null, array $params = []) use ($failure, $response, $project, $oauthDefaultFailure) { $exception = new Exception($type, $message, $code, $previous, params: $params); if (!empty($failure)) { $query = URLParser::parseQuery($failure['query']); @@ -1581,6 +1581,11 @@ Http::get('/v1/account/sessions/oauth2/:provider/redirect') 'type' => $exception->getType(), 'code' => !\is_null($code) ? $code : $exception->getCode(), ]); + // Mirror success path: default OAuth failure relay needs project to deep-link + // back into the native app via appwrite-callback-{project}:// + if (($failure['path'] ?? '') === $oauthDefaultFailure) { + $query['project'] = $project->getId(); + } $failure['query'] = URLParser::unparseQuery($query); $response->redirect(URLParser::unparse($failure), 301); } diff --git a/app/controllers/shared/api.php b/app/controllers/shared/api.php index 65de06c3e8..211128c499 100644 --- a/app/controllers/shared/api.php +++ b/app/controllers/shared/api.php @@ -1130,7 +1130,8 @@ Http::shutdown() ->inject('authorization') ->inject('apiKey') ->inject('mode') - ->action(function (Route $route, Response $response, Document $project, User $user, Database $dbForPlatform, Authorization $authorization, ?Key $apiKey, string $mode) { + ->inject('lock') + ->action(function (Route $route, Response $response, Document $project, User $user, Database $dbForPlatform, Authorization $authorization, ?Key $apiKey, string $mode, Lock $lock) { /** * Persist completed onboarding stage after usage shutdown so a schema/write failure here * cannot suppress RequestCompleted or usage metrics on the same request. @@ -1201,9 +1202,16 @@ Http::shutdown() ]; try { - $authorization->skip(fn () => $dbForPlatform->updateDocument('projects', $project->getId(), new Document([ - 'onboarding' => $byMethod, - ]))); + // last write overwriting the other's stage on multiple request + // onboarding is not a native array attribute, it is a string with json filter. + // we do not have a query operator for array merge keys + $lock->tryWithKey( + 'lock:platform:' . $project->getSequence() . ':onboarding', + fn () => $authorization->skip(fn () => $dbForPlatform->updateDocument('projects', $project->getId(), new Document([ + 'onboarding' => $byMethod, + ]))), + target: 'projects', + ); } catch (\Throwable) { // Missing `onboarding` attribute on upgraded installs must not break the request lifecycle. } diff --git a/app/views/install/env.phtml b/app/views/install/env.phtml index 51bda7a6fb..f956144d97 100644 --- a/app/views/install/env.phtml +++ b/app/views/install/env.phtml @@ -6,7 +6,9 @@ foreach ($vars as $key => $value) { if ($value === null || $value === '') { echo $key . "=\n"; } else { - echo $key . '="' . addcslashes((string) $value, '"\\$`') . '"' . "\n"; + // Escape newlines and special chars so multiline secrets (e.g. GitHub App + // private keys) survive install/upgrade rewrites as a single .env line. + echo $key . '="' . \Appwrite\Docker\Env::encodeValue((string) $value) . '"' . "\n"; } } ?> \ No newline at end of file diff --git a/docker-compose.override.yml b/docker-compose.override.yml index 2613f5f243..b4a2c50993 100644 --- a/docker-compose.override.yml +++ b/docker-compose.override.yml @@ -456,6 +456,14 @@ services: appwrite-embedding: ports: - "9505:3000" + clamav: + profiles: + - antivirus + image: clamav/clamav:1.4-debian + container_name: appwrite-clamav + restart: unless-stopped + networks: + - appwrite configs: redisinsight-connections.json: content: | diff --git a/docker-compose.yml b/docker-compose.yml index a23a2e1b8b..ddcf62e216 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -935,7 +935,7 @@ services: - _APP_MIGRATION_HOST appwrite-task-maintenance: entrypoint: maintenance - restart: on-failure:3 + restart: unless-stopped logging: driver: json-file options: @@ -982,7 +982,7 @@ services: - _APP_DATABASE_SHARED_TABLES appwrite-task-interval: entrypoint: interval - restart: on-failure:3 + restart: unless-stopped logging: driver: json-file options: @@ -1024,7 +1024,7 @@ services: appwrite-task-scheduler-functions: entrypoint: schedule-functions - restart: on-failure:3 + restart: unless-stopped logging: driver: json-file options: @@ -1063,7 +1063,7 @@ services: - _APP_DATABASE_SHARED_TABLES appwrite-task-scheduler-executions: entrypoint: schedule-executions - restart: on-failure:3 + restart: unless-stopped logging: driver: json-file options: @@ -1102,7 +1102,7 @@ services: - _APP_DATABASE_SHARED_TABLES appwrite-task-scheduler-messages: entrypoint: schedule-messages - restart: on-failure:3 + restart: unless-stopped logging: driver: json-file options: @@ -1290,7 +1290,7 @@ services: options: max-file: "5" max-size: 10m - restart: on-failure:3 + restart: unless-stopped networks: - appwrite volumes: diff --git a/docs/tutorials/multi-architecture-support.md b/docs/tutorials/multi-architecture-support.md index 25751ce174..4b632d820c 100644 --- a/docs/tutorials/multi-architecture-support.md +++ b/docs/tutorials/multi-architecture-support.md @@ -1,11 +1,31 @@ # Multi Architecture Support -A list of Appwrite CPU architecture support status. We use this list to track the status of all Appwrite related Docker images and which architecture is supported by each image. +CPU architecture support for Docker images used by the Appwrite stack. Platforms are taken from the published multi-arch manifests on Docker Hub / GHCR (not from historical claims). -| | linux/amd64 | linux/arm64 | linux/arm/v6 | linux/arm/v7 | linux/arm64/v8 | linux/ppc64le | linux/s390x | +| Image | linux/amd64 | linux/arm64 | linux/arm/v6 | linux/arm/v7 | linux/arm64/v8 | linux/ppc64le | linux/s390x | |---|---|---|---|---|---|---|---| -| appwrite/appwrite | 🟢 | 🟢 | 🟢 | 🟢 | 🟢 | 🟢 | 🟢 | -| appwrite/mariadb | 🟢 | 🟢 | 🔴 | 🔴 | 🔴 | 🟢 | 🔴 | -| appwrite/clamav | 🟢 | 🟢 | 🟢 | 🟢 | 🔴 | 🟢 | 🟢 | -| traefik | 🟢 | 🔴 | 🟢 | 🔴 | 🟢 | 🔴 | 🔴 | -| redis | 🟢 | 🟢 | 🟢 | 🟢 | 🟢 | 🟢 | 🟢 | +| **Core** | | | | | | | | +| appwrite/appwrite | 🟢 | 🟢 | 🔴 | 🔴 | 🟢 | 🔴 | 🔴 | +| appwrite/base:1.4.4 | 🟢 | 🟢 | 🔴 | 🔴 | 🟢 | 🔴 | 🔴 | +| appwrite/assistant:0.8.4 | 🟢 | 🟢 | 🔴 | 🔴 | 🟢 | 🔴 | 🔴 | +| appwrite/browser:0.3.3 | 🟢 | 🟢 | 🔴 | 🔴 | 🟢 | 🔴 | 🔴 | +| appwrite/embedding:0.1.0 | 🟢 | 🟢 | 🔴 | 🔴 | 🟢 | 🔴 | 🔴 | +| appwrite/geo:0.3.0 | 🟢 | 🟢 | 🔴 | 🔴 | 🟢 | 🔴 | 🔴 | +| appwrite/postgres:0.1.0 | 🟢 | 🟢 | 🔴 | 🔴 | 🟢 | 🔴 | 🔴 | +| openruntimes/executor:0.25.4 | 🟢 | 🟢 | 🔴 | 🔴 | 🟢 | 🔴 | 🔴 | +| ghcr.io/open-runtimes/orchestrator/jobs-service:0.13.0 | 🟢 | 🟢 | 🔴 | 🔴 | 🟢 | 🔴 | 🔴 | +| traefik:3.6 | 🟢 | 🟢 | 🟢 | 🔴 | 🟢 | 🟢 | 🟢 | +| redis:7.4.7-alpine | 🟢 | 🟢 | 🟢 | 🟢 | 🟢 | 🟢 | 🟢 | +| mariadb:10.11 | 🟢 | 🟢 | 🔴 | 🔴 | 🟢 | 🟢 | 🟢 | +| mongo:8.2.5 | 🟢 | 🟢 | 🔴 | 🔴 | 🟢 | 🔴 | 🔴 | +| **Optional** | | | | | | | | +| clamav/clamav:1.4-debian | 🟢 | 🟢 | 🔴 | 🔴 | 🟢 | 🟢 | 🔴 | +| **Dev / test** | | | | | | | | +| appwrite/mailcatcher:1.1.1 | 🟢 | 🟢 | 🔴 | 🔴 | 🟢 | 🔴 | 🔴 | +| appwrite/requestcatcher:1.1.0 | 🟢 | 🟢 | 🟢 | 🟢 | 🟢 | 🟢 | 🟢 | +| appwrite/altair:0.3.0 | 🟢 | 🟢 | 🔴 | 🟢 | 🟢 | 🔴 | 🔴 | +| coredns/coredns:1.12.4 | 🟢 | 🟢 | 🔴 | 🟢 | 🟢 | 🟢 | 🟢 | + +Notes: + +- `linux/arm64` and `linux/arm64/v8` are treated as the same when the manifest publishes a plain `arm64` platform. diff --git a/src/Appwrite/Docker/Env.php b/src/Appwrite/Docker/Env.php index 7e44a6c5cf..37591b5c86 100644 --- a/src/Appwrite/Docker/Env.php +++ b/src/Appwrite/Docker/Env.php @@ -5,31 +5,168 @@ namespace Appwrite\Docker; class Env { /** - * @var array + * @var array */ protected $vars = []; public function __construct(string $data) { - $data = explode("\n", $data); + $length = \strlen($data); + $offset = 0; - foreach ($data as &$row) { - $row = explode('=', $row, 2); - $key = trim($row[0]); - $value = (isset($row[1])) ? (function (string $v): string { - $v = trim($v); - if ( - (\str_starts_with($v, '"') && \str_ends_with($v, '"')) || - (\str_starts_with($v, "'") && \str_ends_with($v, "'")) - ) { - return \substr($v, 1, -1); - } - return $v; - })(trim($row[1])) : null; - - if ($key) { - $this->vars[$key] = $value; + while ($offset < $length) { + $lineEnd = \strpos($data, "\n", $offset); + if ($lineEnd === false) { + $lineEnd = $length; } + + $line = \substr($data, $offset, $lineEnd - $offset); + // Normalize Windows newlines for line-oriented parsing of keys. + if (\str_ends_with($line, "\r")) { + $line = \substr($line, 0, -1); + } + + $trimmed = \ltrim($line, " \t"); + + // Skip blank lines and comments (including indented comments). + if ($trimmed === '' || $trimmed[0] === '#') { + $offset = $lineEnd < $length ? $lineEnd + 1 : $length; + continue; + } + + $equals = \strpos($line, '='); + if ($equals === false) { + // Skip malformed lines without an assignment on this line. + $offset = $lineEnd < $length ? $lineEnd + 1 : $length; + continue; + } + + $key = \trim(\substr($line, 0, $equals)); + if ($key === '') { + $offset = $lineEnd < $length ? $lineEnd + 1 : $length; + continue; + } + + // Value starts after '=' in the full buffer so quoted values may span lines. + $valueOffset = $offset + $equals + 1; + + // Consume optional whitespace before the value. + while ($valueOffset < $length && ($data[$valueOffset] === ' ' || $data[$valueOffset] === "\t")) { + $valueOffset++; + } + + if ($valueOffset >= $length || $data[$valueOffset] === "\n" || $data[$valueOffset] === "\r") { + $this->vars[$key] = ''; + $offset = $valueOffset < $length ? (($data[$valueOffset] === "\r" && $valueOffset + 1 < $length && $data[$valueOffset + 1] === "\n") ? $valueOffset + 2 : $valueOffset + 1) : $length; + continue; + } + + $quote = $data[$valueOffset]; + if ($quote === '"' || $quote === "'") { + $contentStart = $valueOffset + 1; + $scanOffset = $contentStart; + $value = ''; + $closed = false; + $crossedNewline = false; + + while ($scanOffset < $length) { + $char = $data[$scanOffset]; + + if ($char === "\n" || $char === "\r") { + $crossedNewline = true; + } + + if ($char === $quote) { + // A later line's opening quote (KEY=") must not close an + // earlier unterminated value, or upgrade will drop that key. + if ($crossedNewline && self::isAssignmentOpener($data, $scanOffset)) { + break; + } + $scanOffset++; + $closed = true; + break; + } + + if ($quote === '"' && $char === '\\' && $scanOffset + 1 < $length) { + $next = $data[$scanOffset + 1]; + $value .= match ($next) { + 'n' => "\n", + 'r' => "\r", + 't' => "\t", + '\\' => '\\', + '"' => '"', + "'" => "'", + '$' => '$', + '`' => '`', + // Preserve unknown escapes as a literal backslash + next char + // (e.g. Windows paths or patterns like `\d`, `\q`). + default => '\\' . $next, + }; + $scanOffset += 2; + continue; + } + + $value .= $char; + $scanOffset++; + } + + if (!$closed) { + // Unterminated quote: keep only the remainder of the starting line so + // later assignments are not absorbed into this value during upgrade. + $end = \strpos($data, "\n", $contentStart); + if ($end === false) { + $raw = \substr($data, $contentStart); + $offset = $length; + } else { + $raw = \substr($data, $contentStart, $end - $contentStart); + $offset = $end + 1; + } + if (\str_ends_with($raw, "\r")) { + $raw = \substr($raw, 0, -1); + } + $this->vars[$key] = $raw; + continue; + } + + $valueOffset = $scanOffset; + + // Consume trailing characters until end of line (comments/whitespace). + while ($valueOffset < $length && $data[$valueOffset] !== "\n" && $data[$valueOffset] !== "\r") { + $valueOffset++; + } + if ($valueOffset < $length) { + if ($data[$valueOffset] === "\r" && $valueOffset + 1 < $length && $data[$valueOffset + 1] === "\n") { + $valueOffset += 2; + } else { + $valueOffset++; + } + } + + $this->vars[$key] = $value; + $offset = $valueOffset; + continue; + } + + // Unquoted value: read until end of line, trim trailing whitespace. + $end = \strpos($data, "\n", $valueOffset); + if ($end === false) { + $raw = \substr($data, $valueOffset); + $offset = $length; + } else { + $raw = \substr($data, $valueOffset, $end - $valueOffset); + $offset = $end + 1; + } + + if (\str_ends_with($raw, "\r")) { + $raw = \substr($raw, 0, -1); + } + + // Strip inline comments preceded by whitespace. + if (\preg_match('/\s+#/', $raw, $matches, PREG_OFFSET_CAPTURE) === 1) { + $raw = \substr($raw, 0, $matches[0][1]); + } + + $this->vars[$key] = \rtrim($raw, " \t"); } } @@ -66,6 +203,26 @@ class Env return $this->vars; } + /** + * Encode a value for a double-quoted .env assignment. + */ + public static function encodeValue(string $value): string + { + return \addcslashes($value, "\"\\\n\r\t\$`"); + } + + /** + * True when $quoteOffset points at a quote that opens a new KEY="..." assignment. + */ + private static function isAssignmentOpener(string $data, int $quoteOffset): bool + { + $lineStart = \strrpos(\substr($data, 0, $quoteOffset), "\n"); + $lineStart = $lineStart === false ? 0 : $lineStart + 1; + $before = \substr($data, $lineStart, $quoteOffset - $lineStart); + + return \preg_match('/^\s*[A-Za-z_][A-Za-z0-9_]*\s*=\s*$/', $before) === 1; + } + /** * @return string */ @@ -74,7 +231,12 @@ class Env $output = ''; foreach ($this->vars as $key => $value) { - $output .= $key . '=' . $value . "\n"; + if ($value === null || $value === '') { + $output .= $key . "=\n"; + continue; + } + + $output .= $key . '="' . self::encodeValue((string) $value) . "\"\n"; } return $output; diff --git a/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Files/Create.php b/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Files/Create.php index 3b826c034b..2af26c858b 100644 --- a/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Files/Create.php +++ b/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Files/Create.php @@ -360,7 +360,7 @@ class Create extends Action (int) System::getEnv('_APP_STORAGE_ANTIVIRUS_PORT', 3310) ); - if (!$antivirus->fileScan($path)) { + if (!$antivirus->fileScanInStream($path)) { $deviceForFiles->delete($path); throw new Exception(Exception::STORAGE_INVALID_FILE); } diff --git a/src/Appwrite/Platform/Tasks/Install.php b/src/Appwrite/Platform/Tasks/Install.php index 7462934ec4..ac5b2c1a03 100644 --- a/src/Appwrite/Platform/Tasks/Install.php +++ b/src/Appwrite/Platform/Tasks/Install.php @@ -426,14 +426,8 @@ class Install extends Action } } - foreach ($input as $key => $value) { - if (!is_string($value)) { - continue; - } - if (str_contains($value, "\n") || str_contains($value, "\r")) { - throw new \InvalidArgumentException('Invalid value for ' . $key); - } - } + // Multiline values (e.g. GitHub App PEM private keys) are allowed; env.phtml + // encodes them as escaped single-line double-quoted assignments. // Set database-specific connection details $database = $input['_APP_DB_ADAPTER'] ?? 'postgresql'; diff --git a/src/Appwrite/Platform/Workers/Migrations.php b/src/Appwrite/Platform/Workers/Migrations.php index 3ff9c5600d..ede3360b4c 100644 --- a/src/Appwrite/Platform/Workers/Migrations.php +++ b/src/Appwrite/Platform/Workers/Migrations.php @@ -496,14 +496,14 @@ class Migrations extends Action $transfer = $source = $destination = null; $caughtError = null; - $host = System::getEnv('_APP_MIGRATION_HOST'); - if (empty($host)) { - throw new \Exception('_APP_MIGRATION_HOST is not set'); - } - - $endpoint = 'http://' . $host . '/v1'; - try { + $host = System::getEnv('_APP_MIGRATION_HOST'); + if (empty($host)) { + throw new \Exception('_APP_MIGRATION_HOST is not set'); + } + + $endpoint = 'http://' . $host . '/v1'; + $credentials = $migration->getAttribute('credentials', []); if ($migration->getAttribute('source') === SourceAppwrite::getName()) { diff --git a/src/Appwrite/Screenshots/Client.php b/src/Appwrite/Screenshots/Client.php index 096d737393..f33708090f 100644 --- a/src/Appwrite/Screenshots/Client.php +++ b/src/Appwrite/Screenshots/Client.php @@ -9,6 +9,17 @@ use Utopia\Psr7\Request\Factory; class Client { + public const VIEWPORT_WIDTH = 1280; + + public const VIEWPORT_HEIGHT = 720; + + /** + * Captured pixels per CSS pixel. Above 1 the page is rendered at a higher + * resolution than the viewport, so previews stay sharp when the console + * displays them on high-density screens or scales them up. + */ + public const SCALE = 1.5; + private Factory $factory; /** @@ -33,6 +44,11 @@ class Client 'theme' => $theme, 'headers' => $headers, 'sleep' => $sleep, + 'viewport' => [ + 'width' => self::VIEWPORT_WIDTH, + 'height' => self::VIEWPORT_HEIGHT, + ], + 'deviceScaleFactor' => self::SCALE, ])); $status = $response->getStatusCode(); diff --git a/tests/e2e/Services/Account/AccountCustomClientTest.php b/tests/e2e/Services/Account/AccountCustomClientTest.php index ab25d7b439..a11cdd9b63 100644 --- a/tests/e2e/Services/Account/AccountCustomClientTest.php +++ b/tests/e2e/Services/Account/AccountCustomClientTest.php @@ -3068,6 +3068,148 @@ final class AccountCustomClientTest extends Scope $this->assertEquals(204, $response['headers']['status-code']); } + /** + * Default OAuth failure relay pages need `project` so native apps can deep-link via + * appwrite-callback-{project}://. Without it the UI shows "Missing redirect URL" + * instead of the real OAuth error. + */ + public function testOAuthDefaultFailureRedirectIncludesProject(): void + { + $provider = 'mock'; + $projectId = $this->getProject()['$id']; + + $response = $this->client->call(Client::METHOD_PATCH, '/projects/' . $projectId . '/oauth2', array_merge([ + 'origin' => 'http://localhost', + 'content-type' => 'application/json', + 'x-appwrite-project' => 'console', + 'cookie' => 'a_session_console=' . $this->getRoot()['session'], + ]), [ + 'provider' => $provider, + 'appId' => '1', + 'secret' => '123456', + 'enabled' => true, + ]); + + $this->assertEquals(200, $response['headers']['status-code']); + + // Omit failure so Appwrite uses the default relay URL (/auth/oauth2/failure or legacy /console/...) + $response = $this->client->call(Client::METHOD_GET, '/account/sessions/oauth2/' . $provider, [ + 'origin' => 'http://localhost', + 'content-type' => 'application/json', + 'x-appwrite-project' => $projectId, + ], [ + 'success' => 'http://localhost/v1/mock/tests/general/oauth2/success', + ], followRedirects: false); + + $this->assertEquals(301, $response['headers']['status-code']); + $this->assertStringStartsWith('http://localhost/v1/mock/tests/general/oauth2', $response['headers']['location']); + + $mockQuery = []; + \parse_str((string) \parse_url($response['headers']['location'], PHP_URL_QUERY), $mockQuery); + $this->assertNotEmpty($mockQuery['state'] ?? null); + + // Simulate provider returning an error (same path as a real OAuth denial) + $response = $this->client->call(Client::METHOD_GET, '/account/sessions/oauth2/callback/' . $provider . '/' . $projectId, [ + 'origin' => 'http://localhost', + 'content-type' => 'application/json', + 'x-appwrite-project' => $projectId, + ], [ + 'error' => 'access_denied', + 'error_description' => 'The user denied the request', + 'state' => $mockQuery['state'], + ], followRedirects: false); + + $this->assertEquals(301, $response['headers']['status-code']); + $this->assertStringContainsString('/account/sessions/oauth2/' . $provider . '/redirect?', (string) $response['headers']['location']); + + $oauthClient = new Client(); + $oauthClient->setEndpoint(''); + $response = $oauthClient->call(Client::METHOD_GET, $response['headers']['location'], followRedirects: false); + + $this->assertEquals(301, $response['headers']['status-code']); + + $location = $response['headers']['location']; + $path = \parse_url($location, PHP_URL_PATH); + $query = []; + \parse_str((string) \parse_url($location, PHP_URL_QUERY), $query); + + $this->assertContains($path, ['/auth/oauth2/failure', '/console/auth/oauth2/failure']); + $this->assertEquals($projectId, $query['project'] ?? null); + $this->assertNotEmpty($query['error'] ?? null); + + $error = \json_decode($query['error'], true); + $this->assertIsArray($error); + $this->assertArrayHasKey('message', $error); + $this->assertArrayHasKey('type', $error); + $this->assertStringContainsString('access_denied', (string) $error['message']); + } + + /** + * Custom failure URLs must not have `project` forced onto them — only the default + * Appwrite failure relay path gets that query param (mirrors success-path gating). + */ + public function testOAuthCustomFailureRedirectOmitsProject(): void + { + $provider = 'mock'; + $projectId = $this->getProject()['$id']; + + $response = $this->client->call(Client::METHOD_PATCH, '/projects/' . $projectId . '/oauth2', array_merge([ + 'origin' => 'http://localhost', + 'content-type' => 'application/json', + 'x-appwrite-project' => 'console', + 'cookie' => 'a_session_console=' . $this->getRoot()['session'], + ]), [ + 'provider' => $provider, + 'appId' => '1', + 'secret' => '123456', + 'enabled' => true, + ]); + + $this->assertEquals(200, $response['headers']['status-code']); + + $response = $this->client->call(Client::METHOD_GET, '/account/sessions/oauth2/' . $provider, [ + 'origin' => 'http://localhost', + 'content-type' => 'application/json', + 'x-appwrite-project' => $projectId, + ], [ + 'success' => 'http://localhost/v1/mock/tests/general/oauth2/success', + 'failure' => 'http://localhost/v1/mock/tests/general/oauth2/failure', + ], followRedirects: false); + + $this->assertEquals(301, $response['headers']['status-code']); + + $mockQuery = []; + \parse_str((string) \parse_url($response['headers']['location'], PHP_URL_QUERY), $mockQuery); + $this->assertNotEmpty($mockQuery['state'] ?? null); + + $response = $this->client->call(Client::METHOD_GET, '/account/sessions/oauth2/callback/' . $provider . '/' . $projectId, [ + 'origin' => 'http://localhost', + 'content-type' => 'application/json', + 'x-appwrite-project' => $projectId, + ], [ + 'error' => 'access_denied', + 'error_description' => 'The user denied the request', + 'state' => $mockQuery['state'], + ], followRedirects: false); + + $this->assertEquals(301, $response['headers']['status-code']); + + $oauthClient = new Client(); + $oauthClient->setEndpoint(''); + $response = $oauthClient->call(Client::METHOD_GET, $response['headers']['location'], followRedirects: false); + + $this->assertEquals(301, $response['headers']['status-code']); + + $location = $response['headers']['location']; + $this->assertStringStartsWith('http://localhost/v1/mock/tests/general/oauth2/failure?', $location); + + $query = []; + \parse_str((string) \parse_url($location, PHP_URL_QUERY), $query); + + $this->assertArrayNotHasKey('project', $query); + $this->assertNotEmpty($query['error'] ?? null); + } + public function testOAuthVerifiedEmailCanLinkToExistingAccount(): void { $provider = 'mock'; diff --git a/tests/e2e/Services/Storage/StorageCustomServerTest.php b/tests/e2e/Services/Storage/StorageCustomServerTest.php index a779dce169..39a1b112d7 100644 --- a/tests/e2e/Services/Storage/StorageCustomServerTest.php +++ b/tests/e2e/Services/Storage/StorageCustomServerTest.php @@ -4,14 +4,19 @@ declare(strict_types=1); namespace Tests\E2E\Services\Storage; +use CURLFile; +use PHPUnit\Framework\Attributes\Group; use Tests\E2E\Client; use Tests\E2E\Scopes\ProjectCustom; use Tests\E2E\Scopes\Scope; use Tests\E2E\Scopes\SideServer; use Utopia\Database\Document; use Utopia\Database\Helpers\ID; +use Utopia\Database\Helpers\Permission; +use Utopia\Database\Helpers\Role; use Utopia\Database\Query; use Utopia\Database\Validator\Datetime as DatetimeValidator; +use Utopia\System\System; final class StorageCustomServerTest extends Scope { @@ -365,4 +370,91 @@ final class StorageCustomServerTest extends Scope ); $this->assertEquals(404, $response['headers']['status-code']); } + + /** + * Regression for chunked uploads under the antivirus size limit. + * + * Requires ClamAV (compose profile `antivirus`) and + * `_APP_STORAGE_ANTIVIRUS=enabled`. File must be >5MB (chunked) and + * ≤20MB (`APP_LIMIT_ANTIVIRUS`) so the last chunk triggers a scan. + */ + #[Group('antivirus')] + public function testCreateBucketFileChunkedUploadWithAntivirus(): void + { + if (System::getEnv('_APP_STORAGE_ANTIVIRUS', 'disabled') === 'disabled') { + $this->markTestSkipped('Antivirus is disabled.'); + } + + $health = $this->client->call(Client::METHOD_GET, '/health/anti-virus', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders())); + + $this->assertEquals(200, $health['headers']['status-code']); + $this->assertEquals('pass', $health['body']['status'], 'ClamAV must be reachable when antivirus is enabled.'); + + $bucket = $this->client->call(Client::METHOD_POST, '/storage/buckets', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'bucketId' => ID::unique(), + 'name' => 'Antivirus Chunked Upload Bucket', + 'fileSecurity' => true, + 'antivirus' => true, + 'permissions' => [ + Permission::create(Role::any()), + Permission::read(Role::any()), + ], + ]); + $this->assertEquals(201, $bucket['headers']['status-code']); + + $source = __DIR__ . '/../../../resources/functions/large/blue.mp4'; + $totalSize = \filesize($source); + $this->assertGreaterThan(5 * 1024 * 1024, $totalSize, 'Fixture must be large enough to chunk.'); + $this->assertLessThanOrEqual(20_000_000, $totalSize, 'Fixture must stay under APP_LIMIT_ANTIVIRUS.'); + + $chunkSize = 5 * 1024 * 1024; + $handle = \fopen($source, 'rb'); + $this->assertNotFalse($handle); + + $fileId = 'unique()'; + $mimeType = \mime_content_type($source); + $counter = 0; + $id = ''; + $file = null; + $headers = [ + 'content-type' => 'multipart/form-data', + 'x-appwrite-project' => $this->getProject()['$id'], + ]; + + while (!\feof($handle)) { + $curlFile = new CURLFile( + 'data://' . $mimeType . ';base64,' . \base64_encode(\fread($handle, $chunkSize)), + $mimeType, + 'blue.mp4' + ); + $headers['content-range'] = 'bytes ' . ($counter * $chunkSize) . '-' . \min(((($counter * $chunkSize) + $chunkSize) - 1), $totalSize - 1) . '/' . $totalSize; + if (!empty($id)) { + $headers['x-appwrite-id'] = $id; + } + + $file = $this->client->call(Client::METHOD_POST, '/storage/buckets/' . $bucket['body']['$id'] . '/files', \array_merge($headers, $this->getHeaders()), [ + 'fileId' => $fileId, + 'file' => $curlFile, + 'permissions' => [ + Permission::read(Role::any()), + ], + ]); + $counter++; + $id = $file['body']['$id'] ?? $id; + } + \fclose($handle); + + $this->assertEquals(201, $file['headers']['status-code']); + $this->assertNotEmpty($file['body']['$id']); + $this->assertEquals('blue.mp4', $file['body']['name']); + $this->assertEquals($totalSize, $file['body']['sizeOriginal']); + $this->assertEquals($file['body']['chunksTotal'], $file['body']['chunksUploaded']); + $this->assertNotEmpty($file['body']['mimeType']); + } } diff --git a/tests/resources/sites/static-themed/screenshot-dark.png b/tests/resources/sites/static-themed/screenshot-dark.png index 199a09b902..52b5319b9c 100644 Binary files a/tests/resources/sites/static-themed/screenshot-dark.png and b/tests/resources/sites/static-themed/screenshot-dark.png differ diff --git a/tests/resources/sites/static-themed/screenshot-light.png b/tests/resources/sites/static-themed/screenshot-light.png index 4eae73c5b7..4b451b353f 100644 Binary files a/tests/resources/sites/static-themed/screenshot-light.png and b/tests/resources/sites/static-themed/screenshot-light.png differ diff --git a/tests/unit/Docker/EnvTest.php b/tests/unit/Docker/EnvTest.php index 4f11d31947..7307a6aeb5 100644 --- a/tests/unit/Docker/EnvTest.php +++ b/tests/unit/Docker/EnvTest.php @@ -36,10 +36,110 @@ final class EnvTest extends TestCase public function testExport(): void { - $this->assertSame("_APP_X=value1 -_APP_Y=value2 -_APP_Z=value3 -_APP_W=value5= -", $this->object->export()); + $this->assertSame( + "_APP_X=\"value1\"\n_APP_Y=\"value2\"\n_APP_Z=\"value3\"\n_APP_W=\"value5=\"\n", + $this->object->export() + ); + } + + public function testMultilineDoubleQuotedPrivateKey(): void + { + $pem = "-----BEGIN RSA PRIVATE KEY-----\nABCDEF\n-----END RSA PRIVATE KEY-----"; + $data = "_APP_VCS_GITHUB_PRIVATE_KEY=\"-----BEGIN RSA PRIVATE KEY-----\nABCDEF\n-----END RSA PRIVATE KEY-----\"\n_APP_OTHER=ok\n"; + + $env = new Env($data); + + $this->assertSame($pem, $env->getVar('_APP_VCS_GITHUB_PRIVATE_KEY')); + $this->assertSame('ok', $env->getVar('_APP_OTHER')); + } + + public function testEscapedNewlineInDoubleQuotedValue(): void + { + $data = "_APP_VCS_GITHUB_PRIVATE_KEY=\"-----BEGIN RSA PRIVATE KEY-----\\nABCDEF\\n-----END RSA PRIVATE KEY-----\"\n"; + + $env = new Env($data); + + $this->assertSame( + "-----BEGIN RSA PRIVATE KEY-----\nABCDEF\n-----END RSA PRIVATE KEY-----", + $env->getVar('_APP_VCS_GITHUB_PRIVATE_KEY') + ); + } + + public function testRoundTripPreservesMultilineSecret(): void + { + $pem = "-----BEGIN RSA PRIVATE KEY-----\nLINE1\nLINE2\n-----END RSA PRIVATE KEY-----"; + $env = new Env(''); + $env->setVar('_APP_VCS_GITHUB_PRIVATE_KEY', $pem); + $env->setVar('_APP_DOMAIN', 'example.com'); + + $exported = $env->export(); + $reloaded = new Env($exported); + + $this->assertSame($pem, $reloaded->getVar('_APP_VCS_GITHUB_PRIVATE_KEY')); + $this->assertSame('example.com', $reloaded->getVar('_APP_DOMAIN')); + $this->assertStringNotContainsString("\nLINE1\n", $exported); + $this->assertStringContainsString('\\nLINE1\\n', $exported); + } + + public function testEncodeValueEscapesSpecialCharacters(): void + { + $this->assertSame( + 'say\\"hi\\nnext\\$path', + Env::encodeValue("say\"hi\nnext\$path") + ); + } + + public function testUnknownEscapesPreserveBackslash(): void + { + // .env contains unknown escapes \q and \d; both must keep the backslash. + $data = "_APP_PATTERN=\"prefix\\q and \\d digits\"\n"; + + $env = new Env($data); + + $this->assertSame('prefix\\q and \\d digits', $env->getVar('_APP_PATTERN')); + } + + public function testIndentedCommentDoesNotCaptureNextAssignment(): void + { + $data = " # indented note\n_APP_DOMAIN=example.com\n \n_APP_OPTIONS_FORCE_HTTPS=enabled\n"; + + $env = new Env($data); + + $this->assertSame('example.com', $env->getVar('_APP_DOMAIN')); + $this->assertSame('enabled', $env->getVar('_APP_OPTIONS_FORCE_HTTPS')); + $this->assertArrayNotHasKey(' # indented note', $env->list()); + $this->assertArrayNotHasKey('# indented note', $env->list()); + } + + public function testLineWithoutAssignmentIsSkipped(): void + { + $data = "NOT_AN_ASSIGNMENT\n_APP_DOMAIN=example.com\n"; + + $env = new Env($data); + + $this->assertSame('example.com', $env->getVar('_APP_DOMAIN')); + $this->assertCount(1, $env->list()); + } + + public function testUnterminatedQuoteDoesNotAbsorbLaterAssignments(): void + { + $data = "_APP_BROKEN=\"partial-value\n_APP_DOMAIN=example.com\n_APP_OPTIONS_FORCE_HTTPS=enabled\n"; + + $env = new Env($data); + + $this->assertSame('partial-value', $env->getVar('_APP_BROKEN')); + $this->assertSame('example.com', $env->getVar('_APP_DOMAIN')); + $this->assertSame('enabled', $env->getVar('_APP_OPTIONS_FORCE_HTTPS')); + } + + public function testUnterminatedQuoteDoesNotAbsorbLaterQuotedAssignment(): void + { + $data = "_APP_BROKEN=\"partial-value\n_APP_DOMAIN=\"example.com\"\n_APP_OPTIONS_FORCE_HTTPS=enabled\n"; + + $env = new Env($data); + + $this->assertSame('partial-value', $env->getVar('_APP_BROKEN')); + $this->assertSame('example.com', $env->getVar('_APP_DOMAIN')); + $this->assertSame('enabled', $env->getVar('_APP_OPTIONS_FORCE_HTTPS')); } } diff --git a/tests/unit/Screenshots/ClientTest.php b/tests/unit/Screenshots/ClientTest.php index 6dcfdce28e..ac5efba65f 100644 --- a/tests/unit/Screenshots/ClientTest.php +++ b/tests/unit/Screenshots/ClientTest.php @@ -58,9 +58,41 @@ final class ClientTest extends TestCase 'theme' => 'dark', 'headers' => ['x-appwrite-hostname' => 'example.com'], 'sleep' => 500, + 'viewport' => ['width' => 1280, 'height' => 720], + 'deviceScaleFactor' => 1.5, ], \json_decode((string)$this->request->getBody(), true)); } + public function testCreateCapturesAboveViewportResolution(): void + { + $response = (new Response(200, body: new Stream('png-bytes'))) + ->withHeader('Content-Type', 'image/png'); + + $this->client($response)->create('http://appwrite/', 'light'); + + $body = \json_decode((string)$this->request->getBody(), true); + + // Left unset, the browser service defaults to a scale factor of 1, which + // captures exactly one pixel per CSS pixel and renders soft previews. + $this->assertGreaterThan(1, $body['deviceScaleFactor']); + $this->assertSame(1280, $body['viewport']['width']); + $this->assertSame(720, $body['viewport']['height']); + } + + public function testCreateDoesNotBlockOnSubresources(): void + { + $response = (new Response(200, body: new Stream('png-bytes'))) + ->withHeader('Content-Type', 'image/png'); + + $this->client($response)->create('http://appwrite/', 'light'); + + // The browser service navigates with `domcontentloaded` unless told + // otherwise. Waiting for `load` risks exhausting the request timeout on + // sites with slow or hanging subresources, which yields no screenshot at + // all rather than an early one. + $this->assertArrayNotHasKey('waitUntil', \json_decode((string)$this->request->getBody(), true)); + } + public function testCreateError(): void { $response = (new Response(400, body: new Stream('{"error":"Timeout exceeded"}')))