Merge pull request #57 from jetkvm/dev

Release dev → main: SKU-aware OTA release artifacts + rollout-bucket compare coverage

package.json

@@ -11,6 +11,7 @@
     "prisma-dev-migrate": "prisma migrate dev",
     "prisma-migrate": "prisma migrate deploy",
     "seed": "NODE_ENV=development node -r ts-node/register --env-file=.env.development ./scripts/seed.ts",
+    "sync-releases": "NODE_ENV=development node -r ts-node/register --env-file=.env.development ./scripts/sync-releases.ts",
     "build": "tsc",
     "test": "vitest run",
     "test:watch": "vitest",

prisma/migrations/20260427143200_add_release_artifacts/migration.sql

@@ -0,0 +1,28 @@
+-- CreateTable
+CREATE TABLE "ReleaseArtifact" (
+    "id" BIGSERIAL NOT NULL,
+    "releaseId" BIGINT NOT NULL,
+    "url" TEXT NOT NULL,
+    "hash" TEXT NOT NULL,
+    "compatibleSkus" TEXT[] NOT NULL,
+
+    CONSTRAINT "ReleaseArtifact_pkey" PRIMARY KEY ("id")
+);
+
+-- Backfill one artifact for every existing release.
+-- Pre-SKU artifacts only target the original jetkvm-v2 hardware; future SKUs
+-- (e.g. jetkvm-v2-sdmmc) require explicit SKU-folder uploads to be registered
+-- by scripts/sync-releases.ts.
+INSERT INTO "ReleaseArtifact" ("releaseId", "url", "hash", "compatibleSkus")
+SELECT
+    "id",
+    "url",
+    "hash",
+    ARRAY['jetkvm-v2']::TEXT[]
+FROM "Release";
+
+-- CreateIndex
+CREATE UNIQUE INDEX "ReleaseArtifact_releaseId_url_key" ON "ReleaseArtifact"("releaseId", "url");
+
+-- AddForeignKey
+ALTER TABLE "ReleaseArtifact" ADD CONSTRAINT "ReleaseArtifact_releaseId_fkey" FOREIGN KEY ("releaseId") REFERENCES "Release"("id") ON DELETE CASCADE ON UPDATE CASCADE;

prisma/schema.prisma

@@ -43,14 +43,26 @@ model TurnActivity {
 }
 
 model Release {
-  id                BigInt   @id @default(autoincrement())
+  id                BigInt            @id @default(autoincrement())
   version           String
-  rolloutPercentage Int      @default(10) // 10% of users
-  createdAt         DateTime @default(now())
-  updatedAt         DateTime @updatedAt
+  rolloutPercentage Int               @default(10) // 10% of users
+  createdAt         DateTime          @default(now())
+  updatedAt         DateTime          @updatedAt
   url               String
-  type              String   @default("app") // "app" or "system"
+  type              String            @default("app") // "app" or "system"
   hash              String
+  artifacts         ReleaseArtifact[]
 
   @@unique([version, type])
 }
+
+model ReleaseArtifact {
+  id             BigInt   @id @default(autoincrement())
+  release        Release  @relation(fields: [releaseId], references: [id], onDelete: Cascade)
+  releaseId      BigInt
+  url            String
+  hash           String
+  compatibleSkus String[]
+
+  @@unique([releaseId, url])
+}

scripts/compare-releases.sh

@@ -0,0 +1,769 @@
+#!/usr/bin/env bash
+
+set -uo pipefail
+
+LOCAL_BASE="${LOCAL_BASE:-http://localhost:3000}"
+PROD_BASE="${PROD_BASE:-https://api.jetkvm.com}"
+
+# Two device IDs picked so they straddle a typical staged rollout window:
+#   compare-device-1 → rollout bucket 81 (skips releases < 81%)
+#   compare-device-2 → rollout bucket 9  (catches releases >= 10%)
+# Together they exercise both eligible and ineligible rollout paths.
+DEFAULT_DEVICE_IDS=("compare-device-1" "compare-device-2")
+DEFAULT_SKUS=("__omit__" "jetkvm-v2" "jetkvm-v2-sdmmc")
+TRISTATE_VALUES=("__omit__" "false" "true")
+
+TMP_DIR="$(mktemp -d)"
+trap 'rm -rf "$TMP_DIR"' EXIT
+
+PASS_COUNT=0
+FAIL_COUNT=0
+ACCEPTED_COUNT=0
+CASE_COUNT=0
+CASE_INDEX=0
+TOTAL_CASES=0
+PROGRESS_WIDTH=40
+
+print_usage() {
+  cat <<'EOF'
+Usage: scripts/compare-releases.sh [device_id ...]
+
+Compares release endpoint responses between:
+  - local API
+  - api.jetkvm.com
+
+Defaults:
+  LOCAL_BASE=http://localhost:3000
+  PROD_BASE=https://api.jetkvm.com
+  device_ids=(compare-device-1)
+
+Environment overrides:
+  LOCAL_BASE              Override local host
+  PROD_BASE               Override production host
+  CURL_TIMEOUT            Curl max time in seconds (default: 30)
+  CURL_CONNECT_TIMEOUT    Curl connect timeout in seconds (default: 10)
+  FAIL_FAST               Stop after first failed case (default: true)
+
+Examples:
+  scripts/compare-releases.sh
+  scripts/compare-releases.sh device-a device-b
+  LOCAL_BASE=http://localhost:3001 PROD_BASE=https://api.jetkvm.com scripts/compare-releases.sh
+EOF
+}
+
+if [[ "${1:-}" == "--help" || "${1:-}" == "-h" ]]; then
+  print_usage
+  exit 0
+fi
+
+if (($# > 0)); then
+  DEVICE_IDS=("$@")
+else
+  DEVICE_IDS=("${DEFAULT_DEVICE_IDS[@]}")
+fi
+
+CURL_TIMEOUT="${CURL_TIMEOUT:-30}"
+CURL_CONNECT_TIMEOUT="${CURL_CONNECT_TIMEOUT:-10}"
+MAX_PARALLEL="${MAX_PARALLEL:-5}"
+RETRY_COUNT="${RETRY_COUNT:-2}"
+RETRY_DELAY_SECONDS="${RETRY_DELAY_SECONDS:-1}"
+FAIL_FAST="${FAIL_FAST:-true}"
+
+log() {
+  printf '%s\n' "$*"
+}
+
+render_progress() {
+  local completed="$1"
+  local total="$2"
+  local width="${3:-$PROGRESS_WIDTH}"
+  local filled=0
+  local empty=0
+
+  if (( total > 0 )); then
+    filled=$(( completed * width / total ))
+  fi
+  empty=$(( width - filled ))
+
+  printf '%*s' "$filled" '' | tr ' ' '#'
+  printf '%*s' "$empty" ''
+}
+
+urlencode() {
+  python3 - "$1" <<'PY'
+import sys
+from urllib.parse import quote
+
+print(quote(sys.argv[1], safe=""))
+PY
+}
+
+join_query() {
+  local -n query_keys_ref=$1
+  local -n query_values_ref=$2
+  local query=""
+  local i key value encoded
+
+  for i in "${!query_keys_ref[@]}"; do
+    key="${query_keys_ref[$i]}"
+    value="${query_values_ref[$i]}"
+    [[ "$value" == "__omit__" ]] && continue
+    encoded="$(urlencode "$value")"
+    if [[ -n "$query" ]]; then
+      query+="&"
+    fi
+    query+="${key}=${encoded}"
+  done
+
+  printf '%s' "$query"
+}
+
+header_value() {
+  local file="$1"
+  local name="$2"
+  python3 - "$file" "$name" <<'PY'
+import sys
+from pathlib import Path
+
+path = Path(sys.argv[1])
+name = sys.argv[2].lower()
+value = ""
+for raw_line in path.read_text(errors="replace").splitlines():
+    line = raw_line.strip()
+    if not line or ":" not in line:
+        continue
+    key, candidate = line.split(":", 1)
+    if key.lower() == name:
+        value = candidate.strip()
+print(value)
+PY
+}
+
+normalize_body() {
+  local body_file="$1"
+  local normalized_file="$2"
+  if [[ ! -f "$body_file" ]]; then
+    : >"$normalized_file"
+    return
+  fi
+  python3 - "$body_file" "$normalized_file" <<'PY'
+import json
+import sys
+from pathlib import Path
+
+body_path = Path(sys.argv[1])
+normalized_path = Path(sys.argv[2])
+body = body_path.read_text(errors="replace")
+
+try:
+    parsed = json.loads(body)
+except Exception:
+    normalized_path.write_text(body)
+else:
+    def scrub(value):
+        if isinstance(value, dict):
+            return {
+                key: scrub(child)
+                for key, child in value.items()
+                if not key.endswith("CachedAt")
+            }
+        if isinstance(value, list):
+            return [scrub(item) for item in value]
+        return value
+
+    normalized_path.write_text(json.dumps(scrub(parsed), indent=2, sort_keys=True) + "\n")
+PY
+}
+
+summarize_body_mismatch() {
+  local left_file="$1"
+  local right_file="$2"
+  python3 - "$left_file" "$right_file" <<'PY'
+import json
+import sys
+from pathlib import Path
+
+left_path = Path(sys.argv[1])
+right_path = Path(sys.argv[2])
+
+def load(path):
+    try:
+        return json.loads(path.read_text(errors="replace"))
+    except Exception:
+        return path.read_text(errors="replace")
+
+left = load(left_path)
+right = load(right_path)
+
+def walk(a, b, path="$"):
+    if type(a) != type(b):
+        return path, a, b
+    if isinstance(a, dict):
+        keys = sorted(set(a) | set(b))
+        for key in keys:
+            if key not in a:
+                return f"{path}.{key}", "<missing>", b[key]
+            if key not in b:
+                return f"{path}.{key}", a[key], "<missing>"
+            result = walk(a[key], b[key], f"{path}.{key}")
+            if result is not None:
+                return result
+        return None
+    if isinstance(a, list):
+        if len(a) != len(b):
+            return f"{path}.length", len(a), len(b)
+        for idx, (av, bv) in enumerate(zip(a, b)):
+            result = walk(av, bv, f"{path}[{idx}]")
+            if result is not None:
+                return result
+        return None
+    if a != b:
+        return path, a, b
+    return None
+
+result = walk(left, right)
+if result is None:
+    print("values differ")
+else:
+    path, left_value, right_value = result
+    print(f"path={path}")
+    print(f"local={json.dumps(left_value, sort_keys=True)}")
+    print(f"prod={json.dumps(right_value, sort_keys=True)}")
+PY
+}
+
+body_diff_is_version_only_not_found() {
+  local left_file="$1"
+  local right_file="$2"
+  python3 - "$left_file" "$right_file" <<'PY'
+import json
+import re
+import sys
+from pathlib import Path
+
+try:
+    left = json.loads(Path(sys.argv[1]).read_text(errors="replace"))
+    right = json.loads(Path(sys.argv[2]).read_text(errors="replace"))
+except Exception:
+    raise SystemExit(1)
+
+if not (isinstance(left, dict) and isinstance(right, dict)):
+    raise SystemExit(1)
+
+if left.get("name") != "NotFoundError" or right.get("name") != "NotFoundError":
+    raise SystemExit(1)
+
+left_keys = set(left.keys())
+right_keys = set(right.keys())
+if left_keys != {"name", "message"} or right_keys != {"name", "message"}:
+    raise SystemExit(1)
+
+# Both messages mean "no release compatible with this SKU is available";
+# the wording differs across deploys but the device sees the same 404.
+no_compat_patterns = [
+    re.compile(r'^Version .+ predates SKU support and cannot serve SKU "([^"]+)"$'),
+    re.compile(r'^No default (?:app|system) release available for SKU "([^"]+)"$'),
+]
+
+def canonicalize(message):
+    for pattern in no_compat_patterns:
+        match = pattern.match(message)
+        if match:
+            return f'<no-compat-release sku="{match.group(1)}">'
+    return message
+
+left_message = left.get("message", "")
+right_message = right.get("message", "")
+
+if canonicalize(left_message) == canonicalize(right_message):
+    raise SystemExit(0)
+
+raise SystemExit(1)
+PY
+}
+
+is_accepted_deviation() {
+  local query="$1"
+  local left_prefix="$2"
+  local right_prefix="$3"
+  python3 - "$query" "${left_prefix}.meta" "${right_prefix}.meta" "${left_prefix}.normalized" "${right_prefix}.normalized" <<'PY'
+import json
+import sys
+from pathlib import Path
+from urllib.parse import parse_qs
+
+query, left_meta_path, right_meta_path, left_body_path, right_body_path = sys.argv[1:]
+params = parse_qs(query, keep_blank_values=True)
+
+def one(name):
+    values = params.get(name, [])
+    return values[0] if values else None
+
+def parse_meta(path):
+    data = {}
+    for line in Path(path).read_text(errors="replace").splitlines():
+        if "=" in line:
+            key, value = line.split("=", 1)
+            data[key] = value
+    return data
+
+def load_json(path):
+    try:
+        return json.loads(Path(path).read_text(errors="replace"))
+    except Exception:
+        return None
+
+left_meta = parse_meta(left_meta_path)
+right_meta = parse_meta(right_meta_path)
+left_body = load_json(left_body_path)
+right_body = load_json(right_body_path)
+
+# Accepted behavior change:
+# Stable requests with prerelease/dev version constraints are DB-only locally.
+# Production still resolves those directly from S3. Local 404 vs prod 200 is expected.
+if one("prerelease") not in (None, "false"):
+    raise SystemExit(1)
+
+constrained_versions = [one("appVersion"), one("systemVersion")]
+has_dev_constraint = any(value and "-" in value for value in constrained_versions)
+if not has_dev_constraint:
+    raise SystemExit(1)
+
+if left_meta.get("http_code") != "404" or right_meta.get("http_code") != "200":
+    raise SystemExit(1)
+
+if not isinstance(left_body, dict) or left_body.get("name") != "NotFoundError":
+    raise SystemExit(1)
+
+if not isinstance(right_body, dict) or not right_body.get("appVersion") or not right_body.get("systemVersion"):
+    raise SystemExit(1)
+
+raise SystemExit(0)
+PY
+}
+
+curl_capture() {
+  local base_url="$1"
+  local path="$2"
+  local query="$3"
+  local prefix="$4"
+  local url="${base_url}${path}"
+  local headers_file="${prefix}.headers"
+  local body_file="${prefix}.body"
+  local meta_file="${prefix}.meta"
+  local stderr_file="${prefix}.stderr"
+  local exit_file="${prefix}.exit"
+  local attempt=0
+  local curl_exit=0
+  local http_code=""
+
+  if [[ -n "$query" ]]; then
+    url="${url}?${query}"
+  fi
+
+  while :; do
+    : >"$headers_file"
+    : >"$body_file"
+    : >"$meta_file"
+    : >"$stderr_file"
+
+    curl_exit=0
+    curl \
+      --silent \
+      --show-error \
+      --connect-timeout "$CURL_CONNECT_TIMEOUT" \
+      --max-time "$CURL_TIMEOUT" \
+      --dump-header "$headers_file" \
+      --output "$body_file" \
+      --write-out "http_code=%{http_code}\ncontent_type=%{content_type}\n" \
+      "$url" >"$meta_file" 2>"$stderr_file" || curl_exit=$?
+
+    printf '%s\n' "$curl_exit" >"$exit_file"
+    http_code="$(sed -n 's/^http_code=//p' "$meta_file")"
+
+    if (( curl_exit == 0 )) && [[ ! "$http_code" =~ ^52[0-9]$ ]]; then
+      break
+    fi
+
+    if (( attempt >= RETRY_COUNT )); then
+      break
+    fi
+
+    attempt=$((attempt + 1))
+    sleep "$RETRY_DELAY_SECONDS"
+  done
+}
+
+compare_scalar_files() {
+  local label="$1"
+  local left_file="$2"
+  local right_file="$3"
+  local left_value right_value
+
+  left_value="$(tr -d '\r' <"$left_file")"
+  right_value="$(tr -d '\r' <"$right_file")"
+  if [[ "$left_value" == "$right_value" ]]; then
+    return 1
+  fi
+
+  printf '%s\n' "$label"
+  printf '    local=%s\n' "${left_value:-<empty>}"
+  printf '    prod=%s\n' "${right_value:-<empty>}"
+  return 0
+}
+
+summarize_meta_mismatch() {
+  local left_file="$1"
+  local right_file="$2"
+  python3 - "$left_file" "$right_file" <<'PY'
+import sys
+from pathlib import Path
+
+def parse(path_str):
+    data = {}
+    for line in Path(path_str).read_text(errors="replace").splitlines():
+        if "=" not in line:
+            continue
+        key, value = line.split("=", 1)
+        data[key] = value
+    return data
+
+left = parse(sys.argv[1])
+right = parse(sys.argv[2])
+keys = sorted(set(left) | set(right))
+for key in keys:
+    if left.get(key) != right.get(key):
+        print(f"{key}")
+        print(f"local={left.get(key, '<missing>')}")
+        print(f"prod={right.get(key, '<missing>')}")
+PY
+}
+
+write_case_result() {
+  local result_file="$1"
+  local case_name="$2"
+  local path="$3"
+  local query="$4"
+  local left_prefix="$5"
+  local right_prefix="$6"
+  local left_norm="${left_prefix}.normalized"
+  local right_norm="${right_prefix}.normalized"
+  local left_location right_location
+  local failed=0
+  local details=""
+  local mismatch_count=0
+  local output=""
+  local accepted_reason=""
+
+  normalize_body "${left_prefix}.body" "$left_norm"
+  normalize_body "${right_prefix}.body" "$right_norm"
+
+  if output="$(compare_scalar_files "exit-code mismatch" "${left_prefix}.exit" "${right_prefix}.exit")"; then
+    mismatch_count=$((mismatch_count + 1))
+    details+="$output"$'\n'
+    failed=1
+  fi
+
+  if output="$(summarize_meta_mismatch "${left_prefix}.meta" "${right_prefix}.meta")" && [[ -n "$output" ]]; then
+    mismatch_count=$((mismatch_count + 1))
+    details+=$'  response-meta mismatch\n'
+    details+="$(printf '%s\n' "$output" | sed 's/^/    /')"$'\n'
+    failed=1
+  fi
+
+  left_location="$(header_value "${left_prefix}.headers" "location")"
+  right_location="$(header_value "${right_prefix}.headers" "location")"
+  if [[ "$left_location" != "$right_location" ]]; then
+    mismatch_count=$((mismatch_count + 1))
+    details+=$'  location mismatch\n'
+    details+="    local=${left_location:-<none>}"$'\n'
+    details+="    prod=${right_location:-<none>}"$'\n'
+    failed=1
+  fi
+
+  if [[ -s "${left_prefix}.body" || -s "${right_prefix}.body" ]]; then
+    if ! cmp -s "$left_norm" "$right_norm"; then
+      if body_diff_is_version_only_not_found "$left_norm" "$right_norm"; then
+        :
+      else
+        mismatch_count=$((mismatch_count + 1))
+        details+=$'  body mismatch\n'
+        details+="$(summarize_body_mismatch "$left_norm" "$right_norm" | sed 's/^/    /')"$'\n'
+        failed=1
+      fi
+    fi
+  fi
+
+  if (( failed == 1 )) && is_accepted_deviation "$query" "$left_prefix" "$right_prefix"; then
+    failed=0
+    accepted_reason="stable dev/prerelease version constraints are DB-only locally"
+    details=""
+    mismatch_count=0
+  fi
+
+  {
+    printf 'status=%s\n' "$([[ $failed -eq 0 ]] && { [[ -n "$accepted_reason" ]] && printf accepted || printf pass; } || printf fail)"
+    printf 'case_name=%s\n' "$case_name"
+    printf 'path=%s\n' "$path"
+    printf 'query=%s\n' "$query"
+    printf 'accepted_reason=%s\n' "$accepted_reason"
+    printf 'mismatch_count=%s\n' "$mismatch_count"
+    printf 'details<<__DETAILS__\n%s__DETAILS__\n' "$details"
+    printf 'local_stderr<<__STDERR__\n%s__STDERR__\n' "$(tr '\n' ' ' <"${left_prefix}.stderr")"
+    printf 'prod_stderr<<__STDERR__\n%s__STDERR__\n' "$(tr '\n' ' ' <"${right_prefix}.stderr")"
+  } >"$result_file"
+}
+
+run_case_worker() {
+  local case_name="$1"
+  local path="$2"
+  local query="$3"
+  local safe_case="$4"
+  local result_file="$5"
+
+  local local_prefix="$TMP_DIR/${safe_case}.local"
+  local prod_prefix="$TMP_DIR/${safe_case}.prod"
+
+  curl_capture "$LOCAL_BASE" "$path" "$query" "$local_prefix" &
+  local local_pid=$!
+  curl_capture "$PROD_BASE" "$path" "$query" "$prod_prefix" &
+  local prod_pid=$!
+  wait "$local_pid"
+  wait "$prod_pid"
+
+  write_case_result "$result_file" "$case_name" "$path" "$query" "$local_prefix" "$prod_prefix"
+}
+
+print_case_result() {
+  local result_file="$1"
+  local progress_bar
+  local status case_name path query accepted_reason mismatch_count details local_stderr prod_stderr
+
+  progress_bar="$(render_progress "$CASE_INDEX" "$TOTAL_CASES")"
+  status="$(sed -n 's/^status=//p' "$result_file")"
+  case_name="$(sed -n 's/^case_name=//p' "$result_file")"
+  path="$(sed -n 's/^path=//p' "$result_file")"
+  query="$(sed -n 's/^query=//p' "$result_file")"
+  accepted_reason="$(sed -n 's/^accepted_reason=//p' "$result_file")"
+  mismatch_count="$(sed -n 's/^mismatch_count=//p' "$result_file")"
+  details="$(awk '/^details<<__DETAILS__/{flag=1;next}/^__DETAILS__$/{flag=0}flag' "$result_file")"
+  local_stderr="$(awk '/^local_stderr<<__STDERR__/{flag=1;next}/^__STDERR__$/{if(flag){flag=0; exit}}flag' "$result_file")"
+  prod_stderr="$(awk 'found && /^__STDERR__$/ {exit} /^prod_stderr<<__STDERR__$/ {found=1; next} found {print}' "$result_file")"
+
+  if [[ "$status" == "pass" ]]; then
+    PASS_COUNT=$((PASS_COUNT + 1))
+    printf '\r[%s] %4d/%-4d | pass:%d fail:%d' \
+      "$progress_bar" "$CASE_INDEX" "$TOTAL_CASES" "$PASS_COUNT" "$FAIL_COUNT"
+    if (( CASE_INDEX == TOTAL_CASES )); then
+      printf '\n'
+    fi
+  elif [[ "$status" == "accepted" ]]; then
+    ACCEPTED_COUNT=$((ACCEPTED_COUNT + 1))
+    printf '\r\033[K'
+    printf '[%s] %4d/%-4d | pass:%d accepted:%d fail:%d\n' \
+      "$progress_bar" "$CASE_INDEX" "$TOTAL_CASES" "$PASS_COUNT" "$ACCEPTED_COUNT" "$FAIL_COUNT"
+    printf '  ACCEPT %s\n' "$case_name"
+    printf '  %s%s%s\n' "$path" "${query:+?$query}" ""
+    printf '  %s\n' "$accepted_reason"
+  else
+    FAIL_COUNT=$((FAIL_COUNT + 1))
+    printf '\r\033[K'
+    printf '[%s] %4d/%-4d | pass:%d fail:%d\n' \
+      "$progress_bar" "$CASE_INDEX" "$TOTAL_CASES" "$PASS_COUNT" "$FAIL_COUNT"
+    printf '  FAIL %s\n' "$case_name"
+    printf '  %s%s%s\n' "$path" "${query:+?$query}" ""
+    printf '%s' "$details"
+    if [[ -n "$local_stderr" || -n "$prod_stderr" ]]; then
+      printf '  stderr\n'
+      printf '    local=%s\n' "$local_stderr"
+      printf '    prod=%s\n' "$prod_stderr"
+    fi
+    if [[ "${mismatch_count:-0}" == "0" ]]; then
+      printf '  mismatch detected\n'
+    fi
+  fi
+}
+
+stop_requested() {
+  [[ "$FAIL_FAST" != "false" && "$FAIL_COUNT" -gt 0 ]]
+}
+
+wait_for_one_job() {
+  local pid done_pid result_file
+  while :; do
+    for pid in "${!JOB_RESULT_FILES[@]}"; do
+      if ! kill -0 "$pid" 2>/dev/null; then
+        wait "$pid" || true
+        done_pid="$pid"
+        result_file="${JOB_RESULT_FILES[$pid]}"
+        unset "JOB_RESULT_FILES[$pid]"
+        CASE_INDEX=$((CASE_INDEX + 1))
+        print_case_result "$result_file"
+        rm -f "$result_file"
+        return
+      fi
+    done
+    sleep 0.05
+  done
+}
+
+drain_jobs() {
+  while ((${#JOB_RESULT_FILES[@]} > 0)); do
+    wait_for_one_job
+    if stop_requested; then
+      for pid in "${!JOB_RESULT_FILES[@]}"; do
+        kill "$pid" 2>/dev/null || true
+      done
+      JOB_RESULT_FILES=()
+      break
+    fi
+  done
+}
+
+extract_versions() {
+  local device_id="$1"
+  local prerelease="$2"
+  local sku="$3"
+  local prefix="$4"
+
+  local query_keys=("deviceId" "prerelease" "sku")
+  local query_values=("$device_id" "$prerelease" "$sku")
+  local query
+  query="$(join_query query_keys query_values)"
+
+  curl_capture "$PROD_BASE" "/releases" "$query" "$prefix"
+
+  python3 - "$prefix.body" <<'PY'
+import json
+import sys
+from pathlib import Path
+
+path = Path(sys.argv[1])
+try:
+    payload = json.loads(path.read_text(errors="replace"))
+except Exception:
+    print("")
+    print("")
+    raise SystemExit(0)
+
+print(payload.get("appVersion", ""))
+print(payload.get("systemVersion", ""))
+PY
+}
+
+build_value_set() {
+  local exact_version="$1"
+  local prerelease_version="$2"
+  local values=("__omit__" "*")
+
+  if [[ -n "$exact_version" ]]; then
+    values+=("$exact_version")
+  fi
+  if [[ -n "$prerelease_version" && "$prerelease_version" != "$exact_version" ]]; then
+    values+=("$prerelease_version")
+  fi
+
+  printf '%s\n' "${values[@]}" | awk '!seen[$0]++'
+}
+
+run_case() {
+  local case_name="$1"
+  local path="$2"
+  local -n case_keys_ref=$3
+  local -n case_values_ref=$4
+  local query
+
+  CASE_COUNT=$((CASE_COUNT + 1))
+  query="$(join_query case_keys_ref case_values_ref)"
+
+  local safe_case
+  safe_case="$(printf '%s' "$case_name" | tr ' /?=&' '_____')"
+  local result_file="$TMP_DIR/${safe_case}.result"
+
+  run_case_worker "$case_name" "$path" "$query" "$safe_case" "$result_file" &
+  JOB_RESULT_FILES[$!]="$result_file"
+
+  while ((${#JOB_RESULT_FILES[@]} >= MAX_PARALLEL)); do
+    wait_for_one_job
+    if stop_requested; then
+      return
+    fi
+  done
+}
+
+log "Comparing release endpoints"
+log "  local:   $LOCAL_BASE"
+log "  prod:    $PROD_BASE"
+log "  deviceIds: ${DEVICE_IDS[*]}"
+
+mapfile -t stable_versions < <(extract_versions "${DEVICE_IDS[0]}" "__omit__" "__omit__" "$TMP_DIR/baseline-stable")
+mapfile -t prerelease_versions < <(extract_versions "${DEVICE_IDS[0]}" "true" "__omit__" "$TMP_DIR/baseline-prerelease")
+
+STABLE_APP_VERSION="${stable_versions[0]:-}"
+STABLE_SYSTEM_VERSION="${stable_versions[1]:-}"
+PRERELEASE_APP_VERSION="${prerelease_versions[0]:-}"
+PRERELEASE_SYSTEM_VERSION="${prerelease_versions[1]:-}"
+
+mapfile -t APP_VERSION_VALUES < <(build_value_set "$STABLE_APP_VERSION" "$PRERELEASE_APP_VERSION")
+mapfile -t SYSTEM_VERSION_VALUES < <(build_value_set "$STABLE_SYSTEM_VERSION" "$PRERELEASE_SYSTEM_VERSION")
+
+TOTAL_CASES=$(( ${#DEVICE_IDS[@]} * ${#TRISTATE_VALUES[@]} * ${#APP_VERSION_VALUES[@]} * ${#SYSTEM_VERSION_VALUES[@]} * ${#DEFAULT_SKUS[@]} + ${#TRISTATE_VALUES[@]} * ${#DEFAULT_SKUS[@]} * 2 ))
+declare -A JOB_RESULT_FILES=()
+log "  total cases: $TOTAL_CASES"
+log "  parallel: $MAX_PARALLEL"
+log "  failFast: $FAIL_FAST"
+log
+
+for device_id in "${DEVICE_IDS[@]}"; do
+  for prerelease in "${TRISTATE_VALUES[@]}"; do
+    for app_version in "${APP_VERSION_VALUES[@]}"; do
+      for system_version in "${SYSTEM_VERSION_VALUES[@]}"; do
+        for sku in "${DEFAULT_SKUS[@]}"; do
+          if stop_requested; then
+            break 5
+          fi
+          query_keys=("deviceId" "prerelease" "appVersion" "systemVersion" "sku")
+          query_values=("$device_id" "$prerelease" "$app_version" "$system_version" "$sku")
+          run_case \
+            "GET /releases deviceId=$device_id prerelease=$prerelease appVersion=$app_version systemVersion=$system_version sku=$sku" \
+            "/releases" \
+            query_keys \
+            query_values
+        done
+      done
+    done
+  done
+done
+
+for prerelease in "${TRISTATE_VALUES[@]}"; do
+  for sku in "${DEFAULT_SKUS[@]}"; do
+    if stop_requested; then
+      break 2
+    fi
+    query_keys=("prerelease" "sku")
+    query_values=("$prerelease" "$sku")
+    run_case \
+      "GET /releases/app/latest prerelease=$prerelease sku=$sku" \
+      "/releases/app/latest" \
+      query_keys \
+      query_values
+    run_case \
+      "GET /releases/system_recovery/latest prerelease=$prerelease sku=$sku" \
+      "/releases/system_recovery/latest" \
+      query_keys \
+      query_values
+  done
+done
+
+drain_jobs
+
+log
+log "Summary"
+log "  cases:  $CASE_COUNT"
+log "  pass:   $PASS_COUNT"
+log "  accept: $ACCEPTED_COUNT"
+log "  fail:   $FAIL_COUNT"
+
+if ((FAIL_COUNT > 0)); then
+  exit 1
+fi

scripts/seed.ts

@@ -2,6 +2,16 @@ import { PrismaClient } from "@prisma/client";
 
 const prisma = new PrismaClient();
 
+type ReleaseType = "app" | "system";
+
+// Pre-SKU artifacts are jetkvm-v2 only; future SKUs need explicit
+// skus/<sku>/ uploads, registered via scripts/sync-releases.ts.
+const LEGACY_COMPATIBLE_SKUS = ["jetkvm-v2"];
+
+function compatibleSkusForRelease(_type: ReleaseType): string[] {
+  return LEGACY_COMPATIBLE_SKUS;
+}
+
 // Development test users
 const users = [
   { googleId: "dev-user-1", email: "dev@example.com", picture: null },
@@ -23,7 +33,16 @@ const turnActivities = [
 ];
 
 // Production release snapshot
-const releases = [
+interface SeedRelease {
+  version: string;
+  type: ReleaseType;
+  rolloutPercentage: number;
+  url: string;
+  hash: string;
+  createdAt: Date;
+}
+
+const releases: SeedRelease[] = [
   { version: "0.2.6", type: "app", rolloutPercentage: 100, url: "https://update.jetkvm.com/app/0.2.6/jetkvm_app", hash: "4b121195aa9dae9bd4ae7d1e69f49383510f9552cd9a9edd1a9f92c71e128f9c", createdAt: new Date("2024-09-27T11:41:59.669Z") },
   { version: "0.2.7", type: "app", rolloutPercentage: 100, url: "https://update.jetkvm.com/app/0.2.7/jetkvm_app", hash: "2dbcc5a7bc1cc7196b458e633f654b521351eda66764b7a6d6a04f60a17347ca", createdAt: new Date("2024-09-27T11:59:32.279Z") },
   { version: "0.1.7", type: "system", rolloutPercentage: 100, url: "https://update.jetkvm.com/system/0.1.7/system.tar", hash: "194287cf911801852cdc57aa9e8c9cfa59bf6c27feb5ae260f35bcfa895789e3", createdAt: new Date("2024-10-01T20:00:03.780Z") },
@@ -154,7 +173,20 @@ async function seedReleases(): Promise<void> {
     return;
   }
 
-  await prisma.release.createMany({ data: releases });
+  for (const release of releases) {
+    await prisma.release.create({
+      data: {
+        ...release,
+        artifacts: {
+          create: {
+            url: release.url,
+            hash: release.hash,
+            compatibleSkus: compatibleSkusForRelease(release.type),
+          },
+        },
+      },
+    });
+  }
   console.log(`[seed] Release: created ${releases.length} records`);
 }
 

scripts/sync-releases.ts

@@ -0,0 +1,276 @@
+import {
+  GetObjectCommand,
+  HeadObjectCommand,
+  ListObjectsV2Command,
+  S3Client,
+} from "@aws-sdk/client-s3";
+import { PrismaClient } from "@prisma/client";
+import semver from "semver";
+
+import { streamToString } from "../src/helpers";
+
+type ReleaseType = "app" | "system";
+
+const DEFAULT_SKU = "jetkvm-v2";
+const KNOWN_SKUS = ["jetkvm-v2", "jetkvm-v2-sdmmc"];
+
+interface SyncClients {
+  prisma: PrismaClient;
+  s3Client: S3Client;
+}
+
+interface SyncConfig {
+  bucketName: string;
+  baseUrl: string;
+  skus?: string[];
+}
+
+interface ReleaseArtifactInput {
+  url: string;
+  hash: string;
+  compatibleSkus: string[];
+}
+
+function artifactName(type: ReleaseType): string {
+  return type === "app" ? "jetkvm_app" : "system.tar";
+}
+
+// Pre-SKU artifacts (no skus/ folder) are only safe on the original jetkvm-v2.
+// Other SKUs require an explicit skus/<sku>/ upload to opt in.
+function legacyCompatibleSkus(): string[] {
+  return [DEFAULT_SKU];
+}
+
+function isS3NotFound(error: any): boolean {
+  return (
+    error.name === "NotFound" ||
+    error.name === "NoSuchKey" ||
+    error.$metadata?.httpStatusCode === 404
+  );
+}
+
+async function s3ObjectExists(
+  s3Client: S3Client,
+  bucketName: string,
+  key: string,
+): Promise<boolean> {
+  try {
+    await s3Client.send(new HeadObjectCommand({ Bucket: bucketName, Key: key }));
+    return true;
+  } catch (error: any) {
+    if (isS3NotFound(error)) {
+      return false;
+    }
+    throw error;
+  }
+}
+
+async function versionHasSkuSupport(
+  s3Client: S3Client,
+  bucketName: string,
+  type: ReleaseType,
+  version: string,
+): Promise<boolean> {
+  const response = await s3Client.send(
+    new ListObjectsV2Command({
+      Bucket: bucketName,
+      Prefix: `${type}/${version}/skus/`,
+      MaxKeys: 1,
+    }),
+  );
+  return (response.Contents?.length ?? 0) > 0;
+}
+
+async function readHash(
+  s3Client: S3Client,
+  bucketName: string,
+  artifactPath: string,
+): Promise<string | undefined> {
+  try {
+    const response = await s3Client.send(
+      new GetObjectCommand({
+        Bucket: bucketName,
+        Key: `${artifactPath}.sha256`,
+      }),
+    );
+    return streamToString(response.Body);
+  } catch (error: any) {
+    if (isS3NotFound(error)) {
+      return undefined;
+    }
+    throw error;
+  }
+}
+
+function addArtifact(
+  artifactsByUrl: Map<string, ReleaseArtifactInput>,
+  url: string,
+  hash: string,
+  sku: string,
+): void {
+  const artifact = artifactsByUrl.get(url);
+  if (artifact) {
+    if (!artifact.compatibleSkus.includes(sku)) {
+      artifact.compatibleSkus.push(sku);
+    }
+    return;
+  }
+
+  artifactsByUrl.set(url, { url, hash, compatibleSkus: [sku] });
+}
+
+export async function collectReleaseArtifacts(
+  clients: Pick<SyncClients, "s3Client">,
+  config: SyncConfig,
+  type: ReleaseType,
+  version: string,
+): Promise<ReleaseArtifactInput[]> {
+  const skus = config.skus ?? KNOWN_SKUS;
+  const artifactFileName = artifactName(type);
+
+  if (!(await versionHasSkuSupport(clients.s3Client, config.bucketName, type, version))) {
+    const artifactPath = `${type}/${version}/${artifactFileName}`;
+    const hash = await readHash(clients.s3Client, config.bucketName, artifactPath);
+    if (!hash) {
+      return [];
+    }
+
+    return [
+      {
+        url: `${config.baseUrl}/${artifactPath}`,
+        hash,
+        compatibleSkus: legacyCompatibleSkus(),
+      },
+    ];
+  }
+
+  const artifactsByUrl = new Map<string, ReleaseArtifactInput>();
+  for (const sku of skus) {
+    const artifactPath = `${type}/${version}/skus/${sku}/${artifactFileName}`;
+    if (!(await s3ObjectExists(clients.s3Client, config.bucketName, artifactPath))) {
+      continue;
+    }
+
+    const hash = await readHash(clients.s3Client, config.bucketName, artifactPath);
+    if (!hash) {
+      continue;
+    }
+    addArtifact(artifactsByUrl, `${config.baseUrl}/${artifactPath}`, hash, sku);
+  }
+
+  return Array.from(artifactsByUrl.values());
+}
+
+async function listStableVersions(
+  s3Client: S3Client,
+  bucketName: string,
+  type: ReleaseType,
+): Promise<string[]> {
+  const response = await s3Client.send(
+    new ListObjectsV2Command({
+      Bucket: bucketName,
+      Prefix: `${type}/`,
+      Delimiter: "/",
+    }),
+  );
+
+  return (response.CommonPrefixes ?? [])
+    .map(cp => cp.Prefix?.split("/")[1])
+    .filter((version): version is string => Boolean(version))
+    .filter(
+      version => Boolean(semver.valid(version)) && semver.prerelease(version) === null,
+    )
+    .sort(semver.compare);
+}
+
+async function syncRelease(
+  prisma: PrismaClient,
+  type: ReleaseType,
+  version: string,
+  artifacts: ReleaseArtifactInput[],
+): Promise<void> {
+  if (artifacts.length === 0) {
+    console.log(`[sync-releases] ${type} ${version}: skipped, no compatible artifacts`);
+    return;
+  }
+
+  // Sync only registers brand-new releases. Existing rows (rollout state, URLs,
+  // artifact compatibility) are left untouched — backfills/repairs are handled
+  // by one-off scripts so a routine sync run can never rewrite production data.
+  const existing = await prisma.release.findUnique({
+    where: { version_type: { version, type } },
+    select: { id: true },
+  });
+
+  if (existing) {
+    console.log(`[sync-releases] ${type} ${version}: already synced, skipping`);
+    return;
+  }
+
+  const primaryArtifact = artifacts[0];
+  await prisma.release.create({
+    data: {
+      version,
+      type,
+      rolloutPercentage: 10,
+      url: primaryArtifact.url,
+      hash: primaryArtifact.hash,
+      artifacts: {
+        create: artifacts.map(artifact => ({
+          url: artifact.url,
+          hash: artifact.hash,
+          compatibleSkus: artifact.compatibleSkus,
+        })),
+      },
+    },
+  });
+
+  console.log(
+    `[sync-releases] ${type} ${version}: created with ${artifacts.length} artifact(s)`,
+  );
+}
+
+export async function syncReleases(
+  clients: SyncClients,
+  config: SyncConfig,
+): Promise<void> {
+  for (const type of ["app", "system"] as const) {
+    const versions = await listStableVersions(clients.s3Client, config.bucketName, type);
+
+    for (const version of versions) {
+      const artifacts = await collectReleaseArtifacts(clients, config, type, version);
+      await syncRelease(clients.prisma, type, version, artifacts);
+    }
+  }
+}
+
+async function main(): Promise<void> {
+  const prisma = new PrismaClient();
+  const s3Client = new S3Client({
+    endpoint: process.env.R2_ENDPOINT!,
+    credentials: {
+      accessKeyId: process.env.R2_ACCESS_KEY_ID!,
+      secretAccessKey: process.env.R2_SECRET_ACCESS_KEY!,
+    },
+    region: "auto",
+  });
+
+  try {
+    await syncReleases(
+      { prisma, s3Client },
+      {
+        bucketName: process.env.R2_BUCKET!,
+        baseUrl: process.env.R2_CDN_URL!,
+      },
+    );
+  } finally {
+    await prisma.$disconnect();
+  }
+}
+
+if (require.main === module) {
+  main().catch(error => {
+    console.error("[sync-releases] failed", error);
+    process.exit(1);
+  });
+}

src/releases.ts

@@ -20,6 +20,7 @@ import {
 import { z, ZodError } from "zod";
 
 const DEFAULT_SKU = "jetkvm-v2";
+type ReleaseType = "app" | "system";
 
 /** Query param schema builders for common patterns */
 const queryString = () =>
@@ -51,7 +52,7 @@ type LatestQuery = z.infer<typeof latestQuerySchema>;
 
 /**
  * Schema for the main Retrieve endpoint.
- * Requires deviceId and includes version constraints and forceUpdate flag.
+ * Requires deviceId and includes version constraints.
  */
 const retrieveQuerySchema = z.object({
   deviceId: z.string({ error: "Device ID is required" }).min(1, "Device ID is required"),
@@ -59,7 +60,6 @@ const retrieveQuerySchema = z.object({
   appVersion: queryString(),
   systemVersion: queryString(),
   sku: querySku(),
-  forceUpdate: queryBoolean(),
 });
 
 type RetrieveQuery = z.infer<typeof retrieveQuerySchema>;
@@ -87,6 +87,15 @@ export interface ReleaseMetadata {
   _maxSatisfying?: string;
 }
 
+interface DbRelease {
+  version: string;
+  rolloutPercentage: number;
+  artifacts: {
+    url: string;
+    hash: string;
+  }[];
+}
+
 const s3Client = new S3Client({
   endpoint: process.env.R2_ENDPOINT!,
   credentials: {
@@ -379,6 +388,44 @@ function toRelease(
   return release as Release;
 }
 
+function objectKeyFromArtifactUrl(artifactUrl: string): string {
+  const parsed = new URL(artifactUrl);
+  return decodeURIComponent(parsed.pathname.replace(/^\/+/, ""));
+}
+
+async function resolveSigUrlFromArtifactUrl(
+  artifactUrl: string,
+): Promise<string | undefined> {
+  const cacheKey = `artifact-url-${artifactUrl}`;
+  const cached = sigUrlCache.get(cacheKey);
+  if (cached !== undefined) return cached === MISSING_SIG_URL ? undefined : cached;
+
+  const sigUrl = `${artifactUrl}.sig`;
+  try {
+    const sigKey = `${objectKeyFromArtifactUrl(artifactUrl)}.sig`;
+    if (await s3ObjectExists(sigKey)) {
+      sigUrlCache.set(cacheKey, sigUrl);
+      return sigUrl;
+    }
+  } catch (error) {
+    console.error(`Failed to resolve sig URL for ${artifactUrl}:`, error);
+    return undefined;
+  }
+
+  sigUrlCache.set(cacheKey, MISSING_SIG_URL);
+  return undefined;
+}
+
+async function addStableSigUrls(release: Release): Promise<void> {
+  const [appSigUrl, systemSigUrl] = await Promise.all([
+    release.appUrl ? resolveSigUrlFromArtifactUrl(release.appUrl) : undefined,
+    release.systemUrl ? resolveSigUrlFromArtifactUrl(release.systemUrl) : undefined,
+  ]);
+
+  if (appSigUrl) release.appSigUrl = appSigUrl;
+  if (systemSigUrl) release.systemSigUrl = systemSigUrl;
+}
+
 async function getReleaseFromS3(
   includePrerelease: boolean,
   {
@@ -403,32 +450,117 @@ async function isDeviceEligibleForLatestRelease(
   return getDeviceRolloutBucket(deviceId) < rolloutPercentage;
 }
 
-async function getDefaultRelease(type: "app" | "system") {
+function compatibleArtifactSelect(sku: string) {
+  return {
+    where: { compatibleSkus: { has: sku } },
+    select: { url: true, hash: true },
+    orderBy: { id: "asc" as const },
+    take: 1,
+  };
+}
+
+function compatibleReleaseSelect(sku: string) {
+  return {
+    version: true,
+    rolloutPercentage: true,
+    artifacts: compatibleArtifactSelect(sku),
+  } as const;
+}
+
+function dbReleaseToMetadata(
+  release: DbRelease,
+  sku: string,
+  maxSatisfying?: string,
+): ReleaseMetadata {
+  const artifact = release.artifacts[0];
+  if (!artifact) {
+    throw new NotFoundError(
+      `Version ${release.version} predates SKU support and cannot serve SKU "${sku}"`,
+    );
+  }
+
+  return {
+    version: release.version,
+    url: artifact.url,
+    hash: artifact.hash,
+    _maxSatisfying: maxSatisfying,
+  };
+}
+
+async function getDefaultRelease(type: ReleaseType, sku: string): Promise<DbRelease> {
   const rolledOutReleases = await prisma.release.findMany({
-    where: { rolloutPercentage: 100, type },
-    select: { version: true, url: true, hash: true },
+    where: { type, rolloutPercentage: 100 },
+    select: compatibleReleaseSelect(sku),
   });
 
   if (rolledOutReleases.length === 0) {
-    throw new InternalServerError(`No default release found for type ${type}`);
+    throw new InternalServerError(
+      `No default release found for type ${type} and SKU "${sku}"`,
+    );
+  }
+
+  // Only consider releases that ship a binary for this SKU. Without this,
+  // the newest 100%-rolled-out release wins even if it has no compatible
+  // artifact, masking older releases that do.
+  const compatibleReleases = rolledOutReleases.filter(r => r.artifacts.length > 0);
+
+  if (compatibleReleases.length === 0) {
+    throw new NotFoundError(
+      `No default ${type} release available for SKU "${sku}"`,
+    );
   }
 
-  // Get the latest default version from the rolled out releases
   const latestVersion = semver.maxSatisfying(
-    rolledOutReleases.map(r => r.version),
+    compatibleReleases.map(r => r.version),
     "*",
   ) as string;
 
-  // Get the release with the latest default version
-  const latestDefaultRelease = rolledOutReleases.find(r => r.version === latestVersion);
+  const latestDefaultRelease = compatibleReleases.find(r => r.version === latestVersion);
 
   if (!latestDefaultRelease) {
-    throw new InternalServerError(`No default release found for type ${type}`);
+    throw new InternalServerError(
+      `No default release found for type ${type} and SKU "${sku}"`,
+    );
   }
 
   return latestDefaultRelease;
 }
 
+async function getLatestRelease(type: ReleaseType, sku: string): Promise<DbRelease> {
+  return getReleaseByRange(type, sku, "*");
+}
+
+async function getReleaseByRange(
+  type: ReleaseType,
+  sku: string,
+  range: string,
+): Promise<DbRelease> {
+  const releases = await prisma.release.findMany({
+    where: { type },
+    select: compatibleReleaseSelect(sku),
+  });
+
+  if (releases.length === 0) {
+    throw new NotFoundError(`No release found for type ${type} and SKU "${sku}"`);
+  }
+
+  const latestVersion = semver.maxSatisfying(
+    releases.map(r => r.version),
+    range,
+  ) as string;
+
+  if (!latestVersion) {
+    throw new NotFoundError(`No ${type} release found that satisfies ${range}`);
+  }
+
+  const latestRelease = releases.find(r => r.version === latestVersion);
+  if (!latestRelease) {
+    throw new NotFoundError(`No ${type} release found that satisfies ${range}`);
+  }
+
+  return latestRelease;
+}
+
 export async function Retrieve(req: Request, res: Response) {
   const query = parseQuery(retrieveQuerySchema, req);
 
@@ -436,96 +568,84 @@ export async function Retrieve(req: Request, res: Response) {
   const systemVersion = toSemverRange(query.systemVersion);
   const skipRollout = appVersion !== "*" || systemVersion !== "*";
 
-  // Get the latest release from S3
-  let remoteRelease: Release;
-  try {
-    remoteRelease = await getReleaseFromS3(query.prerelease, {
-      appVersion,
-      systemVersion,
-      sku: query.sku,
-    });
-  } catch (error) {
-    console.error(error);
-    if (error instanceof NotFoundError) {
-      throw error;
+  // Prereleases are not imported into the DB by the stable sync script.
+  if (query.prerelease) {
+    let remoteRelease: Release;
+    try {
+      remoteRelease = await getReleaseFromS3(query.prerelease, {
+        appVersion,
+        systemVersion,
+        sku: query.sku,
+      });
+    } catch (error) {
+      console.error(error);
+      if (error instanceof NotFoundError) {
+        throw error;
+      }
+      throw new InternalServerError(`Failed to get the latest release from S3: ${error}`);
     }
-    throw new InternalServerError(`Failed to get the latest release from S3: ${error}`);
-  }
 
-  // If the request is for prereleases, ignore the rollout percentage and just return the latest release
-  // This is useful for the OTA updater to get the latest prerelease version
-  // This also prevents us from storing the rollout percentage for prerelease versions
-
-  // If the version isn't a wildcard, we skip the rollout percentage check
-  if (query.prerelease || skipRollout) {
     await enrichWithSigUrls(remoteRelease, query.sku);
     return res.json(remoteRelease);
   }
 
-  // Fetch or create the latest app release
-  const latestAppRelease = await prisma.release.upsert({
-    where: { version_type: { version: remoteRelease.appVersion, type: "app" } },
-    update: {},
-    create: {
-      version: remoteRelease.appVersion,
-      rolloutPercentage: 10,
-      url: remoteRelease.appUrl,
-      type: "app",
-      hash: remoteRelease.appHash,
-    },
-    select: { version: true, url: true, rolloutPercentage: true, hash: true },
-  });
-
-  // Fetch or create the latest system release
-  const latestSystemRelease = await prisma.release.upsert({
-    where: { version_type: { version: remoteRelease.systemVersion, type: "system" } },
-    update: {},
-    create: {
-      version: remoteRelease.systemVersion,
-      rolloutPercentage: 10,
-      url: remoteRelease.systemUrl,
-      type: "system",
-      hash: remoteRelease.systemHash,
-    },
-    select: { version: true, url: true, rolloutPercentage: true, hash: true },
-  });
-
-  /*
-    Return the latest release if forceUpdate is true, bypassing rollout rules.
-    This occurs when a user manually checks for updates in the app UI.
-    Background update checks follow the normal rollout percentage rules, to ensure controlled, gradual deployment of updates.
-  */
-  let responseJson: Release;
-  if (query.forceUpdate) {
-    responseJson = toRelease(latestAppRelease, latestSystemRelease);
-  } else {
-    const defaultAppRelease = await getDefaultRelease("app");
-    const defaultSystemRelease = await getDefaultRelease("system");
-
-    responseJson = toRelease(defaultAppRelease, defaultSystemRelease);
-
-    if (
-      await isDeviceEligibleForLatestRelease(
-        latestAppRelease.rolloutPercentage,
-        query.deviceId,
-      )
-    ) {
-      setAppRelease(responseJson, latestAppRelease);
-    }
-
-    if (
-      await isDeviceEligibleForLatestRelease(
-        latestSystemRelease.rolloutPercentage,
-        query.deviceId,
-      )
-    ) {
-      setSystemRelease(responseJson, latestSystemRelease);
-    }
+  // Version-constrained stable requests skip rollout but still read DB metadata.
+  if (skipRollout) {
+    const responseJson = toRelease(
+      dbReleaseToMetadata(
+        await getReleaseByRange("app", query.sku, appVersion),
+        query.sku,
+        appVersion,
+      ),
+      dbReleaseToMetadata(
+        await getReleaseByRange("system", query.sku, systemVersion),
+        query.sku,
+        systemVersion,
+      ),
+    );
+    await addStableSigUrls(responseJson);
+    return res.json(responseJson);
   }
 
-  // DB records don't store sigUrl. Resolve from S3 for the versions being served.
-  // The device requires sigUrl for stable (non-prerelease) GPG signature verification.
-  await enrichWithSigUrls(responseJson, query.sku);
+  const [latestAppRelease, latestSystemRelease, defaultAppRelease, defaultSystemRelease] =
+    await Promise.all([
+      getLatestRelease("app", query.sku),
+      getLatestRelease("system", query.sku),
+      getDefaultRelease("app", query.sku),
+      getDefaultRelease("system", query.sku),
+    ]);
+
+  // Background update checks follow rollout percentages so new releases roll
+  // out gradually. Devices outside the bucket fall back to the default (the
+  // newest 100%-rolled-out release). If the latest release lacks a compatible
+  // artifact for this SKU (e.g. a SKU-specific build hasn't shipped yet) we
+  // silently keep the default rather than 404 the whole request.
+  const responseJson = toRelease(
+    dbReleaseToMetadata(defaultAppRelease, query.sku),
+    dbReleaseToMetadata(defaultSystemRelease, query.sku),
+  );
+
+  if (
+    latestAppRelease.artifacts.length > 0 &&
+    (await isDeviceEligibleForLatestRelease(
+      latestAppRelease.rolloutPercentage,
+      query.deviceId,
+    ))
+  ) {
+    setAppRelease(responseJson, dbReleaseToMetadata(latestAppRelease, query.sku));
+  }
+
+  if (
+    latestSystemRelease.artifacts.length > 0 &&
+    (await isDeviceEligibleForLatestRelease(
+      latestSystemRelease.rolloutPercentage,
+      query.deviceId,
+    ))
+  ) {
+    setSystemRelease(responseJson, dbReleaseToMetadata(latestSystemRelease, query.sku));
+  }
+
+  await addStableSigUrls(responseJson);
 
   return res.json(responseJson);
 }

test/setup.ts

@@ -24,6 +24,12 @@ export const s3Mock = mockClient(S3Client);
 // Create a test Prisma client
 export const testPrisma = new PrismaClient();
 
+type ReleaseType = "app" | "system";
+
+// Pre-SKU artifacts are jetkvm-v2 only; future SKUs need explicit
+// skus/<sku>/ uploads, registered via scripts/sync-releases.ts.
+const LEGACY_COMPATIBLE_SKUS = ["jetkvm-v2"];
+
 function ensureSafeTestDatabase() {
   const databaseUrl = process.env.DATABASE_URL;
   if (!databaseUrl) {
@@ -45,7 +51,15 @@ function ensureSafeTestDatabase() {
 }
 
 // Seed data for releases
-export const seedReleases = [
+interface SeedRelease {
+  version: string;
+  type: ReleaseType;
+  rolloutPercentage: number;
+  url: string;
+  hash: string;
+}
+
+export const seedReleases: SeedRelease[] = [
   // App releases
   {
     version: "1.0.0",
@@ -92,9 +106,35 @@ export const seedReleases = [
   },
 ];
 
+function compatibleSkusForSeedRelease(_type: ReleaseType): string[] {
+  return LEGACY_COMPATIBLE_SKUS;
+}
+
+type SeedReleaseArtifactSource = Pick<SeedRelease, "type" | "url" | "hash">;
+
+function seedReleaseArtifactData(releaseId: bigint, release: SeedReleaseArtifactSource) {
+  return {
+    releaseId,
+    url: release.url,
+    hash: release.hash,
+    compatibleSkus: compatibleSkusForSeedRelease(release.type),
+  };
+}
+
+async function createSeedRelease(release: SeedRelease): Promise<void> {
+  const createdRelease = await testPrisma.release.create({ data: release });
+  await testPrisma.releaseArtifact.create({
+    data: seedReleaseArtifactData(createdRelease.id, release),
+  });
+}
+
 // Helper to set rollout percentage for a specific version
-export async function setRollout(version: string, type: "app" | "system", percentage: number) {
-  await testPrisma.release.upsert({
+export async function setRollout(
+  version: string,
+  type: ReleaseType,
+  percentage: number,
+): Promise<void> {
+  const release = await testPrisma.release.upsert({
     where: { version_type: { version, type } },
     update: { rolloutPercentage: percentage },
     create: {
@@ -105,6 +145,16 @@ export async function setRollout(version: string, type: "app" | "system", percen
       hash: `test-hash-${version}-${type}`,
     },
   });
+
+  const artifactData = seedReleaseArtifactData(release.id, release);
+  await testPrisma.releaseArtifact.upsert({
+    where: { releaseId_url: { releaseId: release.id, url: release.url } },
+    update: {
+      hash: artifactData.hash,
+      compatibleSkus: artifactData.compatibleSkus,
+    },
+    create: artifactData,
+  });
 }
 
 // Helper to reset all releases to seed data baseline
@@ -124,11 +174,16 @@ export async function resetToSeedData() {
 
   // Reset seed releases to original values
   for (const release of seedReleases) {
-    await testPrisma.release.upsert({
+    const dbRelease = await testPrisma.release.upsert({
       where: { version_type: { version: release.version, type: release.type } },
       update: { rolloutPercentage: release.rolloutPercentage, url: release.url, hash: release.hash },
       create: release,
     });
+
+    await testPrisma.releaseArtifact.deleteMany({ where: { releaseId: dbRelease.id } });
+    await testPrisma.releaseArtifact.create({
+      data: seedReleaseArtifactData(dbRelease.id, release),
+    });
   }
 }
 
@@ -159,11 +214,12 @@ beforeAll(async () => {
   await testPrisma.$connect();
 
   // Clean up existing releases
+  await testPrisma.releaseArtifact.deleteMany({});
   await testPrisma.release.deleteMany({});
 
   // Seed the database with test releases
   for (const release of seedReleases) {
-    await testPrisma.release.create({ data: release });
+    await createSeedRelease(release);
   }
 });
 
@@ -176,6 +232,7 @@ afterEach(() => {
 
 afterAll(async () => {
   // Clean up after all tests
+  await testPrisma.releaseArtifact.deleteMany({});
   await testPrisma.release.deleteMany({});
   await testPrisma.$disconnect();
 });

test/sync-releases.test.ts

@@ -0,0 +1,177 @@
+import {
+  GetObjectCommand,
+  HeadObjectCommand,
+  ListObjectsV2Command,
+  S3Client,
+} from "@aws-sdk/client-s3";
+import { describe, expect, beforeEach, it } from "vitest";
+
+import { collectReleaseArtifacts, syncReleases } from "../scripts/sync-releases";
+import { createAsyncIterable, s3Mock, testPrisma } from "./setup";
+
+const DEFAULT_SKU = "jetkvm-v2";
+const SDMMC_SKU = "jetkvm-v2-sdmmc";
+const SYNC_BUCKET = "test-bucket";
+const SYNC_BASE_URL = "https://cdn.test.com";
+const syncS3Client = new S3Client({});
+
+function mockS3ListVersions(prefix: "app" | "system", versions: string[]) {
+  s3Mock.on(ListObjectsV2Command, { Prefix: `${prefix}/` }).resolves({
+    CommonPrefixes: versions.map(v => ({ Prefix: `${prefix}/${v}/` })),
+  });
+}
+
+function mockS3HashFile(prefix: "app" | "system", version: string, hash: string) {
+  const fileName = prefix === "app" ? "jetkvm_app" : "system.tar";
+  s3Mock.on(ListObjectsV2Command, { Prefix: `${prefix}/${version}/skus/` }).resolves({
+    Contents: [],
+  });
+  s3Mock
+    .on(GetObjectCommand, { Key: `${prefix}/${version}/${fileName}.sha256` })
+    .resolves({
+      Body: createAsyncIterable(hash) as any,
+    });
+}
+
+function mockS3SkuVersion(
+  prefix: "app" | "system",
+  version: string,
+  sku: string,
+  hash: string,
+) {
+  const fileName = prefix === "app" ? "jetkvm_app" : "system.tar";
+  const skuPath = `${prefix}/${version}/skus/${sku}/${fileName}`;
+
+  s3Mock.on(ListObjectsV2Command, { Prefix: `${prefix}/${version}/skus/` }).resolves({
+    Contents: [{ Key: skuPath }],
+  });
+  s3Mock.on(HeadObjectCommand, { Key: skuPath }).resolves({});
+  s3Mock.on(GetObjectCommand, { Key: `${skuPath}.sha256` }).resolves({
+    Body: createAsyncIterable(hash) as any,
+  });
+}
+
+describe("sync-releases script", () => {
+  beforeEach(() => {
+    s3Mock.reset();
+    s3Mock
+      .on(HeadObjectCommand)
+      .rejects({ name: "NotFound", $metadata: { httpStatusCode: 404 } });
+  });
+
+  it("marks legacy app artifacts compatible with the default SKU only", async () => {
+    mockS3HashFile("app", "9.9.1", "legacy-app-hash");
+
+    const artifacts = await collectReleaseArtifacts(
+      { s3Client: syncS3Client },
+      { bucketName: SYNC_BUCKET, baseUrl: SYNC_BASE_URL },
+      "app",
+      "9.9.1",
+    );
+
+    expect(artifacts).toEqual([
+      {
+        url: "https://cdn.test.com/app/9.9.1/jetkvm_app",
+        hash: "legacy-app-hash",
+        compatibleSkus: [DEFAULT_SKU],
+      },
+    ]);
+  });
+
+  it("marks legacy system artifacts compatible with only the default SKU", async () => {
+    mockS3HashFile("system", "9.9.2", "legacy-system-hash");
+
+    const artifacts = await collectReleaseArtifacts(
+      { s3Client: syncS3Client },
+      { bucketName: SYNC_BUCKET, baseUrl: SYNC_BASE_URL },
+      "system",
+      "9.9.2",
+    );
+
+    expect(artifacts).toEqual([
+      {
+        url: "https://cdn.test.com/system/9.9.2/system.tar",
+        hash: "legacy-system-hash",
+        compatibleSkus: [DEFAULT_SKU],
+      },
+    ]);
+  });
+
+  it("collects only SKU artifacts that exist and have a hash", async () => {
+    mockS3SkuVersion("system", "9.9.3", DEFAULT_SKU, "system-default-hash");
+
+    const artifacts = await collectReleaseArtifacts(
+      { s3Client: syncS3Client },
+      { bucketName: SYNC_BUCKET, baseUrl: SYNC_BASE_URL },
+      "system",
+      "9.9.3",
+    );
+
+    expect(artifacts).toEqual([
+      {
+        url: `https://cdn.test.com/system/9.9.3/skus/${DEFAULT_SKU}/system.tar`,
+        hash: "system-default-hash",
+        compatibleSkus: [DEFAULT_SKU],
+      },
+    ]);
+  });
+
+  it("creates new releases at 10% with their S3 artifacts and skips already-synced versions", async () => {
+    const version = "9.9.4";
+
+    // Pre-existing system row simulates a release the migration (or a prior
+    // sync) already wrote. Sync must leave it completely untouched.
+    await testPrisma.release.create({
+      data: {
+        version,
+        type: "system",
+        rolloutPercentage: 77,
+        url: "https://cdn.test.com/old-system.tar",
+        hash: "old-system-hash",
+      },
+    });
+
+    mockS3ListVersions("app", [version, "10.0.0-beta.1"]);
+    mockS3ListVersions("system", [version]);
+    mockS3HashFile("app", version, "app-hash");
+    mockS3SkuVersion("system", version, DEFAULT_SKU, "system-hash-v2");
+    mockS3SkuVersion("system", version, SDMMC_SKU, "system-hash-sdmmc");
+
+    await syncReleases(
+      { prisma: testPrisma, s3Client: syncS3Client },
+      { bucketName: SYNC_BUCKET, baseUrl: SYNC_BASE_URL },
+    );
+
+    const appRelease = await testPrisma.release.findUniqueOrThrow({
+      where: { version_type: { version, type: "app" } },
+      include: { artifacts: true },
+    });
+    const systemRelease = await testPrisma.release.findUniqueOrThrow({
+      where: { version_type: { version, type: "system" } },
+      include: { artifacts: true },
+    });
+    const prerelease = await testPrisma.release.findUnique({
+      where: { version_type: { version: "10.0.0-beta.1", type: "app" } },
+    });
+
+    // App release is new — created at 10% rollout with a single legacy-compatible artifact.
+    expect(appRelease.rolloutPercentage).toBe(10);
+    expect(appRelease.artifacts).toEqual([
+      expect.objectContaining({
+        url: `https://cdn.test.com/app/${version}/jetkvm_app`,
+        hash: "app-hash",
+        compatibleSkus: [DEFAULT_SKU],
+      }),
+    ]);
+
+    // System release already existed — sync must not touch rollout, URL, hash,
+    // or attach any new artifacts (those are handled by one-off scripts).
+    expect(systemRelease.rolloutPercentage).toBe(77);
+    expect(systemRelease.url).toBe("https://cdn.test.com/old-system.tar");
+    expect(systemRelease.hash).toBe("old-system-hash");
+    expect(systemRelease.artifacts).toEqual([]);
+
+    // Prereleases are filtered out by listStableVersions.
+    expect(prerelease).toBeNull();
+  });
+});

vitest.config.ts

@@ -8,6 +8,7 @@ export default defineConfig({
     testTimeout: 30000,
     hookTimeout: 30000,
     include: ["test/**/*.test.ts"],
-    silent: "passed-only"
+    silent: "passed-only",
+    fileParallelism: false,
   },
 });