From c08f14ff3f7d300a342677d295629a377c531e58 Mon Sep 17 00:00:00 2001 From: Adam Shiervani Date: Sat, 28 Mar 2026 23:14:02 +0100 Subject: [PATCH] fix: send mouse button state changes via reliable WebRTC channel (#695) (#1338) * fix: send mouse button state changes via reliable WebRTC channel (#695) When holding one mouse button and pressing another without moving the mouse, only pointerdown/pointerup events fire (no mousemove to self-correct). These button-only state changes were sent via the unreliable WebRTC data channel (maxRetransmits: 0), and lost packets were never recovered. Changes: - useHidRpc.ts: Track last button state and send button changes via the reliable channel. Movement-only events continue using the unreliable channel for low latency, since lost movement packets self-correct via subsequent mousemove events. - e2e/remote-agent/main.go: Fix omitempty on InputEvent.Value so button release events (value=0) are included in JSON responses. - ra-all.spec.ts: Add E2E test that holds left mouse button, presses and releases right mouse button, and verifies all 4 button events arrive on the remote host (20 iterations). * fix: rebuild remote agent when source changes to prevent stale deploys ensureDeployed() skipped rebuild and redeploy when the agent was already running, causing the omitempty fix on InputEvent.Value to never reach the remote host. Now compares source mtime against binary mtime and forces redeploy when a rebuild occurs. --- e2e/remote-agent/main.go | 2 +- ui/e2e/remote-agent/ra-all.spec.ts | 48 +++++++++++++++++++++++++++++ ui/e2e/remote-agent/remote-agent.ts | 29 ++++++++++------- ui/src/hooks/useHidRpc.ts | 14 +++++++-- 4 files changed, 79 insertions(+), 14 deletions(-) diff --git a/e2e/remote-agent/main.go b/e2e/remote-agent/main.go index 00f5207d..47387f10 100644 --- a/e2e/remote-agent/main.go +++ b/e2e/remote-agent/main.go @@ -62,7 +62,7 @@ type InputEvent struct { Time int64 `json:"time_ms"` Type string `json:"type"` // "key_press", "key_release", "key_repeat", "mouse_move_rel", "mouse_move_abs", "mouse_button" Code uint16 `json:"code"` // Linux key code or axis - Value int32 `json:"value,omitempty"` // Key: 0/1/2, Mouse: delta or position + Value int32 `json:"value"` // Key: 0/1/2, Mouse: delta or position X int32 `json:"x"` // Mouse X (for move events) Y int32 `json:"y"` // Mouse Y (for move events) Device string `json:"device,omitempty"` // Source device name diff --git a/ui/e2e/remote-agent/ra-all.spec.ts b/ui/e2e/remote-agent/ra-all.spec.ts index 956d21c2..5043f346 100644 --- a/ui/e2e/remote-agent/ra-all.spec.ts +++ b/ui/e2e/remote-agent/ra-all.spec.ts @@ -901,6 +901,54 @@ test.describe("Remote Host Agent", () => { } }); + test("mouse: multi-button combinations (hold left, press right)", async () => { + const BTN_LEFT = 0x110; // 272 + const BTN_RIGHT = 0x111; // 273 + + // Move mouse to a stable position first + await sendAbsMouseMove(sharedPage, 16384, 16384); + await new Promise(r => setTimeout(r, 100)); + + let successes = 0; + const attempts = 20; + + for (let i = 0; i < attempts; i++) { + await agent!.clearMouseEvents(); + + // Press left button (buttons bitmask: 1 = left) + await sendAbsMouseMove(sharedPage, 16384, 16384, 1); + await new Promise(r => setTimeout(r, 100)); + + // While holding left, press right button (buttons bitmask: 3 = left + right) + await sendAbsMouseMove(sharedPage, 16384, 16384, 3); + await new Promise(r => setTimeout(r, 100)); + + // Release right button, left still held (buttons bitmask: 1 = left) + await sendAbsMouseMove(sharedPage, 16384, 16384, 1); + await new Promise(r => setTimeout(r, 100)); + + // Release all buttons (buttons bitmask: 0) + await sendAbsMouseMove(sharedPage, 16384, 16384, 0); + await new Promise(r => setTimeout(r, 150)); + + const events = await agent!.getMouseEvents(); + const buttonEvents = events.filter(ev => ev.type === "mouse_button"); + + // We expect to see button events for both left and right buttons + const leftPress = buttonEvents.find(ev => ev.code === BTN_LEFT && ev.value === 1); + const rightPress = buttonEvents.find(ev => ev.code === BTN_RIGHT && ev.value === 1); + const rightRelease = buttonEvents.find(ev => ev.code === BTN_RIGHT && ev.value === 0); + const leftRelease = buttonEvents.find(ev => ev.code === BTN_LEFT && ev.value === 0); + + if (leftPress && rightPress && rightRelease && leftRelease) { + successes++; + } + } + + // All attempts must succeed — button state changes must be reliably delivered + expect(successes, `Multi-button succeeded ${successes}/${attempts} times`).toBe(attempts); + }); + // ═══════════════════════════════════════════ // MOUSE: BLUR DOES NOT JUMP TO TOP-LEFT (#392) // ═══════════════════════════════════════════ diff --git a/ui/e2e/remote-agent/remote-agent.ts b/ui/e2e/remote-agent/remote-agent.ts index 9e7d1241..eb7d63c8 100644 --- a/ui/e2e/remote-agent/remote-agent.ts +++ b/ui/e2e/remote-agent/remote-agent.ts @@ -498,11 +498,9 @@ export class RemoteAgent { /** * Ensure the remote agent is running on the target host. - * If the agent isn't responding, build (if needed) and deploy it via SSH. + * Always rebuilds from source when it has changed and redeploys. */ async ensureDeployed(sshTarget?: string): Promise { - if (await this.health()) return; - const host = this.baseUrl.replace(/^https?:\/\//, "").replace(/:\d+$/, ""); const port = parseInt(this.baseUrl.replace(/.*:/, ""), 10); const target = sshTarget ?? process.env.JETKVM_REMOTE_HOST ?? `tony@${host}`; @@ -512,13 +510,18 @@ export class RemoteAgent { const binary = path.join(agentDir, "remote-agent"); const goSource = path.join(agentDir, "main.go"); - if (!fs.existsSync(binary)) { - if (!fs.existsSync(goSource)) { - throw new Error( - `Remote agent source not found at ${goSource}. ` + - `Restore it with: git checkout e2e-remote-host-agent -- e2e/remote-agent/`, - ); - } + if (!fs.existsSync(goSource)) { + throw new Error( + `Remote agent source not found at ${goSource}. ` + + `Restore it with: git checkout e2e-remote-host-agent -- e2e/remote-agent/`, + ); + } + + // Rebuild if source is newer than binary (or binary doesn't exist) + const needsBuild = + !fs.existsSync(binary) || fs.statSync(goSource).mtimeMs > fs.statSync(binary).mtimeMs; + + if (needsBuild) { console.log("[remote-agent] Building remote-agent binary..."); execSync("GOOS=linux GOARCH=amd64 go build -o remote-agent .", { cwd: agentDir, @@ -526,9 +529,13 @@ export class RemoteAgent { }); } - console.log(`[remote-agent] Deploying to ${target}...`); + // Skip deploy only when the binary hasn't changed and the agent is already running + if (!needsBuild && (await this.health())) return; + const sshOpts = "-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o ConnectTimeout=30 -o ServerAliveInterval=5 -o ServerAliveCountMax=3"; + + console.log(`[remote-agent] Deploying to ${target}...`); execSync(`scp ${sshOpts} "${binary}" ${target}:/tmp/remote-agent`, { stdio: "inherit" }); console.log(`[remote-agent] Starting on port ${port}...`); diff --git a/ui/src/hooks/useHidRpc.ts b/ui/src/hooks/useHidRpc.ts index 76267e55..ae3055c5 100644 --- a/ui/src/hooks/useHidRpc.ts +++ b/ui/src/hooks/useHidRpc.ts @@ -1,4 +1,4 @@ -import { useCallback, useEffect, useMemo } from "react"; +import { useCallback, useEffect, useMemo, useRef } from "react"; import { Logger } from "tslog"; import { useRTCStore } from "@hooks/stores"; @@ -247,10 +247,20 @@ export function useHidRpc(onHidRpcMessage?: (payload: RpcMessage) => void) { [sendMessage], ); + const lastAbsButtons = useRef(0); + const reportAbsMouseEvent = useCallback( (x: number, y: number, buttons: number) => { + const buttonsChanged = buttons !== lastAbsButtons.current; + lastAbsButtons.current = buttons; + sendMessage(new PointerReportMessage(x, y, buttons), { - useUnreliableChannel: true, + // Use the reliable channel for button state changes to guarantee delivery. + // Movement-only events use the unreliable channel for lower latency; + // lost movement packets self-correct via subsequent mousemove events, + // but button-only changes (pointerdown/pointerup without movement) have + // no such redundancy and must not be dropped. + useUnreliableChannel: !buttonsChanged, }); }, [sendMessage],