diff --git a/audio.go b/audio.go index 4c03c267..a6985cc6 100644 --- a/audio.go +++ b/audio.go @@ -3,7 +3,6 @@ package kvm import ( "context" "errors" - "fmt" "os" "path/filepath" "strconv" @@ -16,25 +15,23 @@ import ( "github.com/pion/webrtc/v4/pkg/media" ) -var audioCancel context.CancelFunc -var audioStopped chan struct{} -var audioRuntimeMu sync.Mutex - -type audioCaptureSource struct { - name string - device string -} +var ( + audioCancel context.CancelFunc + audioStopped chan struct{} + audioMu sync.Mutex +) +// startAudio stops any running capture and, if track is non-nil, starts a new +// capture writing to it. Calling with nil simply stops the current capture. func startAudio(track *webrtc.TrackLocalStaticSample) { + audioMu.Lock() + defer audioMu.Unlock() + stopAudioLocked() + if track == nil { return } - audioRuntimeMu.Lock() - defer audioRuntimeMu.Unlock() - - stopAudioUnderLock() - ctx, cancel := context.WithCancel(context.Background()) audioCancel = cancel audioStopped = make(chan struct{}) @@ -43,17 +40,15 @@ func startAudio(track *webrtc.TrackLocalStaticSample) { } func stopAudio() { - audioRuntimeMu.Lock() - defer audioRuntimeMu.Unlock() - - stopAudioUnderLock() + audioMu.Lock() + defer audioMu.Unlock() + stopAudioLocked() } -func stopAudioUnderLock() { +func stopAudioLocked() { if audioCancel == nil { return } - audioCancel() <-audioStopped audioCancel = nil @@ -63,27 +58,21 @@ func stopAudioUnderLock() { func runAudioCapture(ctx context.Context, track *webrtc.TrackLocalStaticSample, stopped chan<- struct{}) { defer close(stopped) + device := alsaCaptureDevice() codec := audioCodecForTrack(track) - sources := audioCaptureSources() - capture, source, sourceIndex, err := openAudioCaptureFrom(sources, 0) + + capture, err := audio.OpenALSACapture(device) if err != nil { - audioLogger.Error().Err(err).Msg("audio capture unavailable") + audioLogger.Error().Err(err).Str("device", device).Msg("audio capture unavailable") return } - defer func() { - _ = capture.Close() - }() + defer capture.Close() - sample := media.Sample{Duration: 20 * time.Millisecond} - audioLogger.Info(). - Str("source", source.name). - Str("device", source.device). - Str("codec", codec.String()). - Msg("audio capture started") + audioLogger.Info().Str("device", device).Str("codec", codec.String()).Msg("audio capture started") defer audioLogger.Info().Msg("audio capture stopped") - noDataReads := 0 - wroteFirstSample := false + sample := media.Sample{Duration: 20 * time.Millisecond} + idleReads := 0 for { select { @@ -95,51 +84,25 @@ func runAudioCapture(ctx context.Context, track *webrtc.TrackLocalStaticSample, payload, err := capture.ReadEncoded(codec) if err != nil { if errors.Is(err, audio.ErrNoAudioData) { - noDataReads++ - if noDataReads == 50 || noDataReads%500 == 0 { - audioLogger.Info().Int("reads", noDataReads).Msg("audio capture has no data") - } - if noDataReads == 50 { - _ = capture.Close() - reopened, reopenedSource, reopenedSourceIndex, openErr := openAudioCaptureFrom(sources, sourceIndex+1) - if openErr != nil { - audioLogger.Error().Err(openErr).Msg("audio capture reopen failed") - time.Sleep(500 * time.Millisecond) - noDataReads = 0 - continue - } - capture = reopened - source = reopenedSource - sourceIndex = reopenedSourceIndex - noDataReads = 0 - audioLogger.Info(). - Str("source", source.name). - Str("device", source.device). - Msg("audio capture reopened after no data") + if idleReads++; idleReads%500 == 0 { + audioLogger.Debug().Int("reads", idleReads).Msg("audio capture idle") } continue } - audioLogger.Error().Err(err).Msg("audio capture read failed") + audioLogger.Warn().Err(err).Msg("audio capture read failed") time.Sleep(100 * time.Millisecond) continue } + + idleReads = 0 if len(payload) == 0 { continue } - noDataReads = 0 sample.Data = payload if err := track.WriteSample(sample); err != nil { - select { - case <-ctx.Done(): - return - default: - audioLogger.Warn().Err(err).Msg("audio sample write failed") - time.Sleep(100 * time.Millisecond) - } - } else if !wroteFirstSample { - wroteFirstSample = true - audioLogger.Info().Str("codec", codec.String()).Int("bytes", len(payload)).Msg("audio sample written") + audioLogger.Warn().Err(err).Msg("audio sample write failed") + time.Sleep(100 * time.Millisecond) } } } @@ -151,36 +114,13 @@ func audioCodecForTrack(track *webrtc.TrackLocalStaticSample) audio.Codec { return audio.CodecPCMU } -func openAudioCaptureFrom(sources []audioCaptureSource, startIndex int) (audio.Reader, audioCaptureSource, int, error) { - var errs []string - if len(sources) == 0 { - return nil, audioCaptureSource{}, 0, fmt.Errorf("no ALSA capture sources configured") - } - - for offset := range sources { - index := (startIndex + offset) % len(sources) - source := sources[index] - capture, err := audio.OpenALSACapture(source.device) - if err == nil { - return capture, source, index, nil - } - errs = append(errs, fmt.Sprintf("%s %s: %v", source.name, source.device, err)) - } - return nil, audioCaptureSource{}, 0, fmt.Errorf("no ALSA capture device opened (%s)", strings.Join(errs, "; ")) -} - -func audioCaptureSources() []audioCaptureSource { - var sources []audioCaptureSource +// alsaCaptureDevice returns the ALSA device string for the UAC1 gadget card. +// Falls back to hw:1,0 if the sysfs lookup fails (typically only during early boot). +func alsaCaptureDevice() string { if card, ok := findALSACard("UAC1Gadget"); ok { - sources = append(sources, audioCaptureSource{ - name: "uac1", - device: "hw:" + strconv.Itoa(card) + ",0", - }) + return "hw:" + strconv.Itoa(card) + ",0" } - - return append(sources, - audioCaptureSource{name: "uac1-default", device: "hw:1,0"}, - ) + return "hw:1,0" } func findALSACard(cardID string) (int, bool) { @@ -194,14 +134,11 @@ func findALSACard(cardID string) (int, bool) { if !strings.HasPrefix(name, "card") { continue } - id, err := os.ReadFile(filepath.Join("/sys/class/sound", name, "id")) if err != nil || strings.TrimSpace(string(id)) != cardID { continue } - - card, err := strconv.Atoi(strings.TrimPrefix(name, "card")) - if err == nil { + if card, err := strconv.Atoi(strings.TrimPrefix(name, "card")); err == nil { return card, true } } diff --git a/e2e/remote-agent/main.go b/e2e/remote-agent/main.go index cb5cc86d..99d97cbb 100644 --- a/e2e/remote-agent/main.go +++ b/e2e/remote-agent/main.go @@ -147,14 +147,12 @@ func (b *EventBuffer) prune() { // Agent holds the state of the remote agent. type Agent struct { - keyboardEvents *EventBuffer - mouseEvents *EventBuffer - audioMu sync.Mutex - audioToneCmd *exec.Cmd - audioToneCancel context.CancelFunc - audioToneDone chan error - monitorMu sync.Mutex - absMouseState struct { + keyboardEvents *EventBuffer + mouseEvents *EventBuffer + audioMu sync.Mutex + audioToneCmd *exec.Cmd + monitorMu sync.Mutex + absMouseState struct { mu sync.Mutex x int32 y int32 @@ -509,90 +507,13 @@ func getDisplayInfo() []DisplayInfo { return displays } -var ( - aplayDeviceRE = regexp.MustCompile(`^card ([0-9]+): ([^ ]+) \[(.*)\], device ([0-9]+): .*\[(.*)\]`) - pipeWireSinkIDRE = regexp.MustCompile(`([0-9]+)\.`) -) +var aplayDeviceRE = regexp.MustCompile(`^card ([0-9]+): ([^ ]+) \[(.*)\], device ([0-9]+): .*\[(.*)\]`) -func firstLine(value string) string { - if value == "" { - return "" - } - line, _, _ := strings.Cut(value, "\n") - return strings.TrimSpace(line) -} - -func alsaCardDescription(card int) string { - return firstLine(readSysFile(filepath.Join("/proc/asound", "card"+strconv.Itoa(card), "stream0"))) -} - -func alsaCardUSBID(card int) string { - return readSysFile(filepath.Join("/proc/asound", "card"+strconv.Itoa(card), "usbid")) -} - -func isJetKVMUSBAudioDevice(device AudioDeviceInfo) bool { - if strings.EqualFold(strings.TrimSpace(device.USBID), "1d6b:0104") { +func isJetKVMUSBAudioDevice(d AudioDeviceInfo) bool { + if strings.EqualFold(strings.TrimSpace(d.USBID), "1d6b:0104") { return true } - lower := strings.ToLower(device.Name + " " + device.Description) - return strings.Contains(lower, "jetkvm usb emulation device") -} - -func startToneCommand(cmd *exec.Cmd) (chan error, error) { - if err := cmd.Start(); err != nil { - return nil, err - } - - done := make(chan error, 1) - go func() { - done <- cmd.Wait() - close(done) - }() - - select { - case err := <-done: - if err == nil { - err = fmt.Errorf("audio tone exited immediately") - } - return nil, err - case <-time.After(300 * time.Millisecond): - return done, nil - } -} - -func findJetKVMPipeWireSinkID() string { - if _, err := exec.LookPath("wpctl"); err != nil { - return "" - } - - out, err := exec.Command("wpctl", "status").Output() - if err != nil { - return "" - } - - inSinks := false - for _, line := range strings.Split(string(out), "\n") { - lower := strings.ToLower(line) - if strings.Contains(lower, "sinks:") { - inSinks = true - continue - } - if inSinks && strings.Contains(lower, "sink endpoints:") { - return "" - } - if !inSinks { - continue - } - if !strings.Contains(lower, "multifunction composite gadget") && - !strings.Contains(lower, "jetkvm") && - !strings.Contains(lower, "usb emulation device") { - continue - } - if m := pipeWireSinkIDRE.FindStringSubmatch(line); m != nil { - return m[1] - } - } - return "" + return strings.Contains(strings.ToLower(d.Name+" "+d.Description), "jetkvm usb emulation device") } func listAudioDevices() []AudioDeviceInfo { @@ -609,110 +530,70 @@ func listAudioDevices() []AudioDeviceInfo { } card, _ := strconv.Atoi(m[1]) - cardName := strings.TrimSpace(m[3]) device, _ := strconv.Atoi(m[4]) - deviceName := strings.TrimSpace(m[5]) - description := alsaCardDescription(card) - usbID := alsaCardUSBID(card) - nameParts := []string{cardName, deviceName} - if description != "" { - nameParts = append(nameParts, description) - } - name := strings.Join(nameParts, " / ") - pcm := "plughw:" + strconv.Itoa(card) + "," + strconv.Itoa(device) + streamFile := filepath.Join("/proc/asound", "card"+strconv.Itoa(card), "stream0") + description, _, _ := strings.Cut(readSysFile(streamFile), "\n") + description = strings.TrimSpace(description) - deviceInfo := AudioDeviceInfo{ + d := AudioDeviceInfo{ Card: card, Device: device, - Name: name, - PCM: pcm, + Name: strings.TrimSpace(m[3]) + " / " + strings.TrimSpace(m[5]), + PCM: fmt.Sprintf("plughw:%d,%d", card, device), Description: description, - USBID: usbID, + USBID: readSysFile(filepath.Join("/proc/asound", "card"+strconv.Itoa(card), "usbid")), } - deviceInfo.IsJetKVM = isJetKVMUSBAudioDevice(deviceInfo) - devices = append(devices, deviceInfo) + d.IsJetKVM = isJetKVMUSBAudioDevice(d) + devices = append(devices, d) } return devices } -func selectAudioDevice() (AudioDeviceInfo, bool) { - if override := strings.TrimSpace(os.Getenv("JETKVM_AUDIO_DEVICE")); override != "" { - return AudioDeviceInfo{Card: -1, Device: -1, Name: "override", PCM: override, IsJetKVM: true}, true - } - - devices := listAudioDevices() - for _, device := range devices { - if isJetKVMUSBAudioDevice(device) { - return device, true - } - } - return AudioDeviceInfo{}, false -} - func (a *Agent) startAudioTone() (AudioDeviceInfo, error) { a.audioMu.Lock() defer a.audioMu.Unlock() a.stopAudioToneLocked() - killStaleAudioToneProcesses() - device, ok := selectAudioDevice() - if !ok { - return AudioDeviceInfo{}, os.ErrNotExist - } - - playbackDevice := device.PCM - if sinkID := findJetKVMPipeWireSinkID(); sinkID != "" { - if err := exec.Command("wpctl", "set-default", sinkID).Run(); err == nil { - playbackDevice = "default" + var device AudioDeviceInfo + if override := strings.TrimSpace(os.Getenv("JETKVM_AUDIO_DEVICE")); override != "" { + device = AudioDeviceInfo{Card: -1, Device: -1, Name: "override", PCM: override, IsJetKVM: true} + } else { + for _, d := range listAudioDevices() { + if d.IsJetKVM { + device = d + break + } + } + if !device.IsJetKVM { + return AudioDeviceInfo{}, os.ErrNotExist } } - ctx, cancel := context.WithCancel(context.Background()) - cmd := exec.CommandContext(ctx, "speaker-test", "-D", playbackDevice, "-t", "sine", "-f", "997", "-r", "48000", "-c", "2", "-p", "20000", "-b", "80000") - done, err := startToneCommand(cmd) - if err != nil { - cancel() + // 997 Hz / 48 kHz stereo sine; -p 20000 sets period so tone runs ~indefinitely + // (long enough for the spec's 12s deadline) without re-arming. + cmd := exec.Command("speaker-test", "-D", device.PCM, "-t", "sine", "-f", "997", "-r", "48000", "-c", "2", "-p", "20000", "-b", "80000") + if err := cmd.Start(); err != nil { return device, err } a.audioToneCmd = cmd - a.audioToneCancel = cancel - a.audioToneDone = done return device, nil } func (a *Agent) stopAudioTone() { a.audioMu.Lock() defer a.audioMu.Unlock() - a.stopAudioToneLocked() } func (a *Agent) stopAudioToneLocked() { - if a.audioToneCancel != nil { - a.audioToneCancel() - a.audioToneCancel = nil - } - if a.audioToneCmd == nil || a.audioToneCmd.Process == nil { a.audioToneCmd = nil - a.audioToneDone = nil return } _ = a.audioToneCmd.Process.Kill() - if a.audioToneDone != nil { - select { - case <-a.audioToneDone: - case <-time.After(time.Second): - } - } + _ = a.audioToneCmd.Wait() a.audioToneCmd = nil - a.audioToneDone = nil - killStaleAudioToneProcesses() -} - -func killStaleAudioToneProcesses() { - _ = exec.Command("pkill", "-f", "speaker-test -D .* -t sine -f 997 -r 48000 -c 2 -p 20000 -b 80000").Run() } // listMounts returns current mount points, filtered to interesting ones. diff --git a/internal/audio/alsa_capture_stub.go b/internal/audio/alsa_capture_stub.go index ef295c67..b71c3370 100644 --- a/internal/audio/alsa_capture_stub.go +++ b/internal/audio/alsa_capture_stub.go @@ -4,6 +4,12 @@ package audio import "fmt" -func OpenALSACapture(device string) (*unavailableCapture, error) { +type ALSACapture struct{} + +func OpenALSACapture(device string) (*ALSACapture, error) { return nil, fmt.Errorf("ALSA audio capture is not available for this build: %s", device) } + +func (*ALSACapture) ReadEncoded(Codec) ([]byte, error) { return nil, ErrNoAudioData } + +func (*ALSACapture) Close() error { return nil } diff --git a/internal/audio/capture.go b/internal/audio/capture.go index baa4078b..9091b172 100644 --- a/internal/audio/capture.go +++ b/internal/audio/capture.go @@ -30,21 +30,6 @@ func (c Codec) String() string { } } -type Reader interface { - ReadEncoded(codec Codec) ([]byte, error) - Close() error -} - -type unavailableCapture struct { - err error -} - -func (c *unavailableCapture) ReadEncoded(Codec) ([]byte, error) { - return nil, c.err -} - -func (c *unavailableCapture) Close() error { - return nil -} - +// ErrNoAudioData is returned when the underlying capture device is idle. +// The caller should retry; it is not a fatal error. var ErrNoAudioData = io.ErrNoProgress diff --git a/ui/e2e/remote-agent/ra-audio.spec.ts b/ui/e2e/remote-agent/ra-audio.spec.ts index dbb71e6c..c5c42d22 100644 --- a/ui/e2e/remote-agent/ra-audio.spec.ts +++ b/ui/e2e/remote-agent/ra-audio.spec.ts @@ -1,12 +1,5 @@ -import { test, expect, type Page, type TestInfo } from "@playwright/test"; -import * as fs from "fs"; -import { - callJsonRpc, - getDeviceHost, - restartAppViaSSH, - sshExec, - waitForWebRTCReady, -} from "../helpers"; +import { test, expect } from "@playwright/test"; +import { getDeviceHost, waitForWebRTCReady } from "../helpers"; import { createRemoteAgent } from "./remote-agent"; const agent = createRemoteAgent(); @@ -16,174 +9,14 @@ async function ensureNoPasswordViaAPI() { const status = await fetch(`http://${host}/device/status`).then( r => r.json() as Promise<{ isSetup: boolean }>, ); + if (status.isSetup) return; - if (!status.isSetup) { - const res = await fetch(`http://${host}/device/setup`, { - method: "POST", - headers: { "Content-Type": "application/json" }, - body: JSON.stringify({ localAuthMode: "noPassword" }), - }); - if (!res.ok) throw new Error(`Setup POST failed: ${res.status}`); - return; - } - - const probe = await fetch(`http://${host}/device`); - if (probe.status !== 401) return; - - await sshExec("rm -f /userdata/kvm_config.json && sync"); - await restartAppViaSSH(); const res = await fetch(`http://${host}/device/setup`, { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ localAuthMode: "noPassword" }), }); - if (!res.ok) throw new Error(`Setup POST after reset failed: ${res.status}`); -} - -async function openReadyPage(page: Page) { - await page.goto("/", { waitUntil: "networkidle" }); - - if (page.url().includes("/welcome")) { - const host = getDeviceHost(); - await fetch(`http://${host}/device/setup`, { - method: "POST", - headers: { "Content-Type": "application/json" }, - body: JSON.stringify({ localAuthMode: "noPassword" }), - }); - await page.goto("/", { waitUntil: "networkidle" }); - } - - await waitForWebRTCReady(page); -} - -async function attachAudioDiagnostics(testInfo: TestInfo, page: Page): Promise { - const sections: string[] = []; - - const browserStats = await page - .evaluate(() => ({ - active: window.__kvmTestHooks?.isAudioStreamActive(), - stats: window.__kvmTestHooks?.getInboundAudioStats(), - })) - .catch(error => ({ error: error instanceof Error ? error.message : String(error) })); - sections.push(`browser:\n${JSON.stringify(browserStats, null, 2)}`); - - const audioDevices = agent - ? await agent.getAudioDevices().catch(error => ({ - error: error instanceof Error ? error.message : String(error), - })) - : { error: "remote agent unavailable" }; - sections.push(`remote audio devices:\n${JSON.stringify(audioDevices, null, 2)}`); - - const deviceSnapshot = await sshExec( - [ - "echo '=== date ==='", - "date", - "echo '=== uptime ==='", - "uptime", - "echo '=== jetkvm_app pid ==='", - "pidof jetkvm_app || true", - "echo '=== jetkvm_app process ==='", - "ps -w | grep jetkvm_app | grep -v grep || true", - "echo '=== snd nodes ==='", - "ls -l /dev/snd 2>/dev/null || true", - "echo '=== proc asound ==='", - "cat /proc/asound/cards 2>/dev/null || true", - "cat /proc/asound/pcm 2>/dev/null || true", - "echo '=== recent app log ==='", - "tail -300 /userdata/jetkvm/last.log 2>/dev/null || true", - "echo '=== recent dmesg ==='", - "dmesg | tail -120", - ].join("; "), - true, - ); - sections.push(`device snapshot:\n${deviceSnapshot}`); - - const diagnosticsPath = testInfo.outputPath("audio-usb-diagnostics.txt"); - fs.writeFileSync(diagnosticsPath, sections.join("\n\n")); - await testInfo.attach("audio-usb-diagnostics.txt", { - path: diagnosticsPath, - contentType: "text/plain", - }); -} - -async function expectRemoteToneReachesBrowser(page: Page, testInfo: TestInfo) { - const audioDevices = await agent!.getAudioDevices(); - test.skip( - !audioDevices.some(device => device.is_jetkvm), - `No JetKVM USB ALSA playback device found: ${JSON.stringify(audioDevices)}`, - ); - - await openReadyPage(page); - const originalLogLevel = (await callJsonRpc(page, "getDefaultLogLevel")) as string; - let diagnosticsAttached = false; - const attachDiagnosticsOnce = async () => { - if (diagnosticsAttached) return; - diagnosticsAttached = true; - await attachAudioDiagnostics(testInfo, page); - }; - - try { - await callJsonRpc(page, "setDefaultLogLevel", { level: "INFO" }); - - await expect - .poll(() => page.evaluate(() => window.__kvmTestHooks?.isAudioStreamActive()), { - message: "Waiting for browser audio track", - timeout: 10_000, - intervals: [250, 500], - }) - .toBe(true); - - const before = (await page.evaluate(() => window.__kvmTestHooks?.getInboundAudioStats())) ?? { - bytesReceived: 0, - packetsReceived: 0, - totalAudioEnergy: 0, - codecMimeType: "", - }; - - const toneDevice = await agent!.startAudioTone(); - expect( - toneDevice.is_jetkvm, - `Remote agent selected non-JetKVM audio device: ${JSON.stringify(toneDevice)}`, - ).toBe(true); - - try { - await expect - .poll( - async () => { - const stats = await page.evaluate(() => window.__kvmTestHooks?.getInboundAudioStats()); - if (!stats) return false; - const bytesDelta = stats.bytesReceived - before.bytesReceived; - const packetsDelta = stats.packetsReceived - before.packetsReceived; - const energyDelta = stats.totalAudioEnergy - before.totalAudioEnergy; - const codec = stats.codecMimeType.toLowerCase(); - return ( - bytesDelta > 800 && - packetsDelta > 10 && - energyDelta > 0.0001 && - codec.includes("g722") - ); - }, - { - message: "Waiting for captured USB G.722 audio energy", - timeout: 12_000, - intervals: [500, 1000], - }, - ) - .toBe(true); - } catch (error) { - await attachDiagnosticsOnce(); - throw error; - } finally { - await agent!.stopAudioTone(); - } - } catch (error) { - await attachDiagnosticsOnce(); - throw error; - } finally { - await callJsonRpc(page, "setDefaultLogLevel", { level: originalLogLevel }).catch(() => { - /* ignore */ - }); - } + if (!res.ok) throw new Error(`Setup POST failed: ${res.status}`); } test.beforeAll(async () => { @@ -192,13 +25,56 @@ test.beforeAll(async () => { }); test.afterEach(async () => { - if (!agent) return; - await agent.stopAudioTone().catch(() => { - /* ignore */ - }); + await agent?.stopAudioTone().catch(() => undefined); }); -test("audio: USB remote tone reaches browser WebRTC track as G.722", async ({ page }, testInfo) => { +test("audio: USB remote tone reaches browser WebRTC track as G.722", async ({ page }) => { test.setTimeout(35_000); - await expectRemoteToneReachesBrowser(page, testInfo); + + const devices = await agent!.getAudioDevices(); + test.skip( + !devices.some(d => d.is_jetkvm), + `No JetKVM USB ALSA playback device on remote host: ${JSON.stringify(devices)}`, + ); + + await page.goto("/", { waitUntil: "networkidle" }); + await waitForWebRTCReady(page); + + await expect + .poll(() => page.evaluate(() => window.__kvmTestHooks?.isAudioStreamActive()), { + message: "browser audio track did not become live", + timeout: 10_000, + intervals: [250, 500], + }) + .toBe(true); + + const before = (await page.evaluate(() => window.__kvmTestHooks?.getInboundAudioStats())) ?? { + bytesReceived: 0, + packetsReceived: 0, + totalAudioEnergy: 0, + codecMimeType: "", + }; + + const tone = await agent!.startAudioTone(); + expect(tone.is_jetkvm, `selected non-JetKVM playback device: ${JSON.stringify(tone)}`).toBe(true); + + await expect + .poll( + async () => { + const stats = await page.evaluate(() => window.__kvmTestHooks?.getInboundAudioStats()); + if (!stats) return false; + return ( + stats.bytesReceived - before.bytesReceived > 800 && + stats.packetsReceived - before.packetsReceived > 10 && + stats.totalAudioEnergy - before.totalAudioEnergy > 0.0001 && + stats.codecMimeType.toLowerCase().includes("g722") + ); + }, + { + message: "USB G.722 audio energy never reached browser", + timeout: 12_000, + intervals: [500, 1000], + }, + ) + .toBe(true); }); diff --git a/ui/src/components/WebRTCVideo.tsx b/ui/src/components/WebRTCVideo.tsx index 1797a4f0..49fa3d4b 100644 --- a/ui/src/components/WebRTCVideo.tsx +++ b/ui/src/components/WebRTCVideo.tsx @@ -30,8 +30,8 @@ export default function WebRTCVideo({ }) { // Video and stream related refs and states const videoElm = useRef(null); + const audioElm = useRef(null); const fullscreenContainerRef = useRef(null); - const audioElementsRef = useRef([]); const { mediaStream, peerConnectionState } = useRTCStore(); const [isPlaying, setIsPlaying] = useState(false); const [audioAutoplayBlocked, setAudioAutoplayBlocked] = useState(false); @@ -439,63 +439,36 @@ export default function WebRTCVideo({ [updateVideoSizeStore], ); - useEffect( - function updateVideoStreamOnNewTrack() { - if (!peerConnection) return; - const abortController = new AbortController(); - const signal = abortController.signal; - - peerConnection.addEventListener( - "track", - (e: RTCTrackEvent) => { - if (e.track.kind === "video") { - addStreamToVideoElm(e.streams[0]); - return; - } - - if (e.track.kind === "audio") { - const audioElm = document.createElement("audio"); - audioElm.srcObject = new MediaStream([e.track]); - audioElm.style.display = "none"; - document.body.appendChild(audioElm); - audioElementsRef.current.push(audioElm); - - audioElm - .play() - .then(() => { - setAudioAutoplayBlocked(false); - }) - .catch(() => { - console.debug("[Audio] Autoplay blocked, will be started by user interaction"); - setAudioAutoplayBlocked(true); - }); - } - }, - { signal }, - ); - - return () => { - abortController.abort(); - audioElementsRef.current.forEach(audioElm => { - audioElm.srcObject = null; - audioElm.remove(); - }); - audioElementsRef.current = []; - setAudioAutoplayBlocked(false); - }; - }, - [addStreamToVideoElm, peerConnection], - ); - useEffect( function updateVideoStream() { if (!mediaStream) return; - // We set the as early as possible addStreamToVideoElm(mediaStream); }, [addStreamToVideoElm, mediaStream], ); + useEffect( + function updateAudioStream() { + const elm = audioElm.current; + if (!elm || !mediaStream) return; + + elm.srcObject = mediaStream; + elm + .play() + .then(() => setAudioAutoplayBlocked(false)) + .catch(() => { + console.debug("[Audio] Autoplay blocked; waiting for user interaction"); + setAudioAutoplayBlocked(true); + }); + + return () => { + elm.srcObject = null; + setAudioAutoplayBlocked(false); + }; + }, + [mediaStream], + ); + // Setup Keyboard Events useEffect( function setupKeyboardEvents() { @@ -693,6 +666,7 @@ export default function WebRTCVideo({ "animate-slideUpFade": isPlaying, })} /> +