Improve audio video append sync (#7842)

* Improve AV sync in buffer-controller
Resolves #7808
* Fix backfilling of #EXT-X-PROGRAM-DATE-TIME with only one PDT tag after segments
* Fix Low-Latency part/fragment toggle when the selected fragment is changed after initial selection
This commit is contained in:
Rob Walch
2026-05-11 16:01:12 -07:00
committed by GitHub
parent 1214aa0242
commit 987320e051
12 changed files with 412 additions and 51 deletions
@@ -762,7 +762,6 @@ class AudioStreamController
return;
}
if (isMediaFragment(frag)) {
this.fragPrevious = frag;
const track = this.switchingTrack;
if (track) {
this.bufferedTrack = track;
+16 -2
View File
@@ -421,8 +421,9 @@ export default class BaseStreamController
);
fragCurrent.abortRequests();
this.resetLoadingState();
} else {
this.fragPrevious = null;
}
this.fragPrevious = null;
}
}
}
@@ -912,7 +913,6 @@ export default class BaseStreamController
`Loading key for ${frag.sn} of [${details.startSN}-${details.endSN}], ${this.playlistLabel()} ${frag.level}`,
);
this.state = State.KEY_LOADING;
this.fragCurrent = frag;
keyLoadingPromise = this.keyLoader.load(frag).then((keyLoadedData) => {
if (!this.fragContextChanged(keyLoadedData.frag)) {
this.hls.trigger(Events.KEY_LOADED, keyLoadedData);
@@ -961,6 +961,16 @@ export default class BaseStreamController
if (partIndex > -1) {
const part = partList[partIndex];
frag = this.fragCurrent = part.fragment;
if (
!mediaFragmentsAreEqual(frag, fragPrevious) &&
!this.shouldLoadParts(level.details, frag.end)
) {
this.log(
`LL-Part loading OFF @${this.playhead} next part: ${this.fragInfo(frag, false, part)}`,
);
this.loadingParts = false;
return Promise.resolve(null);
}
this.log(
`Loading ${frag.type} sn: ${frag.sn} part: ${part.index} (${partIndex}/${partList.length - 1}) of ${this.fragInfo(frag, false, part)} cc: ${
frag.cc
@@ -1248,6 +1258,10 @@ export default class BaseStreamController
return;
}
if (isMediaFragment(frag) && !this.fragContextChanged(frag)) {
this.fragPrevious = frag;
}
const { data1, data2 } = data;
let buffer = data1;
if (data2) {
+35 -37
View File
@@ -99,11 +99,6 @@ export default class BufferController extends Logger implements ComponentAPI {
// Last MP3 audio chunk appended
private lastMpegAudioChunk: ChunkMetadata | null = null;
// Audio fragment blocked from appending until corresponding video appends or context changes
private blockedAudioAppend: {
op: BufferOperation;
frag: MediaFragment | Part;
} | null = null;
// Keep track of video append position for unblocking audio
private lastVideoAppendEnd: number = 0;
// Whether or not to use ManagedMediaSource API and append source element to media element.
@@ -149,7 +144,7 @@ export default class BufferController extends Logger implements ComponentAPI {
public destroy() {
this.unregisterListeners();
this.details = null;
this.lastMpegAudioChunk = this.blockedAudioAppend = null;
this.lastMpegAudioChunk = null;
this.transferData = this.overrides = undefined;
if (this.operationQueue) {
this.operationQueue.destroy();
@@ -250,7 +245,7 @@ export default class BufferController extends Logger implements ComponentAPI {
];
this.tracks = tracks;
this.resetQueue();
this.lastMpegAudioChunk = this.blockedAudioAppend = null;
this.lastMpegAudioChunk = null;
this.lastVideoAppendEnd = 0;
}
@@ -750,16 +745,14 @@ transfer tracks: ${stringify(transferredTracks, (key, value) => (key === 'initSe
const op: BufferOperation = {
label: 'block-audio',
execute: () => {
const videoTrack = this.tracks.video;
const videoEnd = this.lastVideoAppendEnd;
if (
this.lastVideoAppendEnd > pTime ||
(videoTrack?.buffer &&
BufferHelper.isBuffered(videoTrack.buffer, pTime)) ||
videoEnd > pTime ||
BufferHelper.isBuffered(this.tracks.video?.buffer, pTime) ||
this.fragmentTracker.getAppendedFrag(pTime, PlaylistLevelType.MAIN)
?.gap === true
) {
this.blockedAudioAppend = null;
this.shiftAndExecuteNext('audio');
this.unblockAudio();
}
},
onStart: () => {},
@@ -768,15 +761,12 @@ transfer tracks: ${stringify(transferredTracks, (key, value) => (key === 'initSe
this.warn('Error executing block-audio operation', error);
},
};
this.blockedAudioAppend = { op, frag: partOrFrag };
this.append(op, 'audio', true);
}
private unblockAudio() {
const { blockedAudioAppend, operationQueue } = this;
if (blockedAudioAppend && operationQueue) {
this.blockedAudioAppend = null;
operationQueue.unblockAudio(blockedAudioAppend.op);
if (this.operationQueue) {
this.operationQueue.unblockAudio();
}
}
@@ -819,11 +809,9 @@ transfer tracks: ${stringify(transferredTracks, (key, value) => (key === 'initSe
const videoSb = videoTrack?.buffer;
if (videoSb && sn !== 'initSegment' && offset !== undefined) {
const partOrFrag = part || (frag as MediaFragment);
const blockedAudioAppend = this.blockedAudioAppend;
if (
type === 'audio' &&
parent !== 'main' &&
!this.blockedAudioAppend &&
!(videoTrack.ending || videoTrack.ended)
) {
const pStart = partOrFrag.start;
@@ -834,7 +822,6 @@ transfer tracks: ${stringify(transferredTracks, (key, value) => (key === 'initSe
// wait for video before appending audio
this.blockAudio(partOrFrag);
} else if (
!vappending &&
!BufferHelper.isBuffered(videoSb, pTime) &&
this.lastVideoAppendEnd < pTime
) {
@@ -843,17 +830,15 @@ transfer tracks: ${stringify(transferredTracks, (key, value) => (key === 'initSe
}
} else if (type === 'video') {
const videoAppendEnd = partOrFrag.end;
if (blockedAudioAppend) {
const audioStart = blockedAudioAppend.frag.start;
if (
videoAppendEnd > audioStart ||
videoAppendEnd < this.lastVideoAppendEnd ||
BufferHelper.isBuffered(videoSb, audioStart)
) {
const diff = this.lastVideoAppendEnd - videoAppendEnd;
this.lastVideoAppendEnd = videoAppendEnd;
if (this.isAudioBlocked()) {
if (diff < 0 || diff > partOrFrag.duration) {
this.unblockAudio();
} else {
this.executeNext('audio');
}
}
this.lastVideoAppendEnd = videoAppendEnd;
}
}
@@ -1075,6 +1060,9 @@ transfer tracks: ${stringify(transferredTracks, (key, value) => (key === 'initSe
data: BufferFlushingData,
) {
const { type, startOffset, endOffset } = data;
if (!type || type === 'audio') {
this.unblockAudio();
}
if (type) {
this.append(this.getFlushOp(type, startOffset, endOffset), type);
} else {
@@ -1212,6 +1200,7 @@ transfer tracks: ${stringify(transferredTracks, (key, value) => (key === 'initSe
!this.sourceBuffers.some(([type]) => type && !this.tracks[type]?.ended);
if (allTracksEnding) {
this.unblockAudio();
if (allowEndOfStream) {
this.log(`Queueing EOS`);
this.blockUntilOpen(() => {
@@ -1996,6 +1985,17 @@ transfer tracks: ${stringify(transferredTracks, (key, value) => (key === 'initSe
return !!track && !track.buffer;
}
private isAudioBlocked(): boolean {
return this.currentOp('audio')?.label === 'block-audio';
}
private isAudioBlocking(): boolean {
if (this.operationQueue) {
return this.operationQueue.audioBlocking();
}
return false;
}
// Enqueues an operation to each SourceBuffer queue which, upon execution, resolves a promise. When all promises
// resolve, the onUnblocked function is executed. Functions calling this method do not need to unblock the queue
// upon completion, since we already do it here
@@ -2009,14 +2009,12 @@ transfer tracks: ${stringify(transferredTracks, (key, value) => (key === 'initSe
}
const { operationQueue } = this;
const audioAlreadyBlocked =
bufferNames.length === 2 && this.isAudioBlocking();
// logger.debug(`[buffer-controller]: Blocking ${buffers} SourceBuffer`);
const blockingOperations = bufferNames.map((type) =>
this.appendBlocker(type),
);
const audioBlocked = bufferNames.length > 1 && !!this.blockedAudioAppend;
if (audioBlocked) {
this.unblockAudio();
}
const blockingOperations = audioAlreadyBlocked
? [this.appendBlocker('video')]
: bufferNames.map((type) => this.appendBlocker(type));
return Promise.all(blockingOperations).then((result) => {
if (operationQueue !== this.operationQueue) {
return;
@@ -2033,7 +2031,7 @@ transfer tracks: ${stringify(transferredTracks, (key, value) => (key === 'initSe
// Only cycle the queue if the SB is not updating. There's a bug in Chrome which sets the SB updating flag to
// true when changing the MediaSource duration (https://bugs.chromium.org/p/chromium/issues/detail?id=959359&can=2&q=mediasource%20duration)
// While this is a workaround, it's probably useful to have around
if (!sb || sb.updating) {
if (!sb || sb.updating || (type === 'audio' && this.isAudioBlocked())) {
return;
}
this.shiftAndExecuteNext(type);
+11 -3
View File
@@ -74,6 +74,8 @@ export default class BufferOperationQueue {
if (label === 'async-blocker' || label === 'async-blocker-prepend') {
queue[0].execute();
queue.splice(0, 1);
} else if (label === 'block-audio') {
queue.splice(0, 1);
}
},
);
@@ -87,16 +89,22 @@ export default class BufferOperationQueue {
this.queues[type].splice(1, 0, ...operations);
}
public unblockAudio(op: BufferOperation) {
public unblockAudio() {
if (this.queues === null) {
return;
}
const queue = this.queues.audio;
if (queue[0] === op) {
if (this.current('audio')?.label === 'block-audio') {
this.shiftAndExecuteNext('audio');
}
}
public audioBlocking(): boolean {
if (this.queues === null) {
return false;
}
return this.queues.audio.some((op) => op.label === 'block-audio');
}
public executeNext(type: SourceBufferName) {
if (this.queues === null || this.tracks === null) {
return;
-1
View File
@@ -966,7 +966,6 @@ export default class StreamController
}
let fragError = false;
if (isMediaFragment(frag)) {
this.fragPrevious = frag;
fragError =
!!frag.gap && !frag.tagList.some((tags) => tags[0] === 'GAP');
}
+2 -4
View File
@@ -156,10 +156,8 @@ export class SubtitleStreamController
}
if (!part || end >= frag.end) {
this.fragmentTracker.fragBuffered(frag as MediaFragment);
if (!this.fragContextChanged(frag)) {
if (isMediaFragment(frag)) {
this.fragPrevious = frag;
}
if (isMediaFragment(frag) && !this.fragContextChanged(frag)) {
this.fragPrevious = frag;
}
this.fragBufferedComplete(frag, part);
if (this.media) {
+10 -2
View File
@@ -772,7 +772,11 @@ export default class M3U8Parser {
* computed.
*/
if (firstPdtIndex > 0) {
backfillProgramDateTimes(fragments, firstPdtIndex);
backfillProgramDateTimes(
fragments,
firstPdtIndex,
firstPdtIndex > fragmentLength - 1 ? frag : null,
);
if (firstFragment) {
programDateTimes.unshift(firstFragment as MediaFragment);
}
@@ -996,8 +1000,12 @@ function assignCodec(
function backfillProgramDateTimes(
fragments: M3U8ParserFragments,
firstPdtIndex: number,
fragPrev: Fragment | null,
) {
let fragPrev = fragments[firstPdtIndex] as Fragment;
fragPrev ||= fragments[firstPdtIndex];
if (!fragPrev) {
return;
}
for (let i = firstPdtIndex; i--; ) {
const frag = fragments[i];
// Exit on delta-playlist skipped segments
+5
View File
@@ -114,6 +114,11 @@ export default class TaskLoop extends Logger {
this.tickImmediate();
}
this._tickCallCount = 0;
} else {
this.log(
`possible exception thrown in task-loop (${this.constructor.name}.doTick)`,
);
this._tickCallCount = 0;
}
}
+4 -1
View File
@@ -36,7 +36,10 @@ export class BufferHelper {
/**
* Return true if `media`'s buffered include `position`
*/
static isBuffered(media: Bufferable | null, position: number): boolean {
static isBuffered(
media: Bufferable | null | undefined,
position: number,
): boolean {
if (media) {
const buffered = BufferHelper.getBuffered(media);
for (let i = buffered.length; i--; ) {
@@ -1104,4 +1104,212 @@ describe('BufferController with attached media', function () {
});
});
});
describe('blockAudio / unblockAudio', function () {
function triggerAudioAppend(start: number, duration: number) {
const frag = new Fragment(PlaylistLevelType.AUDIO, '');
frag.start = start;
frag.duration = duration;
const chunkMeta = new ChunkMetadata(frag.start, 0, 0, 0);
const data: BufferAppendingData = {
parent: PlaylistLevelType.AUDIO,
type: 'audio',
data: new Uint8Array(),
frag,
part: null,
chunkMeta,
offset: start,
};
hls.trigger(Events.BUFFER_APPENDING, data);
return frag;
}
function triggerVideoAppend(start: number, duration: number) {
const frag = new Fragment(PlaylistLevelType.MAIN, '');
frag.start = start;
frag.duration = duration;
const chunkMeta = new ChunkMetadata(frag.start, 0, 0, 0);
const data: BufferAppendingData = {
parent: PlaylistLevelType.MAIN,
type: 'video',
data: new Uint8Array(),
frag,
part: null,
chunkMeta,
offset: start,
};
hls.trigger(Events.BUFFER_APPENDING, data);
return frag;
}
function getAudioQueue(): BufferOperation[] {
return (operationQueue as any).queues.audio;
}
function completeAppend(type: SourceBufferName) {
const track = getSourceBufferTrack(bufferController, type);
track?.buffer?.dispatchEvent(new Event('updateend'));
}
it('blocks audio append when video buffer is empty', function () {
const audioQueue = getAudioQueue();
// Audio arrives with no video buffered - should be blocked
triggerAudioAppend(0, 2);
// Queue should have block-audio op followed by the audio append
expect(audioQueue.length).to.equal(2);
expect(audioQueue[0].label).to.equal('block-audio');
expect(audioQueue[1].label).to.equal('append-audio');
});
it('does not block audio append when video is already buffered at audio start', function () {
// Buffer video first
setSourceBufferBufferedRange(bufferController, 'video', 0, 5);
const audioQueue = getAudioQueue();
triggerAudioAppend(0, 2);
// Should not be blocked - just the append
expect(audioQueue.length).to.equal(1);
expect(audioQueue[0].label).to.equal('append-audio');
});
it('blocks audio when audio is ahead of video', function () {
// Video has buffered 0-5, but audio segment starts at 6 (ahead of video)
setSourceBufferBufferedRange(bufferController, 'video', 0, 5);
(bufferController as any).lastVideoAppendEnd = 5;
const audioQueue = getAudioQueue();
triggerAudioAppend(6, 2);
expect(audioQueue.length).to.equal(2);
expect(audioQueue[0].label).to.equal('block-audio');
expect(audioQueue[1].label).to.equal('append-audio');
});
it('unblocks audio when video append advances past blocked audio position', function () {
const audioQueue = getAudioQueue();
// Audio arrives first, gets blocked (no video buffered)
triggerAudioAppend(0, 2);
expect(audioQueue[0].label).to.equal('block-audio');
// Video append arrives — advances lastVideoAppendEnd past audio pTime
// pTime = 0 + 2 * 0.05 = 0.1, videoAppendEnd = 4
triggerVideoAppend(0, 4);
// block-audio execute handler re-runs via executeNext('audio'),
// sees lastVideoAppendEnd (4) > pTime (0.1), calls unblockAudio
expect(audioQueue[0].label).to.equal('append-audio');
});
it('executes audio append after video unblocks it', function () {
const audioBuffer = getSourceBufferTrack(
bufferController,
'audio',
)?.buffer;
expect(audioBuffer).to.exist;
// Audio arrives first, gets blocked
triggerAudioAppend(0, 2);
// Video append arrives and unblocks audio
triggerVideoAppend(0, 4);
// Audio append should have executed (appendBuffer called)
expect(audioBuffer!.appendBuffer).to.have.been.calledOnce;
});
it('blocks each audio segment independently and releases as video advances', function () {
const audioBuffer = getSourceBufferTrack(
bufferController,
'audio',
)?.buffer;
const audioQueue = getAudioQueue();
// First audio triggers block (no video buffered)
triggerAudioAppend(0, 2);
expect(audioQueue[0].label).to.equal('block-audio');
// Second audio also triggers its own block (guard removed)
triggerAudioAppend(2, 2);
expect(audioQueue.length).to.equal(4);
expect(audioQueue[0].label).to.equal('block-audio');
expect(audioQueue[1].label).to.equal('append-audio');
expect(audioQueue[2].label).to.equal('block-audio');
expect(audioQueue[3].label).to.equal('append-audio');
// Video arrives and unblocks first block-audio
triggerVideoAppend(0, 4);
// First block-audio removed, first append-audio executes
expect(audioQueue[0].label).to.equal('append-audio');
expect(audioBuffer!.appendBuffer).to.have.been.calledOnce;
// Complete first audio append — second block-audio becomes head,
// its execute runs and self-resolves (lastVideoAppendEnd=4 > pTime=2.1)
completeAppend('audio');
expect(audioBuffer!.appendBuffer).to.have.been.calledTwice;
});
it('audio queue does not lock when block-audio is behind an in-flight append (#7808)', function () {
const audioBuffer = getSourceBufferTrack(
bufferController,
'audio',
)?.buffer;
const audioQueue = getAudioQueue();
// Video is buffered so first audio append goes through without blocking
setSourceBufferBufferedRange(bufferController, 'video', 0, 4);
(bufferController as any).lastVideoAppendEnd = 4;
triggerAudioAppend(0, 2);
expect(audioQueue.length).to.equal(1);
expect(audioQueue[0].label).to.equal('append-audio');
expect(audioBuffer!.appendBuffer).to.have.been.calledOnce;
// While audio[0] is still in-flight (no updateend yet), second audio
// arrives ahead of video — block-audio lands behind the in-flight append
triggerAudioAppend(6, 2);
expect(audioQueue.length).to.equal(3);
expect(audioQueue[0].label).to.equal('append-audio');
expect(audioQueue[1].label).to.equal('block-audio');
expect(audioQueue[2].label).to.equal('append-audio');
// Video append arrives — advances lastVideoAppendEnd to 8
triggerVideoAppend(4, 4);
// block-audio is not at head so isAudioBlocked() is false;
// video handler just updates lastVideoAppendEnd
expect(audioQueue[1].label).to.equal('block-audio');
// First audio completes — shiftAndExecuteNext makes block-audio the head,
// its execute handler sees lastVideoAppendEnd(8) > pTime(6.1) and self-resolves
completeAppend('audio');
expect(
audioQueue.some((op) => op.label === 'block-audio'),
'block-audio should have self-resolved',
).to.be.false;
expect(audioBuffer!.appendBuffer).to.have.been.calledTwice;
});
it('does not block audio from main parent (muxed content)', function () {
const audioQueue = getAudioQueue();
// Audio with parent='main' should never be blocked
const frag = new Fragment(PlaylistLevelType.MAIN, '');
frag.start = 0;
frag.duration = 2;
const chunkMeta = new ChunkMetadata(0, 0, 0, 0);
hls.trigger(Events.BUFFER_APPENDING, {
parent: PlaylistLevelType.MAIN,
type: 'audio',
data: new Uint8Array(),
frag,
part: null,
chunkMeta,
offset: 0,
});
expect(audioQueue.length).to.equal(1);
expect(audioQueue[0].label).to.equal('append-audio');
});
});
});
@@ -167,4 +167,77 @@ describe('BufferOperationQueue tests', function () {
});
});
});
describe('unblockAudio', function () {
it('removes block-audio op from head and executes next', function () {
const nextExecute = sandbox.spy();
const blockOp: BufferOperation = {
label: 'block-audio',
execute: () => {},
onStart: () => {},
onComplete: () => {},
onError: () => {},
};
const nextOp: BufferOperation = {
label: 'append',
execute: nextExecute,
onStart: () => {},
onComplete: () => {},
onError: () => {},
};
operationQueue.queues.audio.push(blockOp, nextOp);
operationQueue.unblockAudio();
expect(operationQueue.queues.audio).to.have.length(1);
expect(operationQueue.queues.audio[0]).to.equal(nextOp);
expect(nextExecute).to.have.been.calledOnce;
});
it('does not remove head op if it is not block-audio', function () {
const headOp: BufferOperation = {
label: 'append',
execute: () => {},
onStart: () => {},
onComplete: () => {},
onError: () => {},
};
operationQueue.queues.audio.push(headOp);
operationQueue.unblockAudio();
expect(operationQueue.queues.audio).to.have.length(1);
expect(operationQueue.queues.audio[0]).to.equal(headOp);
});
it('does nothing if queue is destroyed', function () {
operationQueue.destroy();
expect(() => operationQueue.unblockAudio()).to.not.throw();
});
});
describe('removeBlockers', function () {
it('removes block-audio ops from audio queue head', function () {
const blockOp: BufferOperation = {
label: 'block-audio',
execute: () => {},
onStart: () => {},
onComplete: () => {},
onError: () => {},
};
const nextOp: BufferOperation = {
label: 'append',
execute: () => {},
onStart: () => {},
onComplete: () => {},
onError: () => {},
};
operationQueue.queues.audio.push(blockOp, nextOp);
operationQueue.removeBlockers();
expect(operationQueue.queues.audio).to.have.length(1);
expect(operationQueue.queues.audio[0]).to.equal(nextOp);
});
});
});
+48
View File
@@ -1328,6 +1328,54 @@ Rollover38803/20160525T064049-01-69844069.ts
expect(result.fragments[2].programDateTime).to.equal(1464366904000);
});
it('parses delta playlists with one #EXT-X-PROGRAM-DATE-TIME after segments', function () {
const level = `#EXTM3U
#EXT-X-TARGETDURATION:6
#EXT-X-VERSION:9
#EXT-X-MAP:URI="fileSequence1.mp4"
#EXT-X-SKIP:SKIPPED-SEGMENTS=17
#EXTINF:6.00000,
fileSequence18.m4s
#EXTINF:6.00000,
fileSequence19.m4s
#EXTINF:6.00000,
fileSequence20.m4s
#EXTINF:6.00000,
fileSequence21.m4s
#EXTINF:6.00000,
fileSequence22.m4s
#EXTINF:6.00000,
fileSequence23.m4s
#EXTINF:6.00000,
fileSequence24.m4s
#EXT-X-PROGRAM-DATE-TIME:2026-05-11T19:03:26.000Z
`;
const result = M3U8Parser.parseLevelPlaylist(
level,
'http://video.example.com/disc.m3u8',
0,
PlaylistLevelType.MAIN,
0,
null,
);
expect(result.playlistParsingError).to.be.null;
expect(result.fragments).to.have.lengthOf(24);
expect(result.hasProgramDateTime).to.be.true;
expect(result.totalduration).to.equal(144);
expect(result.fragments[23].url).to.equal(
'http://video.example.com/fileSequence24.m4s',
);
expect(
new Date(result.fragments[23].programDateTime as number).toISOString(),
).to.equal('2026-05-11T19:03:20.000Z');
expect(
new Date(result.fragments[22].programDateTime as number).toISOString(),
).to.equal('2026-05-11T19:03:14.000Z');
expect(
new Date(result.fragments[21].programDateTime as number).toISOString(),
).to.equal('2026-05-11T19:03:08.000Z');
});
it('parses #EXTINF without a leading digit', function () {
const level = `#EXTM3U
#EXT-X-VERSION:3