diff --git a/.eslintrc.js b/.eslintrc.js index 1a0d2722a..ccce4ebb4 100644 --- a/.eslintrc.js +++ b/.eslintrc.js @@ -26,6 +26,7 @@ module.exports = { __USE_M2TS_ADVANCED_CODECS__: true, __USE_MEDIA_CAPABILITIES__: true, __USE_INTERSTITIALS__: true, + __USE_IFRAMES__: true, }, // see https://github.com/standard/eslint-config-standard // 'prettier' (https://github.com/prettier/eslint-config-prettier) must be last diff --git a/README.md b/README.md index 5bc7158f2..d381a63f8 100644 --- a/README.md +++ b/README.md @@ -79,6 +79,7 @@ For details on the HLS format and these tags' meanings, see https://datatracker. - `#EXT-X-STREAM-INF:` `` +- `#EXT-X-I-FRAME-STREAM-INF` I-frame Media Playlist files - `#EXT-X-MEDIA:` - `#EXT-X-SESSION-DATA:` - `#EXT-X-SESSION-KEY:` EME Key-System selection and preloading @@ -89,8 +90,9 @@ For details on the HLS format and these tags' meanings, see https://datatracker. #### Media Playlist tags - `#EXTM3U` (ignored) -- `#EXT-X-INDEPENDENT-SEGMENTS` (ignored) - `#EXT-X-VERSION=` (value is ignored) +- `#EXT-X-INDEPENDENT-SEGMENTS` (ignored) +- `#EXT-X-I-FRAMES-ONLY` - `#EXTINF:,[]` - `#EXT-X-ENDLIST` - `#EXT-X-MEDIA-SEQUENCE=<n>` @@ -122,7 +124,7 @@ Parsed but missing feature support: For a complete list of issues, see ["Top priorities" in the Release Planning and Backlog project tab](https://github.com/video-dev/hls.js/projects/6). Codec support is dependent on the runtime environment (for example, not all browsers on the same OS support HEVC). -- `#EXT-X-I-FRAME-STREAM-INF` I-frame Media Playlist files +- #EXT-X-PLAYLIST-TYPE is not used to determine if media playlists should be reloaded based on "Expires" header value (#7082) - `REQ-VIDEO-LAYOUT` is not used in variant filtering or selection - "identity" format `SAMPLE-AES` method keys with fmp4, aac, mp3, vtt... segments (MPEG-2 TS only) - MPEG-2 TS segments with FairPlay Streaming, PlayReady, or Widevine encryption diff --git a/api-extractor/report/hls.js.api.md b/api-extractor/report/hls.js.api.md index 915b76cd0..106553d62 100644 --- a/api-extractor/report/hls.js.api.md +++ b/api-extractor/report/hls.js.api.md @@ -102,8 +102,6 @@ export interface AssetListLoadedData { assetListResponse: AssetListJSON; // (undocumented) event: InterstitialEventWithAssetList; - // Warning: (ae-forgotten-export) The symbol "NullableNetworkDetails" needs to be exported by the entry point hls.d.ts - // // (undocumented) networkDetails: NullableNetworkDetails; } @@ -207,7 +205,7 @@ export class AudioStreamController extends BaseStreamController implements Netwo // (undocumented) protected onHandlerDestroying(): void; // (undocumented) - onInitPtsFound(event: Events.INIT_PTS_FOUND, { frag, id, initPTS, timescale, trackId }: InitPTSFoundData): void; + onInitPtsFound(event: Events.INIT_PTS_FOUND, { frag, id, initPTS, timescale, trackId, timestampOffsets, }: InitPTSFoundData): void; // (undocumented) protected onManifestLoading(): void; // (undocumented) @@ -309,6 +307,13 @@ export interface BackBufferData { bufferEnd: number; } +// Warning: (ae-missing-release-tag) "Base" is part of the package's API, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public (undocumented) +export type Base = { + url: string; +}; + // Warning: (ae-missing-release-tag) "BaseData" is part of the package's API, but it is missing a release tag (@alpha, @beta, @public, or @internal) // // @public (undocumented) @@ -396,8 +401,6 @@ export class BasePlaylistController extends Logger implements NetworkComponentAP // @public (undocumented) export class BaseSegment { constructor(base: Base | string); - // Warning: (ae-forgotten-export) The symbol "Base" needs to be exported by the entry point hls.d.ts - // // (undocumented) readonly base: Base; // (undocumented) @@ -535,6 +538,8 @@ export class BaseStreamController extends TaskLoop implements NetworkComponentAP // (undocumented) protected hls: Hls; // (undocumented) + protected get iframesOnly(): boolean | undefined; + // (undocumented) get inFlightFrag(): InFlightData; // (undocumented) protected initPTS: TimestampOffset[]; @@ -588,6 +593,8 @@ export class BaseStreamController extends TaskLoop implements NetworkComponentAP // (undocumented) pauseBuffering(): void; // (undocumented) + protected get playhead(): number; + // (undocumented) protected playlistType: PlaylistLevelType; // (undocumented) protected get primaryPrefetch(): boolean; @@ -958,14 +965,18 @@ export class CaptionScreen { // // @public (undocumented) export class ChunkMetadata { - constructor(level: number, sn: number, id: number, size?: number, part?: number, partial?: boolean); + constructor(level: number, sn: number, id: number, size?: number, part?: number, partial?: boolean, duration?: number, iframe?: boolean); // (undocumented) readonly buffering: { [key in SourceBufferName]: HlsChunkPerformanceTiming; }; // (undocumented) + readonly duration: number; + // (undocumented) readonly id: number; // (undocumented) + readonly iframe: boolean; + // (undocumented) readonly level: number; // (undocumented) readonly part: number; @@ -1106,8 +1117,6 @@ export class DateRange { get startDate(): Date; // (undocumented) get startTime(): number; - // Warning: (ae-forgotten-export) The symbol "MediaFragmentRef" needs to be exported by the entry point hls.d.ts - // // (undocumented) tagAnchor: MediaFragmentRef | null; // (undocumented) @@ -1153,11 +1162,9 @@ export interface DecryptData { // // @public (undocumented) export class Decrypter { - constructor(config: HlsConfig, { removePKCS7Padding }?: { - removePKCS7Padding?: boolean | undefined; - }); + constructor(config: HlsConfig, useSoftware?: boolean); // (undocumented) - decrypt(data: Uint8Array | ArrayBuffer, key: ArrayBuffer, iv: ArrayBuffer, aesMode: DecrypterAesMode): Promise<ArrayBuffer>; + decrypt(data: Uint8Array | ArrayBuffer, key: ArrayBuffer, iv: ArrayBuffer, aesMode: DecrypterAesMode, plainTextLength?: number): Promise<ArrayBuffer>; // (undocumented) destroy(): void; // (undocumented) @@ -1166,10 +1173,6 @@ export class Decrypter { isSync(): boolean; // (undocumented) reset(): void; - // (undocumented) - softwareDecrypt(data: Uint8Array, key: ArrayBuffer, iv: ArrayBuffer, aesMode: DecrypterAesMode): ArrayBuffer | null; - // (undocumented) - webCryptoDecrypt(data: Uint8Array<ArrayBuffer>, key: ArrayBuffer, iv: ArrayBuffer, aesMode: DecrypterAesMode): Promise<ArrayBuffer>; } // Warning: (ae-missing-release-tag) "DecrypterAesMode" is part of the package's API, but it is missing a release tag (@alpha, @beta, @public, or @internal) @@ -1885,7 +1888,7 @@ export class Fragment extends BaseSegment { // (undocumented) cc: number; // (undocumented) - data?: Uint8Array; + data?: Uint8Array<ArrayBuffer>; // (undocumented) get decryptdata(): LevelKey | null; // (undocumented) @@ -1967,7 +1970,7 @@ export class FragmentLoader { // (undocumented) destroy(): void; // (undocumented) - load(frag: Fragment, onProgress?: FragmentLoadProgressCallback): Promise<FragLoadedData>; + load(frag: Fragment, isIFrame?: boolean, onProgress?: FragmentLoadProgressCallback): Promise<FragLoadedData>; // (undocumented) loadPart(frag: Fragment, part: Part, onProgress: FragmentLoadProgressCallback): Promise<FragLoadedData>; } @@ -2100,6 +2103,24 @@ export interface FragParsingUserdataData { samples: UserdataSample[]; } +// Warning: (ae-missing-release-tag) "GapController" is part of the package's API, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public (undocumented) +export class GapController extends TaskLoop { + constructor(hls: Hls, fragmentTracker: FragmentTracker); + // (undocumented) + destroy(): void; + // (undocumented) + ended: number; + // (undocumented) + get hasBuffered(): boolean; + poll(currentTime: number, lastCurrentTime: number): void; + // (undocumented) + tick(): void; + // (undocumented) + waiting: number; +} + // Warning: (ae-missing-release-tag) "GapControllerConfig" is part of the package's API, but it is missing a release tag (@alpha, @beta, @public, or @internal) // // @public (undocumented) @@ -2154,8 +2175,7 @@ class Hls implements HlsEventEmitter { // Warning: (ae-setter-with-docs) The doc comment for the property "capLevelToPlayerSize" must appear on the getter, not the setter. set capLevelToPlayerSize(shouldStartCapping: boolean); readonly config: HlsConfig; - // (undocumented) - createController(ControllerClass: any, components: any): any; + createIFramePlayer(configOverride?: Partial<HlsConfig>): HlsIFramesOnly | null; get currentLevel(): number; // Warning: (ae-setter-with-docs) The doc comment for the property "currentLevel" must appear on the getter, not the setter. set currentLevel(newLevel: number); @@ -2182,6 +2202,7 @@ class Hls implements HlsEventEmitter { getMediaDecodingInfo(level: Level, audioTracks?: MediaPlaylist[]): Promise<MediaDecodingInfo>; static getMediaSource(): typeof MediaSource | undefined; get hasEnoughToStart(): boolean; + get iframeVariants(): LevelParsed[]; // (undocumented) get inFlightFragments(): InFlightFragments; get interstitialsManager(): InterstitialsManager | null; @@ -2262,6 +2283,8 @@ class Hls implements HlsEventEmitter { startLoad(startPosition?: number, skipSeekToStartPosition?: boolean): void; get startPosition(): number; stopLoad(): void; + // (undocumented) + protected streamController: StreamController; get subtitleDisplay(): boolean; // Warning: (ae-setter-with-docs) The doc comment for the property "subtitleDisplay" must appear on the getter, not the setter. set subtitleDisplay(value: boolean); @@ -2277,6 +2300,8 @@ class Hls implements HlsEventEmitter { trigger<E extends keyof HlsListeners>(event: E, eventObject: Parameters<HlsListeners[E]>[1]): boolean; get ttfbEstimate(): number; get url(): string | null; + // (undocumented) + protected _url: string | null; readonly userConfig: Partial<HlsConfig>; static get version(): string; } @@ -2393,17 +2418,23 @@ export type HlsConfig = { cmcd?: CMCDControllerConfig; cmcdController?: typeof CMCDController; contentSteeringController?: typeof ContentSteeringController; + iframeController?: typeof IFrameController; + id3TrackController?: typeof ID3TrackController; + gapController?: typeof GapController; + latencyController?: typeof LatencyController; interstitialsController?: typeof InterstitialsController; enableInterstitialPlayback: boolean; interstitialAppendInPlace: boolean; interstitialLiveLookAhead: number; + loggerId?: string; assetPlayerId?: string; useMediaCapabilities: boolean; + streamController: typeof StreamController; abrController: typeof AbrController; bufferController: typeof BufferController; capLevelController: typeof CapLevelController; errorController: typeof ErrorController; - fpsController: typeof FPSController; + fpsController?: typeof FPSController; progressive: boolean; lowLatencyMode: boolean; primarySessionId?: string; @@ -2429,6 +2460,16 @@ export interface HlsEventEmitter { removeAllListeners<E extends keyof HlsListeners>(event?: E): void; } +// Warning: (ae-missing-release-tag) "HlsIFramesOnly" is part of the package's API, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public (undocumented) +export interface HlsIFramesOnly extends Hls { + // Warning: (ae-forgotten-export) The symbol "LoadMediaAtOptions" needs to be exported by the entry point hls.d.ts + // + // (undocumented) + loadMediaAt(time: number, options?: Partial<LoadMediaAtOptions>): void; +} + // Warning: (ae-missing-release-tag) "HlsListeners" is part of the package's API, but it is missing a release tag (@alpha, @beta, @public, or @internal) // // @public @@ -2641,6 +2682,15 @@ export class HlsUrlParameters { skip?: HlsSkip; } +// Warning: (ae-missing-release-tag) "ID3TrackController" is part of the package's API, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public (undocumented) +export class ID3TrackController implements ComponentAPI { + constructor(hls: any); + // (undocumented) + destroy(): void; +} + // Warning: (ae-missing-release-tag) "IErrorAction" is part of the package's API, but it is missing a release tag (@alpha, @beta, @public, or @internal) // // @public (undocumented) @@ -2654,6 +2704,15 @@ export type IErrorAction = { resolved?: boolean; }; +// Warning: (ae-missing-release-tag) "IFrameController" is part of the package's API, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public (undocumented) +export class IFrameController extends Logger { + constructor(hls: Hls, HlsPlayerClass: typeof Hls); + // (undocumented) + createIFramePlayer(configOverride?: Partial<HlsConfig> | undefined): HlsIFramesOnly | null; +} + // Warning: (ae-missing-release-tag) "ILogFunction" is part of the package's API, but it is missing a release tag (@alpha, @beta, @public, or @internal) // // @public (undocumented) @@ -2710,6 +2769,8 @@ export interface InitPTSFoundData { // (undocumented) timescale: number; // (undocumented) + timestampOffsets: TimestampOffset[]; + // (undocumented) trackId: number; } @@ -3195,6 +3256,28 @@ export type KeyTimeouts = { [keyId: string]: number; }; +// Warning: (ae-missing-release-tag) "LatencyController" is part of the package's API, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public (undocumented) +export class LatencyController implements ComponentAPI { + constructor(hls: Hls); + // (undocumented) + destroy(): void; + // (undocumented) + get drift(): number; + // (undocumented) + get edgeStalled(): number; + // (undocumented) + get latency(): number; + // (undocumented) + get liveSyncPosition(): number | null; + // (undocumented) + get maxLatency(): number; + // (undocumented) + get targetLatency(): number | null; + set targetLatency(latency: number); +} + // Warning: (ae-missing-release-tag) "LatencyControllerConfig" is part of the package's API, but it is missing a release tag (@alpha, @beta, @public, or @internal) // // @public (undocumented) @@ -3251,6 +3334,8 @@ export class Level { // (undocumented) readonly id: number; // (undocumented) + readonly iframes?: boolean; + // (undocumented) loaded?: { bytes: number; duration: number; @@ -3409,6 +3494,8 @@ export class LevelDetails { // (undocumented) holdBack: number; // (undocumented) + iframesOnly: boolean; + // (undocumented) get lastPartIndex(): number; // (undocumented) get lastPartSn(): number; @@ -3570,6 +3657,8 @@ export interface LevelParsed extends CodecsParsed { // (undocumented) id?: number; // (undocumented) + iframes?: boolean; + // (undocumented) name: string; // (undocumented) supplemental?: CodecsParsed; @@ -3963,8 +4052,6 @@ export class M3U8Parser { static parseLevelPlaylist(string: string, baseurl: string, id: number, type: PlaylistLevelType, levelUrlId: number, multivariantVariableList: VariableMap | null): LevelDetails; // (undocumented) static parseMasterPlaylist(string: string, baseurl: string): ParsedMultivariantPlaylist; - // Warning: (ae-forgotten-export) The symbol "ParsedMultivariantMediaOptions" needs to be exported by the entry point hls.d.ts - // // (undocumented) static parseMasterPlaylistMedia(string: string, baseurl: string, parsed: ParsedMultivariantPlaylist): ParsedMultivariantMediaOptions; // (undocumented) @@ -3987,6 +4074,8 @@ export interface ManifestLoadedData { // (undocumented) contentSteering: ContentSteeringOptions | null; // (undocumented) + iframeVariants: LevelParsed[]; + // (undocumented) levels: LevelParsed[]; // (undocumented) networkDetails: NullableNetworkDetails; @@ -4027,6 +4116,8 @@ export interface ManifestParsedData { // (undocumented) firstLevel: number; // (undocumented) + iframeVariants: LevelParsed[]; + // (undocumented) levels: Level[]; // (undocumented) sessionData: Record<string, AttrList> | null; @@ -4158,6 +4249,17 @@ export interface MediaFragment extends Fragment { sn: number; } +// Warning: (ae-missing-release-tag) "MediaFragmentRef" is part of the package's API, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public (undocumented) +export type MediaFragmentRef = { + base: Base; + start: number; + duration: number; + sn: number; + programDateTime: number | null; +}; + // Warning: (ae-missing-release-tag) "MediaKeyFunc" is part of the package's API, but it is missing a release tag (@alpha, @beta, @public, or @internal) // // @public (undocumented) @@ -4310,6 +4412,11 @@ export interface NetworkComponentAPI extends ComponentAPI { stopLoad(): void; } +// Warning: (ae-missing-release-tag) "NetworkDetails" is part of the package's API, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public (undocumented) +export type NetworkDetails = Response | XMLHttpRequest; + // Warning: (ae-missing-release-tag) "NetworkErrorAction" is part of the package's API, but it is missing a release tag (@alpha, @beta, @public, or @internal) // // @public (undocumented) @@ -4354,12 +4461,27 @@ export interface NonNativeTextTracksData { tracks: Array<NonNativeTextTrack>; } +// Warning: (ae-missing-release-tag) "NullableNetworkDetails" is part of the package's API, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public (undocumented) +export type NullableNetworkDetails = NetworkDetails | null; + +// Warning: (ae-missing-release-tag) "ParsedMultivariantMediaOptions" is part of the package's API, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public (undocumented) +export type ParsedMultivariantMediaOptions = { + AUDIO?: MediaPlaylist[]; + SUBTITLES?: MediaPlaylist[]; + 'CLOSED-CAPTIONS'?: MediaPlaylist[]; +}; + // Warning: (ae-missing-release-tag) "ParsedMultivariantPlaylist" is part of the package's API, but it is missing a release tag (@alpha, @beta, @public, or @internal) // // @public (undocumented) export type ParsedMultivariantPlaylist = { contentSteering: ContentSteeringOptions | null; levels: LevelParsed[]; + iframeVariants: LevelParsed[]; playlistParsingError: Error | null; sessionData: Record<string, AttrList> | null; sessionKeys: LevelKey[] | null; diff --git a/build-config.js b/build-config.js index cdcd5f9b3..54a318733 100644 --- a/build-config.js +++ b/build-config.js @@ -62,12 +62,8 @@ const flags = { variableSubstitution: readFeatureFlag('VARIABLE_SUBSTITUTION'), m2tsAdvancedCodecs: readFeatureFlag('M2TS_ADVANCED_CODECS'), mediaCapabilities: readFeatureFlag('MEDIA_CAPABILITIES'), - interstitials: readFeatureFlag( - 'INTERSTITIALS', - // Backward-compatible support for legacy misspelled env vars - 'USE_INTERSTITALS', - 'INTERSTITALS', - ), + interstitials: readFeatureFlag('INTERSTITIALS'), + iframes: readFeatureFlag('IFRAMES'), }; function getFeatureSupport(type) { @@ -81,6 +77,7 @@ function getFeatureSupport(type) { m2tsAdvancedCodecs: isFeatureEnabled(type, flags.m2tsAdvancedCodecs), mediaCapabilities: isFeatureEnabled(type, flags.mediaCapabilities), interstitials: isFeatureEnabled(type, flags.interstitials), + iframes: isFeatureEnabled(type, flags.iframes), }; } @@ -105,6 +102,7 @@ const buildConstants = ( __USE_M2TS_ADVANCED_CODECS__: JSON.stringify(features.m2tsAdvancedCodecs), __USE_MEDIA_CAPABILITIES__: JSON.stringify(features.mediaCapabilities), __USE_INTERSTITIALS__: JSON.stringify(features.interstitials), + __USE_IFRAMES__: JSON.stringify(features.iframes), ...additional, }, @@ -319,6 +317,13 @@ function getAliasesForDist(format, features) { }; } + if (!features.iframes) { + aliases = { + ...aliases, + './controller/iframe-controller': `./${emptyFile}`, + }; + } + return aliases; } diff --git a/docs/API.md b/docs/API.md index 5af412f8a..2452dd4b8 100644 --- a/docs/API.md +++ b/docs/API.md @@ -191,6 +191,10 @@ See [API Reference](https://hlsjs-dev.video-dev.org/api-docs/) for a complete li - [`hls.subtitleTracks`](#hlssubtitletracks) - [`hls.subtitleTrack`](#hlssubtitletrack) - [`hls.subtitleDisplay`](#hlssubtitledisplay) +- [I-Frame Variants API](#i-frame-variants-api) + - [`hls.iframeVariants`](#hlsiframevariants) + - [`hls.createIFramePlayer()`](#hlscreateiframeplayer) + - [Example usage](#example-usage) - [Live stream API](#live-stream-api) - [`hls.liveSyncPosition`](#hlslivesyncposition) - [`hls.latency`](#hlslatency) @@ -2160,6 +2164,77 @@ get/set : index of selected subtitle track in `hls.subtitleTracks`. Returns -1 i get/set : if set to true the active subtitle track mode will be set to `showing` and the browser will display the active subtitles. If set to false, the mode will be set to `hidden`. +## I-Frame Variants API + +### `hls.iframeVariants` + +get : array of parsed I-Frame variants. `iframeVariants` are not selectable in the primary instance (use `hls.createIFramePlayer()`). + +### `hls.createIFramePlayer()` + +`createIFramePlayer` returns a new HlsIFramesOnly instance that uses the current instance's `iframeVariants` as its `levels`. Returns `null` when `iframeVariants` is empty and before any levels have loaded. The IFramePlayer instance is configured automatically based on the current instance. This method accepts optional config overrides argument. + +#### Example usage + +IFrame instances are used to load HLS `#EXT-X-I-FRAME-STREAM-INF` variants (HLS media playlists with `#EXT-X-I-FRAMES-ONLY` segments) that best fit a secondary HTMLVideoElement. I-Frames are buffered and then seeked to (one at a time) using `hlsIframesOnly.loadMediaAt(time)`. There is no need to call `loadSource` on the IFrame instances. The media attached to an IFrame instance will only buffer video I-Frames. Any audio in muxed segments is dropped. Calling `loadMediaAt` while one loading operation is active and another is pending will cancel the latter. + +`hlsIframesOnly` instances do not respond to external seeking or setting for `currentTime` on the attached HTMLVideoElement. Use `loadMediaAt` to buffer frames before they are seeked to. This ensures the last rendered frame is displayed until the next requested one is ready. An I-Frame can be considered appended on FRAG_BUFFERED and rendered on HTMLVideoElement "seeked". + +The I-Frame playlist selection is driven by the video element's dimensions with `capLevelToPlayerSize: true` set in the config. Ensure that the element is styled and sized before calling `startLoad` or `loadMediaAt` to avoid loading additional playlists. + +Note that each time `hls.createIFramePlayer()` is called, it will return a new instance or null. While you may instantiate more than one instance it is not recommended. + +```ts +const mainVideo = document.getElementById('video_1'); +const iframeVideo = document.getElementById('video_2'); +const hls = new Hls(); + +let hlsIframesOnly: HlsIFramesOnly | null = null; + +hls.loadSource('http://example.com/primary.m3u8'); +hls.attachMedia(mainVideo); +// IFrame players can be created as early as MANIFEST_LOADED, but it is best to wait until after media is loaded to make sure frames are synched. +hls.once(Events.INIT_PTS_FOUND, createHlsIframesOnlyIfNeeded); + +function createHlsIframesOnlyIfNeeded() { + if (hls.url !== hlsIframesOnly?.url) { + // If player was destroyed or asset url changed, remove reference. + // (IFrames instance is destroyed when another source is loaded by parent Hls instance.) + hlsIframesOnly = null; + } + if (!hlsIframesOnly && hls.iframeVariants.length) { + hlsIframesOnly = hls.createIFramePlayer(); + if (hlsIframesOnly) { + hlsIframesOnly.attachMedia(iframeVideo); + // Load the level that matches the current video element dimensions. + hlsIframesOnly.startLoad(); + hlsIframesOnly.once( + Events.LEVEL_UPDATED, + (name, { details: { fragments } }) => { + /* fragments contains all iframe start times and durations */ + }, + ); + hlsIframesOnly.on(Events.FRAG_BUFFERED, (name, { frag }) => { + /* iframe buffered */ + }); + hlsIframesOnly.on(Events.ERROR, (name, { error }) => { + if (error.name == 'QuotaExceededError') { + /* MSE buffer is full */ + } + }); + } + } +} +function renderIFrame(currentTime) { + iframeVideo.onseeked = () => + null /* iframe rendered > show iframe video element and remove seeked listener */; + hlsIframesOnly?.loadMediaAt(currentTime); +} +function preloadIFrame(time) { + hlsIframesOnly?.loadMediaAt(time, { seekOnAppend: false }); +} +``` + ## Live stream API ### `hls.liveSyncPosition` diff --git a/src/config.ts b/src/config.ts index a4609ff44..1c18840be 100644 --- a/src/config.ts +++ b/src/config.ts @@ -8,7 +8,12 @@ import ContentSteeringController from './controller/content-steering-controller' import EMEController from './controller/eme-controller'; import ErrorController from './controller/error-controller'; import FPSController from './controller/fps-controller'; +import GapController from './controller/gap-controller'; +import ID3TrackController from './controller/id3-track-controller'; +import { IFrameController } from './controller/iframe-controller'; import InterstitialsController from './controller/interstitials-controller'; +import LatencyController from './controller/latency-controller'; +import StreamController from './controller/stream-controller'; import { SubtitleStreamController } from './controller/subtitle-stream-controller'; import SubtitleTrackController from './controller/subtitle-track-controller'; import { TimelineController } from './controller/timeline-controller'; @@ -333,6 +338,14 @@ export type HlsConfig = { cmcdController?: typeof CMCDController; // Content Steering contentSteeringController?: typeof ContentSteeringController; + // IFrame Controller (setting to null disables I-Frame instance support) + iframeController?: typeof IFrameController; + // ID3 Track Controller + id3TrackController?: typeof ID3TrackController; + // Gap Controller + gapController?: typeof GapController; + // Latency Controller + latencyController?: typeof LatencyController; // Interstitial Controller (setting to null disables Interstitials parsing and playback) interstitialsController?: typeof InterstitialsController; // Option to disable internal playback handling of Interstitials (set to false to disable Interstitials playback without disabling parsing and schedule events) @@ -342,15 +355,18 @@ export type HlsConfig = { // How many seconds past the end of a live playlist to preload Interstitial assets interstitialLiveLookAhead: number; // An optional `Hls` instance ID prefixed to debug logs + loggerId?: string; + // Identifies the `Hls` instance as an Interstitial asset player (also fills in for `loggerId`) assetPlayerId?: string; // MediaCapabilies API for level, track, and switch filtering useMediaCapabilities: boolean; + streamController: typeof StreamController; abrController: typeof AbrController; bufferController: typeof BufferController; capLevelController: typeof CapLevelController; errorController: typeof ErrorController; - fpsController: typeof FPSController; + fpsController?: typeof FPSController; progressive: boolean; lowLatencyMode: boolean; primarySessionId?: string; @@ -443,11 +459,6 @@ export const hlsDefaultConfig: HlsConfig = { xhrSetup: undefined, // used by xhr-loader licenseXhrSetup: undefined, // used by eme-controller licenseResponseCallback: undefined, // used by eme-controller - abrController: AbrController, - bufferController: BufferController, - capLevelController: CapLevelController, - errorController: ErrorController, - fpsController: FPSController, stretchShortVideoTrack: false, // used by mp4-remuxer maxAudioFramesDrift: 1, // used by mp4-remuxer forceKeyFrameOnDiscontinuity: true, // used by ts-demuxer @@ -610,6 +621,16 @@ export const hlsDefaultConfig: HlsConfig = { fragLoadingRetryDelay: 1000, fragLoadingMaxRetryTimeout: 64000, + streamController: StreamController, + abrController: AbrController, + bufferController: BufferController, + capLevelController: CapLevelController, + errorController: ErrorController, + fpsController: FPSController, + id3TrackController: ID3TrackController, + gapController: GapController, + latencyController: LatencyController, + // Dynamic Modules ...timelineConfig(), subtitleStreamController: __USE_SUBTITLES__ @@ -626,6 +647,7 @@ export const hlsDefaultConfig: HlsConfig = { contentSteeringController: __USE_CONTENT_STEERING__ ? ContentSteeringController : undefined, + iframeController: __USE_IFRAMES__ ? IFrameController : undefined, interstitialsController: __USE_INTERSTITIALS__ ? InterstitialsController : undefined, diff --git a/src/controller/audio-stream-controller.ts b/src/controller/audio-stream-controller.ts index 565904ce7..50cfa736f 100644 --- a/src/controller/audio-stream-controller.ts +++ b/src/controller/audio-stream-controller.ts @@ -141,14 +141,21 @@ class AudioStreamController // INIT_PTS_FOUND is triggered when the video track parsed in the stream-controller has a new PTS value onInitPtsFound( event: Events.INIT_PTS_FOUND, - { frag, id, initPTS, timescale, trackId }: InitPTSFoundData, + { + frag, + id, + initPTS, + timescale, + trackId, + timestampOffsets, + }: InitPTSFoundData, ) { // Always update the new INIT PTS // Can change due level switch if (id === PlaylistLevelType.MAIN) { const cc = frag.cc; const inFlightFrag = this.fragCurrent; - this.initPTS[cc] = { baseTime: initPTS, timescale, trackId }; + this.initPTS = timestampOffsets; this.log( `InitPTS for cc: ${cc} found from main: ${initPTS / timescale} (${initPTS}/${timescale}) trackId: ${trackId}`, ); @@ -663,6 +670,7 @@ class AudioStreamController payload.byteLength, partIndex, partial, + frag.duration, ); transmuxer.push( payload, diff --git a/src/controller/base-stream-controller.ts b/src/controller/base-stream-controller.ts index af5317e80..19263a23a 100644 --- a/src/controller/base-stream-controller.ts +++ b/src/controller/base-stream-controller.ts @@ -43,7 +43,7 @@ import { } from '../utils/level-helper'; import { estimatedAudioBitrate } from '../utils/mediacapabilities-helper'; import { appendUint8Array } from '../utils/mp4-tools'; -import TimeRanges from '../utils/time-ranges'; +import { timeRangesToString } from '../utils/time-ranges'; import type { FragmentTracker } from './fragment-tracker'; import type { HlsConfig } from '../config'; import type TransmuxerInterface from '../demux/transmuxer-interface'; @@ -397,12 +397,12 @@ export default class BaseStreamController currentTime, config.maxBufferHole, ); - const noFowardBuffer = !bufferInfo.len; + const fowardBuffer = bufferInfo.len; this.log( `Media seeking to ${ Number.isFinite(currentTime) ? currentTime.toFixed(3) : currentTime - }, state: ${state}, ${noFowardBuffer ? 'out of' : 'in'} buffer`, + }, state: ${state}, ${!fowardBuffer ? 'out of' : 'in'} buffer`, ); if (this.state === State.ENDED) { @@ -415,7 +415,7 @@ export default class BaseStreamController fragCurrent.start + fragCurrent.duration + tolerance; // if seeking out of buffered range or into new one if ( - noFowardBuffer || + !fowardBuffer || fragEndOffset < bufferInfo.start || fragStartOffset > bufferInfo.end ) { @@ -463,8 +463,8 @@ export default class BaseStreamController if (shouldLoadParts) { this.log( `LL-Part loading ON after seeking to ${currentTime.toFixed( - 2, - )} with buffer @${bufferEnd.toFixed(2)}`, + 3, + )} with buffer @${bufferEnd.toFixed(3)}`, ); this.loadingParts = shouldLoadParts; } @@ -483,7 +483,7 @@ export default class BaseStreamController } } - if (noFowardBuffer && this.state === State.IDLE) { + if (fowardBuffer < 1 && this.state === State.IDLE) { // Async tick to speed up processing this.tickImmediate(); } @@ -558,12 +558,17 @@ export default class BaseStreamController this.warn( `${frag.type} sn: ${frag.sn}${ data.part ? ' part: ' + data.part.index : '' - } of ${this.fragInfo(frag, false, data.part)}) was dropped during download.`, + } of ${this.fragInfo(frag, false, data.part)} was dropped during download.`, ); this.fragmentTracker.removeFragment(frag); return; } - frag.stats.chunkCount++; + const chunkCount = ++frag.stats.chunkCount; + this.log( + `load progress ${frag.type} sn: ${frag.sn}${ + data.part ? ' part: ' + data.part.index : '' + } of ${this.fragInfo(frag, false, data.part)} chunk: ${chunkCount}`, + ); this._handleFragmentLoadProgress(data); }; @@ -711,12 +716,17 @@ export default class BaseStreamController ) { const startTime = self.performance.now(); // decrypt init segment data + const byteRange = frag.byteRange; + const plainTextLength = byteRange.length + ? byteRange[1] - byteRange[0] + : 0; return this.decrypter .decrypt( new Uint8Array(payload), decryptData.key.buffer, decryptData.iv.buffer, getAesModeFromFullSegmentMethod(decryptData.method), + plainTextLength, ) .catch((err) => { hls.trigger(Events.ERROR, { @@ -816,7 +826,7 @@ export default class BaseStreamController part ? ' part: ' + part.index : '' } of ${this.fragInfo(frag, false, part)} > buffer:${ media - ? TimeRanges.toString(BufferHelper.getBuffered(media)) + ? timeRangesToString(BufferHelper.getBuffered(media)) : '(detached)' })`, ); @@ -845,7 +855,8 @@ export default class BaseStreamController if (!transmuxer) { return; } - const { frag, part, partsLoaded } = fragLoadedEndData; + const frag = fragLoadedEndData.frag as MediaFragment; + const { part, partsLoaded } = fragLoadedEndData; // If we did not load parts, or loaded all parts, we have complete (not partial) fragment data const complete = !partsLoaded || @@ -853,11 +864,18 @@ export default class BaseStreamController partsLoaded.some((fragLoaded) => !fragLoaded); const chunkMeta = new ChunkMetadata( frag.level, - frag.sn as number, + frag.sn, frag.stats.chunkCount + 1, 0, part ? part.index : -1, !complete, + frag.duration, + this.iframesOnly, + ); + this.log( + `load complete ${frag.type} sn: ${frag.sn}${ + part ? ' part: ' + part.index : '' + } of ${this.fragInfo(frag, false, part)}`, ); transmuxer.flush(chunkMeta); } @@ -936,7 +954,7 @@ export default class BaseStreamController const part = partList[partIndex]; frag = this.fragCurrent = part.fragment; this.log( - `Loading ${frag.type} sn: ${frag.sn} part: ${part.index} (${partIndex}/${partList.length - 1}) of ${this.fragInfo(frag, false, part)}) cc: ${ + `Loading ${frag.type} sn: ${frag.sn} part: ${part.index} (${partIndex}/${partList.length - 1}) of ${this.fragInfo(frag, false, part)} cc: ${ frag.cc } [${details.startSN}-${details.endSN}], target: ${parseFloat( targetBufferTime.toFixed(3), @@ -998,7 +1016,7 @@ export default class BaseStreamController if (isMediaFragment(frag) && this.loadingParts) { this.log( `LL-Part loading OFF after next part miss @${targetBufferTime.toFixed( - 2, + 3, )} Check buffer at sn: ${frag.sn} loaded parts: ${details.partList?.filter((p) => p.loaded).map((p) => `[${p.start}-${p.end}]`)}`, ); this.loadingParts = false; @@ -1008,7 +1026,7 @@ export default class BaseStreamController } this.log( - `Loading ${frag.type} sn: ${frag.sn} of ${this.fragInfo(frag, false)}) cc: ${frag.cc} ${ + `Loading ${frag.type} sn: ${frag.sn} of ${this.fragInfo(frag, false)} cc: ${frag.cc} ${ '[' + details.startSN + '-' + details.endSN + ']' }, target: ${parseFloat(targetBufferTime.toFixed(3))}${ frag.byteRange.length ? ` range:${frag.byteRange.join('-')}` : '' @@ -1030,7 +1048,11 @@ export default class BaseStreamController if (!keyLoadedData || this.fragContextChanged(keyLoadedData.frag)) { return null; } - return this.fragmentLoader.load(frag, progressCallback); + return this.fragmentLoader.load( + frag, + this.iframesOnly, + progressCallback, + ); }) .catch((error) => this.handleFragLoadError(error)); } else { @@ -1039,6 +1061,7 @@ export default class BaseStreamController result = Promise.all([ this.fragmentLoader.load( frag, + this.iframesOnly, dataOnProgress ? progressCallback : undefined, ), keyLoadingPromise, @@ -1150,7 +1173,7 @@ export default class BaseStreamController this.log( `LL-Part loading ${ shouldLoadParts ? 'ON' : 'OFF' - } after parsing segment ending @${frag.end.toFixed(2)}`, + } after parsing segment ending @${frag.end.toFixed(3)}`, ); this.loadingParts = shouldLoadParts; } @@ -1165,16 +1188,14 @@ export default class BaseStreamController if (!details) { return this.loadingParts; } - if (details.partList) { + if (details.partList && !this.iframesOnly) { // Buffer must be ahead of first part + duration of parts after last segment // and playback must be at or past segment adjacent to part list const firstPart = details.partList[0]; const safePartStart = firstPart.end + (details.fragmentHint?.duration || 0); if (bufferEnd >= safePartStart) { - const playhead = this.hls.hasEnoughToStart - ? this.media?.currentTime || this.lastCurrentTime - : this.getLoadPosition(); + const playhead = this.playhead; if (playhead > firstPart.start - firstPart.fragment.duration) { return true; } @@ -1290,10 +1311,7 @@ export default class BaseStreamController bufferable: Bufferable | null, type: PlaylistLevelType, ): BufferInfo | null { - const pos = this.getLoadPosition(); - if (!Number.isFinite(pos)) { - return null; - } + const pos = this.playhead; const backwardSeek = this.lastCurrentTime > pos; const maxBufferHole = backwardSeek || this.media?.paused ? 0 : this.config.maxBufferHole; @@ -1381,7 +1399,7 @@ export default class BaseStreamController // find fragment index, contiguous with end of buffer position const { config } = this; - const start = fragments[0].start; + const playlistStart = levelDetails.fragmentStart; const canLoadParts = config.lowLatencyMode && !!levelDetails.partList; let frag: MediaFragment | null = null; @@ -1401,26 +1419,44 @@ export default class BaseStreamController (!levelDetails.PTSKnown && !this.startFragRequested && this.startPosition === -1) || - pos < start + pos < playlistStart ) { if (canLoadParts && !this.loadingParts) { this.log(`LL-Part loading ON for initial live fragment`); this.loadingParts = true; } frag = this.getInitialLiveFragment(levelDetails); + const configValue = this.config.startPosition; const mainStart = this.hls.startPosition; const liveSyncPosition = this.hls.liveSyncPosition; - const startPosition = frag - ? (mainStart !== -1 && mainStart >= start - ? mainStart - : liveSyncPosition) || frag.start - : pos; + const fragStart = frag?.start || 0; + + let startPosition: number | undefined; + let reason: string | undefined; + if (mainStart !== -1 && mainStart >= playlistStart) { + startPosition = mainStart; + reason = mainStart === configValue ? 'config' : 'next load start'; + } else if (liveSyncPosition) { + startPosition = liveSyncPosition; + reason = 'live edge'; + } else { + startPosition = pos; + reason = 'buffer pos'; + } + if (startPosition < fragStart) { + startPosition = fragStart; + reason = 'live frag start'; + } + if (startPosition < playlistStart) { + startPosition = playlistStart; + reason = 'playlist start'; + } this.log( - `Setting startPosition to ${startPosition} to match start frag at live edge. mainStart: ${mainStart} liveSyncPosition: ${liveSyncPosition} frag.start: ${frag?.start}`, + `Setting startPosition to ${startPosition.toFixed(3)} ${mainStart === -1 ? '' : `(from ${configValue}) `}based on ${reason}. live edge: ${liveSyncPosition} live frag start: ${fragStart.toFixed(3)} playlist start: ${playlistStart.toFixed(3)} buffer pos: ${pos}`, ); this.startPosition = this.nextLoadPosition = startPosition; } - } else if (pos <= start) { + } else if (pos <= playlistStart) { // VoD playlist: if loadPosition before start of playlist, load first fragment frag = fragments[0]; } @@ -1796,7 +1832,7 @@ export default class BaseStreamController alignStream(switchDetails, details, this); const alignedSlidingStart = details.fragmentStart; this.log( - `Live playlist sliding: ${alignedSlidingStart.toFixed(2)} start-sn: ${ + `Live playlist sliding: ${alignedSlidingStart.toFixed(3)} start-sn: ${ previousDetails ? previousDetails.startSN : 'na' }->${details.startSN} fragments: ${length}`, ); @@ -1849,9 +1885,10 @@ export default class BaseStreamController } else if (details.live) { // Leave this.startPosition at -1, so that we can use `getInitialLiveFragment` logic when startPosition has // not been specified via the config or an as an argument to startLoad (#3736). - startPosition = this.hls.liveSyncPosition || sliding; + const liveSyncPosition = this.hls.liveSyncPosition; + startPosition = liveSyncPosition || sliding; this.log( - `Setting startPosition to -1 to start at live edge ${startPosition}`, + `Setting startPosition to -1 to start at ${liveSyncPosition ? 'live edge' : 'playlist start'} ${startPosition.toFixed(3)}`, ); this.startPosition = -1; } else { @@ -1867,7 +1904,7 @@ export default class BaseStreamController const { media } = this; // if we have not yet loaded any fragment, start loading from start position let pos = 0; - if (this.hls?.hasEnoughToStart && media) { + if (this.hls?.hasEnoughToStart && media && !this.iframesOnly) { pos = media.currentTime; } else if (this.nextLoadPosition >= 0) { pos = this.nextLoadPosition; @@ -1876,6 +1913,23 @@ export default class BaseStreamController return pos; } + protected get playhead(): number { + return this.hls?.hasEnoughToStart + ? this.media?.currentTime || this.lastCurrentTime + : this.getLoadPosition(); + } + + protected get iframesOnly(): boolean | undefined { + if (this.playlistType !== PlaylistLevelType.MAIN) { + return false; + } + const details = this.getLevelDetails(); + if (details) { + return details.iframesOnly; + } + return this.levelLastLoaded?.iframes; + } + private handleFragLoadAborted(frag: Fragment, part: Part | null | undefined) { if ( this.transmuxer && @@ -2036,7 +2090,7 @@ export default class BaseStreamController // this happens on IE/Edge, refer to https://github.com/video-dev/hls.js/pull/708 // in that case flush the whole audio buffer to recover this.warn( - `Buffer full error while media.currentTime (${this.getLoadPosition()}) is not buffered, flush ${playlistType} buffer`, + `Buffer full error while media.currentTime (${this.playhead}) is not buffered, flush ${playlistType} buffer`, ); } else if ( bufferedInfo.nextStart && @@ -2152,6 +2206,11 @@ export default class BaseStreamController this.warn('level.details undefined'); return; } + this.log( + `update level timing ${frag.type} sn: ${frag.sn}${ + part ? ' part: ' + part.index : '' + } of ${this.fragInfo(frag, false, part)}`, + ); const parsed = Object.keys(frag.elementaryStreams).reduce( (result, type) => { const info = frag.elementaryStreams[type]; @@ -2175,6 +2234,7 @@ export default class BaseStreamController info.endPTS, info.startDTS, info.endDTS, + this.iframesOnly, this, ); this.hls.trigger(Events.LEVEL_PTS_UPDATED, { @@ -2225,7 +2285,7 @@ export default class BaseStreamController this.log( `Parsed ${frag.type} sn: ${frag.sn}${ part ? ' part: ' + part.index : '' - } of ${this.fragInfo(frag, false, part)})`, + } of ${this.fragInfo(frag, false, part)}`, ); this.hls.trigger(Events.FRAG_PARSED, { frag, part }); } @@ -2243,7 +2303,7 @@ export default class BaseStreamController (pts && !part ? frag.endPTS : (part || frag).end) ?? NaN ).toFixed( 3, - )}]${part && frag.type === 'main' ? 'INDEPENDENT=' + (part.independent ? 'YES' : 'NO') : ''}`; + )}]${part && frag.type === 'main' ? 'INDEPENDENT=' + (part.independent ? 'YES' : 'NO') : ''})`; } private treatAsGap(frag: MediaFragment, level?: Level) { @@ -2331,7 +2391,7 @@ export default class BaseStreamController } } - const currentTime = this.media?.currentTime || this.getLoadPosition(); + const currentTime = this.playhead; // Do not flush in live stream with low buffer const okToFlushForwardBuffer = !levelDetails?.live || bufferInfo.end - currentTime > fetchdelay * 1.5; @@ -2356,7 +2416,7 @@ export default class BaseStreamController // find buffer range that will be reached once new fragment will be fetched const bufferedFrag = okToFlushForwardBuffer - ? this.getBufferedFrag(this.getLoadPosition() + fetchdelay) + ? this.getBufferedFrag(this.playhead + fetchdelay) : null; if (bufferedFrag) { @@ -2401,7 +2461,7 @@ export default class BaseStreamController } // remove back-buffer - const fragPlayingCurrent = this.getAppendedFrag(this.getLoadPosition()); + const fragPlayingCurrent = this.getAppendedFrag(this.playhead); if (fragPlayingCurrent && fragPlayingCurrent.start > 1) { const isAudio = playlistType === PlaylistLevelType.AUDIO; // flush buffer preceding current fragment (flush until current fragment start offset) @@ -2452,7 +2512,7 @@ export default class BaseStreamController this.state = State.IDLE; break; } - this.nextLoadPosition = this.getLoadPosition(); + this.nextLoadPosition = this.playhead; } protected checkFragmentChanged(): boolean { diff --git a/src/controller/buffer-controller.ts b/src/controller/buffer-controller.ts index 02e04efa9..1bdd5c589 100755 --- a/src/controller/buffer-controller.ts +++ b/src/controller/buffer-controller.ts @@ -23,6 +23,7 @@ import { isManagedMediaSource, } from '../utils/mediasource-helper'; import { stringify } from '../utils/safe-json-stringify'; +import { timeRangesToString } from '../utils/time-ranges'; import type { FragmentTracker } from './fragment-tracker'; import type { HlsConfig } from '../config'; import type Hls from '../hls'; @@ -860,7 +861,11 @@ transfer tracks: ${stringify(transferredTracks, (key, value) => (key === 'initSe label: `append-${type}`, execute: () => { chunkStats.executeStart = self.performance.now(); - + // this.log( + // `appending "${type}" sn: ${sn}${part ? ' p: ' + part.index : ''} of ${ + // parent === PlaylistLevelType.MAIN ? 'level' : 'track' + // } ${frag.level} cc: ${cc} offset: ${offset} bytes: ${data.byteLength}`, + // ); const sb = this.tracks[type]?.buffer; if (sb) { if (checkTimestampOffset) { @@ -872,11 +877,19 @@ transfer tracks: ${stringify(transferredTracks, (key, value) => (key === 'initSe this.appendExecutor(data, type); }, onStart: () => { - // logger.debug(`[buffer-controller]: ${type} SourceBuffer updatestart`); + // this.log( + // `updatestart "${type}" sn: ${sn}${part ? ' p: ' + part.index : ''} of ${ + // parent === PlaylistLevelType.MAIN ? 'level' : 'track' + // } ${frag.level} cc: ${cc} offset: ${offset} bytes: ${data.byteLength}`, + // ); }, onComplete: () => { this.clearBufferAppendTimeoutId(this.tracks[type]); - // logger.debug(`[buffer-controller]: ${type} SourceBuffer updateend`); + // this.log( + // `appended "${type}" sn: ${sn}${part ? ' p: ' + part.index : ''} of ${ + // parent === PlaylistLevelType.MAIN ? 'level' : 'track' + // } ${frag.level} cc: ${cc} offset: ${offset} bytes: ${data.byteLength}`, + // ); const end = self.performance.now(); chunkStats.executeEnd = chunkStats.end = end; if (fragBuffering.first === 0) { @@ -904,6 +917,12 @@ transfer tracks: ${stringify(transferredTracks, (key, value) => (key === 'initSe onError: (error: Error) => { this.clearBufferAppendTimeoutId(this.tracks[type]); + // this.log( + // `append-error "${type}" sn: ${sn}${part ? ' p: ' + part.index : ''} of ${ + // parent === PlaylistLevelType.MAIN ? 'level' : 'track' + // } ${frag.level} cc: ${cc} offset: ${offset} bytes: ${data.byteLength}`, + // ); + const isQuotaError = (error as DOMException).code === DOMException.QUOTA_EXCEEDED_ERR || error.name == 'QuotaExceededError' || @@ -952,13 +971,6 @@ transfer tracks: ${stringify(transferredTracks, (key, value) => (key === 'initSe // QuotaExceededError: http://www.w3.org/TR/html5/infrastructure.html#quotaexceedederror // Eviction was already attempted or not possible — report BUFFER_FULL_ERROR event.details = ErrorDetails.BUFFER_FULL_ERROR; - } else if ( - (error as DOMException).code === DOMException.INVALID_STATE_ERR && - this.mediaSourceOpenOrEnded && - !mediaError - ) { - // Allow retry for "Failed to execute 'appendBuffer' on 'SourceBuffer': This SourceBuffer is still processing" errors - event.errorAction = createDoNothingErrorAction(true); } else if ( error.name === TRACK_REMOVED_ERROR_NAME && this.sourceBufferCount === 0 @@ -970,10 +982,11 @@ transfer tracks: ${stringify(transferredTracks, (key, value) => (key === 'initSe /* with UHD content, we could get loop of quota exceeded error until browser is able to evict some data from sourcebuffer. Retrying can help recover. */ + const appendErrorMaxRetry = this.hls.config.appendErrorMaxRetry; this.warn( - `Failed ${appendErrorCount}/${this.hls.config.appendErrorMaxRetry} times to append segment in "${type}" sourceBuffer with error: ${error.message}`, + `Failed ${appendErrorCount}/${appendErrorMaxRetry + 1} times to append segment in "${type}" sourceBuffer with error: ${error.message}`, ); - if (appendErrorCount >= this.hls.config.appendErrorMaxRetry) { + if (appendErrorCount >= appendErrorMaxRetry) { event.fatal = true; } const readyState = this.mediaSource?.readyState; @@ -1653,6 +1666,9 @@ transfer tracks: ${stringify(transferredTracks, (key, value) => (key === 'initSe // If media was ejected check for a change. Added ranges are redundant with changes on 'updateend' event. const removedRanges = event.removedRanges; if (removedRanges?.length) { + this.log( + `${type} buffer removed ${timeRangesToString(removedRanges)}`, + ); this.hls.trigger(Events.BUFFER_FLUSHED, { type: type, start: removedRanges.start(0), diff --git a/src/controller/cap-level-controller.ts b/src/controller/cap-level-controller.ts index adb0e48c2..4ec154dec 100644 --- a/src/controller/cap-level-controller.ts +++ b/src/controller/cap-level-controller.ts @@ -111,7 +111,7 @@ class CapLevelController implements ComponentAPI { } else { this.media = null; } - if ((this.timer || this.observer) && this.hls.levels.length) { + if ((this.timer || this.observer) && this.hls.levels.length > 1) { this.detectPlayerSize(); } } @@ -120,9 +120,8 @@ class CapLevelController implements ComponentAPI { event: Events.MANIFEST_PARSED, data: ManifestParsedData, ) { - const hls = this.hls; this.restrictedLevels = []; - if (hls?.config.capLevelToPlayerSize && data.video) { + if (data.video) { // Start capping immediately if the manifest has signaled video codecs this.startCapping(); } @@ -130,13 +129,19 @@ class CapLevelController implements ComponentAPI { private onLevelsUpdated( event: Events.LEVELS_UPDATED, - data: LevelsUpdatedData, + { levels }: LevelsUpdatedData, ) { if ( (this.timer || this.observer) && Number.isFinite(this.autoLevelCapping) ) { + // Update capped level index when levels change (or stop observing if length is 1) this.detectPlayerSize(); + } else if (this.observer === undefined && this.timer === undefined) { + // Restart observing if length increases + if (levels.length > 1 && levels.some((level) => !!level.videoCodec)) { + this.startCapping(); + } } } @@ -146,8 +151,7 @@ class CapLevelController implements ComponentAPI { event: Events.BUFFER_CODECS, data: BufferCodecsData, ) { - const hls = this.hls; - if (hls?.config.capLevelToPlayerSize && data.video) { + if (data.video) { // If the manifest did not signal a video codec capping has been deferred until we're certain video is present this.startCapping(); } @@ -165,7 +169,7 @@ class CapLevelController implements ComponentAPI { return; } const levels = this.hls.levels; - if (levels.length) { + if (levels.length > 1) { const hls = this.hls; const maxLevel = this.getMaxLevel(levels.length - 1); if (maxLevel !== this.autoLevelCapping) { @@ -184,6 +188,8 @@ class CapLevelController implements ComponentAPI { this.streamController.nextLevelSwitch(); } this.autoLevelCapping = hls.autoLevelCapping; + } else if (levels.length === 1) { + this.stopCapping(); } } } @@ -230,7 +236,7 @@ class CapLevelController implements ComponentAPI { } startCapping() { - if (this.timer || this.observer) { + if (this.timer || this.observer || !this.hls?.config.capLevelToPlayerSize) { // Don't reset capping if started twice; this can happen if the manifest signals a video codec return; } diff --git a/src/controller/gap-controller.ts b/src/controller/gap-controller.ts index 3bd51a45a..653e0a25a 100644 --- a/src/controller/gap-controller.ts +++ b/src/controller/gap-controller.ts @@ -293,8 +293,12 @@ export default class GapController extends TaskLoop { const stalledDuration = tnow - stalled; if ( + // not seeking !seeking && - (stalledDuration >= detectStallWithCurrentTimeMs || tWaiting) && + // currentTime has not advanced after threshold or "waiting" event after start + (stalledDuration >= detectStallWithCurrentTimeMs || + (tWaiting && this.moved)) && + // not destroyed this.hls ) { // Dispatch MEDIA_ENDED when media.ended/ended event is not signalled at end of stream diff --git a/src/controller/id3-track-controller.ts b/src/controller/id3-track-controller.ts index 9bd02ab6f..e48597fcd 100644 --- a/src/controller/id3-track-controller.ts +++ b/src/controller/id3-track-controller.ts @@ -71,7 +71,7 @@ const MAX_CUE_ENDTIME = (() => { return Number.POSITIVE_INFINITY; })(); -class ID3TrackController implements ComponentAPI { +export class ID3TrackController implements ComponentAPI { private hls: Hls | null; private id3Track: HTMLTrackElement | null = null; private media: HTMLMediaElement | null = null; diff --git a/src/controller/iframe-controller.ts b/src/controller/iframe-controller.ts new file mode 100644 index 000000000..2f77449db --- /dev/null +++ b/src/controller/iframe-controller.ts @@ -0,0 +1,424 @@ +import { State } from './base-stream-controller'; +import { Events } from '../events'; +import { type LoaderStats, PlaylistLevelType } from '../types/loader'; +import { BufferHelper } from '../utils/buffer-helper'; +import { Logger } from '../utils/logger'; +import { getVideoPreference } from '../utils/rendition-helper'; +import type { HlsConfig } from '../config'; +import type Hls from '../hls'; +import type StreamController from './stream-controller'; +import type { Fragment, MediaFragment, Part } from '../loader/fragment'; +import type { LevelDetails } from '../loader/level-details'; +import type { + InitPTSFoundData, + LevelsUpdatedData, + ManifestLoadedData, +} from '../types/events'; +import type { Level, VariableMap } from '../types/level'; +import type { TimestampOffset } from '../utils/timescale-conversion'; + +type Constructor<T = object, A extends any[] = any[], Static = {}> = (new ( + ...a: A +) => T) & + Static; + +export type LoadMediaAtOptions = { seekOnAppend: boolean }; + +const loadMediaAtOptionsDefault: LoadMediaAtOptions = { + seekOnAppend: true, +}; + +export interface HlsIFramesOnly extends Hls { + loadMediaAt(time: number, options?: Partial<LoadMediaAtOptions>): void; +} +interface IFrameStreamController extends StreamController { + initDetails?: LevelDetails | null; + setInitPts(initPTS: TimestampOffset[]): void; + loadMediaAt(time: number, options: LoadMediaAtOptions): void; +} + +let HlsIFramesOnlyClass: ReturnType<typeof createHlsIFramesOnly>; + +export class IFrameController extends Logger { + private hls: Hls | undefined; + + // ManifestLoadedData forwarded to iframe instances + private stats?: LoaderStats; + private variableList: VariableMap | null = null; + + private initPTS: TimestampOffset[] = []; + + private iframeInstances: HlsIFramesOnly[]; + private instanceCounter: number = 0; + + constructor(hls: Hls, HlsPlayerClass: typeof Hls) { + super('iframes', hls.logger); + HlsIFramesOnlyClass ||= createHlsIFramesOnly(HlsPlayerClass); + this.hls = hls; + this.iframeInstances = []; + this.registerListeners(); + } + + private registerListeners() { + const hls = this.hls; + if (hls) { + hls.on(Events.MANIFEST_LOADING, this.clearAsset, this); + hls.on(Events.MANIFEST_LOADED, this.onManifestLoaded, this); + hls.on(Events.LEVELS_UPDATED, this.onLevelsUpdated, this); + hls.on(Events.INIT_PTS_FOUND, this.onInitPtsFound, this); + hls.on(Events.DESTROYING, this.onDestroying, this); + } + } + + private unregisterListeners() { + const hls = this.hls; + if (hls) { + hls.off(Events.MANIFEST_LOADING, this.clearAsset, this); + hls.off(Events.MANIFEST_LOADED, this.onManifestLoaded, this); + hls.off(Events.LEVELS_UPDATED, this.onLevelsUpdated, this); + hls.off(Events.INIT_PTS_FOUND, this.onInitPtsFound, this); + hls.off(Events.DESTROYING, this.onDestroying, this); + } + } + + private clearAsset() { + this.stats = undefined; + this.variableList = null; + this.initPTS = []; + this.iframeInstances.forEach((instance) => instance.destroy()); + this.iframeInstances.length = 0; + } + + private onDestroying() { + this.unregisterListeners(); + this.clearAsset(); + this.hls = undefined; + } + + private onManifestLoaded( + event: Events.MANIFEST_LOADED, + data: ManifestLoadedData, + ) { + this.stats = data.stats; + this.variableList = data.variableList; + } + + private onLevelsUpdated( + event: Events.LEVELS_UPDATED, + { levels }: LevelsUpdatedData, + ) { + // Check for Pathway switch and priority change + const pathwayPriority = this.hls?.pathwayPriority; + if (levels.length && pathwayPriority) { + this.iframeInstances.forEach((instance) => { + instance.pathwayPriority = pathwayPriority; + }); + } + } + + private onInitPtsFound( + event: Events.INIT_PTS_FOUND, + { id, timestampOffsets }: InitPTSFoundData, + ) { + if (id === PlaylistLevelType.MAIN) { + this.initPTS = timestampOffsets; + } + } + + public createIFramePlayer( + configOverride?: Partial<HlsConfig> | undefined, + ): HlsIFramesOnly | null { + const { hls } = this; + if (!hls) { + return null; + } + const { + iframeVariants, + url, + userConfig, + latestLevelDetails, + loadLevelObj, + loadLevel, + bandwidthEstimate, + sessionId, + } = hls; + const { stats, variableList } = this; + if (!iframeVariants || !stats || !url) { + return null; + } + const loggerId = `iframe-player-${this.instanceCounter++}`; + const levels = hls.levels as (Level | undefined)[]; + const activeLevel = loadLevelObj || levels[loadLevel]; + const videoPreference = getVideoPreference( + activeLevel, + userConfig.videoPreference, + ); + const pathwayId = activeLevel?.pathwayId || '.'; + + const iframeInstance = new HlsIFramesOnlyClass( + userConfig, + { + loggerId, + primarySessionId: sessionId, + videoPreference, + abrEwmaDefaultEstimate: bandwidthEstimate, + streamController: hls.config.streamController, + }, + configOverride, + url, + this.initPTS, + latestLevelDetails, + ); + + // Remove destroyed instanced from list before adding new ones + this.iframeInstances = this.iframeInstances.filter( + (instance) => !instance.url, + ); + this.iframeInstances.push(iframeInstance); + + iframeInstance.trigger(Events.MANIFEST_LOADED, { + levels: iframeVariants, + contentSteering: { uri: '', pathwayId }, + sessionKeys: null, + audioTracks: [], + iframeVariants: [], + sessionData: null, + startTimeOffset: null, + stats, + networkDetails: null, + url, + variableList, + }); + + return iframeInstance; + } +} + +interface HlsIFramesOnlyAlias extends HlsIFramesOnly {} + +function createHlsIFramesOnly(Base: Constructor<Hls>) { + return class HlsIFramesOnly extends Base implements HlsIFramesOnlyAlias { + constructor( + userConfig: Partial<HlsConfig>, + parentConfig: Partial<HlsConfig> & { + streamController: typeof StreamController; + }, + configOverride: Partial<HlsConfig> | undefined, + url: string, + initPTS: TimestampOffset[], + latestLevelDetails: LevelDetails | null, + ) { + const playerConfig: Partial<HlsConfig> = { + ...userConfig, + ...parentConfig, + + streamController: createIFrameStreamController( + parentConfig.streamController, + ), + + // Disable features not essential to streaming video I-Frames + audioStreamController: undefined, + audioTrackController: undefined, + subtitleStreamController: undefined, + subtitleTrackController: undefined, + timelineController: undefined, + id3TrackController: undefined, + fpsController: undefined, + gapController: undefined, + iframeController: undefined, + cmcd: undefined, + // FIXME: Interstitials must not be loaded independently of parent platyer. Schedule should come from parent. (disabled for now) + interstitialsController: undefined, + + // Only load and unload as needed + // maxMaxBufferLength: 8, + backBufferLength: Infinity, + // Adapt to attached HTMLVideoElement dimension + capLevelToPlayerSize: true, + + // Streamline loading + enableWorker: false, + autoStartLoad: false, + startFragPrefetch: false, + testBandwidth: false, + progressive: false, + // startOnSegmentBoundary: true, + ...configOverride, + }; + + super(playerConfig); + + // Hls.url matches the parent player session source url. + // `Hls.url==null` is used in many places to determine if the instance was destroyed. + this._url = url; + + // Align timestamps based on parent initPts (accounts for audio prime offset and parent variant decode time difference) + (this.streamController as IFrameStreamController).setInitPts(initPTS); + (this.streamController as IFrameStreamController).initDetails = + latestLevelDetails; + } + + loadSource(url: string) {} + + loadMediaAt( + time: number, + options: Partial<LoadMediaAtOptions> = loadMediaAtOptionsDefault, + ) { + if (time < 0) { + return; + } + const settings = { + ...loadMediaAtOptionsDefault, + ...options, + }; + if (!this.loadingEnabled) { + this.startLoad(time); + } + (this.streamController as IFrameStreamController).loadMediaAt( + time, + settings, + ); + } + }; +} + +interface IFrameStreamControllerAlias extends IFrameStreamController {} + +function createIFrameStreamController(Base: Constructor<StreamController>) { + return class IFrameStreamController + extends Base + implements IFrameStreamControllerAlias + { + private currentOp?: [time: number, options: LoadMediaAtOptions]; + private nextOp?: [time: number, options: LoadMediaAtOptions]; + private gotNext: boolean = false; + initDetails?: LevelDetails | null; + + setInitPts(initPTS: TimestampOffset[]) { + this.initPTS = initPTS; + } + + loadMediaAt(time: number, options: LoadMediaAtOptions) { + const { seekOnAppend } = options; + const adjustedTime = time + this.timelineOffset; + this.nextLoadPosition = this.lastCurrentTime = adjustedTime; + this.startPosition = time; + switch (this.state) { + case State.STOPPED: + case State.ENDED: + case State.ERROR: + this.state = State.IDLE; + } + if (this.state === State.IDLE) { + this.hls.resumeBuffering(); + this.tick(); + this.currentOp = [adjustedTime, options]; + } else { + const fragCurrent = this.fragCurrent; + if ( + !fragCurrent || + (time >= fragCurrent.start && time < fragCurrent.end) + ) { + this.nextOp = [adjustedTime, options]; + } + } + const media = this.media; + if (seekOnAppend && media) { + const seeking = this.seekTo(adjustedTime); + if (seeking) { + this.currentOp = [adjustedTime, options]; + this.nextOp = undefined; + } + } + } + + private seekTo(time: number): boolean { + const media = this.media; + if (media) { + const bufferInfo = BufferHelper.bufferInfo(media, time, 0); + const hasEnough = bufferInfo.len > 0 && this.getBufferedAt(time); + if (hasEnough) { + media.currentTime = time; + if (this.state === State.IDLE) { + this.tick(); + } + return true; + } + } + return false; + } + + private getBufferedAt(time: number): MediaFragment | null { + return this.fragmentTracker.getBufferedFrag(time, PlaylistLevelType.MAIN); + } + + // overrides + protected fragBufferedComplete(frag: Fragment, part: Part | null) { + super.fragBufferedComplete(frag, part); + + const { currentOp, nextOp } = this; + this.currentOp = this.nextOp = undefined; + this.state = State.STOPPED; + if (currentOp?.[1].seekOnAppend) { + this.seekTo(currentOp[0]); + if (!nextOp && !this.gotNext) { + // repeat op to get next segment (Chrome may require two HEVC frame appends, or one with EoS, before rendering) + this.gotNext = true; + this.loadMediaAt.apply(this, currentOp); + } + } + if (nextOp) { + this.loadMediaAt.apply(this, nextOp); + } + } + + get playhead(): number { + return this.nextLoadPosition; + } + + startLoad() { + if (!this.startFragRequested) { + const hlsIFrames = this.hls; + hlsIFrames.nextLoadLevel = + hlsIFrames.startLevel === -1 ? 0 : hlsIFrames.firstAutoLevel; + } + } + // public getLevelDetails + protected seekToStartPos() {} + protected setStartPosition() {} + protected onMediaSeeking = () => { + this.gotNext = false; + }; + + protected alignPlaylists( + details: LevelDetails, + previousDetails: LevelDetails | undefined, + switchDetails: LevelDetails | undefined, + ): number { + return super.alignPlaylists( + details, + previousDetails, + switchDetails || this.initDetails || undefined, + ); + } + + getMainFwdBufferInfo() { + const t = this.playhead; + const bufferedFragAtPos = this.getBufferedAt(t); + if (bufferedFragAtPos) { + const len = bufferedFragAtPos.duration; + return { len, start: t, end: t + len, bufferedIndex: -1 }; + } + return { len: 0, start: t, end: t, bufferedIndex: -1 }; + } + + protected _streamEnded() { + const t = this.playhead; + const bufferedFragAtPos = this.fragmentTracker.getBufferedFrag( + t, + PlaylistLevelType.MAIN, + ); + return !!bufferedFragAtPos?.endList; + } + }; +} diff --git a/src/controller/interstitials-controller.ts b/src/controller/interstitials-controller.ts index eb50e9b67..7661ab313 100644 --- a/src/controller/interstitials-controller.ts +++ b/src/controller/interstitials-controller.ts @@ -30,7 +30,10 @@ import { import { hash } from '../utils/hash'; import { Logger } from '../utils/logger'; import { isCompatibleTrackChange } from '../utils/mediasource-helper'; -import { getBasicSelectionOption } from '../utils/rendition-helper'; +import { + getBasicSelectionOption, + getVideoPreference, +} from '../utils/rendition-helper'; import { stringify } from '../utils/safe-json-stringify'; import type { HlsAssetPlayerConfig, @@ -2250,20 +2253,12 @@ Schedule: ${scheduleItems.map((seg) => segmentToString(seg))} pos: ${this.timeli ): HlsAssetPlayer { const primary = this.hls; const userConfig = primary.userConfig; - let videoPreference = userConfig.videoPreference; - const currentLevel = + const activeLevel = primary.loadLevelObj || primary.levels[primary.currentLevel]; - // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition - if (videoPreference || currentLevel) { - videoPreference = Object.assign({}, videoPreference); - if (currentLevel.videoCodec) { - videoPreference.videoCodec = currentLevel.videoCodec; - } - // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition - if (currentLevel.videoRange) { - videoPreference.allowedVideoRanges = [currentLevel.videoRange]; - } - } + const videoPreference = getVideoPreference( + activeLevel, + userConfig.videoPreference, + ); const selectedAudio = primary.audioTracks[primary.audioTrack]; const selectedSubtitle = primary.subtitleTracks[primary.subtitleTrack]; let startPosition = 0; diff --git a/src/controller/level-controller.ts b/src/controller/level-controller.ts index 9ac178e04..e8deaec57 100644 --- a/src/controller/level-controller.ts +++ b/src/controller/level-controller.ts @@ -39,6 +39,8 @@ export default class LevelController extends BasePlaylistController { private steering: ContentSteeringController | null; private lastABRSwitchTime: number = -1; + private _iframeVariants: LevelParsed[] = []; + public onParsedComplete!: Function; constructor( @@ -96,6 +98,8 @@ export default class LevelController extends BasePlaylistController { this.currentLevel = null; this._levels = []; this._maxAutoLevel = -1; + + this._iframeVariants = []; } private onManifestLoading( @@ -119,33 +123,18 @@ export default class LevelController extends BasePlaylistController { data.levels.forEach((levelParsed: LevelParsed) => { const attributes = levelParsed.attrs; - let { audioCodec, videoCodec } = levelParsed; - if (audioCodec) { - // Returns empty and set to undefined for 'mp4a.40.34' with fallback to 'audio/mpeg' SourceBuffer - levelParsed.audioCodec = audioCodec = - getCodecCompatibleName(audioCodec, preferManagedMediaSource) || - undefined; - } - - if (videoCodec) { - videoCodec = levelParsed.videoCodec = convertAVC1ToAVCOTI(videoCodec); - } - - // only keep levels with supported audio/video codecs - const { width, height, unknownCodecs } = levelParsed; - const unknownUnsupportedCodecCount = unknownCodecs?.length || 0; - - resolutionFound ||= !!(width && height); - videoCodecFound ||= !!videoCodec; - audioCodecFound ||= !!audioCodec; - if ( - unknownUnsupportedCodecCount || - (audioCodec && !this.isAudioSupported(audioCodec)) || - (videoCodec && !this.isVideoSupported(videoCodec)) - ) { + const supported = setTrackCodecsAndReturnSupported( + levelParsed, + preferManagedMediaSource, + ); + const { audioCodec, videoCodec, width, height } = levelParsed; + if (!supported) { this.log(`Some or all CODECS not supported "${attributes.CODECS}"`); return; } + resolutionFound ||= !!(width && height); + videoCodecFound ||= !!videoCodec; + audioCodecFound ||= !!audioCodec; const { CODECS, @@ -174,7 +163,7 @@ export default class LevelController extends BasePlaylistController { const level = this.createLevel(levelParsed); redundantSet[levelKey] = level; levels.push(level); - } else { + } else if (!levelParsed.iframes) { redundantSet[levelKey].addGroupId('audio', attributes.AUDIO); redundantSet[levelKey].addGroupId('text', attributes.SUBTITLES); } @@ -194,7 +183,10 @@ export default class LevelController extends BasePlaylistController { const supplemental = levelParsed.supplemental; if ( supplemental?.videoCodec && - !this.isVideoSupported(supplemental.videoCodec) + !isVideoSupported( + supplemental.videoCodec, + this.hls.config.preferManagedMediaSource, + ) ) { const error = new Error( `SUPPLEMENTAL-CODECS not supported "${supplemental.videoCodec}"`, @@ -205,22 +197,6 @@ export default class LevelController extends BasePlaylistController { return level; } - private isAudioSupported(codec: string): boolean { - return areCodecsMediaSourceSupported( - codec, - 'audio', - this.hls.config.preferManagedMediaSource, - ); - } - - private isVideoSupported(codec: string): boolean { - return areCodecsMediaSourceSupported( - codec, - 'video', - this.hls.config.preferManagedMediaSource, - ); - } - private filterAndSortMediaOptions( filteredLevels: Level[], data: ManifestLoadedData, @@ -232,6 +208,7 @@ export default class LevelController extends BasePlaylistController { let subtitleTracks: MediaPlaylist[] = []; let levels = filteredLevels; const statsParsing = data.stats?.parsing || {}; + const preferManagedMediaSource = this.hls.config.preferManagedMediaSource; // remove audio-only and invalid video-range levels if we also have levels with video codecs or RESOLUTION signalled if ((resolutionFound || videoCodecFound) && audioCodecFound) { @@ -276,7 +253,9 @@ export default class LevelController extends BasePlaylistController { if (data.audioTracks) { audioTracks = data.audioTracks.filter( - (track) => !track.audioCodec || this.isAudioSupported(track.audioCodec), + (track) => + !track.audioCodec || + isAudioSupported(track.audioCodec, preferManagedMediaSource), ); // Assign ids after filtering as array indices by group-id assignTrackIdsByGroup(audioTracks); @@ -369,6 +348,11 @@ export default class LevelController extends BasePlaylistController { } } + const iframeVariants = data.iframeVariants.filter((parsedVariant) => + setTrackCodecsAndReturnSupported(parsedVariant, preferManagedMediaSource), + ); + this._iframeVariants = iframeVariants; + // Audio is only alternate if manifest include a URI along with the audio group tag, // and this is not an audio-only stream where levels contain audio-only const audioOnly = audioCodecFound && !videoCodecFound; @@ -380,6 +364,7 @@ export default class LevelController extends BasePlaylistController { levels, audioTracks, subtitleTracks, + iframeVariants, sessionData: data.sessionData, sessionKeys: data.sessionKeys, firstLevel: this._firstLevel, @@ -393,6 +378,13 @@ export default class LevelController extends BasePlaylistController { this.hls.trigger(Events.MANIFEST_PARSED, edata); } + get iframeVariants(): LevelParsed[] | null { + if (this._iframeVariants.length === 0) { + return null; + } + return this._iframeVariants; + } + get levels(): Level[] | null { if (this._levels.length === 0) { return null; @@ -766,6 +758,54 @@ export default class LevelController extends BasePlaylistController { } } +function isAudioSupported( + codec: string, + preferManagedMediaSource: boolean, +): boolean { + return areCodecsMediaSourceSupported( + codec, + 'audio', + preferManagedMediaSource, + ); +} + +function isVideoSupported( + codec: string, + preferManagedMediaSource: boolean, +): boolean { + return areCodecsMediaSourceSupported( + codec, + 'video', + preferManagedMediaSource, + ); +} + +function setTrackCodecsAndReturnSupported( + levelParsed: LevelParsed, + preferManagedMediaSource: boolean, +): boolean { + let { audioCodec, videoCodec } = levelParsed; + if (audioCodec) { + // Returns empty and set to undefined for 'mp4a.40.34' with fallback to 'audio/mpeg' SourceBuffer + levelParsed.audioCodec = audioCodec = + getCodecCompatibleName(audioCodec, preferManagedMediaSource) || undefined; + } + if (videoCodec) { + videoCodec = levelParsed.videoCodec = convertAVC1ToAVCOTI(videoCodec); + } + // only keep levels with supported audio/video codecs + const { unknownCodecs } = levelParsed; + const unknownUnsupportedCodecCount = unknownCodecs?.length || 0; + if ( + unknownUnsupportedCodecCount || + (audioCodec && !isAudioSupported(audioCodec, preferManagedMediaSource)) || + (videoCodec && !isVideoSupported(videoCodec, preferManagedMediaSource)) + ) { + return false; + } + return true; +} + function assignTrackIdsByGroup(tracks: MediaPlaylist[]): void { const groups = {}; tracks.forEach((track) => { diff --git a/src/controller/stream-controller.ts b/src/controller/stream-controller.ts index 69fa9cc81..8e39f39f8 100644 --- a/src/controller/stream-controller.ts +++ b/src/controller/stream-controller.ts @@ -669,6 +669,14 @@ export default class StreamController this.synchronizeToLiveEdge(newDetails); } + // Remove timestamp mapping from sparse array for discontinuities no longer present + this.initPTS.some((t, i) => { + if (i >= newDetails.startCC) { + return true; + } + delete this.initPTS[i]; + }); + // trigger handler right now this.tick(); } @@ -679,7 +687,7 @@ export default class StreamController return; } const liveSyncPosition = this.hls.liveSyncPosition; - const currentTime = this.getLoadPosition(); + const currentTime = this.playhead; const start = levelDetails.fragmentStart; const end = levelDetails.edge; const withinSlidingWindow = @@ -790,13 +798,16 @@ export default class StreamController )); const partIndex = part ? part.index : -1; const partial = partIndex !== -1; + const byteRange = frag.byteRange; const chunkMeta = new ChunkMetadata( frag.level, frag.sn, frag.stats.chunkCount, - payload.byteLength, + byteRange.length ? byteRange[1] - byteRange[0] : payload.byteLength, partIndex, partial, + frag.duration, + this.iframesOnly, ); const initPTS = this.initPTS[frag.cc]; @@ -1220,7 +1231,9 @@ export default class StreamController timescale, trackId, }; + const timestampOffsets = this.initPTS.slice(0); hls.trigger(Events.INIT_PTS_FOUND, { + timestampOffsets, frag, id, initPTS: baseTime, @@ -1476,6 +1489,9 @@ export default class StreamController delete tracks.audiovideo; } if (audiovideo) { + if (this.iframesOnly) { + this.logMuxedErr(frag); + } this.log( `Init audiovideo buffer, container:${audiovideo.container}, codecs[level/parsed]=[${currentLevel.codecs}/${audiovideo.codec}]`, ); diff --git a/src/controller/subtitle-stream-controller.ts b/src/controller/subtitle-stream-controller.ts index 36ffd2aba..2eb30d6cd 100644 --- a/src/controller/subtitle-stream-controller.ts +++ b/src/controller/subtitle-stream-controller.ts @@ -394,7 +394,7 @@ export class SubtitleStreamController } doTick() { - if (this.state === State.IDLE) { + if (this.state === State.IDLE && this.levels) { if ( !this.media && !this.primaryPrefetch && @@ -403,7 +403,7 @@ export class SubtitleStreamController return; } const { currentTrackId, levels } = this; - const track = levels?.[currentTrackId]; + const track = levels[currentTrackId]; const trackDetails = track?.details; if ( !trackDetails || @@ -414,7 +414,7 @@ export class SubtitleStreamController return; } const { config } = this; - const currentTime = this.getLoadPosition(); + const currentTime = this.playhead; const bufferedInfo = BufferHelper.bufferedInfo( this.tracksBuffered[this.currentTrackId] || [], currentTime, diff --git a/src/controller/timeline-controller.ts b/src/controller/timeline-controller.ts index 524684c93..43096abcc 100644 --- a/src/controller/timeline-controller.ts +++ b/src/controller/timeline-controller.ts @@ -185,11 +185,11 @@ export class TimelineController implements ComponentAPI { // Triggered when an initial PTS is found; used for synchronisation of WebVTT. private onInitPtsFound( event: Events.INIT_PTS_FOUND, - { frag, id, initPTS, timescale, trackId }: InitPTSFoundData, + { id, timestampOffsets }: InitPTSFoundData, ) { const { unparsedVttFrags } = this; if (id === PlaylistLevelType.MAIN) { - this.initPTS[frag.cc] = { baseTime: initPTS, timescale, trackId }; + this.initPTS = timestampOffsets; } // Due to asynchronous processing, initial PTS may arrive later than the first VTT fragments are loaded. diff --git a/src/crypt/decrypter.ts b/src/crypt/decrypter.ts index f6258a3ef..20d6b1f40 100644 --- a/src/crypt/decrypter.ts +++ b/src/crypt/decrypter.ts @@ -10,7 +10,7 @@ const CHUNK_SIZE = 16; // 16 bytes, 128 bits export default class Decrypter { private logEnabled: boolean = true; - private removePKCS7Padding: boolean; + private plainTextLength: number = 0; private subtle: SubtleCrypto | null = null; private softwareDecrypter: AESDecryptor | null = null; private key: ArrayBuffer | null = null; @@ -21,11 +21,9 @@ export default class Decrypter { private useSoftware: boolean; private enableSoftwareAES: boolean; - constructor(config: HlsConfig, { removePKCS7Padding = true } = {}) { + constructor(config: HlsConfig, useSoftware?: boolean) { this.enableSoftwareAES = config.enableSoftwareAES; - this.removePKCS7Padding = removePKCS7Padding; - // built in decryptor expects PKCS7 padding - if (removePKCS7Padding) { + if (useSoftware !== true) { try { const browserCrypto = self.crypto; if (browserCrypto) { @@ -62,10 +60,10 @@ export default class Decrypter { } const data = new Uint8Array(currentResult); this.reset(); - if (this.removePKCS7Padding) { - return removePadding(data); + if (this.plainTextLength) { + return data.slice(0, this.plainTextLength); } - return data; + return removePadding(data); } public reset() { @@ -82,8 +80,10 @@ export default class Decrypter { key: ArrayBuffer, iv: ArrayBuffer, aesMode: DecrypterAesMode, + plainTextLength: number = 0, ): Promise<ArrayBuffer> { - if (this.useSoftware) { + this.plainTextLength = plainTextLength; + if (this.useSoftware || plainTextLength) { return new Promise((resolve, reject) => { const dataView = ArrayBuffer.isView(data) ? data : new Uint8Array(data); this.softwareDecrypt(dataView, key, iv, aesMode); @@ -100,7 +100,7 @@ export default class Decrypter { // Software decryption is progressive. Progressive decryption may not return a result on each call. Any cached // data is handled in the flush() call - public softwareDecrypt( + private softwareDecrypt( data: Uint8Array, key: ArrayBuffer, iv: ArrayBuffer, @@ -149,7 +149,7 @@ export default class Decrypter { return result; } - public webCryptoDecrypt( + private webCryptoDecrypt( data: Uint8Array<ArrayBuffer>, key: ArrayBuffer, iv: ArrayBuffer, diff --git a/src/define-plugin.d.ts b/src/define-plugin.d.ts index 41d0e4e72..84383514f 100644 --- a/src/define-plugin.d.ts +++ b/src/define-plugin.d.ts @@ -10,6 +10,7 @@ declare const __USE_VARIABLE_SUBSTITUTION__: boolean; declare const __USE_M2TS_ADVANCED_CODECS__: boolean; declare const __USE_MEDIA_CAPABILITIES__: boolean; declare const __USE_INTERSTITIALS__: boolean; +declare const __USE_IFRAMES__: boolean; // __IN_WORKER__ is provided from a closure call around the final UMD bundle. declare const __IN_WORKER__: boolean; diff --git a/src/demux/mp4demuxer.ts b/src/demux/mp4demuxer.ts index 6f044aebd..a4e5588df 100644 --- a/src/demux/mp4demuxer.ts +++ b/src/demux/mp4demuxer.ts @@ -15,7 +15,7 @@ import { import { appendUint8Array, findBox, - hasMoofData, + hasBoxData, parseEmsg, parseInitSegment, parseSamples, @@ -95,7 +95,7 @@ class MP4Demuxer implements Demuxer { } static probe(data: Uint8Array) { - return hasMoofData(data); + return hasBoxData(data, 'moof'); } public demux( diff --git a/src/demux/sample-aes.ts b/src/demux/sample-aes.ts index b1ead4470..7521a1fdf 100644 --- a/src/demux/sample-aes.ts +++ b/src/demux/sample-aes.ts @@ -21,9 +21,7 @@ class SampleAesDecrypter { constructor(observer: HlsEventEmitter, config: HlsConfig, keyData: KeyData) { this.keyData = keyData; - this.decrypter = new Decrypter(config, { - removePKCS7Padding: false, - }); + this.decrypter = new Decrypter(config, true); } decryptBuffer(encryptedData: Uint8Array | ArrayBuffer): Promise<ArrayBuffer> { diff --git a/src/demux/transmuxer.ts b/src/demux/transmuxer.ts index ae70ca164..8b1f8e832 100644 --- a/src/demux/transmuxer.ts +++ b/src/demux/transmuxer.ts @@ -95,7 +95,7 @@ export default class Transmuxer { const stats = chunkMeta.transmuxing; stats.executeStart = now(); - let uintData: Uint8Array<ArrayBuffer> = new Uint8Array(data); + const uintData: Uint8Array<ArrayBuffer> = new Uint8Array(data); const { currentTransmuxState, transmuxConfig } = this; if (state) { this.currentTransmuxState = state; @@ -122,65 +122,59 @@ export default class Transmuxer { const decrypter = this.getDecrypter(); const aesMode = getAesModeFromFullSegmentMethod(keyData.method); - // Software decryption is synchronous; webCrypto is not - if (decrypter.isSync()) { - // Software decryption is progressive. Progressive decryption may not return a result on each call. Any cached - // data is handled in the flush() call - let decryptedData = decrypter.softwareDecrypt( + const plainTextLength = + chunkMeta.size !== data.byteLength && chunkMeta.iframe + ? chunkMeta.size + : 0; + + this.asyncResult = true; + this.decryptionPromise = decrypter + .decrypt( uintData, keyData.key.buffer, keyData.iv.buffer, aesMode, - ); - // For Low-Latency HLS Parts, decrypt in place, since part parsing is expected on push progress - const loadingParts = chunkMeta.part > -1; - if (loadingParts) { - const data = decrypter.flush(); - decryptedData = data ? data.buffer : data; - } - if (!decryptedData) { - stats.executeEnd = now(); - return emptyResult(chunkMeta); - } - uintData = new Uint8Array(decryptedData); - } else { - this.asyncResult = true; - this.decryptionPromise = decrypter - .webCryptoDecrypt( - uintData, - keyData.key.buffer, - keyData.iv.buffer, - aesMode, - ) - .then((decryptedData): TransmuxerResult => { - // Calling push here is important; if flush() is called while this is still resolving, this ensures that - // the decrypted data has been transmuxed - const result = this.push( - decryptedData, - null, - chunkMeta, - ) as TransmuxerResult; - this.decryptionPromise = null; - return result; - }); - return this.decryptionPromise; - } + plainTextLength, + ) + .then((decryptedData): TransmuxerResult => { + // Calling push here is important; if flush() is called while this is still resolving, this ensures that + // the decrypted data has been transmuxed + const result = this.push( + decryptedData, + null, + chunkMeta, + ) as TransmuxerResult; + this.decryptionPromise = null; + return result; + }); + return this.decryptionPromise; } const resetMuxers = this.needsProbing(discontinuity, trackSwitch); + if (resetMuxers) { - const error = this.configureTransmuxer(uintData); - if (error) { - this.logger.warn(`[transmuxer] ${error.message}`); - this.observer.emit(Events.ERROR, Events.ERROR, { - type: ErrorTypes.MEDIA_ERROR, - details: ErrorDetails.FRAG_PARSING_ERROR, - fatal: false, - error, - reason: error.message, - }); - stats.executeEnd = now(); - return emptyResult(chunkMeta); + if (!this.demuxer) { + // Configure the demuxer using an init segment when I-FRAME MPEG2-TS (I-FRAME media segments have no PMT). + // MP4 probing only works on media segments (which have moof data). + const segmentFormatData = + chunkMeta.iframe && + initSegmentData && + TSDemuxer.probe(initSegmentData, this.logger) + ? initSegmentData + : uintData; + const error = this.configureTransmuxer(segmentFormatData); + if (error) { + this.logger.warn(`[transmuxer] ${error.message}`); + this.observer.emit(Events.ERROR, Events.ERROR, { + type: ErrorTypes.MEDIA_ERROR, + details: ErrorDetails.FRAG_PARSING_ERROR, + fatal: false, + error, + reason: error.message, + }); + stats.executeEnd = now(); + return emptyResult(chunkMeta); + } } } @@ -191,6 +185,7 @@ export default class Transmuxer { videoCodec, duration, decryptdata, + chunkMeta, ); } @@ -265,7 +260,7 @@ export default class Transmuxer { return emptyResults; } - const demuxResultOrPromise = demuxer.flush(timeOffset); + const demuxResultOrPromise = demuxer.flush(timeOffset, chunkMeta); if (isPromise(demuxResultOrPromise)) { this.asyncResult = true; // Decrypt final SAMPLE-AES samples @@ -303,6 +298,7 @@ export default class Transmuxer { accurateTimeOffset, true, this.id, + chunkMeta, ); transmuxResults.push({ remuxResult, @@ -336,6 +332,7 @@ export default class Transmuxer { videoCodec: string | undefined, trackDuration: number, decryptdata: DecryptData | null, + chunkMeta: ChunkMetadata, ) { const { demuxer, remuxer } = this; if (!demuxer || !remuxer) { @@ -346,6 +343,8 @@ export default class Transmuxer { audioCodec, videoCodec, trackDuration, + decryptdata, + chunkMeta, ); remuxer.resetInitSegment( initSegmentData, @@ -401,7 +400,7 @@ export default class Transmuxer { ): TransmuxerResult { const { audioTrack, videoTrack, id3Track, textTrack } = ( this.demuxer as Demuxer - ).demux(data, timeOffset, false, !this.config.progressive); + ).demux(data, timeOffset, chunkMeta, false, !this.config.progressive); const remuxResult = this.remuxer!.remux( audioTrack, videoTrack, @@ -411,6 +410,7 @@ export default class Transmuxer { accurateTimeOffset, false, this.id, + chunkMeta, ); return { remuxResult, @@ -426,7 +426,7 @@ export default class Transmuxer { chunkMeta: ChunkMetadata, ): Promise<TransmuxerResult> { return (this.demuxer as Demuxer) - .demuxSampleAes(data, decryptData, timeOffset) + .demuxSampleAes(data, decryptData, timeOffset, chunkMeta) .then((demuxResult) => { const remuxResult = this.remuxer!.remux( demuxResult.audioTrack, @@ -437,6 +437,7 @@ export default class Transmuxer { accurateTimeOffset, false, this.id, + chunkMeta, ); return { remuxResult, @@ -445,10 +446,10 @@ export default class Transmuxer { }); } - private configureTransmuxer(data: Uint8Array): void | Error { + private configureTransmuxer(data: Uint8Array): undefined | Error { const { config, observer, typeSupported } = this; // probe for content type - let mux; + let mux: MuxConfig | undefined; for (let i = 0, len = muxConfig.length; i < len; i++) { if (muxConfig[i].demux?.probe(data, this.logger)) { mux = muxConfig[i]; @@ -461,15 +462,17 @@ export default class Transmuxer { // so let's check that current remuxer and demuxer are still valid const demuxer = this.demuxer; const remuxer = this.remuxer; - const Remuxer: MuxConfig['remux'] = mux.remux; const Demuxer: MuxConfig['demux'] = mux.demux; - if (!remuxer || !(remuxer instanceof Remuxer)) { - this.remuxer = new Remuxer(observer, config, typeSupported, this.logger); - } + const Remuxer: MuxConfig['remux'] = mux.remux; if (!demuxer || !(demuxer instanceof Demuxer)) { + this.logger.log(`Using ${Demuxer.name}`); this.demuxer = new Demuxer(observer, config, typeSupported, this.logger); this.probe = Demuxer.probe; } + if (!remuxer || !(remuxer instanceof Remuxer)) { + this.logger.log(`Using ${Remuxer.name}`); + this.remuxer = new Remuxer(observer, config, typeSupported, this.logger); + } } private needsProbing(discontinuity: boolean, trackSwitch: boolean): boolean { diff --git a/src/demux/tsdemuxer.ts b/src/demux/tsdemuxer.ts index 0bd103814..efca155ef 100644 --- a/src/demux/tsdemuxer.ts +++ b/src/demux/tsdemuxer.ts @@ -33,6 +33,8 @@ import { import { appendUint8Array, RemuxerTrackIdConfig } from '../utils/mp4-tools'; import type { HlsConfig } from '../config'; import type { HlsEventEmitter } from '../events'; +import type { ChunkMetadata } from '../hls'; +import type { DecryptData } from '../loader/level-key'; import type BaseVideoParser from './video/base-video-parser'; import type { AudioFrame, DemuxedAAC } from '../types/demuxer'; import type { TypeSupported } from '../utils/codecs'; @@ -178,6 +180,8 @@ class TSDemuxer implements Demuxer { audioCodec: string, videoCodec: string, trackDuration: number, + decryptdata: DecryptData | null, + chunkMeta: ChunkMetadata, ) { this.pmtParsed = false; this._pmtId = -1; @@ -202,6 +206,15 @@ class TSDemuxer implements Demuxer { this.remainderData = null; this.audioCodec = audioCodec; this.videoCodec = videoCodec; + + if (initSegment) { + this.demux( + initSegment, + 0, + chunkMeta, + decryptdata?.method === 'SAMPLE-AES', + ); + } } public resetTimeStamp() {} @@ -224,6 +237,7 @@ class TSDemuxer implements Demuxer { public demux( data: Uint8Array, timeOffset: number, + chunkMeta: ChunkMetadata, isSampleAes = false, flush = false, ): DemuxerResult { @@ -238,6 +252,7 @@ class TSDemuxer implements Demuxer { const audioTrack = this._audioTrack as DemuxedAudioTrack; const id3Track = this._id3Track as DemuxedMetadataTrack; const textTrack = this._txtTrack as DemuxedUserdataTrack; + const iframe = chunkMeta.iframe; let videoPid = videoTrack.pid; let videoData = videoTrack.pesData; @@ -307,7 +322,13 @@ class TSDemuxer implements Demuxer { ) { this.readyVideoParser(videoTrack.segmentCodec); if (this.videoParser !== null) { - this.videoParser.parsePES(videoTrack, textTrack, pes, false); + this.videoParser.parsePES( + videoTrack, + textTrack, + pes, + false, + chunkMeta, + ); } } @@ -321,6 +342,9 @@ class TSDemuxer implements Demuxer { } break; case audioPid: + if (iframe) { + break; + } if (stt) { if (audioData && (pes = parsePES(audioData, this.logger))) { switch (audioTrack.segmentCodec) { @@ -345,6 +369,9 @@ class TSDemuxer implements Demuxer { } break; case id3Pid: + if (iframe) { + break; + } if (stt) { if (id3Data && (pes = parsePES(id3Data, this.logger))) { this.parseID3PES(id3Track, pes); @@ -358,6 +385,9 @@ class TSDemuxer implements Demuxer { } break; case klvPid: + if (iframe) { + break; + } if (stt) { if (klvData && (pes = parsePES(klvData, this.logger))) { this.parseKlvPES(id3Track, pes); @@ -465,18 +495,21 @@ class TSDemuxer implements Demuxer { }; if (flush) { - this.extractRemainingSamples(demuxResult); + this.extractRemainingSamples(demuxResult, chunkMeta); } return demuxResult; } - public flush(): DemuxerResult | Promise<DemuxerResult> { + public flush( + timeOffset: number, + chunkMeta: ChunkMetadata, + ): DemuxerResult | Promise<DemuxerResult> { const { remainderData } = this; this.remainderData = null; let result: DemuxerResult; if (remainderData) { - result = this.demux(remainderData, -1, false, true); + result = this.demux(remainderData, 0, chunkMeta, false, true); } else { result = { videoTrack: this._videoTrack as DemuxedVideoTrack, @@ -485,14 +518,17 @@ class TSDemuxer implements Demuxer { textTrack: this._txtTrack as DemuxedUserdataTrack, }; } - this.extractRemainingSamples(result); + this.extractRemainingSamples(result, chunkMeta); if (this.sampleAes) { return this.decrypt(result, this.sampleAes); } return result; } - private extractRemainingSamples(demuxResult: DemuxerResult) { + private extractRemainingSamples( + demuxResult: DemuxerResult, + chunkMeta: ChunkMetadata, + ) { const { audioTrack, videoTrack, id3Track, textTrack } = demuxResult; const videoData = videoTrack.pesData; const audioData = audioTrack.pesData; @@ -512,6 +548,7 @@ class TSDemuxer implements Demuxer { textTrack as DemuxedUserdataTrack, pes, true, + chunkMeta, ); videoTrack.pesData = null; } @@ -559,10 +596,12 @@ class TSDemuxer implements Demuxer { data: Uint8Array, keyData: KeyData, timeOffset: number, + chunkMeta: ChunkMetadata, ): Promise<DemuxerResult> { const demuxResult = this.demux( data, timeOffset, + chunkMeta, true, !this.config.progressive, ); diff --git a/src/demux/video/avc-video-parser.ts b/src/demux/video/avc-video-parser.ts index 15ff4dc68..6a82f4e28 100644 --- a/src/demux/video/avc-video-parser.ts +++ b/src/demux/video/avc-video-parser.ts @@ -1,6 +1,7 @@ import BaseVideoParser from './base-video-parser'; import ExpGolomb from './exp-golomb'; import { parseSEIMessageFromNALu } from '../../utils/mp4-tools'; +import type { ChunkMetadata } from '../../hls'; import type { DemuxedUserdataTrack, DemuxedVideoTrack, @@ -13,7 +14,9 @@ class AvcVideoParser extends BaseVideoParser { textTrack: DemuxedUserdataTrack, pes: PES, endOfSegment: boolean, + chunkMeta: ChunkMetadata, ) { + const iframesOnly = chunkMeta.iframe; const units = this.parseNALu(track, pes.data, endOfSegment); let VideoSample = this.VideoSample; let push: boolean; @@ -36,6 +39,9 @@ class AvcVideoParser extends BaseVideoParser { switch (unit.type) { // NDR case 1: { + if (iframesOnly) { + break; + } let iskey = false; push = true; const data = unit.data; @@ -77,8 +83,8 @@ class AvcVideoParser extends BaseVideoParser { VideoSample.key = iskey; break; - // IDR } + // IDR case 5: push = true; // handle PES not starting with AUD @@ -100,6 +106,9 @@ class AvcVideoParser extends BaseVideoParser { break; // SEI case 6: { + if (iframesOnly) { + break; + } push = true; parseSEIMessageFromNALu( unit.data, diff --git a/src/demux/video/base-video-parser.ts b/src/demux/video/base-video-parser.ts index 93cdd34f2..9a665696e 100644 --- a/src/demux/video/base-video-parser.ts +++ b/src/demux/video/base-video-parser.ts @@ -5,6 +5,7 @@ import type { VideoSample, VideoSampleUnit, } from '../../types/demuxer'; +import type { ChunkMetadata } from '../../types/transmuxer'; import type { ParsedVideoSample } from '../tsdemuxer'; import type { PES } from '../tsdemuxer'; @@ -70,6 +71,7 @@ abstract class BaseVideoParser { textTrack: DemuxedUserdataTrack, pes: PES, last: boolean, + chunkMeta?: ChunkMetadata, ); protected abstract getNALuType(data: Uint8Array, offset: number): number; diff --git a/src/hls.ts b/src/hls.ts index d1d661d73..fbab2a673 100644 --- a/src/hls.ts +++ b/src/hls.ts @@ -3,11 +3,7 @@ import { EventEmitter } from 'eventemitter3'; import { buildAbsoluteURL } from 'url-toolkit'; import { enableStreamingMode, hlsDefaultConfig, mergeConfig } from './config'; import { FragmentTracker } from './controller/fragment-tracker'; -import GapController from './controller/gap-controller'; -import ID3TrackController from './controller/id3-track-controller'; -import LatencyController from './controller/latency-controller'; import LevelController from './controller/level-controller'; -import StreamController from './controller/stream-controller'; import { ErrorDetails, ErrorTypes } from './errors'; import { Events } from './events'; import { isMSESupported, isSupported } from './is-supported'; @@ -35,8 +31,15 @@ import type ContentSteeringController from './controller/content-steering-contro import type EMEController from './controller/eme-controller'; import type ErrorController from './controller/error-controller'; import type FPSController from './controller/fps-controller'; +import type GapController from './controller/gap-controller'; +import type { + HlsIFramesOnly, + IFrameController, +} from './controller/iframe-controller'; import type InterstitialsController from './controller/interstitials-controller'; import type { InterstitialsManager } from './controller/interstitials-controller'; +import type LatencyController from './controller/latency-controller'; +import type StreamController from './controller/stream-controller'; import type { SubtitleStreamController } from './controller/subtitle-stream-controller'; import type SubtitleTrackController from './controller/subtitle-track-controller'; import type Decrypter from './crypt/decrypter'; @@ -53,6 +56,7 @@ import type { NetworkComponentAPI, } from './types/component-api'; import type { MediaAttachingData } from './types/events'; +import type { LevelParsed } from './types/level'; import type { AudioSelectionOption, MediaPlaylist, @@ -89,6 +93,8 @@ export default class Hls implements HlsEventEmitter { */ public readonly logger: ILogger; + protected _url: string | null = null; + protected streamController: StreamController; private coreComponents: ComponentAPI[]; private networkControllers: NetworkComponentAPI[]; private _emitter: HlsEventEmitter = new EventEmitter(); @@ -97,19 +103,18 @@ export default class Hls implements HlsEventEmitter { private abrController: AbrComponentAPI; private bufferController: BufferController; private capLevelController: CapLevelController; - private latencyController: LatencyController; + private latencyController?: LatencyController; private levelController: LevelController; - private streamController: StreamController; private audioStreamController?: AudioStreamController; private subtititleStreamController?: SubtitleStreamController; private audioTrackController?: AudioTrackController; private subtitleTrackController?: SubtitleTrackController; private interstitialsController?: InterstitialsController; - private gapController: GapController; + private iframeController?: IFrameController; + private gapController?: GapController; private emeController?: EMEController; private cmcdController?: CMCDController; private _media: HTMLMediaElement | null = null; - private _url: string | null = null; private _sessionId?: string; private triggeringException?: boolean; private started: boolean = false; @@ -184,7 +189,7 @@ export default class Hls implements HlsEventEmitter { const logger = (this.logger = enableLogs( userConfig.debug || false, 'Hls instance', - userConfig.assetPlayerId, + userConfig.loggerId || userConfig.assetPlayerId, )); const config = (this.config = mergeConfig( Hls.DefaultConfig, @@ -199,11 +204,15 @@ export default class Hls implements HlsEventEmitter { // core controllers and network loaders const { + streamController: _StreamController, abrController: _AbrController, bufferController: _BufferController, capLevelController: _CapLevelController, errorController: _ErrorController, fpsController: _FpsController, + id3TrackController: _ID3TrackController, + iframeController: _IFrameController, + gapController: _GapController, } = config; const errorController = new _ErrorController(this); const abrController = (this.abrController = new _AbrController(this)); @@ -220,7 +229,7 @@ export default class Hls implements HlsEventEmitter { const capLevelController = (this.capLevelController = new _CapLevelController(this)); - const fpsController = new _FpsController(this); + const fpsController = _FpsController ? new _FpsController(this) : null; const playListLoader = new PlaylistLoader(this); const _ContentSteeringController = config.contentSteeringController; @@ -233,23 +242,23 @@ export default class Hls implements HlsEventEmitter { contentSteering, )); - const id3TrackController = new ID3TrackController(this); + const id3TrackController = _ID3TrackController + ? new _ID3TrackController(this) + : undefined; + const keyLoader = new KeyLoader(this.config, this.logger); - const streamController = (this.streamController = new StreamController( + const streamController = (this.streamController = new _StreamController( this, fragmentTracker, keyLoader, )); - const gapController = (this.gapController = new GapController( - this, - fragmentTracker, - )); + const gapController = (this.gapController = _GapController + ? new _GapController(this, fragmentTracker) + : undefined); // Cap level controller uses streamController to flush the buffer capLevelController.setStreamController(streamController); - // fpsController uses streamController to switch when frames are being dropped - fpsController.setStreamController(streamController); const networkControllers: NetworkComponentAPI[] = [ playListLoader, @@ -264,15 +273,20 @@ export default class Hls implements HlsEventEmitter { } this.networkControllers = networkControllers; - const coreComponents: ComponentAPI[] = [ - abrController, - bufferController, - gapController, - capLevelController, - fpsController, - id3TrackController, - fragmentTracker, - ]; + const coreComponents: ComponentAPI[] = [abrController, bufferController]; + if (gapController) { + coreComponents.push(gapController); + } + coreComponents.push(capLevelController); + if (fpsController) { + // fpsController uses streamController to switch when frames are being dropped + fpsController.setStreamController(streamController); + coreComponents.push(fpsController); + } + if (id3TrackController) { + coreComponents.push(id3TrackController); + } + coreComponents.push(fragmentTracker); this.audioTrackController = this.createController( config.audioTrackController, @@ -313,12 +327,15 @@ export default class Hls implements HlsEventEmitter { coreComponents, ); this.latencyController = this.createController( - LatencyController, + config.latencyController, coreComponents, ); - this.coreComponents = coreComponents; + this.iframeController = _IFrameController + ? new _IFrameController(this, Hls) + : undefined; + // Error controller handles errors before and after all other controllers // This listener will be invoked after all other controllers error listeners networkControllers.push(errorController); @@ -334,7 +351,7 @@ export default class Hls implements HlsEventEmitter { ); } - createController(ControllerClass, components) { + private createController(ControllerClass, components: ComponentAPI[]) { if (ControllerClass) { const controllerInstance = new ControllerClass(this); if (components) { @@ -443,6 +460,9 @@ export default class Hls implements HlsEventEmitter { this.coreComponents.forEach((component) => component.destroy()); this.coreComponents.length = 0; + + this.iframeController = undefined; + // Remove any references that could be held in config options or callbacks const config = this.config; config.xhrSetup = config.fetchSetup = undefined; @@ -1177,7 +1197,7 @@ export default class Hls implements HlsEventEmitter { * @returns null prior to loading live Playlist */ get liveSyncPosition(): number | null { - return this.latencyController.liveSyncPosition; + return this.latencyController?.liveSyncPosition || null; } /** @@ -1185,7 +1205,7 @@ export default class Hls implements HlsEventEmitter { * @returns 0 before first playlist is loaded */ get latency(): number { - return this.latencyController.latency; + return this.latencyController?.latency || 0; } /** @@ -1194,17 +1214,18 @@ export default class Hls implements HlsEventEmitter { * @returns 0 before first playlist is loaded */ get maxLatency(): number { - return this.latencyController.maxLatency; + return this.latencyController?.maxLatency || 0; } /** * target distance from the edge as calculated by the latency controller */ get targetLatency(): number | null { - return this.latencyController.targetLatency; + return this.latencyController?.targetLatency || null; } set targetLatency(latency: number) { + if (!this.latencyController) return; this.latencyController.targetLatency = latency; } @@ -1212,7 +1233,7 @@ export default class Hls implements HlsEventEmitter { * the rate at which the edge of the current live playlist is advancing or 1 if there is none */ get drift(): number | null { - return this.latencyController.drift; + return this.latencyController?.drift || null; } /** @@ -1251,7 +1272,32 @@ export default class Hls implements HlsEventEmitter { * returns Interstitials Program Manager */ get interstitialsManager(): InterstitialsManager | null { - return this.interstitialsController?.interstitialsManager || null; + if (__USE_INTERSTITIALS__ && this.interstitialsController) { + return this.interstitialsController.interstitialsManager; + } + return null; + } + + /** + * returns an array of parsed iframe variants + */ + get iframeVariants(): LevelParsed[] { + const iframeVariants = this.levelController.iframeVariants; + return iframeVariants ? iframeVariants : []; + } + + /** + * Returns an new iframe focused Hls (HlsIFramesOnly) instance based on `iframeVariants` found in the current asset, + * or null when none are available. An iframe instance uses iframe variants as its `levels`. + * Use HlsIFramesOnly.loadMediaAt(time) to render video IFrames in an attached video element. + */ + createIFramePlayer( + configOverride?: Partial<HlsConfig>, + ): HlsIFramesOnly | null { + if (__USE_IFRAMES__ && this._url && this.iframeController) { + return this.iframeController.createIFramePlayer(configOverride); + } + return null; } /** @@ -1303,7 +1349,11 @@ export type { EMEController, ErrorController, FPSController, + GapController, + IFrameController, + HlsIFramesOnly, InterstitialsController, + LatencyController, StreamController, SubtitleStreamController, SubtitleTrackController, @@ -1374,6 +1424,7 @@ export type { ErrorActionFlags, IErrorAction, } from './controller/error-controller'; +export type { ID3TrackController } from './controller/id3-track-controller'; export type { HlsAssetPlayer, HlsAssetPlayerConfig, @@ -1392,11 +1443,13 @@ export type { DateRange, DateRangeCue } from './loader/date-range'; export type { LoadStats } from './loader/load-stats'; export type { LevelKey } from './loader/level-key'; export type { + Base, BaseSegment, EncryptedFragment, Fragment, MediaFragment, Part, + MediaFragmentRef, ElementaryStreams, ElementaryStreamTypes, ElementaryStreamInfo, @@ -1420,7 +1473,10 @@ export type { SnapOptions, TimelineOccupancy, } from './loader/interstitial-event'; -export type { ParsedMultivariantPlaylist } from './loader/m3u8-parser'; +export type { + ParsedMultivariantPlaylist, + ParsedMultivariantMediaOptions, +} from './loader/m3u8-parser'; export type { AttachMediaSourceData, BaseTrack, @@ -1524,6 +1580,10 @@ export type { RemuxedUserdata, RemuxerResult, } from './types/remuxer'; +export type { + NetworkDetails, + NullableNetworkDetails, +} from './types/network-details'; export type { AttrList } from './utils/attr-list'; export type { Bufferable } from './utils/buffer-helper'; export type { CaptionScreen } from './utils/cea-608-parser'; diff --git a/src/loader/fragment-loader.ts b/src/loader/fragment-loader.ts index 4a02ed85d..cc2c36d53 100644 --- a/src/loader/fragment-loader.ts +++ b/src/loader/fragment-loader.ts @@ -43,6 +43,7 @@ export default class FragmentLoader { load( frag: Fragment, + isIFrame?: boolean, onProgress?: FragmentLoadProgressCallback, ): Promise<FragLoadedData> { const url = frag.url; @@ -82,7 +83,7 @@ export default class FragmentLoader { const loader = (this.loader = FragmentILoader ? new FragmentILoader(config) : (new DefaultILoader(config) as Loader<FragmentLoaderContext>)); - const loaderContext = createLoaderContext(frag); + const loaderContext = createLoaderContext(frag, null, isIFrame); frag.loader = loader; const loadPolicy = getLoaderConfigWithoutReties( config.fragLoadPolicy.default, @@ -319,6 +320,7 @@ export default class FragmentLoader { function createLoaderContext( frag: Fragment, part: Part | null = null, + isIFrame?: boolean, ): FragmentLoaderContext { const segment: BaseSegment = part || frag; const loaderContext: FragmentLoaderContext = { @@ -337,7 +339,7 @@ function createLoaderContext( let byteRangeStart = start; let byteRangeEnd = end; if ( - frag.sn === 'initSegment' && + (frag.sn === 'initSegment' || isIFrame) && isMethodFullSegmentAesCbc(frag.decryptdata?.method) ) { // MAP segment encrypted with method 'AES-128' or 'AES-256' (cbc), when served with HTTP Range, diff --git a/src/loader/fragment.ts b/src/loader/fragment.ts index 6676dba51..58a92f078 100644 --- a/src/loader/fragment.ts +++ b/src/loader/fragment.ts @@ -229,7 +229,7 @@ export class Fragment extends BaseSegment { // The minimum ending Presentation Time Stamp (audio/video PTS) of the fragment. Set after transmux complete. public minEndPTS?: number; // Init Segment bytes (unset for media segments) - public data?: Uint8Array; + public data?: Uint8Array<ArrayBuffer>; // A flag indicating whether the segment was downloaded in order to test bitrate, and was not buffered public bitrateTest: boolean = false; // #EXTINF segment title diff --git a/src/loader/level-details.ts b/src/loader/level-details.ts index ca6dc0f90..15790292d 100644 --- a/src/loader/level-details.ts +++ b/src/loader/level-details.ts @@ -22,6 +22,7 @@ export class LevelDetails { public dateRanges: Record<string, DateRange | undefined>; public dateRangeTagCount: number = 0; public live: boolean = true; + public iframesOnly: boolean = false; public requestScheduled: number = -1; public ageHeader: number = 0; public advancedDateTime?: number; diff --git a/src/loader/m3u8-parser.ts b/src/loader/m3u8-parser.ts index ac2d3c874..bff3d7e6c 100644 --- a/src/loader/m3u8-parser.ts +++ b/src/loader/m3u8-parser.ts @@ -16,6 +16,7 @@ import type { MediaFragment } from './fragment'; import type { ContentSteeringOptions } from '../types/events'; import type { CodecsParsed, + IFrameAttributes, LevelAttributes, LevelParsed, VariableMap, @@ -29,6 +30,7 @@ type M3U8ParserFragments = Array<Fragment | null>; export type ParsedMultivariantPlaylist = { contentSteering: ContentSteeringOptions | null; levels: LevelParsed[]; + iframeVariants: LevelParsed[]; playlistParsingError: Error | null; sessionData: Record<string, AttrList> | null; sessionKeys: LevelKey[] | null; @@ -37,7 +39,7 @@ export type ParsedMultivariantPlaylist = { hasVariableRefs: boolean; }; -type ParsedMultivariantMediaOptions = { +export type ParsedMultivariantMediaOptions = { AUDIO?: MediaPlaylist[]; SUBTITLES?: MediaPlaylist[]; 'CLOSED-CAPTIONS'?: MediaPlaylist[]; @@ -46,7 +48,7 @@ type ParsedMultivariantMediaOptions = { type LevelKeys = { [key: string]: LevelKey | undefined }; const MASTER_PLAYLIST_REGEX = - /#EXT-X-STREAM-INF:([^\r\n]*)(?:[\r\n](?:#[^\r\n]*)?)*([^\r\n]+)|#EXT-X-(SESSION-DATA|SESSION-KEY|DEFINE|CONTENT-STEERING|START):([^\r\n]*)[\r\n]+/g; + /#EXT-X-STREAM-INF:([^\r\n]*)(?:[\r\n](?:#[^\r\n]*)?)*([^\r\n]+)|#EXT-X-(I-FRAME-STREAM-INF|SESSION-DATA|SESSION-KEY|DEFINE|CONTENT-STEERING|START):([^\r\n]*)[\r\n]+/g; const MASTER_PLAYLIST_MEDIA_REGEX = /#EXT-X-MEDIA:(.*)/g; const IS_MEDIA_PLAYLIST = /^#EXT(?:INF|-X-TARGETDURATION):/m; // Handle empty Media Playlist (first EXTINF not signaled, but TARGETDURATION present) @@ -66,7 +68,8 @@ const LEVEL_PLAYLIST_REGEX_SLOW = new RegExp( .source, /#EXT-X-(BITRATE|DISCONTINUITY-SEQUENCE|MEDIA-SEQUENCE|TARGETDURATION|VERSION): *(\d+)/ .source, - /#EXT-X-(DISCONTINUITY|ENDLIST|GAP|INDEPENDENT-SEGMENTS)/.source, + /#EXT-X-(DISCONTINUITY|ENDLIST|GAP|INDEPENDENT-SEGMENTS|I-FRAMES-ONLY)/ + .source, /(#)([^:]*):(.*)/.source, /(#)(.*)(?:.*)\r?\n?/.source, ].join('|'), @@ -109,6 +112,7 @@ export default class M3U8Parser { const parsed: ParsedMultivariantPlaylist = { contentSteering: null, levels: [], + iframeVariants: [], playlistParsingError: null, sessionData: null, sessionKeys: null, @@ -128,40 +132,39 @@ export default class M3U8Parser { if (result[1]) { // '#EXT-X-STREAM-INF' is found, parse level tag in group 1 const attrs = new AttrList(result[1], parsed) as LevelAttributes; - const uri = __USE_VARIABLE_SUBSTITUTION__ - ? substituteVariables(parsed, result[2]) - : result[2]; - const level: LevelParsed = { + const level = createVariant( attrs, - bitrate: - attrs.decimalInteger('BANDWIDTH') || - attrs.decimalInteger('AVERAGE-BANDWIDTH'), - name: attrs.NAME, - url: M3U8Parser.resolve(uri, baseurl), - }; - - const resolution = attrs.decimalResolution('RESOLUTION'); - if (resolution) { - level.width = resolution.width; - level.height = resolution.height; + result[2], + baseurl, + parsed, + levelsWithKnownCodecs, + ); + if (level) { + parsed.levels.push(level); } - - setCodecs(attrs.CODECS, level); - const supplementalCodecs = attrs['SUPPLEMENTAL-CODECS']; - if (supplementalCodecs) { - level.supplemental = {}; - setCodecs(supplementalCodecs, level.supplemental); - } - - if (!level.unknownCodecs?.length) { - levelsWithKnownCodecs.push(level); - } - - parsed.levels.push(level); } else if (result[3]) { const tag = result[3]; const attributes = result[4]; switch (tag) { + case 'I-FRAME-STREAM-INF': + { + const attrs = new AttrList( + attributes, + parsed, + ) as IFrameAttributes; + const iframeVariant = createVariant( + attrs, + attrs.URI, + baseurl, + parsed, + levelsWithKnownCodecs, + true, + ); + if (iframeVariant) { + parsed.iframeVariants.push(iframeVariant); + } + } + break; case 'SESSION-DATA': { // #EXT-X-SESSION-DATA const sessionAttrs = new AttrList(attributes, parsed); @@ -396,7 +399,6 @@ export default class M3U8Parser { frag.sn = currentSN; frag.level = id; frag.cc = discontinuityCounter; - fragments.push(frag); // avoid sliced strings https://github.com/video-dev/hls.js/issues/939 const uri = (' ' + result[3]).slice(1); frag.relurl = __USE_VARIABLE_SUBSTITUTION__ @@ -411,6 +413,36 @@ export default class M3U8Parser { totalduration += frag.duration; currentSN++; currentPart = 0; + + const byteRange = frag.byteRange; + if (byteRange.length === 2) { + frag.bitrate = + (((byteRange[1] - byteRange[0]) * 8) / frag.duration) | 0; + } + + // Create implicit init segment for ByteRange addressed MPEG2-TS I-FRAME segments (PMT bytes) + if ( + level.iframesOnly && + byteRange[0] && + currentInitSegment?.cc !== discontinuityCounter + ) { + const init = new Fragment(type, base); + init.relurl = frag.relurl; + init.setByteRange(`${Math.min(byteRange[0], 7 * 188)}@0`); + init.level = id; + init.sn = 'initSegment'; + if (levelkeys) { + init.levelkeys = levelkeys; + if (levelkeys.identity) { + (init as any)._decryptdata = + levelkeys.identity.getDecryptData(0); + } + } + currentInitSegment = init; + frag.initSegment = currentInitSegment; + } + fragments.push(frag); + createNextFrag = true; } } else { @@ -499,6 +531,9 @@ export default class M3U8Parser { break; case 'INDEPENDENT-SEGMENTS': break; + case 'I-FRAMES-ONLY': + level.iframesOnly = true; + break; case 'ENDLIST': if (!level.live) { assignMultipleMediaPlaylistTagOccuranceError(level, tag, result); @@ -757,6 +792,49 @@ export default class M3U8Parser { } } +function createVariant( + attrs: LevelAttributes | IFrameAttributes, + tUri: string | undefined, + baseurl: string, + parsed: ParsedMultivariantPlaylist, + levelsWithKnownCodecs: LevelParsed[], + iframes?: boolean, +): LevelParsed | null { + if (tUri === undefined) { + // URI line or attribute is required. Ignore entry if missing. + return null; + } + const uri = __USE_VARIABLE_SUBSTITUTION__ + ? substituteVariables(parsed, tUri) + : tUri; + const level: LevelParsed = { + attrs, + bitrate: + attrs.decimalInteger('BANDWIDTH') || + attrs.decimalInteger('AVERAGE-BANDWIDTH'), + name: attrs.NAME, + url: M3U8Parser.resolve(uri, baseurl), + }; + const resolution = attrs.decimalResolution('RESOLUTION'); + if (resolution) { + level.width = resolution.width; + level.height = resolution.height; + } + setCodecs(attrs.CODECS, level); + const supplementalCodecs = attrs['SUPPLEMENTAL-CODECS']; + if (supplementalCodecs) { + level.supplemental = {}; + setCodecs(supplementalCodecs, level.supplemental); + } + if (!level.unknownCodecs?.length) { + levelsWithKnownCodecs.push(level); + } + if (iframes) { + level.iframes = true; + } + return level; +} + export function mapDateRanges( programDateTimes: MediaFragment[], details: LevelDetails, diff --git a/src/loader/playlist-loader.ts b/src/loader/playlist-loader.ts index 4cb0eb6e2..2c980efbd 100644 --- a/src/loader/playlist-loader.ts +++ b/src/loader/playlist-loader.ts @@ -427,6 +427,7 @@ class PlaylistLoader implements NetworkComponentAPI { const { contentSteering, levels, + iframeVariants, sessionData, sessionKeys, startTimeOffset, @@ -511,11 +512,11 @@ class PlaylistLoader implements NetworkComponentAPI { }); } } - hls.trigger(Events.MANIFEST_LOADED, { levels, audioTracks, subtitles, + iframeVariants, captions, contentSteering, url, @@ -572,6 +573,7 @@ class PlaylistLoader implements NetworkComponentAPI { hls.trigger(Events.MANIFEST_LOADED, { levels: [singleLevel], audioTracks: [], + iframeVariants: [], url, stats, networkDetails, @@ -719,6 +721,18 @@ class PlaylistLoader implements NetworkComponentAPI { typeof context.level === 'number' && parent === PlaylistLevelType.MAIN ? (level as number) : undefined; + if ( + __USE_IFRAMES__ && + levelOrTrack && + 'iframes' in levelOrTrack && + levelOrTrack.iframes && + !levelDetails.iframesOnly + ) { + levelDetails.playlistParsingError = new Error( + `EXT-X-I-FRAME-STREAM-INF media playlist MUST contain an EXT-X-I-FRAMES-ONLY tag`, + ); + levelDetails.iframesOnly = true; + } const error = levelDetails.playlistParsingError; if (error) { this.hls.logger.warn(`${error} ${levelDetails.url}`); diff --git a/src/remux/mp4-generator.ts b/src/remux/mp4-generator.ts index 2fe8da9da..844e1b2b9 100644 --- a/src/remux/mp4-generator.ts +++ b/src/remux/mp4-generator.ts @@ -2,7 +2,12 @@ * Generate MP4 Box */ -import { appendUint8Array } from '../utils/mp4-tools'; +import { + appendUint8Array, + types, + UINT32_MAX, + writeUint32, +} from '../utils/mp4-tools'; import type { DemuxedAC3, DemuxedAudioTrack, @@ -10,277 +15,211 @@ import type { DemuxedHEVC, DemuxedVideoTrack, } from '../types/demuxer'; -import type { - Mp4SampleFlags, - RemuxedAudioTrackSamples, - RemuxedVideoTrackSamples, -} from '../types/remuxer'; +import type { Mp4SampleFlags } from '../types/remuxer'; type MediaTrackType = DemuxedAudioTrack | DemuxedVideoTrack; -type RemuxedTrackType = RemuxedAudioTrackSamples | RemuxedVideoTrackSamples; + +export type TrackFragmentSample = { + cts: number; + duration: number; + size: number; + flags: Mp4SampleFlags; +}; + +type TrackFragmentInfo = { + type: 'audio' | 'video'; + id: number; + samples: TrackFragmentSample[]; +}; type HdlrTypes = { video: Uint8Array; audio: Uint8Array; }; -const UINT32_MAX = Math.pow(2, 32) - 1; +const HDLR_TYPES: HdlrTypes = { + video: new Uint8Array([ + 0x00, // version 0 + 0x00, + 0x00, + 0x00, // flags + 0x00, + 0x00, + 0x00, + 0x00, // pre_defined + 0x76, + 0x69, + 0x64, + 0x65, // handler_type: 'vide' + 0x00, + 0x00, + 0x00, + 0x00, // reserved + 0x00, + 0x00, + 0x00, + 0x00, // reserved + 0x00, + 0x00, + 0x00, + 0x00, // reserved + 0x56, + 0x69, + 0x64, + 0x65, + 0x6f, + 0x48, + 0x61, + 0x6e, + 0x64, + 0x6c, + 0x65, + 0x72, + 0x00, // name: 'VideoHandler' + ]), + audio: new Uint8Array([ + 0x00, // version 0 + 0x00, + 0x00, + 0x00, // flags + 0x00, + 0x00, + 0x00, + 0x00, // pre_defined + 0x73, + 0x6f, + 0x75, + 0x6e, // handler_type: 'soun' + 0x00, + 0x00, + 0x00, + 0x00, // reserved + 0x00, + 0x00, + 0x00, + 0x00, // reserved + 0x00, + 0x00, + 0x00, + 0x00, // reserved + 0x53, + 0x6f, + 0x75, + 0x6e, + 0x64, + 0x48, + 0x61, + 0x6e, + 0x64, + 0x6c, + 0x65, + 0x72, + 0x00, // name: 'SoundHandler' + ]), +}; + +const STCO = new Uint8Array([ + 0x00, // version + 0x00, + 0x00, + 0x00, // flags + 0x00, + 0x00, + 0x00, + 0x00, // entry_count +]); +const STSZ = new Uint8Array([ + 0x00, // version + 0x00, + 0x00, + 0x00, // flags + 0x00, + 0x00, + 0x00, + 0x00, // sample_size + 0x00, + 0x00, + 0x00, + 0x00, // sample_count +]); +const VMHD = new Uint8Array([ + 0x00, // version + 0x00, + 0x00, + 0x01, // flags + 0x00, + 0x00, // graphicsmode + 0x00, + 0x00, + 0x00, + 0x00, + 0x00, + 0x00, // opcolor +]); +const SMHD = new Uint8Array([ + 0x00, // version + 0x00, + 0x00, + 0x00, // flags + 0x00, + 0x00, // balance + 0x00, + 0x00, // reserved +]); +const STSD = new Uint8Array([ + 0x00, // version 0 + 0x00, + 0x00, + 0x00, // flags + 0x00, + 0x00, + 0x00, + 0x01, +]); // entry_count + +const majorBrand = new Uint8Array([105, 115, 111, 109]); // isom +const avc1Brand = new Uint8Array([97, 118, 99, 49]); // avc1 +const minorVersion = new Uint8Array([0, 0, 0, 1]); class MP4 { - public static types: Record<string, number[]>; - private static HDLR_TYPES: HdlrTypes; - private static STTS: Uint8Array; - private static STSC: Uint8Array; - private static STCO: Uint8Array; - private static STSZ: Uint8Array; - private static VMHD: Uint8Array; - private static SMHD: Uint8Array; - private static STSD: Uint8Array; - private static FTYP: Uint8Array; - private static DINF: Uint8Array; + public static FTYP = MP4.box( + types.ftyp, + majorBrand, + minorVersion, + majorBrand, + avc1Brand, + ); - static init() { - MP4.types = { - avc1: [], // codingname - avcC: [], - hvc1: [], - hvcC: [], - btrt: [], - dinf: [], - dref: [], - esds: [], - ftyp: [], - hdlr: [], - mdat: [], - mdhd: [], - mdia: [], - mfhd: [], - minf: [], - moof: [], - moov: [], - mp4a: [], - '.mp3': [], - dac3: [], - 'ac-3': [], - mvex: [], - mvhd: [], - pasp: [], - sdtp: [], - stbl: [], - stco: [], - stsc: [], - stsd: [], - stsz: [], - stts: [], - tfdt: [], - tfhd: [], - traf: [], - trak: [], - trun: [], - trex: [], - tkhd: [], - vmhd: [], - smhd: [], - }; + public static DINF = MP4.box( + types.dinf, + MP4.box( + types.dref, + new Uint8Array([ + 0x00, // version 0 + 0x00, + 0x00, + 0x00, // flags + 0x00, + 0x00, + 0x00, + 0x01, // entry_count + 0x00, + 0x00, + 0x00, + 0x0c, // entry_size + 0x75, + 0x72, + 0x6c, + 0x20, // 'url' type + 0x00, // version 0 + 0x00, + 0x00, + 0x01, // entry_flags + ]), + ), + ); - let i: string; - for (i in MP4.types) { - if (MP4.types.hasOwnProperty(i)) { - MP4.types[i] = [ - i.charCodeAt(0), - i.charCodeAt(1), - i.charCodeAt(2), - i.charCodeAt(3), - ]; - } - } - - const videoHdlr = new Uint8Array([ - 0x00, // version 0 - 0x00, - 0x00, - 0x00, // flags - 0x00, - 0x00, - 0x00, - 0x00, // pre_defined - 0x76, - 0x69, - 0x64, - 0x65, // handler_type: 'vide' - 0x00, - 0x00, - 0x00, - 0x00, // reserved - 0x00, - 0x00, - 0x00, - 0x00, // reserved - 0x00, - 0x00, - 0x00, - 0x00, // reserved - 0x56, - 0x69, - 0x64, - 0x65, - 0x6f, - 0x48, - 0x61, - 0x6e, - 0x64, - 0x6c, - 0x65, - 0x72, - 0x00, // name: 'VideoHandler' - ]); - - const audioHdlr = new Uint8Array([ - 0x00, // version 0 - 0x00, - 0x00, - 0x00, // flags - 0x00, - 0x00, - 0x00, - 0x00, // pre_defined - 0x73, - 0x6f, - 0x75, - 0x6e, // handler_type: 'soun' - 0x00, - 0x00, - 0x00, - 0x00, // reserved - 0x00, - 0x00, - 0x00, - 0x00, // reserved - 0x00, - 0x00, - 0x00, - 0x00, // reserved - 0x53, - 0x6f, - 0x75, - 0x6e, - 0x64, - 0x48, - 0x61, - 0x6e, - 0x64, - 0x6c, - 0x65, - 0x72, - 0x00, // name: 'SoundHandler' - ]); - - MP4.HDLR_TYPES = { - video: videoHdlr, - audio: audioHdlr, - }; - - const dref = new Uint8Array([ - 0x00, // version 0 - 0x00, - 0x00, - 0x00, // flags - 0x00, - 0x00, - 0x00, - 0x01, // entry_count - 0x00, - 0x00, - 0x00, - 0x0c, // entry_size - 0x75, - 0x72, - 0x6c, - 0x20, // 'url' type - 0x00, // version 0 - 0x00, - 0x00, - 0x01, // entry_flags - ]); - - const stco = new Uint8Array([ - 0x00, // version - 0x00, - 0x00, - 0x00, // flags - 0x00, - 0x00, - 0x00, - 0x00, // entry_count - ]); - - MP4.STTS = MP4.STSC = MP4.STCO = stco; - - MP4.STSZ = new Uint8Array([ - 0x00, // version - 0x00, - 0x00, - 0x00, // flags - 0x00, - 0x00, - 0x00, - 0x00, // sample_size - 0x00, - 0x00, - 0x00, - 0x00, // sample_count - ]); - MP4.VMHD = new Uint8Array([ - 0x00, // version - 0x00, - 0x00, - 0x01, // flags - 0x00, - 0x00, // graphicsmode - 0x00, - 0x00, - 0x00, - 0x00, - 0x00, - 0x00, // opcolor - ]); - MP4.SMHD = new Uint8Array([ - 0x00, // version - 0x00, - 0x00, - 0x00, // flags - 0x00, - 0x00, // balance - 0x00, - 0x00, // reserved - ]); - - MP4.STSD = new Uint8Array([ - 0x00, // version 0 - 0x00, - 0x00, - 0x00, // flags - 0x00, - 0x00, - 0x00, - 0x01, - ]); // entry_count - - const majorBrand = new Uint8Array([105, 115, 111, 109]); // isom - const avc1Brand = new Uint8Array([97, 118, 99, 49]); // avc1 - const minorVersion = new Uint8Array([0, 0, 0, 1]); - - MP4.FTYP = MP4.box( - MP4.types.ftyp, - majorBrand, - minorVersion, - majorBrand, - avc1Brand, - ); - MP4.DINF = MP4.box(MP4.types.dinf, MP4.box(MP4.types.dref, dref)); - } - - static box(type: number[], ...payload: Uint8Array[]) { + static box(type: number, ...payload: Uint8Array[]) { let size = 8; let i = payload.length; const len = i; @@ -290,11 +229,8 @@ class MP4 { } const result = new Uint8Array(size); - result[0] = (size >> 24) & 0xff; - result[1] = (size >> 16) & 0xff; - result[2] = (size >> 8) & 0xff; - result[3] = size & 0xff; - result.set(type, 4); + writeUint32(result, 0, size); + writeUint32(result, 4, type); // copy the payload into the result for (i = 0, size = 8; i < len; i++) { // copy payload[i] array @ offset size @@ -305,11 +241,11 @@ class MP4 { } static hdlr(type: keyof HdlrTypes) { - return MP4.box(MP4.types.hdlr, MP4.HDLR_TYPES[type]); + return MP4.box(types.hdlr, HDLR_TYPES[type]); } static mdat(data: Uint8Array) { - return MP4.box(MP4.types.mdat, data); + return MP4.box(types.mdat, data); } static mdhd(timescale: number, duration: number) { @@ -317,7 +253,7 @@ class MP4 { const upperWordDuration = Math.floor(duration / (UINT32_MAX + 1)); const lowerWordDuration = Math.floor(duration % (UINT32_MAX + 1)); return MP4.box( - MP4.types.mdhd, + types.mdhd, new Uint8Array([ 0x01, // version 1 0x00, @@ -361,7 +297,7 @@ class MP4 { static mdia(track: MediaTrackType) { return MP4.box( - MP4.types.mdia, + types.mdia, MP4.mdhd(track.timescale || 0, track.duration || 0), MP4.hdlr(track.type), MP4.minf(track), @@ -370,7 +306,7 @@ class MP4 { static mfhd(sequenceNumber: number) { return MP4.box( - MP4.types.mfhd, + types.mfhd, new Uint8Array([ 0x00, 0x00, @@ -387,15 +323,15 @@ class MP4 { static minf(track: MediaTrackType) { if (track.type === 'audio') { return MP4.box( - MP4.types.minf, - MP4.box(MP4.types.smhd, MP4.SMHD), + types.minf, + MP4.box(types.smhd, SMHD), MP4.DINF, MP4.stbl(track), ); } else { return MP4.box( - MP4.types.minf, - MP4.box(MP4.types.vmhd, MP4.VMHD), + types.minf, + MP4.box(types.vmhd, VMHD), MP4.DINF, MP4.stbl(track), ); @@ -405,10 +341,10 @@ class MP4 { static moof( sn: number, baseMediaDecodeTime: number, - track: RemuxedTrackType, + track: TrackFragmentInfo, ) { return MP4.box( - MP4.types.moof, + types.moof, MP4.mfhd(sn), MP4.traf(track, baseMediaDecodeTime), ); @@ -424,10 +360,7 @@ class MP4 { return MP4.box.apply( null, - [ - MP4.types.moov, - MP4.mvhd(tracks[0].timescale || 0, tracks[0].duration || 0), - ] + [types.moov, MP4.mvhd(tracks[0].timescale || 0, tracks[0].duration || 0)] .concat(boxes) .concat(MP4.mvex(tracks)), ); @@ -441,7 +374,7 @@ class MP4 { boxes[i] = MP4.trex(tracks[i]); } - return MP4.box.apply(null, [MP4.types.mvex, ...boxes]); + return MP4.box.apply(null, [types.mvex, ...boxes]); } static mvhd(timescale: number, duration: number) { @@ -562,10 +495,10 @@ class MP4 { 0xff, 0xff, // next_track_ID ]); - return MP4.box(MP4.types.mvhd, bytes); + return MP4.box(types.mvhd, bytes); } - static sdtp(track: RemuxedTrackType) { + static sdtp(track: TrackFragmentInfo) { const samples = track.samples || []; const bytes = new Uint8Array(4 + samples.length); let i: number; @@ -580,17 +513,17 @@ class MP4 { flags.hasRedundancy; } - return MP4.box(MP4.types.sdtp, bytes); + return MP4.box(types.sdtp, bytes); } static stbl(track: MediaTrackType) { return MP4.box( - MP4.types.stbl, + types.stbl, MP4.stsd(track), - MP4.box(MP4.types.stts, MP4.STTS), - MP4.box(MP4.types.stsc, MP4.STSC), - MP4.box(MP4.types.stsz, MP4.STSZ), - MP4.box(MP4.types.stco, MP4.STCO), + MP4.box(types.stts, STCO), + MP4.box(types.stsc, STCO), + MP4.box(types.stsz, STSZ), + MP4.box(types.stco, STCO), ); } @@ -623,7 +556,7 @@ class MP4 { } const avcc = MP4.box( - MP4.types.avcC, + types.avcC, new Uint8Array( [ 0x01, // version @@ -646,7 +579,7 @@ class MP4 { const vSpacing = track.pixelRatio[1]; return MP4.box( - MP4.types.avc1, + types.avc1, new Uint8Array([ 0x00, 0x00, @@ -729,7 +662,7 @@ class MP4 { ]), // pre_defined = -1 avcc, MP4.box( - MP4.types.btrt, + types.btrt, new Uint8Array([ 0x00, 0x1c, @@ -746,7 +679,7 @@ class MP4 { ]), ), // avgBitrate MP4.box( - MP4.types.pasp, + types.pasp, new Uint8Array([ hSpacing >> 24, // hSpacing (hSpacing >> 16) & 0xff, @@ -838,21 +771,21 @@ class MP4 { static mp4a(track: DemuxedAudioTrack) { return MP4.box( - MP4.types.mp4a, + types.mp4a, MP4.audioStsd(track), - MP4.box(MP4.types.esds, MP4.esds(track)), + MP4.box(types.esds, MP4.esds(track)), ); } static mp3(track: DemuxedAudioTrack) { - return MP4.box(MP4.types['.mp3'], MP4.audioStsd(track)); + return MP4.box(types['.mp3'], MP4.audioStsd(track)); } static ac3(track: DemuxedAudioTrack) { return MP4.box( - MP4.types['ac-3'], + types['ac-3'], MP4.audioStsd(track), - MP4.box(MP4.types.dac3, track.config as Uint8Array), + MP4.box(types.dac3, track.config as Uint8Array), ); } @@ -860,37 +793,29 @@ class MP4 { const { segmentCodec } = track; if (track.type === 'audio') { if (segmentCodec === 'aac') { - return MP4.box(MP4.types.stsd, MP4.STSD, MP4.mp4a(track)); + return MP4.box(types.stsd, STSD, MP4.mp4a(track)); } if ( __USE_M2TS_ADVANCED_CODECS__ && segmentCodec === 'ac3' && track.config ) { - return MP4.box(MP4.types.stsd, MP4.STSD, MP4.ac3(track)); + return MP4.box(types.stsd, STSD, MP4.ac3(track)); } if (segmentCodec === 'mp3' && track.codec === 'mp3') { - return MP4.box(MP4.types.stsd, MP4.STSD, MP4.mp3(track)); + return MP4.box(types.stsd, STSD, MP4.mp3(track)); } } else { if (track.pps && track.sps) { if (segmentCodec === 'avc') { - return MP4.box( - MP4.types.stsd, - MP4.STSD, - MP4.avc1(track as DemuxedAVC1), - ); + return MP4.box(types.stsd, STSD, MP4.avc1(track as DemuxedAVC1)); } if ( __USE_M2TS_ADVANCED_CODECS__ && segmentCodec === 'hevc' && track.vps ) { - return MP4.box( - MP4.types.stsd, - MP4.STSD, - MP4.hvc1(track as DemuxedHEVC), - ); + return MP4.box(types.stsd, STSD, MP4.hvc1(track as DemuxedHEVC)); } } else { throw new Error(`video track missing pps or sps`); @@ -910,7 +835,7 @@ class MP4 { const upperWordDuration = Math.floor(duration / (UINT32_MAX + 1)); const lowerWordDuration = Math.floor(duration % (UINT32_MAX + 1)); return MP4.box( - MP4.types.tkhd, + types.tkhd, new Uint8Array([ 0x01, // version 1 0x00, @@ -1012,7 +937,7 @@ class MP4 { ); } - static traf(track: RemuxedTrackType, baseMediaDecodeTime: number) { + static traf(track: TrackFragmentInfo, baseMediaDecodeTime: number) { const sampleDependencyTable = MP4.sdtp(track); const id = track.id; const upperWordBaseMediaDecodeTime = Math.floor( @@ -1022,9 +947,9 @@ class MP4 { baseMediaDecodeTime % (UINT32_MAX + 1), ); return MP4.box( - MP4.types.traf, + types.traf, MP4.box( - MP4.types.tfhd, + types.tfhd, new Uint8Array([ 0x00, // version 0 0x00, @@ -1037,7 +962,7 @@ class MP4 { ]), ), MP4.box( - MP4.types.tfdt, + types.tfdt, new Uint8Array([ 0x01, // version 1 0x00, @@ -1073,13 +998,13 @@ class MP4 { */ static trak(track: MediaTrackType) { track.duration = track.duration || 0xffffffff; - return MP4.box(MP4.types.trak, MP4.tkhd(track), MP4.mdia(track)); + return MP4.box(types.trak, MP4.tkhd(track), MP4.mdia(track)); } static trex(track: MediaTrackType) { const id = track.id; return MP4.box( - MP4.types.trex, + types.trex, new Uint8Array([ 0x00, // version 0 0x00, @@ -1109,7 +1034,7 @@ class MP4 { ); } - static trun(track: MediaTrackType, offset: number) { + static trun(track: TrackFragmentInfo, offset: number) { const samples = track.samples || []; const len = samples.length; const arraylen = 12 + 16 * len; @@ -1169,14 +1094,10 @@ class MP4 { 12 + 16 * i, ); } - return MP4.box(MP4.types.trun, array); + return MP4.box(types.trun, array); } static initSegment(tracks: MediaTrackType[]) { - if (!MP4.types) { - MP4.init(); - } - const movie = MP4.moov(tracks); const result = appendUint8Array(MP4.FTYP, movie); return result; @@ -1254,14 +1175,14 @@ class MP4 { length += units[i][j].length; } } - const hvcc = MP4.box(MP4.types.hvcC, hvcC); + const hvcc = MP4.box(types.hvcC, hvcC); const width = track.width; const height = track.height; const hSpacing = track.pixelRatio[0]; const vSpacing = track.pixelRatio[1]; return MP4.box( - MP4.types.hvc1, + types.hvc1, new Uint8Array([ 0x00, 0x00, @@ -1344,7 +1265,7 @@ class MP4 { ]), // pre_defined = -1 hvcc, MP4.box( - MP4.types.btrt, + types.btrt, new Uint8Array([ 0x00, 0x1c, @@ -1361,7 +1282,7 @@ class MP4 { ]), ), // avgBitrate MP4.box( - MP4.types.pasp, + types.pasp, new Uint8Array([ hSpacing >> 24, // hSpacing (hSpacing >> 16) & 0xff, diff --git a/src/remux/mp4-remuxer.ts b/src/remux/mp4-remuxer.ts index c8affd625..a186dca47 100644 --- a/src/remux/mp4-remuxer.ts +++ b/src/remux/mp4-remuxer.ts @@ -4,12 +4,14 @@ import { ErrorDetails, ErrorTypes } from '../errors'; import { Events } from '../events'; import { PlaylistLevelType } from '../types/loader'; import { type ILogger, Logger } from '../utils/logger'; +import { types, writeUint32 } from '../utils/mp4-tools'; import { timestampToString, toMsFromMpegTsClock, } from '../utils/timescale-conversion'; import type { HlsConfig } from '../config'; import type { HlsEventEmitter } from '../events'; +import type { ChunkMetadata } from '../hls'; import type { SourceBufferName } from '../types/buffer'; import type { AudioSample, @@ -61,6 +63,7 @@ function createMp4Sample( degradPrio: 0, dependsOn: isKeyframe ? 2 : 1, isNonSync: isKeyframe ? 0 : 1, + paddingValue: 0, }, }; } @@ -172,6 +175,7 @@ export default class MP4Remuxer extends Logger implements Remuxer { accurateTimeOffset: boolean, flush: boolean, playlistType: PlaylistLevelType, + chunkMeta: ChunkMetadata, ): RemuxerResult { let video: RemuxedTrack | undefined; let audio: RemuxedTrack | undefined; @@ -187,7 +191,8 @@ export default class MP4Remuxer extends Logger implements Remuxer { // parameter is greater than -1. The pid is set when the PMT is parsed, which contains the tracks list. // However, if the initSegment has already been generated, or we've reached the end of a segment (flush), // then we can remux one track without waiting for the other. - const hasAudio = audioTrack.pid > -1; + const iframesOnly = chunkMeta.iframe; + const hasAudio = !iframesOnly && audioTrack.pid > -1; const hasVideo = videoTrack.pid > -1; const length = videoTrack.samples.length; const enoughAudioSamples = audioTrack.samples.length > 0; @@ -302,6 +307,7 @@ export default class MP4Remuxer extends Logger implements Remuxer { videoTimeOffset, isVideoContiguous, audioTrackLength, + chunkMeta, ); } } else if (enoughVideoSamples) { @@ -310,6 +316,7 @@ export default class MP4Remuxer extends Logger implements Remuxer { videoTimeOffset, isVideoContiguous, 0, + chunkMeta, ); } if (video) { @@ -532,6 +539,7 @@ export default class MP4Remuxer extends Logger implements Remuxer { timeOffset: number, contiguous: boolean, audioTrackLength: number, + chunkMeta: ChunkMetadata, ): RemuxedTrack | undefined { const timeScale: number = track.inputTimeScale; const inputSamples: Array<VideoSample> = track.samples; @@ -542,8 +550,8 @@ export default class MP4Remuxer extends Logger implements Remuxer { let nextVideoTs = this.nextVideoTs; let offset = 8; let mp4SampleDuration = this.videoSampleDuration; - let firstDTS; - let lastDTS; + let firstDTS: number; + let lastDTS: number; let minPTS: number = Number.POSITIVE_INFINITY; let maxPTS: number = Number.NEGATIVE_INFINITY; let sortSamples = false; @@ -566,7 +574,6 @@ export default class MP4Remuxer extends Logger implements Remuxer { nextVideoTs = pts - cts - initTime; } } - // PTS is coded on 33bits, and can loop from -2^32 to 2^32 // PTSNormalize will make PTS/DTS value monotonic, we use last known DTS value as reference value const nextVideoPts = nextVideoTs + initTime; @@ -707,7 +714,7 @@ export default class MP4Remuxer extends Logger implements Remuxer { /* concatenate the video data and construct the mdat in place (need 8 more bytes to fill length and mpdat type) */ const mdatSize = naluLen + 4 * nbNalu + 8; - let mdat; + let mdat: Uint8Array<ArrayBuffer>; try { mdat = new Uint8Array(mdatSize); } catch (err) { @@ -721,9 +728,8 @@ export default class MP4Remuxer extends Logger implements Remuxer { }); return; } - const view = new DataView(mdat.buffer); - view.setUint32(0, mdatSize); - mdat.set(MP4.types.mdat, 4); + writeUint32(mdat, 0, mdatSize); + writeUint32(mdat, 4, types.mdat); let stretchedLastFrame = false; let minDtsDelta = Number.POSITIVE_INFINITY; @@ -739,7 +745,7 @@ export default class MP4Remuxer extends Logger implements Remuxer { const unit = VideoSampleUnits[j]; const unitData = unit.data; const unitDataLen = unit.data.byteLength; - view.setUint32(offset, unitDataLen); + writeUint32(mdat, offset, unitDataLen); offset += 4; mdat.set(unitData, offset); offset += unitDataLen; @@ -860,8 +866,20 @@ export default class MP4Remuxer extends Logger implements Remuxer { this.nextVideoTs = nextVideoTs = endDTS - initTime; this.videoSampleDuration = mp4SampleDuration; this.isVideoContiguous = true; + if (__USE_IFRAMES__) { + if (chunkMeta.iframe && outputSamples.length === 1) { + outputSamples[0].duration = mp4SampleDuration = + chunkMeta.duration * timeScale; + this.nextVideoTs = nextVideoTs = + firstDTS + mp4SampleDuration - initTime; + } else { + this.warn( + `Not adjusting IFrame duration (sample count ${outputSamples.length})`, + ); + } + } const moof = MP4.moof( - track.sequenceNumber++, + chunkMeta.sn, firstDTS, Object.assign(track, { samples: outputSamples, @@ -1071,7 +1089,7 @@ export default class MP4Remuxer extends Logger implements Remuxer { } let firstPTS: number | null = null; let lastPTS: number | null = null; - let mdat: any; + let mdat: Uint8Array<ArrayBuffer> | undefined; let mdatSize: number = 0; let sampleLength: number = inputSamples.length; while (sampleLength--) { @@ -1111,16 +1129,17 @@ export default class MP4Remuxer extends Logger implements Remuxer { return; } if (!rawMPEG) { - const view = new DataView(mdat.buffer); - view.setUint32(0, mdatSize); - mdat.set(MP4.types.mdat, 4); + writeUint32(mdat, 0, mdatSize); + writeUint32(mdat, 4, types.mdat); } } else { // no audio samples return; } } - mdat.set(unit, offset); + if (mdat) { + mdat.set(unit, offset); + } const unitLen = unit.byteLength; offset += unitLen; // Default the sample's duration to the computed mp4SampleDuration, which will either be 1024 for AAC or 1152 for MPEG diff --git a/src/remux/passthrough-remuxer.ts b/src/remux/passthrough-remuxer.ts index fbc3fce7e..d95fd3546 100644 --- a/src/remux/passthrough-remuxer.ts +++ b/src/remux/passthrough-remuxer.ts @@ -1,3 +1,4 @@ +import MP4 from './mp4-generator'; import { flushTextTrackMetadataCueSamples, flushTextTrackUserdataCueSamples, @@ -5,7 +6,7 @@ import { import { ElementaryStreamTypes } from '../loader/fragment'; import { getCodecCompatibleName } from '../utils/codecs'; import { type ILogger, Logger } from '../utils/logger'; -import { patchEncyptionData } from '../utils/mp4-tools'; +import { patchEncyptionData, writeUint32 } from '../utils/mp4-tools'; import { getSampleData, parseInitSegment } from '../utils/mp4-tools'; import type { HlsConfig } from '../config'; import type { HlsEventEmitter } from '../events'; @@ -16,6 +17,7 @@ import type { DemuxedUserdataTrack, PassthroughTrack, } from '../types/demuxer'; +import type { PlaylistLevelType } from '../types/loader'; import type { InitSegmentData, RemuxedTrack, @@ -23,6 +25,7 @@ import type { RemuxerResult, } from '../types/remuxer'; import type { TrackSet } from '../types/track'; +import type { ChunkMetadata } from '../types/transmuxer'; import type { TypeSupported } from '../utils/codecs'; import type { InitData, InitDataTrack, TrackTimes } from '../utils/mp4-tools'; import type { TimestampOffset } from '../utils/timescale-conversion'; @@ -159,6 +162,9 @@ class PassThroughRemuxer extends Logger implements Remuxer { textTrack: DemuxedUserdataTrack, timeOffset: number, accurateTimeOffset: boolean, + flush: boolean, + playlistType: PlaylistLevelType, + chunkMeta: ChunkMetadata, ): RemuxerResult { let { initPTS, lastEndTime } = this; const result: RemuxerResult = { @@ -203,7 +209,7 @@ class PassThroughRemuxer extends Logger implements Remuxer { this.emitInitSegment = false; } - const trackSampleData = getSampleData(data, initData, this); + const trackSampleData = getSampleData(data, initData, chunkMeta, this); const audioSampleTimestamps = initData.audio ? trackSampleData[initData.audio.id] : null; @@ -211,6 +217,52 @@ class PassThroughRemuxer extends Logger implements Remuxer { ? trackSampleData[initData.video.id] : null; + let data1 = data; + let data2: Uint8Array<ArrayBuffer> | undefined; + if (__USE_IFRAMES__) { + if ( + videoSampleTimestamps && + videoSampleTimestamps.sampleCount > 1 && + chunkMeta.iframe + ) { + const { trun, start, duration } = videoSampleTimestamps; + if (trun.length === 1 && trun[0].samples.length) { + const sampleOffset = trun[0].sampleOffset; + const sample = trun[0].samples[0]; + const { cts, size } = sample; + const sampleEndByte = sampleOffset + size; + + // Remux Iframe segments reporting more than one sample (mp4 byte-range contains moof for playback segment) + data1 = MP4.moof(chunkMeta.sn, start, { + type: 'video', + id: videoTrack.id, + samples: [ + { + cts: cts || 0, + duration: duration, + size, + flags: { + isLeading: 0, + isDependedOn: 0, + hasRedundancy: 0, + degradPrio: 0, + dependsOn: 2, // assume independent iframe + isNonSync: 0, + paddingValue: 0, + }, + }, + ], + }); + data2 = data.subarray(sampleOffset - 8, sampleEndByte); + writeUint32(data2, 0, size + 8); + } else { + this.warn( + `Could not remux IFrame track fragment (trun count ${trun.length})`, + ); + } + } + } + const videoStartTime = toStartEndOrDefault(videoSampleTimestamps, Infinity); const audioStartTime = toStartEndOrDefault(audioSampleTimestamps, Infinity); const videoEndTime = toStartEndOrDefault(videoSampleTimestamps, 0, true); @@ -310,7 +362,8 @@ class PassThroughRemuxer extends Logger implements Remuxer { (initData.video ? initData.video.encrypted : false); const track: RemuxedTrack = { - data1: data, + data1, + data2, startPTS: startTime, startDTS: startTime, endPTS: endTime, diff --git a/src/types/demuxer.ts b/src/types/demuxer.ts index 671753f4c..25d787bb8 100644 --- a/src/types/demuxer.ts +++ b/src/types/demuxer.ts @@ -1,9 +1,12 @@ +import type { ChunkMetadata } from './transmuxer'; +import type { DecryptData } from '../loader/level-key'; import type { RationalTimestamp } from '../utils/timescale-conversion'; export interface Demuxer { demux( data: Uint8Array, timeOffset: number, + chunkMeta: ChunkMetadata, isSampleAes?: boolean, flush?: boolean, ): DemuxerResult; @@ -11,14 +14,20 @@ export interface Demuxer { data: Uint8Array, keyData: KeyData, timeOffset: number, + chunkMeta: ChunkMetadata, ): Promise<DemuxerResult>; - flush(timeOffset?: number): DemuxerResult | Promise<DemuxerResult>; + flush( + timeOffset: number, + chunkMeta: ChunkMetadata, + ): DemuxerResult | Promise<DemuxerResult>; destroy(): void; resetInitSegment( initSegment: Uint8Array | undefined, audioCodec: string | undefined, videoCodec: string | undefined, trackDuration: number, + decryptdata: DecryptData | null, + chunkMeta: ChunkMetadata, ); resetTimeStamp(defaultInitPTS?: RationalTimestamp | null): void; resetContiguity(): void; diff --git a/src/types/events.ts b/src/types/events.ts index 56ffa0489..2a21d5789 100644 --- a/src/types/events.ts +++ b/src/types/events.ts @@ -49,6 +49,7 @@ import type { LevelKey } from '../loader/level-key'; import type { LoadStats } from '../loader/load-stats'; import type { AttrList } from '../utils/attr-list'; import type { BufferInfo } from '../utils/buffer-helper'; +import type { TimestampOffset } from '../utils/timescale-conversion'; export interface MediaAttachingData { media: HTMLMediaElement; @@ -135,6 +136,7 @@ export interface ManifestLoadedData { audioTracks: MediaPlaylist[]; captions?: MediaPlaylist[]; contentSteering: ContentSteeringOptions | null; + iframeVariants: LevelParsed[]; levels: LevelParsed[]; networkDetails: NullableNetworkDetails; sessionData: Record<string, AttrList> | null; @@ -150,6 +152,7 @@ export interface ManifestParsedData { levels: Level[]; audioTracks: MediaPlaylist[]; subtitleTracks: MediaPlaylist[]; + iframeVariants: LevelParsed[]; sessionData: Record<string, AttrList> | null; sessionKeys: LevelKey[] | null; firstLevel: number; @@ -381,6 +384,7 @@ export interface NonNativeTextTracksData { } export interface InitPTSFoundData { + timestampOffsets: TimestampOffset[]; id: PlaylistLevelType; frag: MediaFragment; initPTS: number; diff --git a/src/types/level.ts b/src/types/level.ts index d20a93242..b5495e666 100755 --- a/src/types/level.ts +++ b/src/types/level.ts @@ -13,6 +13,7 @@ export interface LevelParsed extends CodecsParsed { supplemental?: CodecsParsed; url: string; width?: number; + iframes?: boolean; } export interface CodecsParsed { @@ -41,6 +42,8 @@ export interface LevelAttributes extends AttrList { 'VIDEO-RANGE'?: VideoRange; } +export type IFrameAttributes = LevelAttributes & { URI: string }; + export const HdcpLevels = ['NONE', 'TYPE-0', 'TYPE-1', null] as const; export type HdcpLevel = (typeof HdcpLevels)[number]; @@ -117,6 +120,7 @@ export class Level { public readonly supplemental: CodecsParsed | undefined; public readonly videoCodec: string | undefined; public readonly width: number; + public readonly iframes?: boolean; public details?: LevelDetails; public fragmentError: number = 0; public loadError: number = 0; @@ -157,8 +161,12 @@ export class Level { this.codecSet += `,${supplementalVideo.substring(0, 4)}`; } } - this.addGroupId('audio', data.attrs.AUDIO); - this.addGroupId('text', data.attrs.SUBTITLES); + if ('iframes' in data && data.iframes) { + this.iframes = true; + } else { + this.addGroupId('audio', data.attrs.AUDIO); + this.addGroupId('text', data.attrs.SUBTITLES); + } } get maxBitrate(): number { diff --git a/src/types/remuxer.ts b/src/types/remuxer.ts index c1e0a0259..1e46e1548 100644 --- a/src/types/remuxer.ts +++ b/src/types/remuxer.ts @@ -10,6 +10,7 @@ import type { } from './demuxer'; import type { PlaylistLevelType } from './loader'; import type { TrackSet } from './track'; +import type { ChunkMetadata } from './transmuxer'; import type { DecryptData } from '../loader/level-key'; import type { TimestampOffset } from '../utils/timescale-conversion'; @@ -23,6 +24,7 @@ export interface Remuxer { accurateTimeOffset: boolean, flush: boolean, playlistType: PlaylistLevelType, + chunkMeta: ChunkMetadata, ): RemuxerResult; resetInitSegment( initSegment: Uint8Array | undefined, @@ -70,6 +72,7 @@ export type Mp4SampleFlags = { degradPrio: 0; dependsOn: 1 | 2; isNonSync: 0 | 1; + paddingValue: 0; }; export type Mp4Sample = { diff --git a/src/types/transmuxer.ts b/src/types/transmuxer.ts index 1553d60da..0eeddd1a9 100644 --- a/src/types/transmuxer.ts +++ b/src/types/transmuxer.ts @@ -14,6 +14,8 @@ export class ChunkMetadata { public readonly id: number; public readonly size: number; public readonly partial: boolean; + public readonly iframe: boolean; + public readonly duration: number; public readonly transmuxing: HlsChunkPerformanceTiming = getNewPerformanceTiming(); public readonly buffering: { @@ -31,6 +33,8 @@ export class ChunkMetadata { size = 0, part = -1, partial = false, + duration?: number, + iframe?: boolean, ) { this.level = level; this.sn = sn; @@ -38,6 +42,8 @@ export class ChunkMetadata { this.size = size; this.part = part; this.partial = partial; + this.duration = duration || 0; + this.iframe = iframe || false; } } diff --git a/src/utils/level-helper.ts b/src/utils/level-helper.ts index 954e2e158..342214af2 100644 --- a/src/utils/level-helper.ts +++ b/src/utils/level-helper.ts @@ -5,6 +5,7 @@ import { stringify } from './safe-json-stringify'; import { DateRange } from '../loader/date-range'; import { assignProgramDateTime, mapDateRanges } from '../loader/m3u8-parser'; +import { PlaylistLevelType } from '../types/loader'; import type { ILogger } from './logger'; import type { Fragment, MediaFragment, Part } from '../loader/fragment'; import type { LevelDetails } from '../loader/level-details'; @@ -67,6 +68,7 @@ export function updateFragPTSDTS( endPTS: number, startDTS: number, endDTS: number, + iframesOnly: boolean | undefined, logger: ILogger, ): number { const parsedMediaDuration = endPTS - startPTS; @@ -105,10 +107,12 @@ export function updateFragPTSDTS( } const drift = startPTS - frag.start; - if (frag.start !== 0) { - frag.setStart(startPTS); + if (!iframesOnly) { + if (frag.start !== 0) { + frag.setStart(startPTS); + } + frag.setDuration(endPTS - frag.start); } - frag.setDuration(endPTS - frag.start); frag.startPTS = startPTS; frag.maxStartPTS = maxStartPTS; frag.startDTS = startDTS; @@ -121,26 +125,28 @@ export function updateFragPTSDTS( if (!details || sn < details.startSN || sn > details.endSN) { return 0; } - let i: number; - const fragIdx = sn - details.startSN; - const fragments = details.fragments; - // update frag reference in fragments array - // rationale is that fragments array might not contain this frag object. - // this will happen if playlist has been refreshed between frag loading and call to updateFragPTSDTS() - // if we don't update frag, we won't be able to propagate PTS info on the playlist - // resulting in invalid sliding computation - fragments[fragIdx] = frag; - // adjust fragment PTS/duration from seqnum-1 to frag 0 - for (i = fragIdx; i > 0; i--) { - updateFromToPTS(fragments[i], fragments[i - 1]); - } + if (!iframesOnly) { + let i: number; + const fragIdx = sn - details.startSN; + const fragments = details.fragments; + // update frag reference in fragments array + // rationale is that fragments array might not contain this frag object. + // this will happen if playlist has been refreshed between frag loading and call to updateFragPTSDTS() + // if we don't update frag, we won't be able to propagate PTS info on the playlist + // resulting in invalid sliding computation + fragments[fragIdx] = frag; + // adjust fragment PTS/duration from seqnum-1 to frag 0 + for (i = fragIdx; i > 0; i--) { + updateFromToPTS(fragments[i], fragments[i - 1]); + } - // adjust fragment PTS/duration from seqnum to last frag - for (i = fragIdx; i < fragments.length - 1; i++) { - updateFromToPTS(fragments[i], fragments[i + 1]); - } - if (details.fragmentHint) { - updateFromToPTS(fragments[fragments.length - 1], details.fragmentHint); + // adjust fragment PTS/duration from seqnum to last frag + for (i = fragIdx; i < fragments.length - 1; i++) { + updateFromToPTS(fragments[i], fragments[i + 1]); + } + if (details.fragmentHint) { + updateFromToPTS(fragments[fragments.length - 1], details.fragmentHint); + } } details.PTSKnown = details.alignedSliding = true; @@ -297,6 +303,8 @@ export function mergeDetails( // if at least one fragment contains PTS info, recompute PTS information for all fragments if (PTSFrag) { + const iframesOnly = + newDetails.iframesOnly && PTSFrag.type === PlaylistLevelType.MAIN; updateFragPTSDTS( newDetails, PTSFrag, @@ -304,6 +312,7 @@ export function mergeDetails( PTSFrag.endPTS as number, PTSFrag.startDTS as number, PTSFrag.endDTS as number, + iframesOnly, logger, ); } else { diff --git a/src/utils/mp4-tools.ts b/src/utils/mp4-tools.ts index 9db7ea211..943a56829 100644 --- a/src/utils/mp4-tools.ts +++ b/src/utils/mp4-tools.ts @@ -2,6 +2,7 @@ import { utf8ArrayToStr } from '@svta/common-media-library/utils/utf8ArrayToStr' import { arrayToHex } from './hex'; import { ElementaryStreamTypes } from '../loader/fragment'; import { logger } from '../utils/logger'; +import type { ChunkMetadata } from '../hls'; import type { KeySystemIds } from './mediakeys-helper'; import type { DecryptData } from '../loader/level-key'; import type { PassthroughTrack, UserdataSample } from '../types/demuxer'; @@ -9,8 +10,52 @@ import type { ILogger } from '../utils/logger'; type BoxDataOrUndefined = Uint8Array<ArrayBuffer> | undefined; -const UINT32_MAX = Math.pow(2, 32) - 1; -const push = [].push; +export const UINT32_MAX = Math.pow(2, 32) - 1; + +export const types = { + avc1: 0x61766331, + avcC: 0x61766343, + hvc1: 0x68766331, + hvcC: 0x68766343, + btrt: 0x62747274, + dinf: 0x64696e66, + dref: 0x64726566, + esds: 0x65736473, + ftyp: 0x66747970, + hdlr: 0x68646c72, + mdat: 0x6d646174, + mdhd: 0x6d646864, + mdia: 0x6d646961, + mfhd: 0x6d666864, + minf: 0x6d696e66, + moof: 0x6d6f6f66, + moov: 0x6d6f6f76, + mp4a: 0x6d703461, + '.mp3': 0x2e6d7033, + dac3: 0x64616333, + 'ac-3': 0x61632d33, + mvex: 0x6d766578, + mvhd: 0x6d766864, + pasp: 0x70617370, + sdtp: 0x73647470, + stbl: 0x7374626c, + stco: 0x7374636f, + stsc: 0x73747363, + stsd: 0x73747364, + stsz: 0x7374737a, + stts: 0x73747473, + tfdt: 0x74666474, + tfhd: 0x74666864, + traf: 0x74726166, + trak: 0x7472616b, + trun: 0x7472756e, + trex: 0x74726578, + tkhd: 0x746b6864, + vmhd: 0x766d6864, + smhd: 0x736d6864, +} as const; + +type BoxType = keyof typeof types; // We are using fixed track IDs for driving the MP4 remuxer // instead of following the TS PIDs. @@ -64,18 +109,11 @@ export function writeUint32(buffer: Uint8Array, offset: number, value: number) { buffer[offset + 3] = value & 0xff; } -// Find "moof" box -export function hasMoofData(data: Uint8Array): boolean { +export function hasBoxData(data: Uint8Array, type: BoxType): boolean { const end = data.byteLength; for (let i = 0; i < end; ) { const size = readUint32(data, i); - if ( - size > 8 && - data[i + 4] === 0x6d && - data[i + 5] === 0x6f && - data[i + 6] === 0x6f && - data[i + 7] === 0x66 - ) { + if (size > 8 && readUint32(data, i + 4) === types[type]) { return true; } i = size > 1 ? i + size : end; @@ -105,7 +143,7 @@ export function findBox(data: Uint8Array, path: string[]): Uint8Array[] { // recursively search for the next box along the path const subresults = findBox(data.subarray(i + 8, endbox), path.slice(1)); if (subresults.length) { - push.apply(results, subresults); + results.push.apply(results, subresults); } } } @@ -250,6 +288,7 @@ export interface InitData extends Array<any> { stsd: StsdData; default?: { duration: number; + sampleSize: number; flags: number; }; } @@ -306,6 +345,7 @@ export function parseInitSegment(initSegment: Uint8Array): InitData { if (track) { track.default = { duration: readUint32(trex, 12), + sampleSize: readUint32(trex, 16), flags: readUint32(trex, 20), }; } @@ -658,12 +698,23 @@ export function parseSinf(sinf: Uint8Array): BoxDataOrUndefined { unsigned int(32) default_sample_flags } */ + +type TrackFragmentRunSample = { + cts?: number; + size: number; +}; +type TrackFragmentRun = { + sampleOffset: number; + samples: TrackFragmentRunSample[]; +}; export type TrackTimes = { + cts?: number; duration: number; keyFrameIndex?: number; keyFrameStart?: number; start: number; sampleCount: number; + trun: TrackFragmentRun[]; timescale: number; type: HdlrType; }; @@ -671,147 +722,178 @@ export type TrackTimes = { export function getSampleData( data: Uint8Array, initData: InitData, + chunkMeta: ChunkMetadata, logger: ILogger, ): Record<number, TrackTimes> { const tracks: Record<number, TrackTimes> = {}; - const trafs = findBox(data, ['moof', 'traf']); - for (let i = 0; i < trafs.length; i++) { - const traf = trafs[i]; - // There is only one tfhd & trun per traf - // This is true for CMAF style content, and we should perhaps check the ftyp - // and only look for a single trun then, but for ISOBMFF we should check - // for multiple track runs. - const tfhd = findBox(traf, ['tfhd'])[0]; - // get the track id from the tfhd - const id = readUint32(tfhd, 4); - const track = initData[id]; - if (!track) { - continue; - } - (tracks[id] as any) ||= { - start: NaN, - duration: 0, - sampleCount: 0, - timescale: track.timescale, - type: track.type, - }; - const trackTimes: TrackTimes = tracks[id]; - // get start DTS - const tfdt = findBox(traf, ['tfdt'])[0]; + const moofs = findBox(data, ['moof']); + const eof = data.byteLength; + for (let mi = 0; mi < moofs.length; mi++) { + const moof = moofs[mi]; + const moofOffset = moof.byteOffset - 8; + const trafs = findBox(moof, ['traf']); + for (let i = 0; i < trafs.length; i++) { + const traf = trafs[i]; + // There is only one tfhd & trun per traf + // This is true for CMAF style content, and we should perhaps check the ftyp + // and only look for a single trun then, but for ISOBMFF we should check + // for multiple track runs. + const tfhd = findBox(traf, ['tfhd'])[0]; + // get the track id from the tfhd + const id = readUint32(tfhd, 4); + const track = initData[id]; + if (!track) { + continue; + } + const trackTimes = (tracks[id] ||= { + start: NaN, + duration: 0, + sampleCount: 0, + trun: [], + timescale: track.timescale, + type: track.type, + }); + // get start DTS + const tfdt = findBox(traf, ['tfdt'])[0]; - if (tfdt as any) { - const version = tfdt[0]; - let baseTime = readUint32(tfdt, 4); - if (version === 1) { - // If value is too large, assume signed 64-bit. Negative track fragment decode times are invalid, but they exist in the wild. - // This prevents large values from being used for initPTS, which can cause playlist sync issues. - // https://github.com/video-dev/hls.js/issues/5303 - if (baseTime === UINT32_MAX) { - logger.warn( - `[mp4-demuxer]: Ignoring assumed invalid signed 64-bit track fragment decode time`, - ); - } else { - baseTime *= UINT32_MAX + 1; - baseTime += readUint32(tfdt, 8); + if (tfdt as any) { + const version = tfdt[0]; + let baseTime = readUint32(tfdt, 4); + if (version === 1) { + // If value is too large, assume signed 64-bit. Negative track fragment decode times are invalid, but they exist in the wild. + // This prevents large values from being used for initPTS, which can cause playlist sync issues. + // https://github.com/video-dev/hls.js/issues/5303 + if (baseTime === UINT32_MAX) { + logger.warn( + `[mp4-demuxer]: Ignoring assumed invalid signed 64-bit track fragment decode time`, + ); + } else { + baseTime *= UINT32_MAX + 1; + baseTime += readUint32(tfdt, 8); + } + } + if ( + Number.isFinite(baseTime) && + (!Number.isFinite(trackTimes.start) || baseTime < trackTimes.start) + ) { + trackTimes.start = baseTime; } } - if ( - Number.isFinite(baseTime) && - (!Number.isFinite(trackTimes.start) || baseTime < trackTimes.start) - ) { - trackTimes.start = baseTime; - } - } - const trackDefault = track.default; - const tfhdFlags = readUint32(tfhd, 0) | trackDefault?.flags!; - let defaultSampleDuration: number = trackDefault?.duration || 0; - if (tfhdFlags & 0x000008) { - // 0x000008 indicates the presence of the default_sample_duration field - if (tfhdFlags & 0x000002) { - // 0x000002 indicates the presence of the sample_description_index field, which precedes default_sample_duration - // If present, the default_sample_duration exists at byte offset 12 - defaultSampleDuration = readUint32(tfhd, 12); - } else { - // Otherwise, the duration is at byte offset 8 - defaultSampleDuration = readUint32(tfhd, 8); - } - } - const truns = findBox(traf, ['trun']); - let sampleDTS = trackTimes.start || 0; - let rawDuration = 0; - let sampleDuration = defaultSampleDuration; - for (let j = 0; j < truns.length; j++) { - const trun = truns[j]; - const sampleCount = readUint32(trun, 4); - const sampleIndex = trackTimes.sampleCount; - trackTimes.sampleCount += sampleCount; - // Get duration from samples - const dataOffsetPresent = trun[3] & 0x01; - const firstSampleFlagsPresent = trun[3] & 0x04; - const sampleDurationPresent = trun[2] & 0x01; - const sampleSizePresent = trun[2] & 0x02; - const sampleFlagsPresent = trun[2] & 0x04; - const sampleCompositionTimeOffsetPresent = trun[2] & 0x08; - let offset = 8; - let remaining = sampleCount; - if (dataOffsetPresent) { - offset += 4; - } - if (firstSampleFlagsPresent && sampleCount) { - const isNonSyncSample = trun[offset + 1] & 0x01; - if (!isNonSyncSample && trackTimes.keyFrameIndex === undefined) { - trackTimes.keyFrameIndex = sampleIndex; - } - offset += 4; - if (sampleDurationPresent) { - sampleDuration = readUint32(trun, offset); - offset += 4; + const trackDefault = track.default; + const tfhdFlags = readUint32(tfhd, 0) | trackDefault?.flags!; + let defaultSampleDuration = trackDefault?.duration || 0; + const defaultSampleSize = trackDefault?.sampleSize || 0; + if (tfhdFlags & 0x000008) { + // 0x000008 indicates the presence of the default_sample_duration field + if (tfhdFlags & 0x000002) { + // 0x000002 indicates the presence of the sample_description_index field, which precedes default_sample_duration + // If present, the default_sample_duration exists at byte offset 12 + defaultSampleDuration = readUint32(tfhd, 12); } else { - sampleDuration = defaultSampleDuration; + // Otherwise, the duration is at byte offset 8 + defaultSampleDuration = readUint32(tfhd, 8); } - if (sampleSizePresent) { - offset += 4; - } - if (sampleCompositionTimeOffsetPresent) { - offset += 4; - } - sampleDTS += sampleDuration; - rawDuration += sampleDuration; - remaining--; } - while (remaining--) { - if (sampleDurationPresent) { - sampleDuration = readUint32(trun, offset); - offset += 4; - } else { - sampleDuration = defaultSampleDuration; - } - if (sampleSizePresent) { + let baseDataOffset = 0; + const baseDataOffsetPresent = (tfhdFlags & 0x000001) !== 0; + const defaultBaseIsMoof = + !baseDataOffsetPresent && (tfhdFlags & 0x020000) !== 0; + if (baseDataOffsetPresent) { + // Should be 64 bit as per 14496-12 standard. + // check for possible overflow, and log for now. + baseDataOffset = readUint32(tfhd, 8); + baseDataOffset *= Math.pow(2, 32); + baseDataOffset += readUint32(tfhd, 12); + } else if (defaultBaseIsMoof) { + baseDataOffset = moofOffset; + } + + const truns = findBox(traf, ['trun']); + let sampleDTS = trackTimes.start || 0; + let rawDuration = 0; + let sampleDuration = defaultSampleDuration; + for (let j = 0; j < truns.length; j++) { + const trun = truns[j]; + // const version = trun[0]; + const sampleCount = readUint32(trun, 4); + const sampleIndex = trackTimes.sampleCount; + trackTimes.sampleCount += sampleCount; + // Get duration from samples + const dataOffsetPresent = trun[3] & 0x01; + let dataOffset = 0; + const firstSampleFlagsPresent = trun[3] & 0x04; + const sampleDurationPresent = trun[2] & 0x01; + const sampleSizePresent = trun[2] & 0x02; + const sampleFlagsPresent = trun[2] & 0x04; + const sampleCompositionTimeOffsetPresent = trun[2] & 0x08; + let offset = 8; + if (dataOffsetPresent) { + dataOffset = readSint32(trun, offset); offset += 4; } - if (sampleFlagsPresent) { + if (firstSampleFlagsPresent) { const isNonSyncSample = trun[offset + 1] & 0x01; - if (!isNonSyncSample) { - if (trackTimes.keyFrameIndex === undefined) { - trackTimes.keyFrameIndex = - trackTimes.sampleCount - (remaining + 1); - trackTimes.keyFrameStart = sampleDTS; - } + if (!isNonSyncSample && trackTimes.keyFrameIndex === undefined) { + trackTimes.keyFrameIndex = sampleIndex; } offset += 4; } - if (sampleCompositionTimeOffsetPresent) { - offset += 4; + let sampleOffset = baseDataOffset + dataOffset; + const samples: TrackFragmentRunSample[] = []; + if (sampleOffset <= eof) { + const fragRun: TrackFragmentRun = { + sampleOffset, + samples, + }; + trackTimes.trun.push(fragRun); + } + for (let ix = 0; ix < sampleCount; ix++) { + const sample: TrackFragmentRunSample = { size: 0 }; + if (sampleDurationPresent) { + sampleDuration = readUint32(trun, offset); + offset += 4; + } else { + sampleDuration = defaultSampleDuration; + } + if (sampleSizePresent) { + sample.size = readUint32(trun, offset); + offset += 4; + } else { + sample.size = defaultSampleSize; + } + if (sampleOffset <= eof) { + samples[ix] = sample; + } + sampleOffset += sample.size; + if (sampleFlagsPresent) { + const isNonSyncSample = trun[offset + 1] & 0x01; + if (!isNonSyncSample) { + if (trackTimes.keyFrameIndex === undefined) { + trackTimes.keyFrameIndex = ix; + trackTimes.keyFrameStart = sampleDTS; + } + } + offset += 4; + } + if (sampleCompositionTimeOffsetPresent) { + const version = trun[0]; + if (version === 0) { + sample.cts = readUint32(trun, offset); + } else { + sample.cts = readSint32(trun, offset); + } + offset += 4; + } + sampleDTS += sampleDuration; + rawDuration += sampleDuration; + } + if (!rawDuration && defaultSampleDuration) { + rawDuration += defaultSampleDuration * sampleCount; } - sampleDTS += sampleDuration; - rawDuration += sampleDuration; - } - if (!rawDuration && defaultSampleDuration) { - rawDuration += defaultSampleDuration * sampleCount; } + trackTimes.duration += rawDuration; } - trackTimes.duration += rawDuration; } if (!Object.keys(tracks).some((trackId) => tracks[trackId].duration)) { // If duration samples are not available in the traf use sidx subsegment_duration @@ -925,17 +1007,26 @@ export function parseSamples( const id = readUint32(tfhd, 4); const tfhdFlags = readUint32(tfhd, 0) & 0xffffff; const baseDataOffsetPresent = (tfhdFlags & 0x000001) !== 0; + let baseDataOffset = 0; const sampleDescriptionIndexPresent = (tfhdFlags & 0x000002) !== 0; const defaultSampleDurationPresent = (tfhdFlags & 0x000008) !== 0; let defaultSampleDuration = 0; const defaultSampleSizePresent = (tfhdFlags & 0x000010) !== 0; let defaultSampleSize = 0; const defaultSampleFlagsPresent = (tfhdFlags & 0x000020) !== 0; + const defaultBaseIsMoof = + !baseDataOffsetPresent && (tfhdFlags & 0x020000) !== 0; let tfhdOffset = 8; if (id === trackId) { if (baseDataOffsetPresent) { - tfhdOffset += 8; + baseDataOffset = readUint32(tfhd, tfhdOffset); + tfhdOffset += 4; + baseDataOffset *= Math.pow(2, 32); + baseDataOffset += readUint32(tfhd, tfhdOffset); + tfhdOffset += 4; + } else if (defaultBaseIsMoof) { + baseDataOffset = moofOffset; } if (sampleDescriptionIndexPresent) { tfhdOffset += 4; @@ -978,9 +1069,7 @@ export function parseSamples( if (firstSampleFlagsPresent) { trunOffset += 4; } - - let sampleOffset = dataOffset + moofOffset; - + let sampleOffset = baseDataOffset + dataOffset; for (let ix = 0; ix < sampleCount; ix++) { if (sampleDurationPresent) { sampleDuration = readUint32(trun, trunOffset); diff --git a/src/utils/rendition-helper.ts b/src/utils/rendition-helper.ts index 0edb8fdc7..b7eae7b2a 100644 --- a/src/utils/rendition-helper.ts +++ b/src/utils/rendition-helper.ts @@ -511,3 +511,21 @@ export function inGroupOrNone( } return groupIds.indexOf(groupId) !== -1; } + +export function getVideoPreference( + level: Level | null | undefined, + basePreference?: VideoSelectionOption, +): VideoSelectionOption | undefined { + if (basePreference || level) { + const videoPreference = Object.assign({}, basePreference); + if (level) { + if (level.videoCodec) { + videoPreference.videoCodec = level.videoCodec; + } + if (level.videoRange) { + videoPreference.allowedVideoRanges = [level.videoRange]; + } + } + return videoPreference; + } +} diff --git a/src/utils/time-ranges.ts b/src/utils/time-ranges.ts index 29d48a1de..b707b35c0 100644 --- a/src/utils/time-ranges.ts +++ b/src/utils/time-ranges.ts @@ -2,16 +2,12 @@ * TimeRanges to string helper */ -const TimeRanges = { - toString: function (r: TimeRanges) { - let log = ''; - const len = r.length; - for (let i = 0; i < len; i++) { - log += `[${r.start(i).toFixed(3)}-${r.end(i).toFixed(3)}]`; - } +export function timeRangesToString(r: TimeRanges) { + let log = ''; + const len = r.length; + for (let i = 0; i < len; i++) { + log += `[${r.start(i).toFixed(3)}-${r.end(i).toFixed(3)}]`; + } - return log; - }, -}; - -export default TimeRanges; + return log; +} diff --git a/tests/index.js b/tests/index.js index fe6e9249d..f298561db 100644 --- a/tests/index.js +++ b/tests/index.js @@ -18,6 +18,7 @@ import './unit/controller/fragment-finders'; import './unit/controller/fragment-tracker'; import './unit/controller/gap-controller'; import './unit/controller/id3-track-controller'; +import './unit/controller/iframe-controller'; import './unit/controller/interstitials-controller'; import './unit/controller/latency-controller'; import './unit/controller/level-controller'; diff --git a/tests/unit/controller/base-stream-controller.ts b/tests/unit/controller/base-stream-controller.ts index 5fcae0613..065f92791 100644 --- a/tests/unit/controller/base-stream-controller.ts +++ b/tests/unit/controller/base-stream-controller.ts @@ -959,6 +959,7 @@ describe('BaseStreamController', function () { (baseStreamController as any).getLevelDetails = () => ({ live: false }); mockBufferInfo.end = 20; media.currentTime = 5; + ((hls as any).streamController as any)._hasEnoughToStart = true; const result = (baseStreamController as any).calculateOptimalSwitchPoint( mockLevel, @@ -972,6 +973,7 @@ describe('BaseStreamController', function () { (baseStreamController as any).getLevelDetails = () => ({ live: true }); mockBufferInfo.end = 10; media.currentTime = 9.5; + ((hls as any).streamController as any)._hasEnoughToStart = true; const result = (baseStreamController as any).calculateOptimalSwitchPoint( mockLevel, diff --git a/tests/unit/controller/content-steering-controller.ts b/tests/unit/controller/content-steering-controller.ts index 6b3663ccd..026e695c1 100644 --- a/tests/unit/controller/content-steering-controller.ts +++ b/tests/unit/controller/content-steering-controller.ts @@ -100,6 +100,7 @@ describe('ContentSteeringController', function () { pathwayId: 'pathway-2', }, levels: [], + iframeVariants: [], audioTracks: [], subtitles: [], networkDetails: new Response('ok'), @@ -124,6 +125,7 @@ describe('ContentSteeringController', function () { pathwayId: 'pathway-2', }, levels: [], + iframeVariants: [], audioTracks: [], subtitles: [], networkDetails: new Response('ok'), @@ -148,6 +150,7 @@ describe('ContentSteeringController', function () { pathwayId: 'pathway-2', }, levels: [], + iframeVariants: [], audioTracks: [], subtitles: [], networkDetails: new Response('ok'), @@ -179,6 +182,7 @@ describe('ContentSteeringController', function () { pathwayId: 'pathway-2', }, levels: [], + iframeVariants: [], audioTracks: [], subtitles: [], networkDetails: new Response('ok'), @@ -206,6 +210,7 @@ describe('ContentSteeringController', function () { pathwayId: 'pathway-2', }, levels: [], + iframeVariants: [], audioTracks: [], subtitles: [], networkDetails: new Response('ok'), @@ -261,6 +266,7 @@ http://a.example.com/md/prog_index.m3u8`; const manifestLoadedData: ManifestLoadedData = { contentSteering: parsedMultivariant.contentSteering, levels: parsedMultivariant.levels, + iframeVariants: parsedMultivariant.iframeVariants, audioTracks: parsedMediaOptions.AUDIO!, subtitles: parsedMediaOptions.SUBTITLES, networkDetails: new Response('ok'), @@ -375,6 +381,7 @@ https://backup.example.com/video12/hi/video.m3u8`; const manifestLoadedData: ManifestLoadedData = { contentSteering: parsedMultivariant.contentSteering, levels: parsedMultivariant.levels, + iframeVariants: parsedMultivariant.iframeVariants, audioTracks: parsedMediaOptions.AUDIO!, subtitles: parsedMediaOptions.SUBTITLES, networkDetails: new Response('ok'), @@ -533,6 +540,7 @@ https://backup.example.com/video12/hi/video.m3u8`; const manifestLoadedData: ManifestLoadedData = { contentSteering: parsedMultivariant.contentSteering, levels: parsedMultivariant.levels, + iframeVariants: parsedMultivariant.iframeVariants, audioTracks: parsedMediaOptions.AUDIO!, subtitles: parsedMediaOptions.SUBTITLES, networkDetails: new Response('ok'), diff --git a/tests/unit/controller/iframe-controller.ts b/tests/unit/controller/iframe-controller.ts new file mode 100644 index 000000000..3872c7a7d --- /dev/null +++ b/tests/unit/controller/iframe-controller.ts @@ -0,0 +1,188 @@ +import { config as chaiConfig, expect, use } from 'chai'; +import sinon from 'sinon'; +import sinonChai from 'sinon-chai'; +import { Events } from '../../../src/events'; +import Hls from '../../../src/hls'; +import { LoadStats } from '../../../src/loader/load-stats'; +import { PlaylistLevelType } from '../../../src/types/loader'; +import type { HlsConfig } from '../../../src/config'; +import type { + HlsIFramesOnly, + IFrameController, +} from '../../../src/controller/iframe-controller'; +import type PlaylistLoader from '../../../src/loader/playlist-loader'; +import type { + ComponentAPI, + NetworkComponentAPI, +} from '../../../src/types/component-api'; +import type { TimestampOffset } from '../../../src/utils/timescale-conversion'; + +use(sinonChai); +chaiConfig.truncateThreshold = 0; + +type HlsTestable = Omit<Hls, 'networkControllers' | 'coreComponents'> & { + coreComponents: ComponentAPI[]; + networkControllers: NetworkComponentAPI[]; + iframeController: IFrameController; + trigger: Hls['trigger'] & sinon.SinonSpy; +}; + +type PlaylistLoaderTestable = Omit<PlaylistLoader, 'handleMasterPlaylist'> & { + handleMasterPlaylist: ( + response: { data: string; url: string }, + stats: LoadStats, + ) => void; +}; + +class HLSTestPlayer extends Hls { + constructor(config: Partial<HlsConfig>) { + super(config); + } +} + +const playlistWithIFrameVariants = `#EXTM3U +#EXT-X-MEDIA:TYPE=AUDIO,GROUP-ID="audio",LANGUAGE="en",NAME="English",AUTOSELECT=YES,DEFAULT=YES,CHANNELS="2",URI="audio/en/mp4a.40.2/media.m3u8" +#EXT-X-STREAM-INF:AUDIO="audio",AVERAGE-BANDWIDTH=6383725,BANDWIDTH=7495785,CODECS="avc1.64002A,mp4a.40.2",RESOLUTION=1920x1080 +video/avc1/1/media.m3u8 +#EXT-X-STREAM-INF:AUDIO="audio",AVERAGE-BANDWIDTH=2131576,BANDWIDTH=2653633,CODECS="avc1.64002A,mp4a.40.2",RESOLUTION=960x540 +video/avc1/3/media.m3u8 +#EXT-X-STREAM-INF:AUDIO="audio",AVERAGE-BANDWIDTH=3552258,BANDWIDTH=4191484,CODECS="avc1.64002A,mp4a.40.2",RESOLUTION=1280x720 +video/avc1/4/media.m3u8 +#EXT-X-I-FRAME-STREAM-INF:AVERAGE-BANDWIDTH=309502,BANDWIDTH=481062,CODECS="avc1.64002A",RESOLUTION=1920x1080,URI="video/avc1/1/iframes.m3u8" +#EXT-X-I-FRAME-STREAM-INF:AVERAGE-BANDWIDTH=65770,BANDWIDTH=110693,CODECS="avc1.64002A",RESOLUTION=960x540,URI="video/avc1/3/iframes.m3u8" +#EXT-X-I-FRAME-STREAM-INF:AVERAGE-BANDWIDTH=169447,BANDWIDTH=271378,CODECS="avc1.64002A",RESOLUTION=1280x720,URI="video/avc1/4/iframes.m3u8" +`; + +const playlistWithSteeringPathways = `#EXTM3U +#EXT-X-STREAM-INF:PATHWAY-ID=".",BANDWIDTH=7495785,CODECS="avc1.64002A,mp4a.40.2",RESOLUTION=1920x1080 +video/avc1/1/media.m3u8 +#EXT-X-STREAM-INF:PATHWAY-ID="..",BANDWIDTH=7495785,CODECS="avc1.64002A,mp4a.40.2",RESOLUTION=1920x1080 +https://b.com/video/avc1/1/media.m3u8 +#EXT-X-I-FRAME-STREAM-INF:PATHWAY-ID=".",BANDWIDTH=481062,CODECS="avc1.64002A",RESOLUTION=1920x1080,URI="video/avc1/1/iframes.m3u8" +#EXT-X-I-FRAME-STREAM-INF:PATHWAY-ID="..",BANDWIDTH=481062,CODECS="avc1.64002A",RESOLUTION=1920x1080,URI="https://b.com/video/avc1/1/media.m3u8" +`; + +describe('IFrameController', function () { + const sandbox = sinon.createSandbox(); + let hls: HlsTestable; + let playlistLoader: PlaylistLoaderTestable; + let iframeController: IFrameController; + + beforeEach(function () { + hls = new HLSTestPlayer({ + autoStartLoad: false, + // debug: true, + debug: { + trace: () => null, + debug: () => null, + log: () => null, + warn: () => null, + info: () => null, + error: () => null, + }, + }) as unknown as HlsTestable; + playlistLoader = hls.networkControllers[0] as PlaylistLoaderTestable; + sandbox.spy(hls, 'trigger'); + iframeController = hls.iframeController; + }); + + afterEach(function () { + hls.destroy(); + sandbox.restore(); + }); + + function loadManifest(data: string) { + const url = 'main.m3u8'; + (iframeController as any).clearAsset(); + (hls as any)._url = url; + playlistLoader.handleMasterPlaylist({ data, url }, new LoadStats()); + expect(hls.trigger).to.be.calledWith(Events.MANIFEST_LOADED); + } + + function loadedIFramePlayer(data: string): HlsIFramesOnly { + loadManifest(data); + const iframePlayer = iframeController.createIFramePlayer(); + expect(iframePlayer).to.be.an.instanceOf(Hls); + return iframePlayer as HlsIFramesOnly; + } + + it('Does not return IFramePlayers before iframes are loaded', function () { + const iframePlayer = iframeController.createIFramePlayer(); + expect(iframePlayer).to.be.null; + }); + + it('creates IFramePlayer instances once iframes are loaded', function () { + loadManifest(playlistWithIFrameVariants); + expect(hls.iframeVariants).to.have.lengthOf(3); + const iframePlayer = iframeController.createIFramePlayer(); + expect(iframePlayer).to.be.an.instanceOf(Hls); + }); + + it('configures IFramePlayer instances to only handle iframe video variants', function () { + const iframePlayer = loadedIFramePlayer(playlistWithIFrameVariants); + expect(iframePlayer.levels).to.have.to.have.lengthOf(3); + expect(iframePlayer.iframeVariants).to.have.lengthOf(0); + }); + + it('destroys child IFramePlayer instances when destroyed', function () { + const iframePlayer = loadedIFramePlayer(playlistWithIFrameVariants); + expect(hls.url).to.eql('main.m3u8'); + expect(iframePlayer.url).to.eql('main.m3u8'); + hls.destroy(); + expect(hls.url).to.eql(null); + expect(iframePlayer.url).to.eql(null); + }); + + it('destroys child IFramePlayer instances when loading a new asset', function () { + const iframePlayer = loadedIFramePlayer(playlistWithIFrameVariants); + expect(hls.url).to.eql('main.m3u8'); + expect(iframePlayer.url).to.eql('main.m3u8'); + hls.loadSource('another.m3u8'); + expect(iframePlayer.url).to.eql(null); + }); + + it('sets Initial Pathway of IFramePlayer to parent instances active variant pathway', function () { + loadManifest(playlistWithSteeringPathways); + expect(hls.iframeVariants).to.have.lengthOf(2); + expect(hls.pathways).to.deep.eq(['.', '..']); + + hls.loadLevel = 0; + expect(hls.loadLevel).to.eq(0); + expect(hls.levels[hls.loadLevel].pathwayId).to.eq('.'); + + hls.pathwayPriority = ['..', '.']; + expect(hls.pathwayPriority).to.deep.eq(['..', '.']); + expect(hls.levels[hls.loadLevel].pathwayId).to.eq('..'); + + const iframePlayer = iframeController.createIFramePlayer(); + expect(iframePlayer).to.be.an.instanceOf(Hls); + if (iframePlayer) { + expect(iframePlayer.levels[0].pathwayId).to.eq('..'); + } + }); + + it('updates IFramePlayer Pathway priorty once it has changed on the parent instance', function () { + const iframePlayer = loadedIFramePlayer(playlistWithSteeringPathways); + expect(hls.pathways).to.deep.eq(['.', '..']); + expect(iframePlayer.levels[0].pathwayId).to.eq('.'); + hls.pathwayPriority = ['..', '.']; + expect(iframePlayer.levels[0].pathwayId).to.eq('..'); + }); + + it('passes current initPTS to a newly created IFramePlayer', function () { + loadManifest(playlistWithIFrameVariants); + const timestamps: TimestampOffset[] = [ + { baseTime: 900000, timescale: 90000, trackId: 0 }, + ]; + hls.trigger(Events.INIT_PTS_FOUND, { + id: PlaylistLevelType.MAIN, + timestampOffsets: timestamps, + frag: { cc: 0 } as any, + initPTS: 900000, + timescale: 90000, + }); + const iframePlayer = iframeController.createIFramePlayer(); + const iframeStreamController = (iframePlayer as any).streamController; + expect(iframeStreamController.initPTS).to.equal(timestamps); + }); +}); diff --git a/tests/unit/controller/level-controller.ts b/tests/unit/controller/level-controller.ts index 2705888c6..596b70d72 100755 --- a/tests/unit/controller/level-controller.ts +++ b/tests/unit/controller/level-controller.ts @@ -117,6 +117,7 @@ describe('LevelController', function () { name: '1080', }), ], + iframeVariants: [], networkDetails: new Response('ok'), sessionData: null, sessionKeys: null, @@ -186,6 +187,7 @@ describe('LevelController', function () { levelController.onManifestLoaded(Events.MANIFEST_LOADED, { audioTracks: [], levels: [], + iframeVariants: [], networkDetails: new Response('ok'), subtitles: [], url: 'https://test-streams.mux.dev/x36xhzz/x36xhzz.m3u8', @@ -239,6 +241,7 @@ describe('LevelController', function () { name: '1080', }), ], + iframeVariants: [], networkDetails: new Response('ok'), subtitles: [], sessionData: null, @@ -254,6 +257,7 @@ describe('LevelController', function () { expect(hls.trigger).to.have.been.calledWith(Events.MANIFEST_PARSED, { levels: data.levels.map((levelParsed) => new Level(levelParsed)), + iframeVariants: [], audioTracks: [], subtitleTracks: [], sessionData: null, @@ -286,6 +290,7 @@ http://bar.example.com/audio-only/prog_index.m3u8`, expect(parsedLevels).to.have.lengthOf(6, 'MANIFEST_LOADED levels'); levelController.onManifestLoaded(Events.MANIFEST_LOADED, { levels: parsedLevels, + iframeVariants: [], audioTracks: [], subtitles: [], networkDetails: new Response('ok'), @@ -334,6 +339,7 @@ http://bar.example.com/audio-only/prog_index.m3u8`, name: '144', }), ], + iframeVariants: [], networkDetails: new Response('ok'), subtitles: [], sessionData: null, @@ -347,6 +353,7 @@ http://bar.example.com/audio-only/prog_index.m3u8`, const parsedData: ManifestParsedData = { levels: data.levels.map((levelParsed) => new Level(levelParsed)), + iframeVariants: [], audioTracks: data.audioTracks, subtitleTracks: [], sessionData: null, @@ -379,6 +386,7 @@ http://bar.example.com/audio-only/prog_index.m3u8`, audioCodec: 'mp4a.40.2', }), ], + iframeVariants: [], networkDetails: new Response('ok'), subtitles: [], sessionData: null, @@ -393,6 +401,7 @@ http://bar.example.com/audio-only/prog_index.m3u8`, levelController.onManifestLoaded(Events.MANIFEST_LOADED, data); expect(hls.trigger).to.have.been.calledWith(Events.MANIFEST_PARSED, { levels: data.levels.map((levelParsed) => new Level(levelParsed)), + iframeVariants: [], audioTracks: data.audioTracks, subtitleTracks: [], sessionData: null, @@ -419,6 +428,7 @@ http://bar.example.com/audio-only/prog_index.m3u8`, audioCodec: 'mp4a.40.2', }), ], + iframeVariants: [], networkDetails: new Response('ok'), subtitles: [], sessionData: null, @@ -433,6 +443,7 @@ http://bar.example.com/audio-only/prog_index.m3u8`, levelController.onManifestLoaded(Events.MANIFEST_LOADED, data); expect(hls.trigger).to.have.been.calledWith(Events.MANIFEST_PARSED, { levels: data.levels.map((levelParsed) => new Level(levelParsed)), + iframeVariants: [], audioTracks: data.audioTracks, subtitleTracks: [], sessionData: null, @@ -457,6 +468,7 @@ http://bar.example.com/audio-only/prog_index.m3u8`, name: '144', }), ], + iframeVariants: [], networkDetails: new Response('ok'), subtitles: [], sessionData: null, @@ -682,6 +694,7 @@ http://bar.example.com/md/prog_index.m3u8`, expect(parsedLevels).to.have.lengthOf(4, 'MANIFEST_LOADED levels'); levelController.onManifestLoaded(Events.MANIFEST_LOADED, { levels: parsedLevels, + iframeVariants: [], audioTracks: [], subtitles: [], networkDetails: new Response('ok'), @@ -746,6 +759,7 @@ http://bar.example.com/md/prog_index.m3u8`; expect(parsedSubs).to.be.undefined; levelController.onManifestLoaded(Events.MANIFEST_LOADED, { levels: parsedLevels, + iframeVariants: [], audioTracks: parsedAudio!, subtitles: [], networkDetails: new Response('ok'), @@ -809,6 +823,7 @@ http://bar.example.com/md/prog_index.m3u8`; expect(parsedSubs).to.have.lengthOf(9, 'MANIFEST_LOADED subtitles'); levelController.onManifestLoaded(Events.MANIFEST_LOADED, { levels: parsedLevels, + iframeVariants: [], audioTracks: parsedAudio, subtitles: parsedSubs, networkDetails: new Response('ok'), @@ -952,6 +967,7 @@ http://bar.example.com/md/prog_index.m3u8`; expect(parsedSubs).to.have.lengthOf(3, 'MANIFEST_LOADED subtitles'); levelController.onManifestLoaded(Events.MANIFEST_LOADED, { levels: parsedLevels, + iframeVariants: [], audioTracks: parsedAudio, subtitles: parsedSubs, networkDetails: new Response('ok'), @@ -1046,6 +1062,7 @@ http://bar.example.com/md/prog_index.m3u8`; expect(parsedSubs).to.have.lengthOf(6, 'MANIFEST_LOADED subtitles'); levelController.onManifestLoaded(Events.MANIFEST_LOADED, { levels: parsedLevels, + iframeVariants: [], audioTracks: parsedAudio, subtitles: parsedSubs, networkDetails: new Response('ok'), @@ -1114,6 +1131,7 @@ describe('LevelController - nextLoadLevel', function () { parsedLevel({ bitrate: 2149280, name: '720' }), parsedLevel({ bitrate: 6221600, name: '1080' }), ], + iframeVariants: [], networkDetails: new Response('ok'), sessionData: null, sessionKeys: null, diff --git a/tests/unit/controller/level-helper.ts b/tests/unit/controller/level-helper.ts index 4042febdb..04596b7fb 100644 --- a/tests/unit/controller/level-helper.ts +++ b/tests/unit/controller/level-helper.ts @@ -1494,6 +1494,7 @@ audio_5441.m4s`; hls.trigger(Events.MANIFEST_PARSED, { levels: [levelInfo], audioTracks: [trackInfo], + iframeVariants: [], subtitleTracks: [], sessionData: null, sessionKeys: null, diff --git a/tests/unit/controller/stream-controller.ts b/tests/unit/controller/stream-controller.ts index f7a07768c..9b0643a18 100644 --- a/tests/unit/controller/stream-controller.ts +++ b/tests/unit/controller/stream-controller.ts @@ -87,6 +87,7 @@ describe('StreamController', function () { } = result; hls.trigger(Events.MANIFEST_LOADED, { levels, + iframeVariants: [], audioTracks: [], contentSteering, url: 'http://www.example.com', @@ -166,6 +167,7 @@ describe('StreamController', function () { } = result; hls.trigger(Events.MANIFEST_LOADED, { levels, + iframeVariants: [], audioTracks: [], contentSteering, url: 'http://www.example.com', diff --git a/tests/unit/controller/subtitle-track-controller.ts b/tests/unit/controller/subtitle-track-controller.ts index fb770f64c..b4df9114d 100644 --- a/tests/unit/controller/subtitle-track-controller.ts +++ b/tests/unit/controller/subtitle-track-controller.ts @@ -156,6 +156,7 @@ describe('SubtitleTrackController', function () { hls.trigger(Events.MANIFEST_PARSED, { subtitleTracks, levels, + iframeVariants: [], audioTracks: [], sessionData: null, sessionKeys: null, diff --git a/tests/unit/crypt/decrypter.js b/tests/unit/crypt/decrypter.js index 35f20fee9..0dffffc0f 100644 --- a/tests/unit/crypt/decrypter.js +++ b/tests/unit/crypt/decrypter.js @@ -6,7 +6,7 @@ describe('Decrypter', function () { const data = get128cbcData(); const config = { enableSoftwareAES: true }; - const decrypter = new Decrypter(config, { removePKCS7Padding: true }); + const decrypter = new Decrypter(config); const cbcMode = 0; decrypter.softwareDecrypt(data.encrypted, data.key, data.iv, cbcMode); diff --git a/tests/unit/loader/fragment-loader.ts b/tests/unit/loader/fragment-loader.ts index 838829ebb..dbb796595 100644 --- a/tests/unit/loader/fragment-loader.ts +++ b/tests/unit/loader/fragment-loader.ts @@ -47,7 +47,7 @@ describe('FragmentLoader tests', function () { const onProgress = sinon.spy(); const fragmentLoaderPrivates = fragmentLoader as any; fragmentLoader - .load(frag, onProgress) + .load(frag, false, onProgress) .then((data) => { expect(data).to.deep.equal({ frag, diff --git a/tests/unit/loader/fragment.ts b/tests/unit/loader/fragment.ts index 6533f1e61..94336e417 100644 --- a/tests/unit/loader/fragment.ts +++ b/tests/unit/loader/fragment.ts @@ -86,7 +86,7 @@ describe('Fragment class tests', function () { expect(frag.byteRange).to.deep.equal([0, 5000]); }); - it('set byte range with no offset and uses 0 as offset', function () { + it('set byte range with no offset and uses previous segment end as offset', function () { const prevFrag = new Fragment(PlaylistLevelType.MAIN, ''); prevFrag.setByteRange('1000@10000'); frag.setByteRange('5000', prevFrag); diff --git a/tests/unit/loader/m3u8-parser.ts b/tests/unit/loader/m3u8-parser.ts index fa24476e2..bf1025dec 100644 --- a/tests/unit/loader/m3u8-parser.ts +++ b/tests/unit/loader/m3u8-parser.ts @@ -1147,6 +1147,7 @@ https://sample-host/segment1.m4a`; { contentSteering: null, levels: [], + iframeVariants: [], playlistParsingError: null, sessionData: null, sessionKeys: null,