diff --git a/api-extractor/report/hls.js.api.md b/api-extractor/report/hls.js.api.md index e51390cb018..537d0c16c14 100644 --- a/api-extractor/report/hls.js.api.md +++ b/api-extractor/report/hls.js.api.md @@ -350,6 +350,10 @@ export class BaseStreamController extends TaskLoop implements NetworkComponentAP // // (undocumented) protected fragmentLoader: FragmentLoader; + // Warning: (ae-forgotten-export) The symbol "FragmentPreloader" needs to be exported by the entry point hls.d.ts + // + // (undocumented) + protected fragmentPreloader: FragmentPreloader; // (undocumented) protected fragmentTracker: FragmentTracker; // (undocumented) @@ -405,6 +409,8 @@ export class BaseStreamController extends TaskLoop implements NetworkComponentAP // (undocumented) protected levels: Array | null; // (undocumented) + protected loadedEndOfParts(partList: Part[], targetBufferTime: number): boolean; + // (undocumented) protected loadedmetadata: boolean; // (undocumented) protected loadFragment(frag: Fragment, level: Level, targetBufferTime: number): void; @@ -2263,6 +2269,11 @@ export class LevelDetails { // (undocumented) playlistParsingError: Error | null; // (undocumented) + preloadData?: { + frag: Fragment; + part?: Part; + }; + // (undocumented) preloadHint?: AttrList; // (undocumented) PTSKnown: boolean; @@ -2635,6 +2646,8 @@ export interface LoaderStats { // (undocumented) aborted: boolean; // (undocumented) + blockingLoad: boolean; + // (undocumented) buffering: HlsProgressivePerformanceTiming; // (undocumented) bwEstimate: number; @@ -2666,6 +2679,8 @@ export class LoadStats implements LoaderStats { // (undocumented) aborted: boolean; // (undocumented) + blockingLoad: boolean; + // (undocumented) buffering: HlsProgressivePerformanceTiming; // (undocumented) bwEstimate: number; diff --git a/src/controller/abr-controller.ts b/src/controller/abr-controller.ts index 9c9b3b64345..31649faba4f 100644 --- a/src/controller/abr-controller.ts +++ b/src/controller/abr-controller.ts @@ -378,7 +378,7 @@ class AbrController extends Logger implements AbrComponentAPI { { frag, part }: FragLoadedData, ) { const stats = part ? part.stats : frag.stats; - if (frag.type === PlaylistLevelType.MAIN) { + if (frag.type === PlaylistLevelType.MAIN && !stats.blockingLoad) { this.bwEstimator.sampleTTFB(stats.loading.first - stats.loading.start); } if (this.ignoreFragment(frag)) { @@ -434,9 +434,12 @@ class AbrController extends Logger implements AbrComponentAPI { // Use the difference between parsing and request instead of buffering and request to compute fragLoadingProcessing; // rationale is that buffer appending only happens once media is attached. This can happen when config.startFragPrefetch // is used. If we used buffering in that case, our BW estimate sample will be very large. + const loadStart = part?.stats.blockingLoad + ? stats.loading.first + : stats.loading.start; const processingMs = stats.parsing.end - - stats.loading.start - + loadStart - Math.min( stats.loading.first - stats.loading.start, this.bwEstimator.getEstimateTTFB(), diff --git a/src/controller/audio-stream-controller.ts b/src/controller/audio-stream-controller.ts index a5fb144daa7..49766f2b96e 100644 --- a/src/controller/audio-stream-controller.ts +++ b/src/controller/audio-stream-controller.ts @@ -536,6 +536,8 @@ class AudioStreamController let sliding = 0; if (newDetails.live || track.details?.live) { this.checkLiveUpdate(newDetails); + // reset the preloader state to IDLE if we have finished loading, never loaded, or have old data + this.fragmentPreloader.revalidate(data); const mainDetails = this.mainDetails; if (newDetails.deltaUpdateFailed || !mainDetails) { return; diff --git a/src/controller/base-stream-controller.ts b/src/controller/base-stream-controller.ts index 20e407e546a..6371e8e28f0 100644 --- a/src/controller/base-stream-controller.ts +++ b/src/controller/base-stream-controller.ts @@ -52,6 +52,7 @@ import type { HlsConfig } from '../config'; import type { NetworkComponentAPI } from '../types/component-api'; import type { SourceBufferName } from '../types/buffer'; import type { RationalTimestamp } from '../utils/timescale-conversion'; +import FragmentPreloader from '../loader/fragment-preloader'; type ResolveFragLoaded = (FragLoadedEndData) => void; type RejectFragLoaded = (LoadError) => void; @@ -95,6 +96,7 @@ export default class BaseStreamController protected retryDate: number = 0; protected levels: Array | null = null; protected fragmentLoader: FragmentLoader; + protected fragmentPreloader: FragmentPreloader; protected keyLoader: KeyLoader; protected levelLastLoaded: Level | null = null; protected startFragRequested: boolean = false; @@ -115,6 +117,7 @@ export default class BaseStreamController this.playlistType = playlistType; this.hls = hls; this.fragmentLoader = new FragmentLoader(hls.config); + this.fragmentPreloader = new FragmentPreloader(hls.config, logPrefix); this.keyLoader = keyLoader; this.fragmentTracker = fragmentTracker; this.config = hls.config; @@ -153,7 +156,8 @@ export default class BaseStreamController return; } this.fragmentLoader.abort(); - this.keyLoader.abort(this.playlistType); + this.fragmentPreloader.abort(); + this.keyLoader.abort(); const frag = this.fragCurrent; if (frag?.loader) { frag.abortRequests(); @@ -363,6 +367,14 @@ export default class BaseStreamController this.startTimeOffset = data.startTimeOffset; } + private cachePreloadHint(details: LevelDetails): void { + const data = details.preloadData; + if (!data) { + return; + } + this.fragmentPreloader.cache(data.frag, data.part); + } + protected onHandlerDestroying() { this.stopLoad(); if (this.transmuxer) { @@ -379,6 +391,9 @@ export default class BaseStreamController if (this.fragmentLoader) { this.fragmentLoader.destroy(); } + if (this.fragmentPreloader) { + this.fragmentPreloader.destroy(); + } if (this.keyLoader) { this.keyLoader.destroy(); } @@ -392,6 +407,7 @@ export default class BaseStreamController this.decrypter = this.keyLoader = this.fragmentLoader = + this.fragmentPreloader = this.fragmentTracker = null as any; super.onHandlerDestroyed(); @@ -765,6 +781,10 @@ export default class BaseStreamController if (targetBufferTime > frag.end && details.fragmentHint) { frag = details.fragmentHint; } + const loadedEndOfParts = this.loadedEndOfParts( + partList, + targetBufferTime, + ); const partIndex = this.getNextPart(partList, frag, targetBufferTime); if (partIndex > -1) { const part = partList[partIndex]; @@ -818,10 +838,15 @@ export default class BaseStreamController ); } return result; - } else if ( - !frag.url || - this.loadedEndOfParts(partList, targetBufferTime) - ) { + } else if (!frag.url || loadedEndOfParts) { + if ( + loadedEndOfParts && + this.hls.lowLatencyMode && + details?.live && + details.canBlockReload + ) { + this.cachePreloadHint(details); + } // Fragment hint has no parts return Promise.resolve(null); } @@ -856,23 +881,28 @@ export default class BaseStreamController let result: Promise; if (dataOnProgress && keyLoadingPromise) { result = keyLoadingPromise - .then((keyLoadedData) => { + .then((keyLoadedData: void | KeyLoadedData) => { if (!keyLoadedData || this.fragContextChanged(keyLoadedData?.frag)) { return null; } - return this.fragmentLoader.load(frag, progressCallback); + return this.getCachedRequestOrLoad( + frag, + /*part*/ undefined, + /*dataOnProgress*/ true, + progressCallback, + ); }) .catch((error) => this.handleFragLoadError(error)); } else { // load unencrypted fragment data with progress event, // or handle fragment result after key and fragment are finished loading - result = Promise.all([ - this.fragmentLoader.load( - frag, - dataOnProgress ? progressCallback : undefined, - ), - keyLoadingPromise, - ]) + const loadRequest = this.getCachedRequestOrLoad( + frag, + /*part*/ undefined, + dataOnProgress, + progressCallback, + ); + result = Promise.all([loadRequest, keyLoadingPromise]) .then(([fragLoadedData]) => { if (!dataOnProgress && fragLoadedData && progressCallback) { progressCallback(fragLoadedData); @@ -901,8 +931,7 @@ export default class BaseStreamController const partsLoaded: FragLoadedData[] = []; const initialPartList = level.details?.partList; const loadPart = (part: Part) => { - this.fragmentLoader - .loadPart(frag, part, progressCallback) + this.getCachedRequestOrLoad(frag, part, true, progressCallback) .then((partLoadedData: FragLoadedData) => { partsLoaded[part.index] = partLoadedData; const loadedPart = partLoadedData.part as Part; @@ -927,6 +956,36 @@ export default class BaseStreamController ); } + private getCachedRequestOrLoad( + frag: Fragment, + part: Part | undefined, + dataOnProgress: boolean, + progressCallback?: FragmentLoadProgressCallback, + ): Promise { + const request = this.fragmentPreloader.getCachedRequest(frag, part); + if (request !== undefined) { + return request.then((data) => { + if (progressCallback) { + progressCallback(data); + } + return data; + }); + } + + if (part) { + return this.fragmentLoader.loadPart( + frag, + part, + progressCallback ?? (() => {}), + ); + } + + return this.fragmentLoader.load( + frag, + dataOnProgress ? progressCallback : undefined, + ); + } + private handleFragLoadError(error: LoadError | Error) { if ('data' in error) { const data = error.data; @@ -1332,7 +1391,7 @@ export default class BaseStreamController return nextPart; } - private loadedEndOfParts( + protected loadedEndOfParts( partList: Part[], targetBufferTime: number, ): boolean { @@ -1650,6 +1709,7 @@ export default class BaseStreamController ) { this.state = State.IDLE; } + this.fragmentPreloader.abort(); } protected onFragmentOrKeyLoadError( @@ -1801,6 +1861,7 @@ export default class BaseStreamController if (this.state !== State.STOPPED) { this.state = State.IDLE; } + this.fragmentPreloader.abort(); } protected resetStartWhenNotLoaded(level: Level | null): void { diff --git a/src/controller/error-controller.ts b/src/controller/error-controller.ts index aca730efba2..8131f2e38c2 100644 --- a/src/controller/error-controller.ts +++ b/src/controller/error-controller.ts @@ -276,6 +276,12 @@ export default class ErrorController } private getFragRetryOrSwitchAction(data: ErrorData): IErrorAction { + if (data.frag?.stats.blockingLoad) { + return { + action: NetworkErrorAction.DoNothing, + flags: ErrorActionFlags.None, + }; + } const hls = this.hls; // Share fragment error count accross media options (main, audio, subs) // This allows for level based rendition switching when media option assets fail diff --git a/src/controller/stream-controller.ts b/src/controller/stream-controller.ts index d2f5bc2a700..95e5658fcdc 100644 --- a/src/controller/stream-controller.ts +++ b/src/controller/stream-controller.ts @@ -652,6 +652,15 @@ export default class StreamController let sliding = 0; if (newDetails.live || curLevel.details?.live) { this.checkLiveUpdate(newDetails); + if ( + this.fragmentPreloader.loading && + this.fragmentPreloader.frag?.level !== data.level + ) { + this.fragmentPreloader.abort(); + } else { + // reset the preloader state if we have finished loading, never loaded, or have old data + this.fragmentPreloader.revalidate(data); + } if (newDetails.deltaUpdateFailed) { return; } diff --git a/src/loader/fragment-preloader.ts b/src/loader/fragment-preloader.ts new file mode 100644 index 00000000000..077cb323e47 --- /dev/null +++ b/src/loader/fragment-preloader.ts @@ -0,0 +1,232 @@ +import FragmentLoader from './fragment-loader'; +import { Fragment, Part } from './fragment'; + +import { + FragLoadedData, + LevelLoadedData, + PartsLoadedData, + TrackLoadedData, +} from '../types/events'; +import { HlsConfig } from '../hls'; +import { logger } from '../utils/logger'; +import FetchLoader from '../utils/fetch-loader'; + +export const enum FragRequestState { + IDLE, + LOADING, +} + +type FragPreloadRequest = { + frag: Fragment; + part: Part | undefined; + loadPromise: Promise; +}; + +type FragPreloaderStorage = { + request: FragPreloadRequest | undefined; + state: FragRequestState; +}; + +export default class FragmentPreloader extends FragmentLoader { + private storage: FragPreloaderStorage = { + request: undefined, + state: FragRequestState.IDLE, + }; + protected log: (msg: any) => void; + private fetchLoader?: FetchLoader | undefined; + + constructor(config: HlsConfig, logPrefix: string) { + super(config); + this.log = logger.log.bind(logger, `${logPrefix}>preloader:`); + } + + private getStateString() { + switch (this.storage.state) { + case FragRequestState.IDLE: + return 'IDLE '; + case FragRequestState.LOADING: + return 'LOADING'; + } + } + + public has(frag: Fragment, part: Part | undefined): boolean { + const { request } = this.storage; + + if (request === undefined || request.frag.sn !== frag.sn) { + return false; + } + + const requestPart = request.part; + // frag preload hint only + if (requestPart === undefined && part === undefined) { + return true; + } + + // mismatched part / frag + if (requestPart === undefined || part === undefined) { + return false; + } + + if (requestPart.index === part.index) { + return true; + } + + // Return true if byterange into requested part AND fetch loader exists + return ( + this.fetchLoader !== undefined && + // request is byterange + requestPart.byteRangeStartOffset !== undefined && + requestPart.byteRangeEndOffset !== undefined && + // part is byterange + part.byteRangeStartOffset !== undefined && + part.byteRangeEndOffset !== undefined && + // part byterange contained within request range + requestPart.byteRangeStartOffset <= part.byteRangeStartOffset && + requestPart.byteRangeEndOffset >= part.byteRangeEndOffset + ); + } + + public get loading(): boolean { + const { request, state } = this.storage; + return request !== undefined && state !== FragRequestState.IDLE; + } + + public cache(frag: Fragment, part: Part | undefined): void { + if (this.has(frag, part)) { + return; + } else { + this.abort(); + } + + let loadPromise: Promise; + if (part !== undefined) { + // TODO: Use fetch loader to progressively load open-ended byterange requests + if (part?.byteRangeEndOffset === Number.MAX_SAFE_INTEGER) { + return; + } else { + loadPromise = this.loadPart(frag, part, noop); + } + } else { + loadPromise = this.load(frag, noop); + } + + this.log( + `[${this.getStateString()}] create request for [${frag.type}] ${ + frag.sn + }:${part?.index}`, + ); + + const request = { + frag, + part, + loadPromise, + }; + + this.storage = { + request: request, + state: FragRequestState.LOADING, + }; + } + + public getCachedRequest( + frag: Fragment, + part: Part | undefined, + ): Promise | undefined { + const request = this.storage.request; + + if (!request) { + return undefined; + } + + const cacheHit = this.has(frag, part); + + this.log( + `[${this.getStateString()}] check cache for [${frag.type}] ${ + frag.sn + }:${part?.index ?? ''} / have: ${request.frag.sn}:${request.part?.index ?? ''} hit=${cacheHit}`, + ); + if (cacheHit) { + return request.loadPromise.then((data) => { + mergeFragData(frag, part, data); + this.reset(); + return data; + }); + } else if (this.loading) { + const { frag: preloadFrag, part: preloadPart } = request; + const haveOldSn = preloadFrag.sn < frag.sn; + const haveOldPart = + preloadPart !== undefined && + part !== undefined && + !haveOldSn && + preloadPart.index < part.index; + + if (haveOldSn || haveOldPart) { + this.reset(); + } + } + + return undefined; + } + + public revalidate(data: LevelLoadedData | TrackLoadedData) { + const partList = data.details.partList ?? []; + if (partList.length === 0) { + this.abort(); + return; + } + } + + public get state() { + return this.storage.state; + } + + public get frag() { + if (this.storage.request) { + return this.storage.request.frag; + } + return undefined; + } + + public reset() { + this.storage = { + request: undefined, + state: FragRequestState.IDLE, + }; + } + + abort(): void { + super.abort(); + this.reset(); + } + + destroy(): void { + this.reset(); + super.destroy(); + } +} + +function noop() {} + +function mergeFragData( + frag: Fragment, + part: Part | undefined, + data: FragLoadedData, +) { + const loadedFrag = data.frag; + const loadedPart = data.part; + + if (frag.stats.loaded === 0) { + frag.stats = loadedFrag.stats; + } else { + const fragStats = frag.stats; + const loadStats = loadedFrag.stats; + fragStats.loading.end = loadStats.loading.end; + } + + if (part && loadedPart) { + part.stats = loadedPart.stats; + part.stats.blockingLoad = true; + } + + frag.stats.blockingLoad = true; +} diff --git a/src/loader/fragment.ts b/src/loader/fragment.ts index 007a32a25d6..979552ba36a 100644 --- a/src/loader/fragment.ts +++ b/src/loader/fragment.ts @@ -57,7 +57,13 @@ export class BaseSegment { } else { start = parseInt(params[1]); } - this._byteRange = [start, parseInt(params[0]) + start]; + const bytelength = parseInt(params[0]); + const offsetEnd = + Number.isSafeInteger(bytelength) && bytelength !== Number.MAX_SAFE_INTEGER + ? start + bytelength + : Number.MAX_SAFE_INTEGER; + + this._byteRange = [start, offsetEnd]; } get byteRange(): [number, number] | [] { diff --git a/src/loader/level-details.ts b/src/loader/level-details.ts index ecb855ce468..8d168e2880b 100644 --- a/src/loader/level-details.ts +++ b/src/loader/level-details.ts @@ -44,6 +44,7 @@ export class LevelDetails { public holdBack: number = 0; public partTarget: number = 0; public preloadHint?: AttrList; + public preloadData?: { frag: Fragment; part?: Part }; public renditionReports?: AttrList[]; public tuneInGoal: number = 0; public deltaUpdateFailed?: boolean; @@ -120,7 +121,8 @@ export class LevelDetails { get partEnd(): number { if (this.partList?.length) { - return this.partList[this.partList.length - 1].end; + const lastPart = this.partList[this.partList.length - 1]; + return lastPart.end; } return this.fragmentEnd; } diff --git a/src/loader/load-stats.ts b/src/loader/load-stats.ts index 0bb0e6adb54..937d32f3178 100644 --- a/src/loader/load-stats.ts +++ b/src/loader/load-stats.ts @@ -14,4 +14,5 @@ export class LoadStats implements LoaderStats { loading: HlsProgressivePerformanceTiming = { start: 0, first: 0, end: 0 }; parsing: HlsPerformanceTiming = { start: 0, end: 0 }; buffering: HlsProgressivePerformanceTiming = { start: 0, first: 0, end: 0 }; + blockingLoad = false; } diff --git a/src/loader/m3u8-parser.ts b/src/loader/m3u8-parser.ts index 65043dab63d..feb041e155a 100644 --- a/src/loader/m3u8-parser.ts +++ b/src/loader/m3u8-parser.ts @@ -690,6 +690,78 @@ export default class M3U8Parser { level.endCC = discontinuityCounter; + const preloadHintAttrs = level.preloadHint; + if (preloadHintAttrs) { + let byteRange: string | undefined; + if ( + preloadHintAttrs['BYTERANGE-START'] || + preloadHintAttrs['BYTERANGE-LENGTH'] + ) { + const byteRangeStartOffset = preloadHintAttrs['BYTERANGE-START']; + + if (isFinite(byteRangeStartOffset)) { + let byteRangeLength = preloadHintAttrs['BYTERANGE-LENGTH']; + if (!isFinite(byteRangeLength) || byteRangeLength <= 0) { + byteRangeLength = Number.MAX_SAFE_INTEGER; + } + byteRange = `${byteRangeLength}@${byteRangeStartOffset}`; + } + } + const preloadType = preloadHintAttrs.TYPE; + if (preloadType === 'PART' && level.partList) { + const lastPart = level.partList[level.partList.length - 1]; + const lastPartSn = lastPart.fragment.sn; + const lastPartPublished = lastPartSn === lastFragment?.sn; + const partIndex = lastPartPublished ? 0 : lastPart.index + 1; + + let preloadFrag = lastPart.fragment; + // Need to construct fake fragment for this part since the fragment this part belongs to + // is not published either. + if (lastPartPublished && level.fragmentHint) { + preloadFrag = level.fragmentHint; + } + const partAttrs = new AttrList({ + DURATION: level.partTarget, + URI: preloadHintAttrs.URI, + BYTERANGE: byteRange, + }); + const preloadPart = new Part( + partAttrs, + preloadFrag, + baseurl, + partIndex, + lastPartPublished ? undefined : lastPart, + ); + level.preloadData = { + frag: preloadFrag, + part: preloadPart, + }; + } else if (preloadType === 'MAP') { + const preloadFrag = new Fragment(type, baseurl); + const mapAttrs = new AttrList({ + URI: preloadHintAttrs.URI, + BYTERANGE: byteRange, + }); + // Initial segment tag is before segment duration tag + setInitSegment(preloadFrag, mapAttrs, id, levelkeys); + level.preloadData = { + frag: preloadFrag, + }; + } + } + + /** + * Backfill any missing PDT values + * "If the first EXT-X-PROGRAM-DATE-TIME tag in a Playlist appears after + * one or more Media Segment URIs, the client SHOULD extrapolate + * backward from that tag (using EXTINF durations and/or media + * timestamps) to associate dates with those segments." + * We have already extrapolated forward, but all fragments up to the first instance of PDT do not have their PDTs + * computed. + */ + if (firstPdtIndex > 0) { + backfillProgramDateTimes(fragments, firstPdtIndex); + } return level; } } diff --git a/src/types/loader.ts b/src/types/loader.ts index fc563688b23..843be86358e 100644 --- a/src/types/loader.ts +++ b/src/types/loader.ts @@ -79,6 +79,7 @@ export interface LoaderStats { loading: HlsProgressivePerformanceTiming; parsing: HlsPerformanceTiming; buffering: HlsProgressivePerformanceTiming; + blockingLoad: boolean; } export interface HlsPerformanceTiming { diff --git a/tests/unit/loader/playlist-loader.ts b/tests/unit/loader/playlist-loader.ts index 28c4263a4b3..4370da0702e 100644 --- a/tests/unit/loader/playlist-loader.ts +++ b/tests/unit/loader/playlist-loader.ts @@ -1587,6 +1587,26 @@ fileSequence1151226.ts`, }); }); + it('Creates preload hinted part', function () { + const details = M3U8Parser.parseLevelPlaylist( + playlist, + 'http://dummy.url.com/playlist.m3u8', + 0, + PlaylistLevelType.MAIN, + 0, + null, + ); + expect(details.preloadData).to.be.an('object'); + const frag = details.preloadData?.frag; + const part = details.preloadData?.part; + + expect(frag?.sn).to.equal(1151234); + + expect(part).to.be.an('object'); + expect(part?.fragment.sn).to.equal(1151234); + expect(part?.index).to.equal(0); + }); + it('Parses EXT-X-RENDITION-REPORT', function () { const details = M3U8Parser.parseLevelPlaylist( playlist,