Skip to content

Commit

Permalink
Support unknown codec as video or audio with passing isTypeSupported …
Browse files Browse the repository at this point in the history 
…checks

Add `getMediaDecodingInfo` to public API (undocumented) and fix issues with video and audio only input
Add codecs-helper to fill in AV1 codec string missing parameters required by MediaCapabilities decodingInfo checks
Use multivariant playlist CODECS when mp4 parsed codec is incomplete
  • Loading branch information
@robwalch
robwalch committed Oct 4, 2024
1 parent 8ae2e98 commit ed32b5b
Show file tree
Hide file tree
Showing 10 changed files with 256 additions and 95 deletions.
9 changes: 5 additions & 4 deletions api-extractor/report/hls.js.api.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1920,13 +1920,13 @@ class Hls implements HlsEventEmitter {
constructor(userConfig?: Partial<HlsConfig>);
// (undocumented)
get abrEwmaDefaultEstimate(): number;
get allAudioTracks(): Array<MediaPlaylist>;
get allSubtitleTracks(): Array<MediaPlaylist>;
get allAudioTracks(): MediaPlaylist[];
get allSubtitleTracks(): MediaPlaylist[];
attachMedia(data: HTMLMediaElement | MediaAttachingData): void;
get audioTrack(): number;
// Warning: (ae-setter-with-docs) The doc comment for the property "audioTrack" must appear on the getter, not the setter.
set audioTrack(audioTrackId: number);
get audioTracks(): Array<MediaPlaylist>;
get audioTracks(): MediaPlaylist[];
get autoLevelCapping(): number;
// Warning: (ae-setter-with-docs) The doc comment for the property "autoLevelCapping" must appear on the getter, not the setter.
set autoLevelCapping(newLevel: number);
Expand Down Expand Up @@ -1964,6 +1964,7 @@ class Hls implements HlsEventEmitter {
// Warning: (ae-setter-with-docs) The doc comment for the property "firstLevel" must appear on the getter, not the setter.
set firstLevel(newLevel: number);
get forceStartLoad(): boolean;
getMediaDecodingInfo(level: Level, audioTracks?: MediaPlaylist[]): Promise<MediaDecodingInfo>;
static getMediaSource(): typeof MediaSource | undefined;
get hasEnoughToStart(): boolean;
get interstitialsManager(): InterstitialsManager | null;
Expand Down Expand Up @@ -2043,7 +2044,7 @@ class Hls implements HlsEventEmitter {
get subtitleTrack(): number;
// Warning: (ae-setter-with-docs) The doc comment for the property "subtitleTrack" must appear on the getter, not the setter.
set subtitleTrack(subtitleTrackId: number);
get subtitleTracks(): Array<MediaPlaylist>;
get subtitleTracks(): MediaPlaylist[];
swapAudioCodec(): void;
get targetLatency(): number | null;
set targetLatency(latency: number);
Expand Down
4 changes: 2 additions & 2 deletions src/controller/buffer-controller.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -634,7 +634,7 @@ transfer tracks: ${JSON.stringify(transferredTracks, (key, value) => (key === 'i
if (trackName.slice(0, 5) === 'audio') {
trackCodec = getCodecCompatibleName(trackCodec, this.appendSource);
}
this.log(`switching codec ${sbCodec} to ${codec}`);
this.log(`switching codec ${sbCodec} to ${trackCodec}`);
if (trackCodec !== (track.pendingCodec || track.codec)) {
track.pendingCodec = trackCodec;
}
Expand Down Expand Up @@ -1431,7 +1431,7 @@ transfer tracks: ${JSON.stringify(transferredTracks, (key, value) => (key === 'i
}

private getTrackCodec(track: BaseTrack, trackName: SourceBufferName): string {
const codec = track.codec || track.levelCodec;
const codec = pickMostCompleteCodecName(track.codec, track.levelCodec);
if (codec) {
if (trackName.slice(0, 5) === 'audio') {
return getCodecCompatibleName(codec, this.appendSource);
Expand Down
65 changes: 44 additions & 21 deletions src/controller/level-controller.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ import {
codecsSetSelectionPreferenceValue,
convertAVC1ToAVCOTI,
getCodecCompatibleName,
sampleEntryCodesISO,
videoCodecPreferenceValue,
} from '../utils/codecs';
import BasePlaylistController from './base-playlist-controller';
Expand Down Expand Up @@ -130,23 +131,36 @@ export default class LevelController extends BasePlaylistController {

// only keep levels with supported audio/video codecs
const { width, height, unknownCodecs } = levelParsed;
let unknownUnsupportedCodecCount = unknownCodecs
? unknownCodecs.length
: 0;
if (unknownCodecs) {
// Treat unknown codec as audio or video codec based on passing `isTypeSupported` check
// (allows for playback of any supported codec even if not indexed in utils/codecs)
for (let i = unknownUnsupportedCodecCount; i--; ) {
const unknownCodec = unknownCodecs[i];
if (this.isAudioSupported(unknownCodec)) {
levelParsed.audioCodec = audioCodec = audioCodec
? `${audioCodec},${unknownCodec}`
: unknownCodec;
unknownUnsupportedCodecCount--;
sampleEntryCodesISO.audio[audioCodec.substring(0, 4)] = 2;
} else if (this.isVideoSupported(unknownCodec)) {
levelParsed.videoCodec = videoCodec = videoCodec
? `${videoCodec},${unknownCodec}`
: unknownCodec;
unknownUnsupportedCodecCount--;
sampleEntryCodesISO.video[videoCodec.substring(0, 4)] = 2;
}
}
}
resolutionFound ||= !!(width && height);
videoCodecFound ||= !!videoCodec;
audioCodecFound ||= !!audioCodec;
if (
unknownCodecs?.length ||
(audioCodec &&
!areCodecsMediaSourceSupported(
audioCodec,
'audio',
preferManagedMediaSource,
)) ||
(videoCodec &&
!areCodecsMediaSourceSupported(
videoCodec,
'video',
preferManagedMediaSource,
))
unknownUnsupportedCodecCount ||
(audioCodec && !this.isAudioSupported(audioCodec)) ||
(videoCodec && !this.isVideoSupported(videoCodec))
) {
return;
}
Expand Down Expand Up @@ -193,6 +207,22 @@ export default class LevelController extends BasePlaylistController {
);
}

private isAudioSupported(codec: string): boolean {
return areCodecsMediaSourceSupported(
codec,
'audio',
this.hls.config.preferManagedMediaSource,
);
}

private isVideoSupported(codec: string): boolean {
return areCodecsMediaSourceSupported(
codec,
'video',
this.hls.config.preferManagedMediaSource,
);
}

private filterAndSortMediaOptions(
filteredLevels: Level[],
data: ManifestLoadedData,
Expand Down Expand Up @@ -240,15 +270,8 @@ export default class LevelController extends BasePlaylistController {
}

if (data.audioTracks) {
const { preferManagedMediaSource } = this.hls.config;
audioTracks = data.audioTracks.filter(
(track) =>
!track.audioCodec ||
areCodecsMediaSourceSupported(
track.audioCodec,
'audio',
preferManagedMediaSource,
),
(track) => !track.audioCodec || this.isAudioSupported(track.audioCodec),
);
// Assign ids after filtering as array indices by group-id
assignTrackIdsByGroup(audioTracks);
Expand Down
33 changes: 30 additions & 3 deletions src/controller/stream-controller.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ import TransmuxerInterface from '../demux/transmuxer-interface';
import { ChunkMetadata } from '../types/transmuxer';
import GapController, { MAX_START_GAP_JUMP } from './gap-controller';
import { ErrorDetails } from '../errors';
import { pickMostCompleteCodecName } from '../utils/codecs';
import type { NetworkComponentAPI } from '../types/component-api';
import type Hls from '../hls';
import type { Level } from '../types/level';
Expand Down Expand Up @@ -1367,7 +1368,16 @@ export default class StreamController
// include levelCodec in audio and video tracks
const { audio, video, audiovideo } = tracks;
if (audio) {
let audioCodec = currentLevel.audioCodec;
let audioCodec = pickMostCompleteCodecName(
audio.codec,
currentLevel.audioCodec,
);
// Add level and profile to make up for passthrough-remuxer not being able to parse full codec
// (logger warning "Unhandled audio codec...")
if (audioCodec === 'mp4a') {
audioCodec = 'mp4a.40.5';
}
// Handle `audioCodecSwitch`
const ua = navigator.userAgent.toLowerCase();
if (this.audioCodecSwitch) {
if (audioCodec) {
Expand Down Expand Up @@ -1420,12 +1430,29 @@ export default class StreamController
if (video) {
video.levelCodec = currentLevel.videoCodec;
video.id = 'main';
const parsedVideoCodec = video.codec;
if (parsedVideoCodec?.length === 4) {
// Make up for passthrough-remuxer not being able to parse full codec
// (logger warning "Unhandled video codec...")
switch (parsedVideoCodec) {
case 'hvc1':
case 'hev1':
video.codec = 'hvc1.1.6.L120.90';
break;
case 'av01':
video.codec = 'av01.0.04M.08';
break;
case 'avc1':
video.codec = 'avc1.42e01e';
break;
}
}
this.log(
`Init video buffer, container:${
video.container
}, codecs[level/parsed]=[${currentLevel.videoCodec || ''}/${
video.codec
}]`,
parsedVideoCodec
}${video.codec !== parsedVideoCodec ? ' parsed-corrected=' + video.codec : ''}}]`,
);
delete tracks.audiovideo;
}
Expand Down
28 changes: 24 additions & 4 deletions src/hls.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,11 @@ import type FragmentLoader from './loader/fragment-loader';
import type { LevelDetails } from './loader/level-details';
import type TaskLoop from './task-loop';
import type TransmuxerInterface from './demux/transmuxer-interface';
import { getAudioTracksByGroup } from './utils/rendition-helper';
import {
getMediaDecodingInfoPromise,
MediaDecodingInfo,
} from './utils/mediacapabilities-helper';

/**
* The `Hls` class is the core of the HLS.js library used to instantiate player instances.
Expand Down Expand Up @@ -951,15 +956,15 @@ export default class Hls implements HlsEventEmitter {
/**
* Get the complete list of audio tracks across all media groups
*/
get allAudioTracks(): Array<MediaPlaylist> {
get allAudioTracks(): MediaPlaylist[] {
const audioTrackController = this.audioTrackController;
return audioTrackController ? audioTrackController.allAudioTracks : [];
}

/**
* Get the list of selectable audio tracks
*/
get audioTracks(): Array<MediaPlaylist> {
get audioTracks(): MediaPlaylist[] {
const audioTrackController = this.audioTrackController;
return audioTrackController ? audioTrackController.audioTracks : [];
}
Expand All @@ -985,7 +990,7 @@ export default class Hls implements HlsEventEmitter {
/**
* get the complete list of subtitle tracks across all media groups
*/
get allSubtitleTracks(): Array<MediaPlaylist> {
get allSubtitleTracks(): MediaPlaylist[] {
const subtitleTrackController = this.subtitleTrackController;
return subtitleTrackController
? subtitleTrackController.allSubtitleTracks
Expand All @@ -995,7 +1000,7 @@ export default class Hls implements HlsEventEmitter {
/**
* get alternate subtitle tracks list from playlist
*/
get subtitleTracks(): Array<MediaPlaylist> {
get subtitleTracks(): MediaPlaylist[] {
const subtitleTrackController = this.subtitleTrackController;
return subtitleTrackController
? subtitleTrackController.subtitleTracks
Expand Down Expand Up @@ -1132,6 +1137,21 @@ export default class Hls implements HlsEventEmitter {
get interstitialsManager(): InterstitialsManager | null {
return this.interstitialsController?.interstitialsManager || null;
}

/**
* returns mediaCapabilities.decodingInfo for a variant/rendition
*/
getMediaDecodingInfo(
level: Level,
audioTracks: MediaPlaylist[] = this.allAudioTracks,
): Promise<MediaDecodingInfo> {
const audioTracksByGroup = getAudioTracksByGroup(audioTracks);
return getMediaDecodingInfoPromise(
level,
audioTracksByGroup,
navigator.mediaCapabilities,
);
}
}

export type {
Expand Down
22 changes: 7 additions & 15 deletions src/remux/passthrough-remuxer.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ import type {
} from '../types/demuxer';
import type { DecryptData } from '../loader/level-key';
import type { TypeSupported } from '../utils/codecs';
import type { ILogger } from '../utils/logger';
import { logger, type ILogger } from '../utils/logger';
import type { RationalTimestamp } from '../utils/timescale-conversion';

class PassThroughRemuxer implements Remuxer {
Expand Down Expand Up @@ -86,7 +86,7 @@ class PassThroughRemuxer implements Remuxer {
}
const initData = (this.initData = parseInitSegment(initSegment));

// Get codec from initSegment or fallback to default
// Get codec from initSegment
if (initData.audio) {
audioCodec = getParsedTrackCodec(
initData.audio,
Expand Down Expand Up @@ -298,21 +298,13 @@ function getParsedTrackCodec(
const preferManagedMediaSource = false;
return getCodecCompatibleName(parsedCodec, preferManagedMediaSource);
}
const result = 'mp4a.40.5';
this.logger.info(
`Parsed audio codec "${parsedCodec}" or audio object type not handled. Using "${result}"`,
);
return result;

logger.warn(`Unhandled audio codec "${parsedCodec}" in mp4 MAP`);
return parsedCodec || 'mp4a';
}
// Provide defaults based on codec type
// This allows for some playback of some fmp4 playlists without CODECS defined in manifest
this.logger.warn(`Unhandled video codec "${parsedCodec}"`);
if (parsedCodec === 'hvc1' || parsedCodec === 'hev1') {
return 'hvc1.1.6.L120.90';
}
if (parsedCodec === 'av01') {
return 'av01.0.04M.08';
}
return 'avc1.42e01e';
logger.warn(`Unhandled video codec "${parsedCodec}" in mp4 MAP`);
return parsedCodec || 'avc1';
}
export default PassThroughRemuxer;
42 changes: 36 additions & 6 deletions src/utils/codecs.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ import { getMediaSource } from './mediasource-helper';

// from http://mp4ra.org/codecs.html
// values indicate codec selection preference (lower is higher priority)
const sampleEntryCodesISO = {
export const sampleEntryCodesISO = {
audio: {
a3ds: 1,
'ac-3': 0.95,
Expand Down Expand Up @@ -107,7 +107,7 @@ function isCodecMediaSourceSupported(
}

export function mimeTypeForCodec(codec: string, type: CodecType): string {
return `${type}/mp4;codecs="${codec}"`;
return `${type}/mp4;codecs=${codec}`;
}

export function videoCodecPreferenceValue(
Expand Down Expand Up @@ -198,16 +198,33 @@ export function pickMostCompleteCodecName(
): string | undefined {
// Parsing of mp4a codecs strings in mp4-tools from media is incomplete as of d8c6c7a
// so use level codec is parsed codec is unavailable or incomplete
if (parsedCodec && parsedCodec !== 'mp4a') {
if (
parsedCodec &&
(parsedCodec.length > 4 ||
['ac-3', 'ec-3', 'alac', 'fLaC', 'Opus'].indexOf(parsedCodec) !== -1)
) {
return parsedCodec;
}
return levelCodec ? levelCodec.split(',')[0] : levelCodec;
if (levelCodec) {
const levelCodecs = levelCodec.split(',');
if (levelCodecs.length > 1) {
if (parsedCodec) {
for (let i = levelCodecs.length; i--; ) {
if (levelCodecs[i].substring(0, 4) === parsedCodec.substring(0, 4)) {
return levelCodecs[i];
}
}
}
return levelCodecs[0];
}
}
return levelCodec || parsedCodec;
}

export function convertAVC1ToAVCOTI(codec: string) {
export function convertAVC1ToAVCOTI(videoCodecs: string): string {
// Convert avc1 codec string from RFC-4281 to RFC-6381 for MediaSource.isTypeSupported
// Examples: avc1.66.30 to avc1.42001e and avc1.77.30,avc1.66.30 to avc1.4d001e,avc1.42001e.
const codecs = codec.split(',');
const codecs = videoCodecs.split(',');
for (let i = 0; i < codecs.length; i++) {
const avcdata = codecs[i].split('.');
if (avcdata.length > 2) {
Expand All @@ -222,6 +239,19 @@ export function convertAVC1ToAVCOTI(codec: string) {
return codecs.join(',');
}

export function fillInMissingAV01Params(videoCodec: string): string {
// Used to fill in incomplete AV1 playlist CODECS strings for mediaCapabilities.decodingInfo queries
if (videoCodec.startsWith('av01.')) {
const av1params = videoCodec.split('.');
const placeholders = ['0', '111', '01', '01', '01', '0'];
for (let i = av1params.length; i > 4 && i < 10; i++) {
av1params[i] = placeholders[i - 4];
}
return av1params.join('.');
}
return videoCodec;
}

export interface TypeSupported {
mpeg: boolean;
mp3: boolean;
Expand Down
Loading

0 comments on commit ed32b5b

Please sign in to comment.