diff --git a/build/types/core b/build/types/core index f73e71f276..2ce8a0d5c2 100644 --- a/build/types/core +++ b/build/types/core @@ -6,6 +6,7 @@ +../../lib/abr/simple_abr_manager.js +../../lib/config/auto_show_text.js ++../../lib/config/codec_switching_strategy.js +../../lib/debug/asserts.js +../../lib/debug/log.js diff --git a/demo/config.js b/demo/config.js index c3256dd4d1..2c30432f75 100644 --- a/demo/config.js +++ b/demo/config.js @@ -448,13 +448,24 @@ shakaDemo.Config = class { /** @private */ addMediaSourceSection_() { + const strategyOptions = shaka.config.CodecSwitchingStrategy; + const strategyOptionsNames = { + 'RELOAD': 'reload', + 'SMOOTH': 'smooth', + }; + const docLink = this.resolveExternLink_('.MediaSourceConfiguration'); this.addSection_('Media source', docLink) .addTextInput_('Source buffer extra features', 'mediaSource.sourceBufferExtraFeatures') .addBoolInput_('Force Transmux', 'mediaSource.forceTransmux') .addBoolInput_('Insert fake encryption in init segments when needed ' + - 'by the platform.', 'mediaSource.insertFakeEncryptionInInit'); + 'by the platform.', 'mediaSource.insertFakeEncryptionInInit') + .addSelectInput_( + 'Codec Switching Strategy', + 'mediaSource.codecSwitchingStrategy', + strategyOptions, + strategyOptionsNames); } /** @private */ diff --git a/externs/shaka/player.js b/externs/shaka/player.js index ad5439671f..532bc70f25 100644 --- a/externs/shaka/player.js +++ b/externs/shaka/player.js @@ -1230,6 +1230,7 @@ shaka.extern.StreamingConfiguration; /** * @typedef {{ + * codecSwitchingStrategy: shaka.config.CodecSwitchingStrategy, * sourceBufferExtraFeatures: string, * forceTransmux: boolean, * insertFakeEncryptionInInit: boolean @@ -1238,6 +1239,11 @@ shaka.extern.StreamingConfiguration; * @description * Media source configuration. * + * @property {shaka.config.CodecSwitchingStrategy} codecSwitchingStrategy + * Allow codec switching strategy. SMOOTH loading uses + * SourceBuffer.changeType. RELOAD uses cycling of MediaSource. + * Defaults to SMOOTH if SMOOTH codec switching is supported, RELOAD + * overwise. * @property {string} sourceBufferExtraFeatures * Some platforms may need to pass features when initializing the * sourceBuffer. diff --git a/karma.conf.js b/karma.conf.js index f1cf52091b..6574d2dd09 100644 --- a/karma.conf.js +++ b/karma.conf.js @@ -233,6 +233,7 @@ module.exports = (config) => { {pattern: 'third_party/**/*.js', included: false}, {pattern: 'test/**/*.js', included: false}, {pattern: 'test/test/assets/*', included: false}, + {pattern: 'test/test/assets/dash-multi-codec/*', included: false}, {pattern: 'test/test/assets/3675/*', included: false}, {pattern: 'test/test/assets/dash-aes-128/*', included: false}, {pattern: 'test/test/assets/hls-raw-aac/*', included: false}, diff --git a/lib/config/codec_switching_strategy.js b/lib/config/codec_switching_strategy.js new file mode 100644 index 0000000000..39845663eb --- /dev/null +++ b/lib/config/codec_switching_strategy.js @@ -0,0 +1,24 @@ +/*! @license + * Shaka Player + * Copyright 2016 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +goog.provide('shaka.config.CodecSwitchingStrategy'); + +/** + * @enum {string} + * @export + */ +shaka.config.CodecSwitchingStrategy = { + // Allow codec switching which will always involve reloading the + // `MediaSource`. + RELOAD: 'reload', + // Allow codec switching; determine if `SourceBuffer.changeType` is available + // and attempt to use this first, but fall back to reloading `MediaSource` if + // not available. + // + // Note: Some devices that support `SourceBuffer.changeType` can become stuck + // in a pause state. + SMOOTH: 'smooth', +}; diff --git a/lib/media/adaptation_set.js b/lib/media/adaptation_set.js index af18e738a1..c4c3c5dbf2 100644 --- a/lib/media/adaptation_set.js +++ b/lib/media/adaptation_set.js @@ -43,10 +43,11 @@ shaka.media.AdaptationSet = class { /** * @param {shaka.extern.Variant} variant + * @param {boolean=} compareCodecs * @return {boolean} */ - add(variant) { - if (this.canInclude(variant)) { + add(variant, compareCodecs = true) { + if (this.canInclude(variant, compareCodecs)) { this.variants_.add(variant); return true; } @@ -62,18 +63,21 @@ shaka.media.AdaptationSet = class { * |false|, calling |add| will result in it being ignored. * * @param {shaka.extern.Variant} variant + * @param {boolean=} compareCodecs * @return {boolean} */ - canInclude(variant) { - return shaka.media.AdaptationSet.areAdaptable(this.root_, variant); + canInclude(variant, compareCodecs = true) { + return shaka.media.AdaptationSet + .areAdaptable(this.root_, variant, compareCodecs); } /** * @param {shaka.extern.Variant} a * @param {shaka.extern.Variant} b + * @param {boolean=} compareCodecs * @return {boolean} */ - static areAdaptable(a, b) { + static areAdaptable(a, b, compareCodecs = true) { const AdaptationSet = shaka.media.AdaptationSet; // All variants should have audio or should all not have audio. @@ -95,7 +99,7 @@ shaka.media.AdaptationSet = class { !!a.audio == !!b.audio, 'Both should either have audio or not have audio.'); if (a.audio && b.audio && - !AdaptationSet.areAudiosCompatible_(a.audio, b.audio)) { + !AdaptationSet.areAudiosCompatible_(a.audio, b.audio, compareCodecs)) { return false; } @@ -103,7 +107,7 @@ shaka.media.AdaptationSet = class { !!a.video == !!b.video, 'Both should either have video or not have video.'); if (a.video && b.video && - !AdaptationSet.areVideosCompatible_(a.video, b.video)) { + !AdaptationSet.areVideosCompatible_(a.video, b.video, compareCodecs)) { return false; } @@ -122,10 +126,11 @@ shaka.media.AdaptationSet = class { * * @param {shaka.extern.Stream} a * @param {shaka.extern.Stream} b + * @param {boolean} compareCodecs * @return {boolean} * @private */ - static areAudiosCompatible_(a, b) { + static areAudiosCompatible_(a, b, compareCodecs) { const AdaptationSet = shaka.media.AdaptationSet; // Don't adapt between channel counts, which could annoy the user @@ -139,7 +144,7 @@ shaka.media.AdaptationSet = class { } // We can only adapt between base-codecs. - if (!AdaptationSet.canTransitionBetween_(a, b)) { + if (compareCodecs && !AdaptationSet.canTransitionBetween_(a, b)) { return false; } @@ -161,14 +166,15 @@ shaka.media.AdaptationSet = class { * * @param {shaka.extern.Stream} a * @param {shaka.extern.Stream} b + * @param {boolean} compareCodecs * @return {boolean} * @private */ - static areVideosCompatible_(a, b) { + static areVideosCompatible_(a, b, compareCodecs) { const AdaptationSet = shaka.media.AdaptationSet; // We can only adapt between base-codecs. - if (!AdaptationSet.canTransitionBetween_(a, b)) { + if (compareCodecs && !AdaptationSet.canTransitionBetween_(a, b)) { return false; } diff --git a/lib/media/adaptation_set_criteria.js b/lib/media/adaptation_set_criteria.js index 9abed02870..29d83bc03b 100644 --- a/lib/media/adaptation_set_criteria.js +++ b/lib/media/adaptation_set_criteria.js @@ -8,8 +8,10 @@ goog.provide('shaka.media.AdaptationSetCriteria'); goog.provide('shaka.media.ExampleBasedCriteria'); goog.provide('shaka.media.PreferenceBasedCriteria'); +goog.require('shaka.config.CodecSwitchingStrategy'); goog.require('shaka.log'); goog.require('shaka.media.AdaptationSet'); +goog.require('shaka.media.Capabilities'); goog.require('shaka.util.LanguageUtils'); goog.require('shaka.util.StreamUtils'); @@ -40,10 +42,14 @@ shaka.media.AdaptationSetCriteria = class { shaka.media.ExampleBasedCriteria = class { /** * @param {shaka.extern.Variant} example + * @param {shaka.config.CodecSwitchingStrategy=} codecSwitchingStrategy */ - constructor(example) { + constructor(example, + codecSwitchingStrategy = shaka.config.CodecSwitchingStrategy.RELOAD) { /** @private {shaka.extern.Variant} */ this.example_ = example; + /** @private {shaka.config.CodecSwitchingStrategy} */ + this.codecSwitchingStrategy_ = codecSwitchingStrategy; // We can't know if role and label are really important, so we don't use // role and label for this. @@ -56,15 +62,20 @@ shaka.media.ExampleBasedCriteria = class { /** @private {!shaka.media.AdaptationSetCriteria} */ this.fallback_ = new shaka.media.PreferenceBasedCriteria( - example.language, role, channelCount, hdrLevel, label); + example.language, role, channelCount, hdrLevel, label, + codecSwitchingStrategy); } /** @override */ create(variants) { + const supportsSmoothCodecTransitions = this.codecSwitchingStrategy_ == + shaka.config.CodecSwitchingStrategy.SMOOTH && + shaka.media.Capabilities.isChangeTypeSupported(); // We can't assume that the example is in |variants| because it could // actually be from another period. const shortList = variants.filter((variant) => { - return shaka.media.AdaptationSet.areAdaptable(this.example_, variant); + return shaka.media.AdaptationSet.areAdaptable(this.example_, variant, + !supportsSmoothCodecTransitions); }); if (shortList.length) { @@ -90,8 +101,10 @@ shaka.media.PreferenceBasedCriteria = class { * @param {number} channelCount * @param {string} hdrLevel * @param {string=} label + * @param {shaka.config.CodecSwitchingStrategy=} codecSwitchingStrategy */ - constructor(language, role, channelCount, hdrLevel, label = '') { + constructor(language, role, channelCount, hdrLevel, label = '', + codecSwitchingStrategy = shaka.config.CodecSwitchingStrategy.RELOAD) { /** @private {string} */ this.language_ = language; /** @private {string} */ @@ -102,6 +115,8 @@ shaka.media.PreferenceBasedCriteria = class { this.hdrLevel_ = hdrLevel; /** @private {string} */ this.label_ = label; + /** @private {shaka.config.CodecSwitchingStrategy} */ + this.codecSwitchingStrategy_ = codecSwitchingStrategy; } /** @override */ @@ -162,12 +177,15 @@ shaka.media.PreferenceBasedCriteria = class { } } + const supportsSmoothCodecTransitions = this.codecSwitchingStrategy_ == + shaka.config.CodecSwitchingStrategy.SMOOTH && + shaka.media.Capabilities.isChangeTypeSupported(); + // Make sure we only return a valid adaptation set. const set = new shaka.media.AdaptationSet(current[0]); for (const variant of current) { - if (set.canInclude(variant)) { - set.add(variant); - } + // `add` checks combatability by calling `canInclude` internally. + set.add(variant, !supportsSmoothCodecTransitions); } return set; diff --git a/lib/media/media_source_engine.js b/lib/media/media_source_engine.js index f2fabda61e..ab61a64d28 100644 --- a/lib/media/media_source_engine.js +++ b/lib/media/media_source_engine.js @@ -8,6 +8,7 @@ goog.provide('shaka.media.MediaSourceEngine'); goog.require('goog.asserts'); goog.require('shaka.log'); +goog.require('shaka.config.CodecSwitchingStrategy'); goog.require('shaka.media.Capabilities'); goog.require('shaka.media.ContentWorkarounds'); goog.require('shaka.media.ClosedCaptionParser'); @@ -115,6 +116,9 @@ shaka.media.MediaSourceEngine = class { /** @private {string} */ this.url_ = ''; + /** @private {boolean} */ + this.playbackHasBegun_ = false; + /** @private {MediaSource} */ this.mediaSource_ = this.createMediaSource(this.mediaSourceOpen_); @@ -159,6 +163,11 @@ shaka.media.MediaSourceEngine = class { this.eventManager_.listenOnce( mediaSource, 'sourceopen', () => this.onSourceOpen_(p)); + // Correctly set when playback has begun. + this.eventManager_.listenOnce(this.video_, 'playing', () => { + this.playbackHasBegun_ = true; + }); + // Store the object URL for releasing it later. this.url_ = shaka.media.MediaSourceEngine.createObjectURL(mediaSource); @@ -534,7 +543,8 @@ shaka.media.MediaSourceEngine = class { * @return {?number} The timestamp in seconds, or null if nothing is buffered. */ bufferStart(contentType) { - if (this.reloadingMediaSource_) { + if (this.reloadingMediaSource_ || + !Object.keys(this.sourceBuffers_).length) { return null; } const ContentType = shaka.util.ManifestParserUtils.ContentType; @@ -790,6 +800,11 @@ shaka.media.MediaSourceEngine = class { return; } + if (!this.sourceBuffers_[contentType]) { + shaka.log.warning('Attempted to restore a non-existent source buffer'); + return; + } + let timestampOffset = this.sourceBuffers_[contentType].timestampOffset; let mimeType = this.sourceBufferTypes_[contentType]; @@ -1641,6 +1656,20 @@ shaka.media.MediaSourceEngine = class { this.reloadingMediaSource_ = true; this.needSplitMuxedContent_ = false; const currentTime = this.video_.currentTime; + + // When codec switching if the user is currently paused we don't want + // to trigger a play when switching codec. + // Playing can also end up in a paused state after a codec switch + // so we need to remember the current states. + const previousAutoPlayState = this.video_.autoplay; + const previousPausedState = this.video_.paused; + if (this.playbackHasBegun_) { + // Only set autoplay to false if the video playback has already begun. + // When a codec switch happens before playback has begun this can cause + // autoplay not to work as expected. + this.video_.autoplay = false; + } + try { this.eventManager_.removeAll(); @@ -1674,7 +1703,6 @@ shaka.media.MediaSourceEngine = class { } await Promise.all(cleanup); this.transmuxers_ = {}; - this.queues_ = {}; this.sourceBuffers_ = {}; const previousDuration = this.mediaSource_.duration; @@ -1713,6 +1741,17 @@ shaka.media.MediaSourceEngine = class { await sourceBufferAdded; } finally { this.reloadingMediaSource_ = false; + + this.destroyer_.ensureNotDestroyed(); + + this.eventManager_.listenOnce(this.video_, 'canplay', () => { + this.destroyer_.ensureNotDestroyed(); + + this.video_.autoplay = previousAutoPlayState; + if (!previousPausedState) { + this.video_.play(); + } + }); } } @@ -1778,8 +1817,10 @@ shaka.media.MediaSourceEngine = class { return false; } - if (shaka.media.Capabilities.isChangeTypeSupported() && - !this.needSplitMuxedContent_) { + if (this.config_.codecSwitchingStrategy === + shaka.config.CodecSwitchingStrategy.SMOOTH && + shaka.media.Capabilities.isChangeTypeSupported() && + !this.needSplitMuxedContent_) { await this.changeType(contentType, newMimeType, transmuxer); } else { if (transmuxer) { @@ -1790,6 +1831,70 @@ shaka.media.MediaSourceEngine = class { return true; } + /** + * Returns true if it's necessary codec switch to load the new stream. + * + * @param {shaka.util.ManifestParserUtils.ContentType} contentType + * @param {shaka.extern.Stream} stream + * @return {boolean} + * @private + */ + isCodecSwitchNecessary_(contentType, stream) { + const MimeUtils = shaka.util.MimeUtils; + const currentCodec = MimeUtils.getCodecBase( + MimeUtils.getCodecs(this.sourceBufferTypes_[contentType])); + const currentBasicType = MimeUtils.getBasicType( + this.sourceBufferTypes_[contentType]); + + let newMimeType = shaka.util.MimeUtils.getFullType( + stream.mimeType, stream.codecs); + let needTransmux = this.config_.forceTransmux; + if (!shaka.media.Capabilities.isTypeSupported(newMimeType) || + (!this.sequenceMode_ && + shaka.util.MimeUtils.RAW_FORMATS.includes(newMimeType))) { + needTransmux = true; + } + const newMimeTypeWithAllCodecs = + shaka.util.MimeUtils.getFullTypeWithAllCodecs( + stream.mimeType, stream.codecs); + const TransmuxerEngine = shaka.transmuxer.TransmuxerEngine; + if (needTransmux) { + const transmuxerPlugin = + TransmuxerEngine.findTransmuxer(newMimeTypeWithAllCodecs); + if (transmuxerPlugin) { + const transmuxer = transmuxerPlugin(); + newMimeType = + transmuxer.convertCodecs(contentType, newMimeTypeWithAllCodecs); + transmuxer.destroy(); + } + } + + const newCodec = MimeUtils.getCodecBase( + MimeUtils.getCodecs(newMimeType)); + const newBasicType = MimeUtils.getBasicType(newMimeType); + + return currentCodec !== newCodec || currentBasicType !== newBasicType; + } + + /** + * Returns true if it's necessary reset the media source to load the + * new stream. + * + * @param {shaka.util.ManifestParserUtils.ContentType} contentType + * @param {shaka.extern.Stream} stream + * @return {boolean} + */ + isResetMediaSourceNecessary(contentType, stream) { + if (!this.isCodecSwitchNecessary_(contentType, stream)) { + return false; + } + + return this.config_.codecSwitchingStrategy !== + shaka.config.CodecSwitchingStrategy.SMOOTH || + !shaka.media.Capabilities.isChangeTypeSupported() || + this.needSplitMuxedContent_; + } + /** * Update LCEVC Decoder object when ready for LCEVC Decode. * @param {?shaka.lcevc.Dec} lcevcDec diff --git a/lib/media/streaming_engine.js b/lib/media/streaming_engine.js index f51d336ba9..691912d155 100644 --- a/lib/media/streaming_engine.js +++ b/lib/media/streaming_engine.js @@ -1684,22 +1684,21 @@ shaka.media.StreamingEngine = class { shaka.log.v1(logPrefix, 'setting append window end to ' + appendWindowEnd); - if ((mediaState.lastCodecs && codecs != mediaState.lastCodecs) || - (mediaState.lastMimeType && mimeType != mediaState.lastMimeType)) { + const isResetMediaSourceNecessary = + mediaState.lastCodecs && mediaState.lastMimeType && + this.playerInterface_.mediaSourceEngine.isResetMediaSourceNecessary( + mediaState.type, mediaState.stream); + if (isResetMediaSourceNecessary) { + let otherState = null; if (mediaState.type === ContentType.VIDEO) { - /** @type {shaka.media.StreamingEngine.MediaState_} */ - const audioState = this.mediaStates_.get(ContentType.AUDIO); - if (audioState) { - audioState.lastInitSegmentReference = null; - this.abortOperations_(audioState); - } + otherState = this.mediaStates_.get(ContentType.AUDIO); } else if (mediaState.type === ContentType.AUDIO) { - /** @type {shaka.media.StreamingEngine.MediaState_} */ - const videoState = this.mediaStates_.get(ContentType.VIDEO); - if (videoState) { - videoState.lastInitSegmentReference = null; - this.abortOperations_(videoState); - } + otherState = this.mediaStates_.get(ContentType.VIDEO); + } + if (otherState) { + otherState.lastInitSegmentReference = null; + this.forceClearBuffer_(otherState); + this.abortOperations_(otherState).catch(() => {}); } } @@ -1729,6 +1728,19 @@ shaka.media.StreamingEngine = class { mediaState.type, timestampOffset, appendWindowStart, appendWindowEnd, ignoreTimestampOffset, mediaState.stream, streamsByType); + if (isResetMediaSourceNecessary) { + let otherState = null; + if (mediaState.type === ContentType.VIDEO) { + otherState = this.mediaStates_.get(ContentType.AUDIO); + } else if (mediaState.type === ContentType.AUDIO) { + otherState = this.mediaStates_.get(ContentType.VIDEO); + } + if (otherState) { + otherState.waitingToClearBuffer = false; + otherState.performingUpdate = false; + this.forceClearBuffer_(otherState); + } + } } catch (error) { mediaState.lastAppendWindowStart = null; mediaState.lastAppendWindowEnd = null; @@ -1738,7 +1750,10 @@ shaka.media.StreamingEngine = class { throw error; } }; - operations.push(setProperties()); + // Dispatching init asynchronously causes the sourceBuffers in + // the MediaSourceEngine to become detached do to race conditions + // with mediaSource and sourceBuffers being created simultaneously. + await setProperties(); } if (!shaka.media.InitSegmentReference.equal( diff --git a/lib/player.js b/lib/player.js index 6df2288d90..67b718d5e7 100644 --- a/lib/player.js +++ b/lib/player.js @@ -623,7 +623,8 @@ shaka.Player = class extends shaka.util.FakeEventTarget { this.config_.preferredAudioLanguage, this.config_.preferredVariantRole, this.config_.preferredAudioChannelCount, - this.config_.preferredVideoHdrLevel); + this.config_.preferredVideoHdrLevel, + this.config_.mediaSource.codecSwitchingStrategy); /** @private {string} */ this.currentTextLanguage_ = this.config_.preferredTextLanguage; @@ -2159,7 +2160,8 @@ shaka.Player = class extends shaka.util.FakeEventTarget { this.config_.preferredVariantRole, this.config_.preferredAudioChannelCount, this.config_.preferredVideoHdrLevel, - this.config_.preferredAudioLabel); + this.config_.preferredAudioLabel, + this.config_.mediaSource.codecSwitchingStrategy); this.currentTextLanguage_ = this.config_.preferredTextLanguage; this.currentTextRole_ = this.config_.preferredTextRole; @@ -2307,8 +2309,6 @@ shaka.Player = class extends shaka.util.FakeEventTarget { this.onAbrStatusChanged_(); } - // Re-filter the manifest after streams have been chosen. - this.filterManifestByCurrentVariant_(); // Dispatch a 'trackschanged' event now that all initial filtering is done. this.onTracksChanged_(); @@ -4256,7 +4256,8 @@ shaka.Player = class extends shaka.util.FakeEventTarget { if (this.manifest_ && this.playhead_) { this.currentAdaptationSetCriteria_ = new shaka.media.PreferenceBasedCriteria(language, role || '', - channelsCount, /* hdrLevel= */ '', /* label= */ ''); + channelsCount, /* hdrLevel= */ '', /* label= */ '', + this.config_.mediaSource.codecSwitchingStrategy); const diff = (a, b) => { if (!a.video && !b.video) { @@ -4373,7 +4374,8 @@ shaka.Player = class extends shaka.util.FakeEventTarget { // label have the same language. this.currentAdaptationSetCriteria_ = new shaka.media.PreferenceBasedCriteria( - firstVariantWithLabel.language, '', 0, '', label); + firstVariantWithLabel.language, '', 0, '', label, + this.config_.mediaSource.codecSwitchingStrategy); this.chooseVariantAndSwitch_(clearBuffer, safeMargin); } else if (this.video_ && this.video_.audioTracks) { @@ -5524,7 +5526,8 @@ shaka.Player = class extends shaka.util.FakeEventTarget { this.streamingEngine_.getCurrentVariant() : null; await shaka.util.StreamUtils.filterManifest( - this.drmEngine_, currentVariant, manifest); + this.drmEngine_, currentVariant, manifest, + this.config_.mediaSource.codecSwitchingStrategy); this.checkPlayableVariants_(manifest); } @@ -5581,21 +5584,6 @@ shaka.Player = class extends shaka.util.FakeEventTarget { } } - /** - * @private - */ - filterManifestByCurrentVariant_() { - goog.asserts.assert(this.manifest_, 'Manifest should be valid'); - goog.asserts.assert(this.streamingEngine_, - 'StreamingEngine should be valid'); - - const currentVariant = this.streamingEngine_ ? - this.streamingEngine_.getCurrentVariant() : null; - shaka.util.StreamUtils.filterManifestByCurrentVariant(currentVariant, - this.manifest_); - this.checkPlayableVariants_(this.manifest_); - } - /** * @param {shaka.extern.Variant} initialVariant * @param {number} time diff --git a/lib/util/platform.js b/lib/util/platform.js index b802099538..92f1ce7eec 100644 --- a/lib/util/platform.js +++ b/lib/util/platform.js @@ -154,6 +154,26 @@ shaka.util.Platform = class { 'Chrome/38.0.2125.122 Safari/537.36'); } + /** + * Check if the current platform is a WebOS 4. + * + * @return {boolean} + */ + static isWebOS4() { + // See: http://webostv.developer.lge.com/discover/specifications/web-engine/ + return !!navigator.userAgent.match(/webOS\/4/i); + } + + /** + * Check if the current platform is a WebOS 4. + * + * @return {boolean} + */ + static isWebOS5() { + // See: http://webostv.developer.lge.com/discover/specifications/web-engine/ + return !!navigator.userAgent.match(/webOS\/5/i); + } + /** * Check if the current platform is a Google Chromecast. * @@ -460,6 +480,26 @@ shaka.util.Platform = class { return true; } + /** + * Returns if codec switching SMOOTH is known reliable device support. + * + * Some devices are known not to support `MediaSource.changeType` + * well. These devices should use the reload strategy. If a device + * reports that it supports `changeType` but support it reliabley + * it should be added to this list. + * + * @return {boolean} + */ + static supportsSmoothCodecSwitching() { + const Platform = shaka.util.Platform; + if (Platform.isTizen2() || Platform.isTizen3() || + Platform.isTizen4() || Platform.isWebOS3() || + Platform.isWebOS4() || Platform.isWebOS5()) { + return false; + } + return true; + } + /** * Returns true if MediaKeys is polyfilled * diff --git a/lib/util/player_configuration.js b/lib/util/player_configuration.js index bd4ee307d8..f808437df2 100644 --- a/lib/util/player_configuration.js +++ b/lib/util/player_configuration.js @@ -9,6 +9,7 @@ goog.provide('shaka.util.PlayerConfiguration'); goog.require('goog.asserts'); goog.require('shaka.abr.SimpleAbrManager'); goog.require('shaka.config.AutoShowText'); +goog.require('shaka.config.CodecSwitchingStrategy'); goog.require('shaka.log'); goog.require('shaka.net.NetworkingEngine'); goog.require('shaka.util.ConfigUtils'); @@ -308,7 +309,13 @@ shaka.util.PlayerConfiguration = class { drawLogo: false, }; + let codecSwitchingStrategy = shaka.config.CodecSwitchingStrategy.RELOAD; + if (shaka.util.Platform.supportsSmoothCodecSwitching()) { + codecSwitchingStrategy = shaka.config.CodecSwitchingStrategy.SMOOTH; + } + const mediaSource = { + codecSwitchingStrategy: codecSwitchingStrategy, sourceBufferExtraFeatures: '', forceTransmux: false, insertFakeEncryptionInInit: true, diff --git a/lib/util/stream_utils.js b/lib/util/stream_utils.js index 6578fe7778..3d1aae036c 100644 --- a/lib/util/stream_utils.js +++ b/lib/util/stream_utils.js @@ -7,6 +7,7 @@ goog.provide('shaka.util.StreamUtils'); goog.require('goog.asserts'); +goog.require('shaka.config.CodecSwitchingStrategy'); goog.require('shaka.log'); goog.require('shaka.media.Capabilities'); goog.require('shaka.text.TextEngine'); @@ -38,103 +39,148 @@ shaka.util.StreamUtils = class { preferredAudioCodecs, preferredAudioChannelCount, preferredDecodingAttributes) { const StreamUtils = shaka.util.StreamUtils; + const MimeUtils = shaka.util.MimeUtils; let variants = manifest.variants; - // To start, choose the codecs based on configured preferences if available. if (preferredVideoCodecs.length || preferredAudioCodecs.length) { variants = StreamUtils.choosePreferredCodecs(variants, preferredVideoCodecs, preferredAudioCodecs); } - // Consider a subset of variants based on audio channel - // preferences. - // For some content (#1013), surround-sound variants will use a different - // codec than stereo variants, so it is important to choose codecs **after** - // considering the audio channel config. - variants = StreamUtils.filterVariantsByAudioChannelCount( - variants, preferredAudioChannelCount); - - // Now organize variants into buckets by codecs. - /** @type {!shaka.util.MultiMap.} */ - let variantsByCodecs = StreamUtils.getVariantsByCodecs_(variants); - variantsByCodecs = StreamUtils.filterVariantsByDensity_(variantsByCodecs); + if (preferredDecodingAttributes.length) { + // group variants by resolution and choose preferred variants only + /** @type {!shaka.util.MultiMap.} */ + const variantsByResolutionMap = new shaka.util.MultiMap(); + for (const variant of variants) { + variantsByResolutionMap + .push(String(variant.video.width || 0), variant); + } + const bestVariants = []; + variantsByResolutionMap.forEach((width, variantsByResolution) => { + let highestMatch = 0; + let matchingVariants = []; + for (const variant of variantsByResolution) { + const matchCount = preferredDecodingAttributes.filter( + (attribute) => variant.decodingInfos[0][attribute], + ).length; + if (matchCount > highestMatch) { + highestMatch = matchCount; + matchingVariants = [variant]; + } else if (matchCount == highestMatch) { + matchingVariants.push(variant); + } + } + bestVariants.push(...matchingVariants); + }); + variants = bestVariants; + } - const bestCodecs = StreamUtils.chooseCodecsByDecodingAttributes_( - variantsByCodecs, preferredDecodingAttributes); + if (preferredAudioChannelCount) { + variants = StreamUtils.filterVariantsByAudioChannelCount( + variants, preferredAudioChannelCount); + } - // Filter out any variants that don't match, forcing AbrManager to choose - // from a single video codec and a single audio codec possible. - manifest.variants = manifest.variants.filter((variant) => { - const codecs = StreamUtils.getVariantCodecs_(variant); - if (codecs == bestCodecs) { - return true; + const audioStreamsSet = new Set(); + const videoStreamsSet = new Set(); + for (const variant of variants) { + if (variant.audio) { + audioStreamsSet.add(variant.audio); + } + if (variant.video) { + videoStreamsSet.add(variant.video); } + } - shaka.log.debug('Dropping Variant (better codec available)', variant); - return false; + const audioStreams = Array.from(audioStreamsSet).sort((v1, v2) => { + return v1.bandwidth - v2.bandwidth; }); - } - - /** - * Get variants by codecs. - * - * @param {!Array} variants - * @return {!shaka.util.MultiMap.} - * @private - */ - static getVariantsByCodecs_(variants) { - const variantsByCodecs = new shaka.util.MultiMap(); - for (const variant of variants) { - const variantCodecs = shaka.util.StreamUtils.getVariantCodecs_(variant); - variantsByCodecs.push(variantCodecs, variant); + const audioStreamsMap = new Map(); + const getAudioId = (stream) => { + return stream.language + (stream.channelsCount || 0) + + stream.roles.join(',') + stream.label; + }; + for (const stream of audioStreams) { + const id = getAudioId(stream); + if (!audioStreamsMap.has(id)) { + audioStreamsMap.set(id, stream); + } else { + const includedStream = audioStreamsMap.get(id); + if (includedStream.bandwidth > stream.bandwidth) { + audioStreamsMap.set(id, stream); + } + } } + const validAudioStreams = [...audioStreamsMap.values()]; - return variantsByCodecs; - } + const videoStreams = Array.from(videoStreamsSet) + .sort((v1, v2) => { + if (!v1.bandwidth || !v2.bandwidth) { + return v1.width - v2.width; + } + return v1.bandwidth - v2.bandwidth; + }); - /** - * Filters variants by density. - * Get variants by codecs map with the max density where all codecs are - * present. - * - * @param {!shaka.util.MultiMap.} variantsByCodecs - * @return {!shaka.util.MultiMap.} - * @private - */ - static filterVariantsByDensity_(variantsByCodecs) { - let maxDensity = 0; - const codecGroupsByDensity = new Map(); - const countCodecs = variantsByCodecs.size(); - - variantsByCodecs.forEach((codecs, variants) => { - for (const variant of variants) { - const video = variant.video; - if (!video || !video.width || !video.height) { - continue; - } + const isChangeTypeSupported = + shaka.media.Capabilities.isChangeTypeSupported(); - const density = video.width * video.height * (video.frameRate || 1); - if (!codecGroupsByDensity.has(density)) { - codecGroupsByDensity.set(density, new shaka.util.MultiMap()); + const validVideoStreams = []; + for (const stream of videoStreams) { + if (!validVideoStreams.length) { + validVideoStreams.push(stream); + } else { + const previousStream = validVideoStreams[validVideoStreams.length - 1]; + if (!isChangeTypeSupported) { + const previousCodec = + MimeUtils.getNormalizedCodec(previousStream.codecs); + const currentCodec = + MimeUtils.getNormalizedCodec(stream.codecs); + if (previousCodec !== currentCodec) { + continue; + } + } + if (stream.width > previousStream.width || + stream.height > previousStream.height) { + validVideoStreams.push(stream); + } else if (stream.width == previousStream.width && + stream.height == previousStream.height) { + const previousCodec = + MimeUtils.getNormalizedCodec(previousStream.codecs); + const currentCodec = + MimeUtils.getNormalizedCodec(stream.codecs); + if (previousCodec == currentCodec) { + if (stream.bandwidth < previousStream.bandwidth) { + validVideoStreams.push(stream); + } + } } + } + } - /** @type {!shaka.util.MultiMap.} */ - const group = codecGroupsByDensity.get(density); - group.push(codecs, variant); - - // We want to look at the groups in which all codecs are present. - // Take the max density from those groups where all codecs are present. - // Later, we will compare bandwidth numbers only within this group. - // Effectively, only the bandwidth differences in the highest-res and - // highest-framerate content will matter in choosing a codec. - if (group.size() === countCodecs) { - maxDensity = Math.max(maxDensity, density); + const validAudioIds = + validAudioStreams.map((audioStream) => audioStream.id); + const validVideoIds = + validVideoStreams.map((videoStream) => videoStream.id); + + // Filter out any variants that don't match, forcing AbrManager to choose + // from a single video codec and a single audio codec possible. + manifest.variants = manifest.variants.filter((variant) => { + const audio = variant.audio; + const video = variant.video; + if (audio) { + if (!validAudioIds.includes(audio.id)) { + shaka.log.debug('Dropping Variant (better codec available)', variant); + return false; + } + } + if (video) { + if (!validVideoIds.includes(video.id)) { + shaka.log.debug('Dropping Variant (better codec available)', variant); + return false; } } + return true; }); - - return maxDensity ? codecGroupsByDensity.get(maxDensity) : variantsByCodecs; } /** @@ -170,135 +216,6 @@ shaka.util.StreamUtils = class { return subset; } - /** - * Choose the codecs by configured preferred decoding attributes. - * - * @param {!shaka.util.MultiMap.} variantsByCodecs - * @param {!Array.} attributes - * @return {string} - * @private - */ - static chooseCodecsByDecodingAttributes_(variantsByCodecs, attributes) { - const StreamUtils = shaka.util.StreamUtils; - - for (const attribute of attributes) { - if (attribute == StreamUtils.DecodingAttributes.SMOOTH || - attribute == StreamUtils.DecodingAttributes.POWER) { - variantsByCodecs = StreamUtils.chooseCodecsByMediaCapabilitiesInfo_( - variantsByCodecs, attribute); - // If we only have one smooth or powerEfficient codecs, choose it as the - // best codecs. - if (variantsByCodecs.size() == 1) { - return variantsByCodecs.keys()[0]; - } - } else if (attribute == StreamUtils.DecodingAttributes.BANDWIDTH) { - return StreamUtils.findCodecsByLowestBandwidth_(variantsByCodecs); - } - } - // If there's no configured decoding preferences, or we have multiple codecs - // that meets the configured decoding preferences, choose the one with - // the lowest bandwidth. - return StreamUtils.findCodecsByLowestBandwidth_(variantsByCodecs); - } - - /** - * Choose the best codecs by configured preferred MediaCapabilitiesInfo - * attributes. - * - * @param {!shaka.util.MultiMap.} variantsByCodecs - * @param {string} attribute - * @return {!shaka.util.MultiMap.} - * @private - */ - static chooseCodecsByMediaCapabilitiesInfo_(variantsByCodecs, attribute) { - let highestScore = 0; - const bestVariantsByCodecs = new shaka.util.MultiMap(); - variantsByCodecs.forEach((codecs, variants) => { - let sum = 0; - let num = 0; - - for (const variant of variants) { - if (variant.decodingInfos.length) { - sum += variant.decodingInfos[0][attribute] ? 1 : 0; - num++; - } - } - - const averageScore = sum / num; - shaka.log.debug('codecs', codecs, 'avg', attribute, averageScore); - - if (averageScore > highestScore) { - bestVariantsByCodecs.clear(); - bestVariantsByCodecs.push(codecs, variants); - highestScore = averageScore; - } else if (averageScore == highestScore) { - bestVariantsByCodecs.push(codecs, variants); - } - }); - return bestVariantsByCodecs; - } - - /** - * Find the lowest-bandwidth (best) codecs. - * Compute the average bandwidth for each group of variants. - * - * @param {!shaka.util.MultiMap.} variantsByCodecs - * @return {string} - * @private - */ - static findCodecsByLowestBandwidth_(variantsByCodecs) { - let bestCodecs = ''; - let lowestAverageBandwidth = Infinity; - - variantsByCodecs.forEach((codecs, variants) => { - let sum = 0; - let num = 0; - for (const variant of variants) { - sum += variant.bandwidth || 0; - ++num; - } - - const averageBandwidth = sum / num; - shaka.log.debug('codecs', codecs, 'avg bandwidth', averageBandwidth); - - if (averageBandwidth < lowestAverageBandwidth) { - bestCodecs = codecs; - lowestAverageBandwidth = averageBandwidth; - } - }); - - goog.asserts.assert(bestCodecs !== '', 'Should have chosen codecs!'); - goog.asserts.assert(!isNaN(lowestAverageBandwidth), - 'Bandwidth should be a number!'); - - return bestCodecs; - } - - /** - * Get a string representing all codecs used in a variant. - * - * @param {!shaka.extern.Variant} variant - * @return {string} - * @private - */ - static getVariantCodecs_(variant) { - // Only consider the base of the codec string. For example, these should - // both be considered the same codec: avc1.42c01e, avc1.4d401f - let baseVideoCodec = ''; - if (variant.video) { - baseVideoCodec = - shaka.util.MimeUtils.getNormalizedCodec(variant.video.codecs); - } - - let baseAudioCodec = ''; - if (variant.audio) { - baseAudioCodec = - shaka.util.MimeUtils.getNormalizedCodec(variant.audio.codecs); - } - - return baseVideoCodec + '-' + baseAudioCodec; - } - /** * Filter the variants in |manifest| to only include the variants that meet * the given restrictions. @@ -404,12 +321,12 @@ shaka.util.StreamUtils = class { * @param {shaka.media.DrmEngine} drmEngine * @param {?shaka.extern.Variant} currentVariant * @param {shaka.extern.Manifest} manifest + * @param {shaka.config.CodecSwitchingStrategy=} codecSwitchingStrategy */ - static async filterManifest(drmEngine, currentVariant, manifest) { + static async filterManifest(drmEngine, currentVariant, manifest, + codecSwitchingStrategy=shaka.config.CodecSwitchingStrategy.RELOAD) { await shaka.util.StreamUtils.filterManifestByMediaCapabilities(manifest, manifest.offlineSessionIds.length > 0); - shaka.util.StreamUtils.filterManifestByCurrentVariant( - currentVariant, manifest); shaka.util.StreamUtils.filterTextStreams_(manifest); await shaka.util.StreamUtils.filterImageStreams_(manifest); } @@ -1510,6 +1427,7 @@ shaka.util.StreamUtils = class { /** @type {!Map.>} */ const variantsByChannelCount = new Map(); + const capableVariants = []; for (const variant of variantsWithChannelCounts) { const count = variant.audio.channelsCount; goog.asserts.assert(count != null, 'Must have count after filtering!'); @@ -1517,6 +1435,14 @@ shaka.util.StreamUtils = class { variantsByChannelCount.set(count, []); } variantsByChannelCount.get(count).push(variant); + if (count <= preferredAudioChannelCount) { + capableVariants.push(variant); + } + } + + // return all capable channelcounts + if (capableVariants.length) { + return capableVariants; } /** @type {!Array.} */ @@ -1527,15 +1453,6 @@ shaka.util.StreamUtils = class { return variants; } - // Choose the variants with the largest number of audio channels less than - // or equal to the configured number of audio channels. - const countLessThanOrEqualtoConfig = - channelCounts.filter((count) => count <= preferredAudioChannelCount); - if (countLessThanOrEqualtoConfig.length) { - return variantsByChannelCount.get( - Math.max(...countLessThanOrEqualtoConfig)); - } - // If all variants have more audio channels than the config, choose the // variants with the fewest audio channels. return variantsByChannelCount.get(Math.min(...channelCounts)); @@ -1762,7 +1679,6 @@ shaka.util.StreamUtils.nextTrackId_ = 0; shaka.util.StreamUtils.DecodingAttributes = { SMOOTH: 'smooth', POWER: 'powerEfficient', - BANDWIDTH: 'bandwidth', }; /** diff --git a/test/codec_switching/codec_switching_integration.js b/test/codec_switching/codec_switching_integration.js new file mode 100644 index 0000000000..841aa2a650 --- /dev/null +++ b/test/codec_switching/codec_switching_integration.js @@ -0,0 +1,181 @@ +/*! @license + * Shaka Player + * Copyright 2016 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +describe('Codec Switching', () => { + /** @type {!HTMLVideoElement} */ + let video; + /** @type {shaka.Player} */ + let player; + /** @type {!shaka.util.EventManager} */ + let eventManager; + + let compiledShaka; + + /** @type {!shaka.test.Waiter} */ + let waiter; + + beforeAll(async () => { + video = shaka.test.UiUtils.createVideoElement(); + document.body.appendChild(video); + compiledShaka = + await shaka.test.Loader.loadShaka(getClientArg('uncompiled')); + }); + + beforeEach(async () => { + await shaka.test.TestScheme.createManifests(compiledShaka, '_compiled'); + player = new compiledShaka.Player(video); + + // Disable stall detection, which can interfere with playback tests. + player.configure('streaming.stallEnabled', false); + + eventManager = new shaka.util.EventManager(); + waiter = new shaka.test.Waiter(eventManager); + waiter.setPlayer(player); + }); + + afterEach(async () => { + eventManager.release(); + await player.destroy(); + }); + + afterAll(() => { + document.body.removeChild(video); + }); + + describe('for audio and only-audio content', () => { + it('can switch codecs RELOAD', async () => { + if (!MediaSource.isTypeSupported('audio/webm; codecs="opus"')) { + pending('Codec OPUS in WEBM is not supported by the platform.'); + } + const preferredAudioLanguage = 'en'; + player.configure({preferredAudioLanguage: preferredAudioLanguage}); + player.configure('manifest.disableVideo', true); + player.configure('streaming.mediaSource.codecSwitchingStrategy', + shaka.config.CodecSwitchingStrategy.RELOAD); + + await player.load('/base/test/test/assets/dash-multi-codec/dash.mpd', 9); + await video.play(); + await waiter.waitForMovementOrFailOnTimeout(video, 10); + + expect(player.isLive()).toBe(false); + + let variants = player.getVariantTracks(); + + expect(variants.length).toBe(2); + expect(variants.find((v) => !!v.active).language).toBe('en'); + + player.selectAudioLanguage('es'); + await waiter.waitUntilPlayheadReachesOrFailOnTimeout(video, 10, 45); + + variants = player.getVariantTracks(); + + expect(variants.find((v) => !!v.active).language).toBe('es'); + + await player.unload(); + }); + + it('can switch codecs SMOOTH', async () => { + if (!shaka.util.Platform.supportsSmoothCodecSwitching()) { + pending('Mediasource.ChangeType is not considered ' + + 'reliable on this device'); + } + if (!MediaSource.isTypeSupported('audio/webm; codecs="opus"')) { + pending('Codec OPUS in WEBM is not supported by the platform.'); + } + const preferredAudioLanguage = 'en'; + player.configure({preferredAudioLanguage: preferredAudioLanguage}); + player.configure('manifest.disableVideo', true); + player.configure('streaming.mediaSource.codecSwitchingStrategy', + shaka.config.CodecSwitchingStrategy.SMOOTH); + + await player.load('/base/test/test/assets/dash-multi-codec/dash.mpd', 9); + await video.play(); + await waiter.waitForMovementOrFailOnTimeout(video, 10); + + expect(player.isLive()).toBe(false); + + let variants = player.getVariantTracks(); + + expect(variants.length).toBe(2); + expect(variants.find((v) => !!v.active).language).toBe('en'); + + player.selectAudioLanguage('es'); + await waiter.waitUntilPlayheadReachesOrFailOnTimeout(video, 10, 45); + + variants = player.getVariantTracks(); + + expect(variants.find((v) => !!v.active).language).toBe('es'); + + await player.unload(); + }); + }); + + describe('for audio', () => { + it('can switch codecs RELOAD', async () => { + if (!MediaSource.isTypeSupported('audio/webm; codecs="opus"')) { + pending('Codec OPUS in WEBM is not supported by the platform.'); + } + const preferredAudioLanguage = 'en'; + player.configure({preferredAudioLanguage: preferredAudioLanguage}); + player.configure('streaming.mediaSource.codecSwitchingStrategy', + shaka.config.CodecSwitchingStrategy.RELOAD); + + await player.load('/base/test/test/assets/dash-multi-codec/dash.mpd', 9); + await video.play(); + await waiter.waitForMovementOrFailOnTimeout(video, 10); + + expect(player.isLive()).toBe(false); + + let variants = player.getVariantTracks(); + + expect(variants.length).toBe(2); + expect(variants.find((v) => !!v.active).language).toBe('en'); + + player.selectAudioLanguage('es'); + await waiter.waitUntilPlayheadReachesOrFailOnTimeout(video, 10, 45); + + variants = player.getVariantTracks(); + + expect(variants.find((v) => !!v.active).language).toBe('es'); + + await player.unload(); + }); + + it('can switch codecs SMOOTH', async () => { + if (!shaka.util.Platform.supportsSmoothCodecSwitching()) { + pending('Mediasource.ChangeType is not considered ' + + 'reliable on this device'); + } + if (!MediaSource.isTypeSupported('audio/webm; codecs="opus"')) { + pending('Codec OPUS in WEBM is not supported by the platform.'); + } + const preferredAudioLanguage = 'en'; + player.configure({preferredAudioLanguage: preferredAudioLanguage}); + player.configure('streaming.mediaSource.codecSwitchingStrategy', + shaka.config.CodecSwitchingStrategy.SMOOTH); + + await player.load('/base/test/test/assets/dash-multi-codec/dash.mpd', 9); + await video.play(); + await waiter.waitForMovementOrFailOnTimeout(video, 10); + + expect(player.isLive()).toBe(false); + + let variants = player.getVariantTracks(); + + expect(variants.length).toBe(2); + expect(variants.find((v) => !!v.active).language).toBe('en'); + + player.selectAudioLanguage('es'); + await waiter.waitUntilPlayheadReachesOrFailOnTimeout(video, 10, 45); + + variants = player.getVariantTracks(); + + expect(variants.find((v) => !!v.active).language).toBe('es'); + + await player.unload(); + }); + }); +}); diff --git a/test/media/adaptation_set_criteria_unit.js b/test/media/adaptation_set_criteria_unit.js index abf156d403..b6a2dd285a 100644 --- a/test/media/adaptation_set_criteria_unit.js +++ b/test/media/adaptation_set_criteria_unit.js @@ -49,6 +49,90 @@ describe('AdaptationSetCriteria', () => { ]); }); + it('should not filter varaints when codec switching startegy is smooth '+ + 'and changeType is supported', () => { + const manifest = shaka.test.ManifestGenerator.generate((manifest) => { + manifest.addVariant(1, (variant) => { + variant.addAudio(10, (stream) => { + stream.codecs = 'mp4a.69'; + }); + variant.addVideo(11, (stream) => { + stream.codecs = 'avc1'; + }); + }); + manifest.addVariant(2, (variant) => { + variant.addAudio(12, (stream) => { + stream.codecs = 'mp4a.66'; + }); + variant.addVideo(13, (stream) => { + stream.codecs = 'hvc1'; + }); + }); + manifest.addVariant(3, (variant) => { + variant.addAudio(14, (stream) => { + stream.codecs = 'mp4a.a6'; + }); + variant.addVideo(14, (stream) => { + stream.codecs = 'dvh1'; + }); + }); + }); + + const originalIsChangeTypeSupported = shaka.media.Capabilities + .isChangeTypeSupported; + + try { + shaka.media.Capabilities.isChangeTypeSupported = () => { + return true; + }; + + const builder = new shaka.media.PreferenceBasedCriteria('en', '', 0, '', + undefined, shaka.config.CodecSwitchingStrategy.SMOOTH); + const set = builder.create(manifest.variants); + + expect(Array.from(set.values()).length).toBe(3); + } finally { + shaka.media.Capabilities + .isChangeTypeSupported = originalIsChangeTypeSupported; + } + }); + + it('should filter varaints when codec switching startegy'+ + 'is not SMOOTH', () => { + const manifest = shaka.test.ManifestGenerator.generate((manifest) => { + manifest.addVariant(1, (variant) => { + variant.addAudio(10, (stream) => { + stream.codecs = 'mp4a.69'; + }); + variant.addVideo(11, (stream) => { + stream.codecs = 'avc1'; + }); + }); + manifest.addVariant(2, (variant) => { + variant.addAudio(12, (stream) => { + stream.codecs = 'mp4a.66'; + }); + variant.addVideo(13, (stream) => { + stream.codecs = 'hvc1'; + }); + }); + manifest.addVariant(3, (variant) => { + variant.addAudio(14, (stream) => { + stream.codecs = 'mp4a.a6'; + }); + variant.addVideo(14, (stream) => { + stream.codecs = 'dvh1'; + }); + }); + }); + + const builder = new shaka.media.PreferenceBasedCriteria('en', '', 0, '', + undefined, shaka.config.CodecSwitchingStrategy.RELOAD); + const set = builder.create(manifest.variants); + + expect(Array.from(set.values()).length).toBe(1); + }); + it('chooses variants in preferred language and role', () => { const manifest = shaka.test.ManifestGenerator.generate((manifest) => { manifest.addVariant(1, (variant) => { diff --git a/test/media/media_source_engine_unit.js b/test/media/media_source_engine_unit.js index 566e4e8326..4a3bb70c8f 100644 --- a/test/media/media_source_engine_unit.js +++ b/test/media/media_source_engine_unit.js @@ -116,7 +116,13 @@ describe('MediaSourceEngine', () => { mockMediaSource = createMockMediaSource(); mockMediaSource.addSourceBuffer.and.callFake((mimeType) => { const type = mimeType.split('/')[0]; - return type == 'audio' ? audioSourceBuffer : videoSourceBuffer; + const buffer = type == 'audio' ? audioSourceBuffer : videoSourceBuffer; + // reset buffer params + buffer.timestampOffset = 0; + buffer.appendWindowEnd = Infinity; + buffer.appendWindowStart = 0; + + return buffer; }); mockTransmuxer = new shaka.test.FakeTransmuxer(); shaka.transmuxer.TransmuxerEngine.findTransmuxer = @@ -167,10 +173,12 @@ describe('MediaSourceEngine', () => { goog.asserts.assert(attr == 'src', 'Unexpected removeAttribute() call'); mockVideo.src = ''; }, + addEventListener: jasmine.createSpy('addVideoEventListener'), load: /** @this {HTMLVideoElement} */ () => { // This assertion alerts us if the requirements for this mock change. goog.asserts.assert(mockVideo.src == '', 'Unexpected load() call'); }, + play: jasmine.createSpy('play'), }; video = /** @type {HTMLMediaElement} */(mockVideo); mockClosedCaptionParser = new shaka.test.FakeClosedCaptionParser(); @@ -1136,6 +1144,161 @@ describe('MediaSourceEngine', () => { }); }); + describe('reload codec switching', () => { + beforeEach( + /** @suppress {visibility, checkTypes} */ + () => { + mediaSourceEngine.eventManager_.listenOnce = + jasmine.createSpy('listener'); + mediaSourceEngine.eventManager_.listen = + jasmine.createSpy('eventListener'); + }); + const initObject = new Map(); + initObject.set(ContentType.VIDEO, fakeVideoStream); + initObject.set(ContentType.AUDIO, fakeAudioStream); + + it('should re-create a new MediaSource', + /** @suppress {visibility} */ async () => { + await mediaSourceEngine.init(initObject, false); + mediaSourceEngine.reset_(initObject); + expect(createMediaSourceSpy).toHaveBeenCalled(); + }); + + it('should re-create the audio & video source buffers', + /** @suppress {invalidCasts, visibility, checkTypes} */ async () => { + await mediaSourceEngine.init(initObject, false); + mediaSourceEngine.reset_(initObject); + expect(mockMediaSource.addSourceBuffer).toHaveBeenCalledTimes(2); + }); + + it('should persist the previous source buffer parameters', + /** @suppress {invalidCasts, visibility, checkTypes} */async () => { + await mediaSourceEngine.init(initObject, false); + + audioSourceBuffer.timestampOffset = 10; + audioSourceBuffer.appendWindowStart = 5; + audioSourceBuffer.appendWindowEnd = 20; + + videoSourceBuffer.timestampOffset = 20; + videoSourceBuffer.appendWindowStart = 15; + videoSourceBuffer.appendWindowEnd = 30; + + mediaSourceEngine.reset_(initObject); + + expect(audioSourceBuffer.timestampOffset).toBe(10); + expect(audioSourceBuffer.appendWindowStart).toBe(5); + expect(audioSourceBuffer.appendWindowEnd).toBe(20); + + expect(videoSourceBuffer.timestampOffset).toBe(20); + expect(videoSourceBuffer.appendWindowStart).toBe(15); + expect(videoSourceBuffer.appendWindowEnd).toBe(30); + }); + + it('should preserve autoplay state', + /** @suppress {invalidCasts, visibility, checkTypes} */ + async () => { + const originalInitSourceBuffer = mediaSourceEngine.initSourceBuffer_; + try { + await mediaSourceEngine.init(initObject, false); + video.autoplay = true; + video.paused = true; + const playSpy = /** @type {jasmine.Spy} */ (video.play); + const addListenOnceSpy = + /** @type {jasmine.Spy} */ + (mediaSourceEngine.eventManager_.listenOnce); + const addEventListenerSpy = + /** @type {jasmine.Spy} */ + (mediaSourceEngine.eventManager_.listen); + mediaSourceEngine.playbackHasBegun_ = true; + mediaSourceEngine.initSourceBuffer_ = + jasmine.createSpy('initSourceBuffer'); + const initSourceBufferSpy = + /** @type {jasmine.Spy} */ + (mediaSourceEngine.initSourceBuffer_); + addEventListenerSpy.and.callFake((o, e, c) => { + c(); // audio + c(); // video + }); + await mediaSourceEngine.reset_(initObject); + const callback = addListenOnceSpy.calls.argsFor(0)[2]; + callback(); + expect(initSourceBufferSpy).toHaveBeenCalled(); + expect(addListenOnceSpy.calls.argsFor(0)[1]).toBe('canplay'); + expect(video.autoplay).toBe(true); + expect(playSpy).not.toHaveBeenCalled(); + } finally { + mediaSourceEngine.initSourceBuffer_ = originalInitSourceBuffer; + } + }); + + it('should not set autoplay to false if playback has not begun', + /** @suppress {invalidCasts, visibility, checkTypes} */ + async () => { + const originalInitSourceBuffer = mediaSourceEngine.initSourceBuffer_; + try { + await mediaSourceEngine.init(initObject, false); + video.autoplay = true; + let setCount = 0; + const addEventListenerSpy = + /** @type {jasmine.Spy} */ + (mediaSourceEngine.eventManager_.listen); + addEventListenerSpy.and.callFake((o, e, c) => { + c(); // audio + c(); // video + }); + mediaSourceEngine.initSourceBuffer_ = + jasmine.createSpy('initSourceBuffer'); + Object.defineProperty(video, 'autoplay', { + get: () => true, + set: () => { + setCount++; + }, + }); + await mediaSourceEngine.reset_(initObject); + expect(setCount).toBe(0); + } finally { + mediaSourceEngine.initSourceBuffer_ = originalInitSourceBuffer; + } + }); + + it('should preserve playing state', + /** @suppress {invalidCasts, visibility, checkTypes} */ + async () => { + const originalInitSourceBuffer = mediaSourceEngine.initSourceBuffer_; + try { + await mediaSourceEngine.init(initObject, false); + video.autoplay = false; + video.paused = false; + const playSpy = /** @type {jasmine.Spy} */ (video.play); + const addListenOnceSpy = + /** @type {jasmine.Spy} */ + (mediaSourceEngine.eventManager_.listenOnce); + const addEventListenerSpy = + /** @type {jasmine.Spy} */ + (mediaSourceEngine.eventManager_.listen); + mediaSourceEngine.playbackHasBegun_ = true; + mediaSourceEngine.initSourceBuffer_ = + jasmine.createSpy('initSourceBuffer'); + const initSourceBufferSpy = + /** @type {jasmine.Spy} */ + (mediaSourceEngine.initSourceBuffer_); + addEventListenerSpy.and.callFake((o, e, c) => { + c(); // audio + c(); // video + }); + await mediaSourceEngine.reset_(initObject); + const callback = addListenOnceSpy.calls.argsFor(0)[2]; + callback(); + expect(initSourceBufferSpy).toHaveBeenCalled(); + expect(addListenOnceSpy.calls.argsFor(0)[1]).toBe('canplay'); + expect(video.autoplay).toBe(false); + expect(playSpy).toHaveBeenCalled(); + } finally { + mediaSourceEngine.initSourceBuffer_ = originalInitSourceBuffer; + } + }); + }); + describe('destroy', () => { beforeEach(async () => { captureEvents(audioSourceBuffer, ['updateend', 'error']); @@ -1259,6 +1422,7 @@ describe('MediaSourceEngine', () => { function createMockMediaSource() { const mediaSource = { readyState: 'open', + sourceBuffers: document.createElement('div'), addSourceBuffer: jasmine.createSpy('addSourceBuffer'), endOfStream: jasmine.createSpy('endOfStream'), durationGetter: jasmine.createSpy('duration getter'), diff --git a/test/offline/storage_integration.js b/test/offline/storage_integration.js index f6de41348d..9e905846aa 100644 --- a/test/offline/storage_integration.js +++ b/test/offline/storage_integration.js @@ -1530,10 +1530,7 @@ filterDescribe('Storage', storageSupport, () => { manifest.addVariant(3, (variant) => { variant.language = frenchCanadian; variant.bandwidth = kbps(13); - variant.addVideo(4, (stream) => { - stream.bandwidth = kbps(10); - stream.size(100, 200); - }); + variant.addExistingStream(1); variant.addAudio(5, (stream) => { stream.language = frenchCanadian; stream.bandwidth = kbps(3); diff --git a/test/player_unit.js b/test/player_unit.js index 4d410c4406..3369a5e1d8 100644 --- a/test/player_unit.js +++ b/test/player_unit.js @@ -92,7 +92,7 @@ describe('Player', () => { variant.addVideo(2); }); manifest.addVariant(1, (variant) => { - variant.addAudio(3); + variant.addAudio(1); variant.addVideo(4); }); }); @@ -184,6 +184,7 @@ describe('Player', () => { describe('destroy', () => { it('cleans up all dependencies', async () => { goog.asserts.assert(manifest, 'Manifest should be non-null'); + await player.load(fakeManifestUri, 0, fakeMimeType); const segmentIndexes = []; for (const variant of manifest.variants) { if (variant.audio) { @@ -340,15 +341,12 @@ describe('Player', () => { }); describe('disableStream', () => { - /** @type {function(function(), number)} */ - let realSetTimeout; /** @type {number} */ let disableTimeInSeconds; /** @type {?jasmine.Spy} */ let getBufferedInfoSpy; beforeAll(() => { - realSetTimeout = window.setTimeout; jasmine.clock().install(); jasmine.clock().mockDate(); }); @@ -381,15 +379,21 @@ describe('Player', () => { stream.mime('audio/mp4', 'mp4a.40.2'); stream.language = 'en'; }); - variant.addVideo(2); + variant.addVideo(2, (stream) => { + stream.size(10, 10); + }); }); manifest.addVariant(1, (variant) => { variant.addExistingStream(1); - variant.addVideo(3); + variant.addVideo(3, (stream) => { + stream.size(20, 20); + }); }); manifest.addVariant(2, (variant) => { variant.addExistingStream(1); - variant.addVideo(4); + variant.addVideo(4, (stream) => { + stream.size(30, 30); + }); }); }); } @@ -498,102 +502,6 @@ describe('Player', () => { expect(fromAdaptation).toBeFalsy(); }); - it('disables all variants containing stream', async () => { - manifest = shaka.test.ManifestGenerator.generate((manifest) => { - manifest.addVariant(0, (variant) => { - variant.addAudio(1, (stream) => { - stream.mime('audio/mp4', 'mp4a.40.2'); - stream.language = 'en'; - stream.bandwidth = 100; - }); - variant.addVideo(2); - }); - manifest.addVariant(1, (variant) => { - variant.addExistingStream(1); - variant.addVideo(3); - }); - - manifest.addVariant(2, (variant) => { - variant.addAudio(4, (stream) => { - stream.mime('audio/mp4', 'mp4a.40.2'); - stream.language = 'en'; - stream.bandwidth = 200; - }); - variant.addExistingStream(2); - }); - manifest.addVariant(3, (variant) => { - variant.addExistingStream(4); - variant.addExistingStream(3); - }); - }); - - await player.load(fakeManifestUri, 0, fakeMimeType); - - const variantCount = manifest.variants.length; - const variantAffected = 2; - const audioStream = - /** @type {shaka.extern.Stream} */ (manifest.variants[0].audio); - - player.disableStream(audioStream, disableTimeInSeconds); - - await shaka.test.Util.shortDelay(realSetTimeout); - - expect(abrManager.setVariants).toHaveBeenCalled(); - - const updatedVariants = - abrManager.setVariants.calls.mostRecent().args[0]; - - expect(updatedVariants.length).toBe(variantCount - variantAffected); - - for (const variant of updatedVariants) { - expect(variant.audio).not.toEqual(audioStream); - } - }); - - it('does not disable streams if abr is disabled', async () => { - await player.load(fakeManifestUri, 0, fakeMimeType); - - player.configure({abr: {enabled: true}}); - - const videoStream = - /** @type {shaka.extern.Stream} */ (manifest.variants[0].video); - let status = player.disableStream(videoStream, 10); - expect(status).toBe(true); - - player.configure({abr: {enabled: false}}); - - status = player.disableStream(videoStream, 10); - expect(status).toBe(false); - }); - - it('disables stream if have alternate stream', async () => { - // Run test with the default manifest - await runTest(0, 'video', true); - - manifest = shaka.test.ManifestGenerator.generate((manifest) => { - manifest.addVariant(0, (variant) => { - variant.addAudio(1, (stream) => { - stream.mime('audio/mp4', 'mp4a.40.2'); - stream.language = 'en'; - stream.bandwidth = 10; - }); - variant.addVideo(2); - }); - manifest.addVariant(1, (variant) => { - variant.addAudio(3, (stream) => { - stream.mime('audio/mp4', 'mp4a.40.2'); - stream.language = 'en'; - stream.bandwidth = 20; - }); - variant.addExistingStream(2); - }); - }); - - player.configure({abr: {enabled: true}}); - - await runTest(0, 'audio', true); - }); - describe('does not disable stream if there not alternate stream', () => { it('single audio multiple videos', async () => { manifest = shaka.test.ManifestGenerator.generate((manifest) => { @@ -2711,15 +2619,15 @@ describe('Player', () => { }); manifest.addVariant(2, (variant) => { variant.bandwidth = 300; - variant.addAudio(4, (stream) => { + variant.addAudio(1, (stream) => { stream.bandwidth = 200; }); variant.addExistingStream(2); // video }); manifest.addVariant(3, (variant) => { variant.bandwidth = 400; - variant.addExistingStream(4); // audio - variant.addExistingStream(3); // video + variant.addExistingStream(1); // audio + variant.addExistingStream(2); // video }); }); @@ -3028,11 +2936,15 @@ describe('Player', () => { manifest = shaka.test.ManifestGenerator.generate((manifest) => { manifest.addVariant(0, (variant) => { variant.bandwidth = 500; - variant.addVideo(1); + variant.addVideo(1, (stream) => { + stream.size(10, 10); + }); }); manifest.addVariant(1, (variant) => { variant.bandwidth = 100; - variant.addVideo(2); + variant.addVideo(2, (stream) => { + stream.size(30, 30); + }); }); }); @@ -3067,11 +2979,15 @@ describe('Player', () => { manifest = shaka.test.ManifestGenerator.generate((manifest) => { manifest.addVariant(1, (variant) => { variant.bandwidth = 500; - variant.addVideo(10); + variant.addVideo(10, (stream) => { + stream.size(10, 10); + }); }); manifest.addVariant(2, (variant) => { variant.bandwidth = 100; - variant.addVideo(20); + variant.addVideo(20, (stream) => { + stream.size(30, 30); + }); }); }); @@ -3101,10 +3017,13 @@ describe('Player', () => { manifest.addVariant(0, (variant) => { variant.addVideo(1, (stream) => { stream.keyIds = new Set(['abc']); + stream.size(10, 10); }); }); manifest.addVariant(1, (variant) => { - variant.addVideo(2); + variant.addVideo(2, (stream) => { + stream.size(20, 20); + }); }); }); @@ -3133,10 +3052,13 @@ describe('Player', () => { manifest.addVariant(0, (variant) => { variant.addVideo(1, (stream) => { stream.keyIds = new Set(['abc']); + stream.size(10, 10); }); }); manifest.addVariant(1, (variant) => { - variant.addVideo(2); + variant.addVideo(2, (stream) => { + stream.size(20, 20); + }); }); }); @@ -3232,10 +3154,13 @@ describe('Player', () => { manifest.addVariant(0, (variant) => { variant.addVideo(1, (stream) => { stream.keyIds = new Set(['abc']); + stream.size(10, 10); }); }); manifest.addVariant(1, (variant) => { - variant.addVideo(2); + variant.addVideo(2, (stream) => { + stream.size(20, 20); + }); }); }); @@ -3254,10 +3179,13 @@ describe('Player', () => { manifest.addVariant(0, (variant) => { variant.addVideo(1, (stream) => { stream.keyIds = new Set(['abc']); + stream.size(10, 10); }); }); manifest.addVariant(1, (variant) => { - variant.addVideo(2); + variant.addVideo(2, (stream) => { + stream.size(20, 20); + }); }); }); @@ -3276,10 +3204,13 @@ describe('Player', () => { manifest.addVariant(0, (variant) => { variant.addVideo(1, (stream) => { stream.keyIds = new Set(['abc']); + stream.size(10, 10); }); }); manifest.addVariant(2, (variant) => { - variant.addVideo(3); + variant.addVideo(3, (stream) => { + stream.size(20, 20); + }); }); }); @@ -3299,16 +3230,19 @@ describe('Player', () => { manifest.addVariant(0, (variant) => { variant.addVideo(1, (stream) => { stream.keyIds = new Set(['abc']); + stream.size(10, 10); }); }); manifest.addVariant(2, (variant) => { variant.addVideo(3, (stream) => { stream.keyIds = new Set(['def']); + stream.size(20, 20); }); }); manifest.addVariant(4, (variant) => { variant.addVideo(5, (stream) => { stream.keyIds = new Set(['abc', 'def']); + stream.size(30, 30); }); }); }); @@ -3326,10 +3260,13 @@ describe('Player', () => { manifest.addVariant(0, (variant) => { variant.addVideo(1, (stream) => { stream.keyIds = new Set(['abc']); + stream.size(10, 10); }); }); manifest.addVariant(2, (variant) => { - variant.addVideo(3); + variant.addVideo(3, (stream) => { + stream.size(20, 20); + }); }); }); @@ -3349,10 +3286,13 @@ describe('Player', () => { manifest.addVariant(0, (variant) => { variant.addVideo(1, (stream) => { stream.keyIds = new Set(['abc']); + stream.size(10, 10); }); }); manifest.addVariant(2, (variant) => { - variant.addVideo(3); + variant.addVideo(3, (stream) => { + stream.size(20, 20); + }); }); }); @@ -3371,15 +3311,19 @@ describe('Player', () => { manifest.addVariant(0, (variant) => { variant.addVideo(1, (stream) => { stream.keyIds = new Set(['abc']); + stream.size(10, 10); }); }); manifest.addVariant(2, (variant) => { variant.addVideo(3, (stream) => { stream.keyIds = new Set(['xyz']); + stream.size(20, 20); }); }); manifest.addVariant(4, (variant) => { - variant.addVideo(5); + variant.addVideo(5, (stream) => { + stream.size(30, 30); + }); }); }); @@ -3433,15 +3377,21 @@ describe('Player', () => { manifest = shaka.test.ManifestGenerator.generate((manifest) => { manifest.addVariant(0, (variant) => { variant.bandwidth = 10; - variant.addVideo(1); + variant.addVideo(1, (stream) => { + stream.size(10, 10); + }); }); manifest.addVariant(1, (variant) => { variant.bandwidth = 1500; - variant.addVideo(2); + variant.addVideo(2, (stream) => { + stream.size(20, 20); + }); }); manifest.addVariant(2, (variant) => { variant.bandwidth = 500; - variant.addVideo(3); + variant.addVideo(3, (stream) => { + stream.size(30, 30); + }); }); }); @@ -3557,7 +3507,7 @@ describe('Player', () => { variant.addVideo(3, (stream) => { stream.size(190, 190); }); - variant.addAudio(4); + variant.addAudio(2); }); }); @@ -3620,18 +3570,21 @@ describe('Player', () => { variant.bandwidth = 100; variant.addVideo(0, (stream) => { stream.codecs = 'good'; + stream.size(10, 10); }); }); manifest.addVariant(1, (variant) => { variant.bandwidth = 200; variant.addVideo(1, (stream) => { stream.codecs = 'good'; + stream.size(20, 20); }); }); manifest.addVariant(2, (variant) => { variant.bandwidth = 300; variant.addVideo(2, (stream) => { stream.codecs = 'good'; + stream.size(30, 30); }); }); @@ -3640,18 +3593,21 @@ describe('Player', () => { variant.bandwidth = 10000; variant.addVideo(3, (stream) => { stream.codecs = 'bad'; + stream.size(10, 10); }); }); manifest.addVariant(4, (variant) => { variant.bandwidth = 20000; variant.addVideo(4, (stream) => { stream.codecs = 'bad'; + stream.size(20, 20); }); }); manifest.addVariant(5, (variant) => { variant.bandwidth = 30000; variant.addVideo(5, (stream) => { stream.codecs = 'bad'; + stream.size(30, 30); }); }); }); @@ -3672,10 +3628,13 @@ describe('Player', () => { manifest.addVariant(0, (variant) => { variant.addVideo(1, (stream) => { stream.keyIds = new Set(['abc']); + stream.size(10, 10); }); }); manifest.addVariant(2, (variant) => { - variant.addVideo(3); + variant.addVideo(3, (stream) => { + stream.size(20, 20); + }); }); }); @@ -3778,19 +3737,25 @@ describe('Player', () => { manifest = shaka.test.ManifestGenerator.generate((manifest) => { manifest.addVariant(0, (variant) => { variant.bandwidth = 100; - variant.addVideo(0); + variant.addVideo(0, (stream) => { + stream.size(10, 10); + }); variant.addAudio(9); }); manifest.addVariant(1, (variant) => { variant.bandwidth = 200; - variant.addVideo(1); + variant.addVideo(1, (stream) => { + stream.size(20, 20); + }); variant.addExistingStream(9); // audio }); manifest.addVariant(2, (variant) => { variant.bandwidth = 300; - variant.addVideo(2); + variant.addVideo(2, (stream) => { + stream.size(30, 30); + }); variant.addExistingStream(9); // audio }); }); @@ -3859,15 +3824,21 @@ describe('Player', () => { manifest = shaka.test.ManifestGenerator.generate((manifest) => { manifest.addVariant(0, (variant) => { variant.bandwidth = /** @type {?} */(undefined); - variant.addVideo(0); + variant.addVideo(0, (stream) => { + stream.size(10, 10); + }); }); manifest.addVariant(1, (variant) => { variant.bandwidth = NaN; - variant.addVideo(1); + variant.addVideo(1, (stream) => { + stream.size(20, 20); + }); }); manifest.addVariant(2, (variant) => { variant.bandwidth = 0; - variant.addVideo(2); + variant.addVideo(2, (stream) => { + stream.size(30, 30); + }); }); }); diff --git a/test/test/assets/dash-multi-codec/audio_en_2c_128k_aac.mp4 b/test/test/assets/dash-multi-codec/audio_en_2c_128k_aac.mp4 new file mode 100644 index 0000000000..cdb4984762 Binary files /dev/null and b/test/test/assets/dash-multi-codec/audio_en_2c_128k_aac.mp4 differ diff --git a/test/test/assets/dash-multi-codec/audio_es_2c_64k_opus.webm b/test/test/assets/dash-multi-codec/audio_es_2c_64k_opus.webm new file mode 100644 index 0000000000..1414e35965 Binary files /dev/null and b/test/test/assets/dash-multi-codec/audio_es_2c_64k_opus.webm differ diff --git a/test/test/assets/dash-multi-codec/dash.mpd b/test/test/assets/dash-multi-codec/dash.mpd new file mode 100644 index 0000000000..3f92430017 --- /dev/null +++ b/test/test/assets/dash-multi-codec/dash.mpd @@ -0,0 +1,32 @@ + + + + + + + video_144p_108k_h264.mp4 + + + + + + + + + audio_en_2c_128k_aac.mp4 + + + + + + + + + audio_es_2c_64k_opus.webm + + + + + + + \ No newline at end of file diff --git a/test/test/assets/dash-multi-codec/video_144p_108k_h264.mp4 b/test/test/assets/dash-multi-codec/video_144p_108k_h264.mp4 new file mode 100644 index 0000000000..e2e10d7b00 Binary files /dev/null and b/test/test/assets/dash-multi-codec/video_144p_108k_h264.mp4 differ diff --git a/test/test/util/fake_media_source_engine.js b/test/test/util/fake_media_source_engine.js index 7d870a314d..04df9e9a9f 100644 --- a/test/test/util/fake_media_source_engine.js +++ b/test/test/util/fake_media_source_engine.js @@ -341,6 +341,12 @@ shaka.test.FakeMediaSourceEngine = class { return Promise.resolve(); } + + /** @override */ + isResetMediaSourceNecessary() { + return false; + } + /** * @param {string} type * @return {!Promise} diff --git a/test/test/util/manifest_generator.js b/test/test/util/manifest_generator.js index 20e56f5018..1a6167cc8e 100644 --- a/test/test/util/manifest_generator.js +++ b/test/test/util/manifest_generator.js @@ -459,9 +459,11 @@ shaka.test.ManifestGenerator.Stream = class { * @param {string=} label */ constructor(manifest, isPartial, id, type, lang, label) { - goog.asserts.assert( - !manifest || !manifest.isIdUsed_(id), - 'Streams should have unique ids!'); + // variants can be made up of different combinations of video + // and audio streams + // goog.asserts.assert( + // !manifest || !manifest.isIdUsed_(id), + // 'Streams should have unique ids!'); const ContentType = shaka.util.ManifestParserUtils.ContentType; /** @const {shaka.test.ManifestGenerator.Manifest} */ diff --git a/test/ui/ui_unit.js b/test/ui/ui_unit.js index 0f475a3622..cb3311a557 100644 --- a/test/ui/ui_unit.js +++ b/test/ui/ui_unit.js @@ -616,19 +616,10 @@ describe('UI', () => { player.configure('abr.enabled', false); const tracks = player.getVariantTracks(); - const en2 = - tracks.find((t) => t.language == 'en' && t.channelsCount == 2); const en1 = tracks.find((t) => t.language == 'en' && t.channelsCount == 1); const es = tracks.find((t) => t.language == 'es'); - // There are 3 variants with English 2-channel, but one is a duplicate - // and shouldn't appear in the list. - goog.asserts.assert(en2, 'Unable to find tracks'); - player.selectVariantTrack(en2, true); - await updateResolutionMenu(); - expect(getResolutions()).toEqual(['240p', '480p']); - // There is 1 variant with English 1-channel. goog.asserts.assert(en1, 'Unable to find tracks'); player.selectVariantTrack(en1, true); @@ -646,7 +637,9 @@ describe('UI', () => { const manifest = shaka.test.ManifestGenerator.generate((manifest) => { manifest.addVariant(0, (variant) => { - variant.addAudio(0); + variant.addAudio(0, (stream) => { + stream.roles = ['main']; + }); variant.bandwidth = 100000; }); manifest.addVariant(1, (variant) => { diff --git a/test/util/stream_utils_unit.js b/test/util/stream_utils_unit.js index a8a8d96cb7..ab3d56badb 100644 --- a/test/util/stream_utils_unit.js +++ b/test/util/stream_utils_unit.js @@ -636,7 +636,8 @@ describe('StreamUtils', () => { const noVariant = null; await shaka.util.StreamUtils.filterManifest( - fakeDrmEngine, noVariant, manifest); + fakeDrmEngine, noVariant, manifest, + shaka.config.CodecSwitchingStrategy.RELOAD); // Covers a regression in which we would remove streams with codecs. // The last two streams should be removed because their full MIME types @@ -673,7 +674,8 @@ describe('StreamUtils', () => { const noVariant = null; await shaka.util.StreamUtils.filterManifest( - fakeDrmEngine, noVariant, manifest); + fakeDrmEngine, noVariant, manifest, + shaka.config.CodecSwitchingStrategy.RELOAD); // Covers a regression in which we would remove streams with codecs. // The first 4 streams should be there because they are always supported. @@ -691,6 +693,38 @@ describe('StreamUtils', () => { jasmine.objectContaining({id: 5})); }); + it('does not filter manifest when codec switching is enabled', async () => { + manifest = shaka.test.ManifestGenerator.generate((manifest) => { + manifest.addVariant(1, (variant) => { + variant.addAudio(10, (stream) => { + stream.codecs = 'mp4a.69'; + }); + variant.addVideo(11, (stream) => { + stream.codecs = 'avc1'; + }); + }); + }); + + const originalFilterManifestByCurrentVariant = + shaka.util.StreamUtils.filterManifestByCurrentVariant; + + try { + const filterManifestByCurrentVariantSpy = + jasmine.createSpy('filterManifestByCurrentVariant'); + shaka.util.StreamUtils.filterManifestByCurrentVariant = + shaka.test.Util.spyFunc(filterManifestByCurrentVariantSpy); + + await shaka.util.StreamUtils.filterManifest( + fakeDrmEngine, /* currentVariant= */ null, manifest, + shaka.config.CodecSwitchingStrategy.RELOAD); + + expect(filterManifestByCurrentVariantSpy).not.toHaveBeenCalled(); + } finally { + shaka.util.StreamUtils.filterManifestByCurrentVariant = + originalFilterManifestByCurrentVariant; + } + }); + it('filters transport streams', async () => { manifest = shaka.test.ManifestGenerator.generate((manifest) => { manifest.addVariant(0, (variant) => { @@ -705,7 +739,8 @@ describe('StreamUtils', () => { }); await shaka.util.StreamUtils.filterManifest( - fakeDrmEngine, /* currentVariant= */ null, manifest); + fakeDrmEngine, /* currentVariant= */ null, manifest, + shaka.config.CodecSwitchingStrategy.RELOAD); // Covers a regression in which we would remove streams with codecs. // The last two streams should be removed because their full MIME types @@ -732,7 +767,8 @@ describe('StreamUtils', () => { }); await shaka.util.StreamUtils.filterManifest( - fakeDrmEngine, /* currentVariant= */ null, manifest); + fakeDrmEngine, /* currentVariant= */ null, manifest, + shaka.config.CodecSwitchingStrategy.RELOAD); expect(manifest.variants.length).toBe(1); }); @@ -749,7 +785,8 @@ describe('StreamUtils', () => { }); await shaka.util.StreamUtils.filterManifest( - fakeDrmEngine, /* currentVariant= */ null, manifest); + fakeDrmEngine, /* currentVariant= */ null, manifest, + shaka.config.CodecSwitchingStrategy.RELOAD); expect(manifest.variants.length).toBe(1); }); @@ -813,148 +850,245 @@ describe('StreamUtils', () => { }); await shaka.util.StreamUtils.filterManifest( - fakeDrmEngine, /* currentVariant= */ null, manifest); + fakeDrmEngine, /* currentVariant= */ null, manifest, + shaka.config.CodecSwitchingStrategy.RELOAD); expect(manifest.variants.length).toBe(1); }); }); describe('chooseCodecsAndFilterManifest', () => { - const avc1Codecs = 'avc1.640028'; - const vp09Codecs = 'vp09.00.40.08.00.02.02.02.00'; - - const addVariant1080Avc1 = (manifest) => { + const addVariant720Avc1 = (manifest) => { manifest.addVariant(0, (variant) => { variant.bandwidth = 5058558; variant.addAudio(1, (stream) => { stream.bandwidth = 129998; - stream.codecs = 'opus'; + stream.mime('audio/mp4', 'mp4a.40.2'); }); variant.addVideo(2, (stream) => { stream.bandwidth = 4928560; - stream.codecs = avc1Codecs; - stream.size(1920, 1080); + stream.size(1280, 720); + stream.mime('video/mp4', 'avc1.640028'); }); }); }; - const addVariant1080Vp9 = (manifest) => { + const addVariant720Vp9 = (manifest) => { manifest.addVariant(3, (variant) => { variant.bandwidth = 4911000; variant.addAudio(4, (stream) => { stream.bandwidth = 129998; - stream.codecs = 'vorbis'; + stream.mime('audio/webm', 'vorbis'); }); variant.addVideo(5, (stream) => { stream.bandwidth = 4781002; - stream.codecs = vp09Codecs; - stream.size(1920, 1080); + stream.size(1280, 720); + stream.mime('video/webm', 'vp9'); }); }); }; - const addVariant2160Vp9 = (manifest) => { + const addVariant1080Vp9 = (manifest) => { manifest.addVariant(6, (variant) => { variant.bandwidth = 10850316; - variant.addAudio(7, (stream) => { + variant.addAudio(1, (stream) => { stream.bandwidth = 129998; - stream.codecs = 'opus'; + stream.mime('audio/mp4', 'mp4a.40.2'); }); variant.addVideo(8, (stream) => { stream.bandwidth = 10784324; - stream.codecs = vp09Codecs; - stream.size(3840, 2160); + stream.size(1920, 1080); + stream.mime('video/webm', 'vp9'); }); }); }; + it('should filter variants by the best available bandwidth' + + ' for video resolution', () => { + manifest = shaka.test.ManifestGenerator.generate((manifest) => { + manifest.addVariant(0, (variant) => { + variant.bandwidth = 4058558; + variant.addVideo(1, (stream) => { + stream.bandwidth = 300000; + stream.size(10, 10); + }); + }); + manifest.addVariant(2, (variant) => { + variant.bandwidth = 4781002; + variant.addVideo(3, (stream) => { + stream.bandwidth = 400000; + stream.size(10, 10); + }); + }); + manifest.addVariant(4, (variant) => { + variant.addVideo(5, (stream) => { + stream.bandwidth = 500000; + stream.size(20, 20); + }); + }); + manifest.addVariant(6, (variant) => { + variant.addVideo(7, (stream) => { + stream.bandwidth = 600000; + stream.size(20, 20); + }); + }); + }); + + shaka.util.StreamUtils.chooseCodecsAndFilterManifest(manifest, + /* preferredVideoCodecs= */[], + /* preferredAudioCodecs= */[], + /* preferredAudioChannelCount= */2, + /* preferredDecodingAttributes= */[]); + + expect(manifest.variants.length).toBe(2); + expect(manifest.variants.every((v) => [300000, 500000].includes( + v.video.bandwidth))).toBeTruthy(); + }); + + it('should filter variants by the best available bandwidth' + + ' for audio language', () => { + manifest = shaka.test.ManifestGenerator.generate((manifest) => { + manifest.addVariant(0, (variant) => { + variant.bandwidth = 4058558; + variant.addAudio(1, (stream) => { + stream.bandwidth = 100000; + stream.language = 'en'; + }); + }); + manifest.addVariant(2, (variant) => { + variant.bandwidth = 4781002; + variant.addAudio(3, (stream) => { + stream.bandwidth = 200000; + stream.language = 'en'; + }); + }); + manifest.addVariant(4, (variant) => { + variant.addAudio(5, (stream) => { + stream.bandwidth = 100000; + stream.language = 'es'; + }); + }); + manifest.addVariant(6, (variant) => { + variant.addAudio(7, (stream) => { + stream.bandwidth = 500000; + stream.language = 'es'; + }); + }); + }); + + shaka.util.StreamUtils.chooseCodecsAndFilterManifest(manifest, + /* preferredVideoCodecs= */[], + /* preferredAudioCodecs= */[], + /* preferredAudioChannelCount= */2, + /* preferredDecodingAttributes= */[]); + + expect(manifest.variants.length).toBe(2); + expect(manifest.variants.every((v) => v.audio.bandwidth == 100000)) + .toBeTruthy(); + }); + + it('should allow multiple codecs for codec switching', () => { + if (!MediaSource.isTypeSupported('video/webm; codecs="vp9"')) { + pending('Codec VP9 is not supported by the platform.'); + } + if (!MediaSource.isTypeSupported('video/webm; codecs="vorbis"')) { + pending('Codec vorbis is not supported by the platform.'); + } + // This test is flaky in some Tizen devices, due to codec restrictions. + if (shaka.util.Platform.isTizen()) { + pending('Skip flaky test in Tizen'); + } + manifest = shaka.test.ManifestGenerator.generate((manifest) => { + addVariant720Avc1(manifest); + addVariant720Vp9(manifest); + addVariant1080Vp9(manifest); + }); + + manifest.variants[0].video.bandwidth = 1; + + shaka.util.StreamUtils.chooseCodecsAndFilterManifest(manifest, + /* preferredVideoCodecs= */[], + /* preferredAudioCodecs= */[], + /* preferredAudioChannelCount= */2, + /* preferredDecodingAttributes= */[]); + + expect(manifest.variants.length).toBe(2); + expect(manifest.variants[0].video.codecs) + .not.toBe(manifest.variants[1].video.codecs); + }); + it('chooses preferred audio and video codecs', () => { + if (!MediaSource.isTypeSupported('video/webm; codecs="vp9"')) { + pending('Codec VP9 is not supported by the platform.'); + } + if (!MediaSource.isTypeSupported('video/webm; codecs="vorbis"')) { + pending('Codec vorbis is not supported by the platform.'); + } manifest = shaka.test.ManifestGenerator.generate((manifest) => { - addVariant1080Avc1(manifest); + addVariant720Avc1(manifest); + addVariant720Vp9(manifest); addVariant1080Vp9(manifest); - addVariant2160Vp9(manifest); }); const variants = shaka.util.StreamUtils.choosePreferredCodecs(manifest.variants, - /* preferredVideoCodecs= */['vp09'], - /* preferredAudioCodecs= */['opus']); + /* preferredVideoCodecs= */['vp9'], + /* preferredAudioCodecs= */['mp4a']); expect(variants.length).toBe(1); - expect(variants[0].video.codecs).toBe(vp09Codecs); - expect(variants[0].audio.codecs).toBe('opus'); + expect(variants[0].video.codecs).toBe('vp9'); + expect(variants[0].audio.codecs).toBe('mp4a.40.2'); }); it('chooses preferred video codecs', () => { + if (!MediaSource.isTypeSupported('video/webm; codecs="vp9"')) { + pending('Codec VP9 is not supported by the platform.'); + } + if (!MediaSource.isTypeSupported('video/webm; codecs="vorbis"')) { + pending('Codec vorbis is not supported by the platform.'); + } // If no preferred audio codecs is specified or can be found, choose the // variants with preferred video codecs. manifest = shaka.test.ManifestGenerator.generate((manifest) => { - addVariant1080Avc1(manifest); + addVariant720Avc1(manifest); + addVariant720Vp9(manifest); addVariant1080Vp9(manifest); - addVariant2160Vp9(manifest); }); const variants = shaka.util.StreamUtils.choosePreferredCodecs(manifest.variants, - /* preferredVideoCodecs= */['vp09'], + /* preferredVideoCodecs= */['vp9'], /* preferredAudioCodecs= */[]); expect(variants.length).toBe(2); - expect(variants[0].video.codecs).toBe(vp09Codecs); + expect(variants[0].video.codecs).toBe('vp9'); expect(variants[0].audio.codecs).toBe('vorbis'); - expect(variants[1].video.codecs).toBe(vp09Codecs); - expect(variants[1].audio.codecs).toBe('opus'); + expect(variants[1].video.codecs).toBe('vp9'); + expect(variants[1].audio.codecs).toBe('mp4a.40.2'); }); it('chooses preferred audio codecs', () => { + if (!MediaSource.isTypeSupported('video/webm; codecs="vp9"')) { + pending('Codec VP9 is not supported by the platform.'); + } + if (!MediaSource.isTypeSupported('video/webm; codecs="vorbis"')) { + pending('Codec vorbis is not supported by the platform.'); + } // If no preferred video codecs is specified or can be found, choose the // variants with preferred audio codecs. manifest = shaka.test.ManifestGenerator.generate((manifest) => { - addVariant1080Avc1(manifest); + addVariant720Avc1(manifest); + addVariant720Vp9(manifest); addVariant1080Vp9(manifest); - addVariant2160Vp9(manifest); }); const variants = shaka.util.StreamUtils.choosePreferredCodecs(manifest.variants, /* preferredVideoCodecs= */['foo'], - /* preferredAudioCodecs= */['opus']); + /* preferredAudioCodecs= */['mp4a.40.2']); expect(variants.length).toBe(2); - expect(variants[0].video.codecs).toBe(avc1Codecs); - expect(variants[0].audio.codecs).toBe('opus'); - expect(variants[1].video.codecs).toBe(vp09Codecs); - expect(variants[1].audio.codecs).toBe('opus'); - }); - - it('chooses variants with different sizes (density) by codecs', () => { - manifest = shaka.test.ManifestGenerator.generate((manifest) => { - addVariant1080Avc1(manifest); - addVariant1080Vp9(manifest); - addVariant2160Vp9(manifest); - }); - - shaka.util.StreamUtils.chooseCodecsAndFilterManifest(manifest, - /* preferredVideoCodecs= */[], - /* preferredAudioCodecs= */[], - /* preferredAudioChannelCount= */2, - /* preferredDecodingAttributes= */[]); - - expect(manifest.variants.length).toBe(1); - expect(manifest.variants[0].video.codecs).toBe(vp09Codecs); - }); - - it('chooses variants with same sizes (density) by codecs', () => { - manifest = shaka.test.ManifestGenerator.generate((manifest) => { - addVariant1080Avc1(manifest); - addVariant1080Vp9(manifest); - }); - - shaka.util.StreamUtils.chooseCodecsAndFilterManifest(manifest, - /* preferredVideoCodecs= */[], - /* preferredAudioCodecs= */[], - /* preferredAudioChannelCount= */2, - /* preferredDecodingAttributes= */[]); - - expect(manifest.variants.length).toBe(1); - expect(manifest.variants[0].video.codecs).toBe(vp09Codecs); + expect(variants[0].video.codecs).toBe('avc1.640028'); + expect(variants[0].audio.codecs).toBe('mp4a.40.2'); + expect(variants[1].video.codecs).toBe('vp9'); + expect(variants[1].audio.codecs).toBe('mp4a.40.2'); }); it('chooses variants by decoding attributes', async () => {