feat: Enable codec switching (#5470)

Closes: #1528 Closes: #1567 Closes: #4379 Closes: #5306 --------- Co-authored-by: Álvaro Velad Galván <[email protected]>
shaka-project · Oct 4, 2023 · 0078137 · 0078137
1 parent c02ccee
commit 0078137
Show file tree

Hide file tree

Showing 26 changed files with 1,200 additions and 498 deletions.
diff --git a/build/types/core b/build/types/core
@@ -6,6 +6,7 @@
 +../../lib/abr/simple_abr_manager.js
 
 +../../lib/config/auto_show_text.js
++../../lib/config/codec_switching_strategy.js
 
 +../../lib/debug/asserts.js
 +../../lib/debug/log.js

diff --git a/demo/config.js b/demo/config.js
@@ -448,13 +448,24 @@ shakaDemo.Config = class {
 
   /** @private */
   addMediaSourceSection_() {
+    const strategyOptions = shaka.config.CodecSwitchingStrategy;
+    const strategyOptionsNames = {
+      'RELOAD': 'reload',
+      'SMOOTH': 'smooth',
+    };
+
     const docLink = this.resolveExternLink_('.MediaSourceConfiguration');
     this.addSection_('Media source', docLink)
         .addTextInput_('Source buffer extra features',
             'mediaSource.sourceBufferExtraFeatures')
         .addBoolInput_('Force Transmux', 'mediaSource.forceTransmux')
         .addBoolInput_('Insert fake encryption in init segments when needed ' +
-            'by the platform.', 'mediaSource.insertFakeEncryptionInInit');
+            'by the platform.', 'mediaSource.insertFakeEncryptionInInit')
+        .addSelectInput_(
+            'Codec Switching Strategy',
+            'mediaSource.codecSwitchingStrategy',
+            strategyOptions,
+            strategyOptionsNames);
   }
 
   /** @private */

diff --git a/externs/shaka/player.js b/externs/shaka/player.js
@@ -1230,6 +1230,7 @@ shaka.extern.StreamingConfiguration;
 
 /**
  * @typedef {{
+ *   codecSwitchingStrategy: shaka.config.CodecSwitchingStrategy,
  *   sourceBufferExtraFeatures: string,
  *   forceTransmux: boolean,
 *    insertFakeEncryptionInInit: boolean
@@ -1238,6 +1239,11 @@ shaka.extern.StreamingConfiguration;
  * @description
  *   Media source configuration.
  *
+ * @property {shaka.config.CodecSwitchingStrategy} codecSwitchingStrategy
+ *   Allow codec switching strategy. SMOOTH loading uses
+ *   SourceBuffer.changeType. RELOAD uses cycling of MediaSource.
+ *   Defaults to SMOOTH if SMOOTH codec switching is supported, RELOAD
+ *   overwise.
  * @property {string} sourceBufferExtraFeatures
  *   Some platforms may need to pass features when initializing the
  *   sourceBuffer.

diff --git a/karma.conf.js b/karma.conf.js
@@ -233,6 +233,7 @@ module.exports = (config) => {
       {pattern: 'third_party/**/*.js', included: false},
       {pattern: 'test/**/*.js', included: false},
       {pattern: 'test/test/assets/*', included: false},
+      {pattern: 'test/test/assets/dash-multi-codec/*', included: false},
       {pattern: 'test/test/assets/3675/*', included: false},
       {pattern: 'test/test/assets/dash-aes-128/*', included: false},
       {pattern: 'test/test/assets/hls-raw-aac/*', included: false},

diff --git a/lib/config/codec_switching_strategy.js b/lib/config/codec_switching_strategy.js
@@ -0,0 +1,24 @@
+/*! @license
+ * Shaka Player
+ * Copyright 2016 Google LLC
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+goog.provide('shaka.config.CodecSwitchingStrategy');
+
+/**
+ * @enum {string}
+ * @export
+ */
+shaka.config.CodecSwitchingStrategy = {
+  // Allow codec switching which will always involve reloading the
+  // `MediaSource`.
+  RELOAD: 'reload',
+  // Allow codec switching; determine if `SourceBuffer.changeType` is available
+  // and attempt to use this first, but fall back to reloading `MediaSource` if
+  // not available.
+  //
+  // Note: Some devices that support `SourceBuffer.changeType` can become stuck
+  // in a pause state.
+  SMOOTH: 'smooth',
+};
diff --git a/lib/media/adaptation_set.js b/lib/media/adaptation_set.js
@@ -43,10 +43,11 @@ shaka.media.AdaptationSet = class {
 
   /**
    * @param {shaka.extern.Variant} variant
+   * @param {boolean=} compareCodecs
    * @return {boolean}
    */
-  add(variant) {
-    if (this.canInclude(variant)) {
+  add(variant, compareCodecs = true) {
+    if (this.canInclude(variant, compareCodecs)) {
       this.variants_.add(variant);
       return true;
     }
@@ -62,18 +63,21 @@ shaka.media.AdaptationSet = class {
    * |false|, calling |add| will result in it being ignored.
    *
    * @param {shaka.extern.Variant} variant
+   * @param {boolean=} compareCodecs
    * @return {boolean}
    */
-  canInclude(variant) {
-    return shaka.media.AdaptationSet.areAdaptable(this.root_, variant);
+  canInclude(variant, compareCodecs = true) {
+    return shaka.media.AdaptationSet
+        .areAdaptable(this.root_, variant, compareCodecs);
   }
 
   /**
    * @param {shaka.extern.Variant} a
    * @param {shaka.extern.Variant} b
+   * @param {boolean=} compareCodecs
    * @return {boolean}
    */
-  static areAdaptable(a, b) {
+  static areAdaptable(a, b, compareCodecs = true) {
     const AdaptationSet = shaka.media.AdaptationSet;
 
     // All variants should have audio or should all not have audio.
@@ -95,15 +99,15 @@ shaka.media.AdaptationSet = class {
         !!a.audio == !!b.audio,
         'Both should either have audio or not have audio.');
     if (a.audio && b.audio &&
-        !AdaptationSet.areAudiosCompatible_(a.audio, b.audio)) {
+        !AdaptationSet.areAudiosCompatible_(a.audio, b.audio, compareCodecs)) {
       return false;
     }
 
     goog.asserts.assert(
         !!a.video == !!b.video,
         'Both should either have video or not have video.');
     if (a.video && b.video &&
-        !AdaptationSet.areVideosCompatible_(a.video, b.video)) {
+        !AdaptationSet.areVideosCompatible_(a.video, b.video, compareCodecs)) {
       return false;
     }
 
@@ -122,10 +126,11 @@ shaka.media.AdaptationSet = class {
    *
    * @param {shaka.extern.Stream} a
    * @param {shaka.extern.Stream} b
+   * @param {boolean} compareCodecs
    * @return {boolean}
    * @private
    */
-  static areAudiosCompatible_(a, b) {
+  static areAudiosCompatible_(a, b, compareCodecs) {
     const AdaptationSet = shaka.media.AdaptationSet;
 
     // Don't adapt between channel counts, which could annoy the user
@@ -139,7 +144,7 @@ shaka.media.AdaptationSet = class {
     }
 
     // We can only adapt between base-codecs.
-    if (!AdaptationSet.canTransitionBetween_(a, b)) {
+    if (compareCodecs && !AdaptationSet.canTransitionBetween_(a, b)) {
       return false;
     }
 
@@ -161,14 +166,15 @@ shaka.media.AdaptationSet = class {
    *
    * @param {shaka.extern.Stream} a
    * @param {shaka.extern.Stream} b
+   * @param {boolean} compareCodecs
    * @return {boolean}
    * @private
    */
-  static areVideosCompatible_(a, b) {
+  static areVideosCompatible_(a, b, compareCodecs) {
     const AdaptationSet = shaka.media.AdaptationSet;
 
     // We can only adapt between base-codecs.
-    if (!AdaptationSet.canTransitionBetween_(a, b)) {
+    if (compareCodecs && !AdaptationSet.canTransitionBetween_(a, b)) {
       return false;
     }
 

diff --git a/lib/media/adaptation_set_criteria.js b/lib/media/adaptation_set_criteria.js
@@ -8,8 +8,10 @@ goog.provide('shaka.media.AdaptationSetCriteria');
 goog.provide('shaka.media.ExampleBasedCriteria');
 goog.provide('shaka.media.PreferenceBasedCriteria');
 
+goog.require('shaka.config.CodecSwitchingStrategy');
 goog.require('shaka.log');
 goog.require('shaka.media.AdaptationSet');
+goog.require('shaka.media.Capabilities');
 goog.require('shaka.util.LanguageUtils');
 goog.require('shaka.util.StreamUtils');
 
@@ -40,10 +42,14 @@ shaka.media.AdaptationSetCriteria = class {
 shaka.media.ExampleBasedCriteria = class {
   /**
    * @param {shaka.extern.Variant} example
+   * @param {shaka.config.CodecSwitchingStrategy=} codecSwitchingStrategy
    */
-  constructor(example) {
+  constructor(example,
+      codecSwitchingStrategy = shaka.config.CodecSwitchingStrategy.RELOAD) {
     /** @private {shaka.extern.Variant} */
     this.example_ = example;
+    /** @private {shaka.config.CodecSwitchingStrategy} */
+    this.codecSwitchingStrategy_ = codecSwitchingStrategy;
 
     // We can't know if role and label are really important, so we don't use
     // role and label for this.
@@ -56,15 +62,20 @@ shaka.media.ExampleBasedCriteria = class {
 
     /** @private {!shaka.media.AdaptationSetCriteria} */
     this.fallback_ = new shaka.media.PreferenceBasedCriteria(
-        example.language, role, channelCount, hdrLevel, label);
+        example.language, role, channelCount, hdrLevel, label,
+        codecSwitchingStrategy);
   }
 
   /** @override */
   create(variants) {
+    const supportsSmoothCodecTransitions = this.codecSwitchingStrategy_ ==
+      shaka.config.CodecSwitchingStrategy.SMOOTH &&
+        shaka.media.Capabilities.isChangeTypeSupported();
     // We can't assume that the example is in |variants| because it could
     // actually be from another period.
     const shortList = variants.filter((variant) => {
-      return shaka.media.AdaptationSet.areAdaptable(this.example_, variant);
+      return shaka.media.AdaptationSet.areAdaptable(this.example_, variant,
+          !supportsSmoothCodecTransitions);
     });
 
     if (shortList.length) {
@@ -90,8 +101,10 @@ shaka.media.PreferenceBasedCriteria = class {
    * @param {number} channelCount
    * @param {string} hdrLevel
    * @param {string=} label
+   * @param {shaka.config.CodecSwitchingStrategy=} codecSwitchingStrategy
    */
-  constructor(language, role, channelCount, hdrLevel, label = '') {
+  constructor(language, role, channelCount, hdrLevel, label = '',
+      codecSwitchingStrategy = shaka.config.CodecSwitchingStrategy.RELOAD) {
     /** @private {string} */
     this.language_ = language;
     /** @private {string} */
@@ -102,6 +115,8 @@ shaka.media.PreferenceBasedCriteria = class {
     this.hdrLevel_ = hdrLevel;
     /** @private {string} */
     this.label_ = label;
+    /** @private {shaka.config.CodecSwitchingStrategy} */
+    this.codecSwitchingStrategy_ = codecSwitchingStrategy;
   }
 
   /** @override */
@@ -162,12 +177,15 @@ shaka.media.PreferenceBasedCriteria = class {
       }
     }
 
+    const supportsSmoothCodecTransitions = this.codecSwitchingStrategy_ ==
+      shaka.config.CodecSwitchingStrategy.SMOOTH &&
+        shaka.media.Capabilities.isChangeTypeSupported();
+
     // Make sure we only return a valid adaptation set.
     const set = new shaka.media.AdaptationSet(current[0]);
     for (const variant of current) {
-      if (set.canInclude(variant)) {
-        set.add(variant);
-      }
+      // `add` checks combatability by calling `canInclude` internally.
+      set.add(variant, !supportsSmoothCodecTransitions);
     }
 
     return set;