Merge branch 'dev' into fix/ISS-30637-ua-upgrade-macos-version

README.md

@@ -31,14 +31,14 @@ The 100ms SDK gives you everything you need to build scalable, high-quality live
 
 **There are two ways you can add 100ms to your apps:**
 
-1. ## Custom UI
+## 1. Custom UI
    - 100ms SDKs are powerful and highly extensible to build and support all custom experiences and UI.
    - **Related packages include:** `@100mslive/react-sdk`, `@100mslive/hms-video-store` and `@100mslive/react-icons`.
    - Get started with integrating the SDK using the [How to Guide](https://www.100ms.live/docs/javascript/v2/how-to-guides/install-the-sdk/integration).
 
 > Navigate to `react-sdk` for the base React Hooks and some commonly used functionalities by clicking [here](./packages/react-sdk).
 
-2.  ## 100ms Prebuilt
+## 2. 100ms Prebuilt
     - 100ms Prebuilt is a high-level abstraction with no-code customization that enables you to embed video conferencing and/or live streaming UI—with a few lines of code.
     - **Related packages include:** `roomkit-react` and `roomkit-web`.
     - Get started with 100ms Prebuilt using the [Prebuilt Quickstart for Web](https://www.100ms.live/docs/javascript/v2/quickstart/prebuilt-quickstart).

packages/hms-video-store/src/audio-sink-manager/AudioSinkManager.ts

@@ -5,10 +5,12 @@ import { ErrorFactory } from '../error/ErrorFactory';
 import { HMSAction } from '../error/HMSAction';
 import { EventBus } from '../events/EventBus';
 import { HMSDeviceChangeEvent, HMSTrackUpdate, HMSUpdateListener } from '../interfaces';
+import { HMSAudioContextHandler } from '../internal';
 import { HMSRemoteAudioTrack } from '../media/tracks';
 import { HMSRemotePeer } from '../sdk/models/peer';
 import { Store } from '../sdk/store';
 import HMSLogger from '../utils/logger';
+import { sleep } from '../utils/timer-utils';
 
 /**
  * Following are the errors thrown when autoplay is blocked in different browsers
@@ -71,8 +73,9 @@ export class AudioSinkManager {
    */
   async unblockAutoplay() {
     if (this.autoPausedTracks.size > 0) {
-      this.unpauseAudioTracks();
+      await this.unpauseAudioTracks();
     }
+    await HMSAudioContextHandler.resumeContext();
   }
 
   init(elementId?: string) {
@@ -137,11 +140,15 @@ export class AudioSinkManager {
       );
       this.eventBus.analytics.publish(AnalyticsEventFactory.audioPlaybackError(ex));
       if (audioEl?.error?.code === MediaError.MEDIA_ERR_DECODE) {
-        await track.setVolume(0);
-        await track.setVolume(this.volume);
-        this.eventBus.analytics.publish(
-          AnalyticsEventFactory.audioRecovered('Audio recovered after media decode error'),
-        );
+        // try to wait for main execution to complete first
+        this.removeAudioElement(audioEl, track);
+        await sleep(500);
+        await this.handleTrackAdd({ track, peer, callListener: false });
+        if (!this.state.autoplayFailed) {
+          this.eventBus.analytics.publish(
+            AnalyticsEventFactory.audioRecovered('Audio recovered after media decode error'),
+          );
+        }
       }
     };
     track.setAudioElement(audioEl);
@@ -179,12 +186,12 @@ export class AudioSinkManager {
     await this.playAudioFor(track);
   };
 
-  private handleAudioDeviceChange = (event: HMSDeviceChangeEvent) => {
+  private handleAudioDeviceChange = async (event: HMSDeviceChangeEvent) => {
     // if there is no selection that means this is an init request. No need to do anything
     if (event.isUserSelection || event.error || !event.selection || event.type === 'video') {
       return;
     }
-    this.unpauseAudioTracks();
+    await this.unpauseAudioTracks();
   };
 
   /**

packages/hms-video-store/src/device-manager/DeviceManager.ts

@@ -106,7 +106,6 @@ export class DeviceManager implements HMSDeviceManager {
     // do it only on initial load.
     if (!force) {
       await this.updateToActualDefaultDevice();
-      await this.autoSelectAudioOutput();
       this.startPollingForDevices();
     }
     this.logDevices('Init');
@@ -134,12 +133,12 @@ export class DeviceManager implements HMSDeviceManager {
   }
 
   cleanup() {
-    this.initialized = false;
-    this.earpieceSelected = false;
     if (this.timer) {
       clearTimeout(this.timer);
       this.timer = null;
     }
+    this.initialized = false;
+    this.earpieceSelected = false;
     this.audioInput = [];
     this.audioOutput = [];
     this.videoInput = [];
@@ -483,8 +482,9 @@ export class DeviceManager implements HMSDeviceManager {
    * Mweb is not able to play via call channel by default, this is to switch from media channel to call channel
    */
   // eslint-disable-next-line complexity
-  private autoSelectAudioOutput = async () => {
-    if ('ondevicechange' in navigator.mediaDevices) {
+  public autoSelectAudioOutput = async () => {
+    // do this only after join so the earpiece would be selected at the right time
+    if ('ondevicechange' in navigator.mediaDevices || !this.store.getLocalPeer()?.joinedAt) {
       return;
     }
     const { bluetoothDevice, earpiece, speakerPhone, wired } = this.categorizeAudioInputDevices();

packages/hms-video-store/src/plugins/audio/HMSAudioPluginsManager.ts

@@ -4,19 +4,13 @@ import AnalyticsEventFactory from '../../analytics/AnalyticsEventFactory';
 import { ErrorFactory } from '../../error/ErrorFactory';
 import { HMSAction } from '../../error/HMSAction';
 import { EventBus } from '../../events/EventBus';
+import { HMSAudioContextHandler } from '../../internal';
 import { HMSAudioTrackSettingsBuilder } from '../../media/settings';
 import { standardMediaConstraints } from '../../media/settings/constants';
 import { HMSLocalAudioTrack } from '../../media/tracks';
 import Room from '../../sdk/models/HMSRoom';
 import HMSLogger from '../../utils/logger';
 
-const DEFAULT_SAMPLE_RATE = 48000;
-
-//Handling sample rate error in case of firefox
-const checkBrowserSupport = () => {
-  return navigator.userAgent.indexOf('Firefox') !== -1;
-};
-
 /**
  * This class manages applying different plugins on a local audio track. Plugins which need to modify the audio
  * are called in the order they were added. Plugins which do not need to modify the audio are called
@@ -50,7 +44,7 @@ export class HMSAudioPluginsManager {
     this.hmsTrack = track;
     this.pluginsMap = new Map();
     this.analytics = new AudioPluginsAnalytics(eventBus);
-    this.createAudioContext();
+    this.audioContext = HMSAudioContextHandler.getAudioContext();
     this.room = room;
   }
 
@@ -234,7 +228,6 @@ export class HMSAudioPluginsManager {
 
   //Keeping it separate since we are initializing context only once
   async closeContext() {
-    this.audioContext?.close();
     this.audioContext = undefined;
   }
 
@@ -248,15 +241,14 @@ export class HMSAudioPluginsManager {
     for (const plugin of plugins) {
       await this.addPlugin(plugin);
     }
-    this.updateProcessedTrack();
+    await this.updateProcessedTrack();
   }
 
   private async initAudioNodes() {
     if (this.audioContext) {
-      if (!this.sourceNode) {
-        const audioStream = new MediaStream([this.hmsTrack.nativeTrack]);
-        this.sourceNode = this.audioContext.createMediaStreamSource(audioStream);
-      }
+      // recreate this again, irrespective of it being already there so that the latest native track is used in source node
+      const audioStream = new MediaStream([this.hmsTrack.nativeTrack]);
+      this.sourceNode = this.audioContext.createMediaStreamSource(audioStream);
       if (!this.destinationNode) {
         this.destinationNode = this.audioContext.createMediaStreamDestination();
         this.outputTrack = this.destinationNode.stream.getAudioTracks()[0];
@@ -316,19 +308,4 @@ export class HMSAudioPluginsManager {
     plugin.stop();
     this.analytics.removed(name);
   }
-
-  private createAudioContext() {
-    if (!this.audioContext) {
-      if (checkBrowserSupport()) {
-        /**
-        Not setting default sample rate for firefox since connecting
-        audio nodes from context with different sample rate is not
-        supported in firefox
- */
-        this.audioContext = new AudioContext();
-      } else {
-        this.audioContext = new AudioContext({ sampleRate: DEFAULT_SAMPLE_RATE });
-      }
-    }
-  }
 }

packages/hms-video-store/src/reactive-store/HMSSDKActions.ts

@@ -1534,24 +1534,29 @@ export class HMSSDKActions<T extends HMSGenericTypes = { sessionStore: Record<st
     }
   }
 
+  //eslint-disable-next-line complexity
   private async addRemoveAudioPlugin(plugin: HMSAudioPlugin, action: 'add' | 'remove') {
-    if (!plugin) {
-      HMSLogger.w('Invalid plugin received in store');
-      return;
-    }
-    const trackID = this.store.getState(selectLocalAudioTrackID);
-    if (trackID) {
-      const sdkTrack = this.getLocalTrack(trackID);
-      if (sdkTrack) {
-        if (action === 'add') {
-          await (sdkTrack as SDKHMSLocalAudioTrack).addPlugin(plugin);
-        } else if (action === 'remove') {
-          await (sdkTrack as SDKHMSLocalAudioTrack).removePlugin(plugin);
+    try {
+      if (!plugin) {
+        HMSLogger.w('Invalid plugin received in store');
+        return;
+      }
+      const trackID = this.store.getState(selectLocalAudioTrackID);
+      if (trackID) {
+        const sdkTrack = this.getLocalTrack(trackID);
+        if (sdkTrack) {
+          if (action === 'add') {
+            await (sdkTrack as SDKHMSLocalAudioTrack).addPlugin(plugin);
+          } else if (action === 'remove') {
+            await (sdkTrack as SDKHMSLocalAudioTrack).removePlugin(plugin);
+          }
+          this.syncRoomState(`${action}AudioPlugin`);
+        } else {
+          this.logPossibleInconsistency(`track ${trackID} not present, unable to ${action} plugin`);
         }
-        this.syncRoomState(`${action}AudioPlugin`);
-      } else {
-        this.logPossibleInconsistency(`track ${trackID} not present, unable to ${action} plugin`);
       }
+    } catch (err) {
+      console.error(err);
     }
   }
 

packages/hms-video-store/src/sdk/index.ts

@@ -92,6 +92,7 @@ import {
   DEFAULT_PLAYLIST_AUDIO_BITRATE,
   DEFAULT_PLAYLIST_VIDEO_BITRATE,
   HAND_RAISE_GROUP_NAME,
+  LEAVE_REASON,
 } from '../utils/constants';
 import { fetchWithRetry } from '../utils/fetch';
 import decodeJWT from '../utils/jwt';
@@ -666,6 +667,11 @@ export class HMSSdk implements HMSInterface {
       await this.notifyJoin();
       this.sdkState.isJoinInProgress = false;
       await this.publish(config.settings, previewRole);
+      await this.deviceManager.autoSelectAudioOutput();
+      // Throw autoplay error even if audio context is suspended as it will be used in Audio Plugins which can lead to no audio
+      if (HMSAudioContextHandler.getAudioContext().state === 'suspended') {
+        this.listener?.onError(ErrorFactory.TracksErrors.AutoplayBlocked(HMSAction.JOIN));
+      }
     } catch (error) {
       this.analyticsTimer.end(TimedEvent.JOIN);
       this.sdkState.isJoinInProgress = false;
@@ -719,6 +725,7 @@ export class HMSSdk implements HMSInterface {
     return this.internalLeave(notifyServer);
   }
 
+  // eslint-disable-next-line complexity
   private async internalLeave(notifyServer = true, error?: HMSException) {
     const room = this.store?.getRoom();
     if (room) {
@@ -739,7 +746,7 @@ export class HMSSdk implements HMSInterface {
       // tab refresh or close. Therefore prioritise the leave action over anything else, if tab is closed/refreshed
       // we would want leave to succeed to stop stucked peer for others. The followup cleanup however is important
       // for cases where uses stays on the page post leave.
-      await this.transport?.leave(notifyServer);
+      await this.transport?.leave(notifyServer, error ? LEAVE_REASON.SDK_REQUEST : LEAVE_REASON.USER_REQUEST);
       this.cleanup();
       HMSLogger.d(this.TAG, `✅ Left room ${roomId}, peerId=${peerId}`);
     }

packages/hms-video-store/src/signal/jsonrpc/index.ts

@@ -9,6 +9,7 @@ import { PeerNotificationInfo, SendMessage } from '../../notification-manager';
 import {
   DEFAULT_SIGNAL_PING_INTERVAL,
   DEFAULT_SIGNAL_PING_TIMEOUT,
+  LEAVE_REASON,
   PONG_RESPONSE_TIMES_SIZE,
 } from '../../utils/constants';
 import HMSLogger from '../../utils/logger';
@@ -307,8 +308,8 @@ export default class JsonRpcSignal {
     return await this.call<BroadcastResponse>(HMSSignalMethod.BROADCAST, message);
   }
 
-  leave() {
-    this.notify(HMSSignalMethod.LEAVE, {});
+  leave(reason: LEAVE_REASON) {
+    this.notify(HMSSignalMethod.LEAVE, { client_reason: reason });
   }
 
   async endRoom(lock: boolean, reason: string) {

packages/hms-video-store/src/transport/index.ts

@@ -36,6 +36,7 @@ import { ISignalEventsObserver } from '../signal/ISignalEventsObserver';
 import JsonRpcSignal from '../signal/jsonrpc';
 import {
   ICE_DISCONNECTION_TIMEOUT,
+  LEAVE_REASON,
   PROTOCOL_SPEC,
   PROTOCOL_VERSION,
   PUBLISH_STATS_PUSH_INTERVAL,
@@ -359,7 +360,7 @@ export default class HMSTransport {
     }
   }
 
-  async leave(notifyServer: boolean): Promise<void> {
+  async leave(notifyServer: boolean, reason = LEAVE_REASON.USER_REQUEST): Promise<void> {
     this.retryScheduler.reset();
     this.joinParameters = undefined;
     HMSLogger.d(TAG, 'leaving in transport');
@@ -375,7 +376,7 @@ export default class HMSTransport {
       this.clearPeerConnections();
       if (notifyServer) {
         try {
-          this.signal.leave();
+          this.signal.leave(reason);
           HMSLogger.d(TAG, 'signal leave done');
         } catch (err) {
           HMSLogger.w(TAG, 'failed to send leave on websocket to server', err);

packages/hms-video-store/src/utils/constants.ts

@@ -71,3 +71,8 @@ export const DEFAULT_PLAYLIST_AUDIO_BITRATE = 64;
 export const WHITEBOARD_ORIGIN = 'https://whiteboard.100ms.live';
 
 export const WHITEBOARD_QA_ORIGIN = 'https://whiteboard-qa.100ms.live';
+
+export enum LEAVE_REASON {
+  USER_REQUEST = 'user request',
+  SDK_REQUEST = 'sdk request',
+}

packages/hms-video-store/src/utils/media.ts

@@ -1,7 +1,11 @@
 import HMSLogger from './logger';
+import { isFirefox } from './support';
 import { BuildGetMediaError } from '../error/utils';
 import { HMSTrackExceptionTrackType } from '../media/tracks/HMSTrackExceptionTrackType';
 
+// discussed with krisp team and this is their recommendation for the sample rate
+const DEFAULT_SAMPLE_RATE = 32000;
+
 export async function getLocalStream(constraints: MediaStreamConstraints): Promise<MediaStream> {
   try {
     const stream = await navigator.mediaDevices.getUserMedia(constraints);
@@ -52,9 +56,10 @@ export const HMSAudioContextHandler: HMSAudioContext = {
   audioContext: null,
   getAudioContext() {
     if (!this.audioContext) {
-      this.audioContext = new AudioContext();
+      this.audioContext = isFirefox ? new AudioContext() : new AudioContext({ sampleRate: DEFAULT_SAMPLE_RATE });
     }
-    return this.audioContext;
+
+    return this.audioContext!;
   },
   async resumeContext() {
     try {