Merge pull request #116 from GetStream/PBE-6272-Reduce-microphone-del…

…ay-attempt

Fix Microphone delay  -> Sync Mic recording & playback buffers + play AudioSource right after recording started

Packages/StreamVideo/Runtime/Core/DeviceManagers/DeviceManagerBase.cs

@@ -7,14 +7,14 @@
 namespace StreamVideo.Core.DeviceManagers
 {
     public delegate void DeviceEnabledChangeHandler(bool isEnabled);
-    
+
     public delegate void
         SelectedDeviceChangeHandler<in TDeviceInfo>(TDeviceInfo previousDevice, TDeviceInfo currentDevice);
 
     internal abstract class DeviceManagerBase<TDeviceInfo> : IDeviceManager<TDeviceInfo> where TDeviceInfo : struct
     {
         public event DeviceEnabledChangeHandler IsEnabledChanged;
-        
+
         public event SelectedDeviceChangeHandler<TDeviceInfo> SelectedDeviceChanged;
 
         public bool IsEnabled
@@ -26,7 +26,7 @@ private set
                 {
                     return;
                 }
-                
+
                 _isEnabled = value;
                 IsEnabledChanged?.Invoke(IsEnabled);
             }
@@ -69,7 +69,7 @@ public Task<bool> TestDeviceAsync(TDeviceInfo device, float timeout = 1f)
         {
             const float MinTimeout = 0f;
             const float MaxTimeout = 20f;
-            
+
             if (timeout <= MinTimeout || timeout > MaxTimeout)
             {
                 throw new ArgumentOutOfRangeException(
@@ -104,15 +104,20 @@ internal DeviceManagerBase(RtcSession rtcSession, IInternalStreamVideoClient cli
             //StreamTodo: react to when video & audio streams become available and disable them if IsEnabled was set to false before the call
         }
 
+        internal void Update() => OnUpdate();
+
         protected RtcSession RtcSession { get; }
         protected IInternalStreamVideoClient Client { get; }
         protected ILogs Logs { get; }
 
-
         protected abstract void OnSetEnabled(bool isEnabled);
 
         protected abstract Task<bool> OnTestDeviceAsync(TDeviceInfo device, int msTimeout);
 
+        protected virtual void OnUpdate()
+        {
+        }
+
         protected virtual void OnDisposing()
         {
         }

Packages/StreamVideo/Runtime/Core/DeviceManagers/StreamAudioDeviceManager.cs

@@ -2,6 +2,7 @@
 using System.Collections.Generic;
 using System.Threading.Tasks;
 using StreamVideo.Core.LowLevelClient;
+using StreamVideo.Core.Utils;
 using StreamVideo.Libs.Logs;
 using UnityEngine;
 using Object = UnityEngine.Object;
@@ -23,7 +24,7 @@ protected override async Task<bool> OnTestDeviceAsync(MicrophoneDeviceInfo devic
         {
             const int sampleRate = 44100;
             var maxRecordingTime = (int)Math.Ceiling(msTimeout / 1000f);
-            
+
             var clip = Microphone.Start(device.Name, true, maxRecordingTime, sampleRate);
             if (clip == null)
             {
@@ -64,29 +65,23 @@ public void SelectDevice(MicrophoneDeviceInfo device, bool enable)
                 throw new ArgumentException($"{nameof(device)} argument is not valid. The device name is empty.");
             }
 
-            TryStopRecording();
+            TryStopRecording(device);
 
             SelectedDevice = device;
-
-            var targetAudioSource = GetOrCreateTargetAudioSource();
-
-            targetAudioSource.clip
-                = Microphone.Start(SelectedDevice.Name, true, 1, AudioSettings.outputSampleRate);
-            targetAudioSource.loop = true;
-
+
 #if STREAM_DEBUG_ENABLED
             Logs.Info($"Changed microphone device to: {SelectedDevice}");
 #endif
-            
+
             //StreamTodo: in some cases starting the mic recording before the call was causing the recorded audio being played in speakers
             //I think the reason was that AudioSource was being captured by an AudioListener but once I've joined the call, this disappeared
             //Check if we can have this AudioSource to be ignored by AudioListener's or otherwise mute it when there is not active call session
 
             SetEnabled(enable);
         }
-        
+
         //StreamTodo: https://docs.unity3d.com/ScriptReference/AudioSource-ignoreListenerPause.html perhaps this should be enabled so that AudioListener doesn't affect recorded audio
-        
+
         internal StreamAudioDeviceManager(RtcSession rtcSession, IInternalStreamVideoClient client, ILogs logs)
             : base(rtcSession, client, logs)
         {
@@ -96,26 +91,51 @@ protected override void OnSetEnabled(bool isEnabled)
         {
             if (isEnabled && SelectedDevice.IsValid && !GetOrCreateTargetAudioSource().isPlaying)
             {
-                GetOrCreateTargetAudioSource().Play();
+                TryStopRecording(SelectedDevice);
+
+                var targetAudioSource = GetOrCreateTargetAudioSource();
+
+                // StreamTodo: use Microphone.GetDeviceCaps to get min/max frequency -> validate it and pass to Microphone.Start
+
+                targetAudioSource.clip
+                    = Microphone.Start(SelectedDevice.Name, loop: true, lengthSec: 10, AudioSettings.outputSampleRate);
+                targetAudioSource.loop = true;
+
+                using (new DebugStopwatchScope(Logs, "Waiting for microphone to start recording"))
+                {
+                    while (!(Microphone.GetPosition(SelectedDevice.Name) > 0))
+                    {
+                        // StreamTodo: add timeout. Otherwise might hang application
+                    }
+                }
+
+                targetAudioSource.Play();
             }
 
             if (!isEnabled)
             {
-                TryStopRecording();
+                TryStopRecording(SelectedDevice);
             }
-            
+
             RtcSession.TrySetAudioTrackEnabled(isEnabled);
         }
 
+        protected override void OnUpdate()
+        {
+            base.OnUpdate();
+
+            TrySyncMicrophoneAudioSourceReadPosWithMicrophoneWritePos();
+        }
+
         protected override void OnDisposing()
         {
-            TryStopRecording();
-            
+            TryStopRecording(SelectedDevice);
+
             if (_targetAudioSourceContainer != null)
             {
                 Object.Destroy(_targetAudioSourceContainer);
             }
-            
+
             base.OnDisposing();
         }
 
@@ -141,22 +161,37 @@ private AudioSource GetOrCreateTargetAudioSource()
                 hideFlags = HideFlags.HideInHierarchy | HideFlags.DontSave
 #endif
             };
-            
+
             _targetAudioSource = _targetAudioSourceContainer.AddComponent<AudioSource>();
             Client.SetAudioInputSource(_targetAudioSource);
             return _targetAudioSource;
         }
+
+        private static void TryStopRecording(MicrophoneDeviceInfo device)
+        {
+            if (!device.IsValid)
+            {
+                return;
+            }
+
+            if (Microphone.IsRecording(device.Name))
+            {
+                Microphone.End(device.Name);
+            }
+        }
 
-        private void TryStopRecording()
+        private void TrySyncMicrophoneAudioSourceReadPosWithMicrophoneWritePos()
         {
-            if (!SelectedDevice.IsValid)
+            var isRecording = IsEnabled && SelectedDevice.IsValid && Microphone.IsRecording(SelectedDevice.Name) && _targetAudioSource != null;
+            if (!isRecording)
             {
                 return;
             }
 
-            if (Microphone.IsRecording(SelectedDevice.Name))
+            var microphonePosition = Microphone.GetPosition(SelectedDevice.Name);
+            if (microphonePosition >= 0 && _targetAudioSource.timeSamples > microphonePosition)
             {
-                Microphone.End(SelectedDevice.Name);
+                _targetAudioSource.timeSamples = microphonePosition;
             }
         }
     }

Packages/StreamVideo/Runtime/Core/StreamVideoClient.cs

@@ -46,9 +46,12 @@ public class StreamVideoClient : IStreamVideoClient, IInternalStreamVideoClient
         public IStreamCall ActiveCall => InternalLowLevelClient.RtcSession.ActiveCall;
 
         public bool IsConnected => InternalLowLevelClient.ConnectionState == ConnectionState.Connected;
+
+        public IStreamVideoDeviceManager VideoDeviceManager => _videoDeviceManager;
+        public IStreamAudioDeviceManager AudioDeviceManager => _audioDeviceManager;
 
-        public IStreamVideoDeviceManager VideoDeviceManager { get; }
-        public IStreamAudioDeviceManager AudioDeviceManager { get; }
+        private StreamVideoDeviceManager _videoDeviceManager;
+        private StreamAudioDeviceManager _audioDeviceManager;
 
         /// <summary>
         /// Use this method to create the Video Client. You should have only one instance of this class
@@ -177,7 +180,12 @@ public async Task<IStreamVideoUser> ConnectUserAsync(AuthCredentials credentials
         }
 
         //StreamTodo: change public to explicit interface
-        public void Update() => InternalLowLevelClient.Update();
+        public void Update()
+        {
+            InternalLowLevelClient.Update();
+            _videoDeviceManager?.Update();
+            _audioDeviceManager?.Update();
+        }
 
         //StreamTodo: change public to explicit interface
         public IEnumerator WebRTCUpdateCoroutine() => WebRTC.Update();
@@ -386,8 +394,8 @@ private StreamVideoClient(IWebsocketClient coordinatorWebSocket, IWebsocketClien
             _cache = new Cache(this, serializer, _logs);
             InternalLowLevelClient.RtcSession.SetCache(_cache);
 
-            VideoDeviceManager = new StreamVideoDeviceManager(InternalLowLevelClient.RtcSession, this, _logs);
-            AudioDeviceManager = new StreamAudioDeviceManager(InternalLowLevelClient.RtcSession, this, _logs);
+            _videoDeviceManager = new StreamVideoDeviceManager(InternalLowLevelClient.RtcSession, this, _logs);
+            _audioDeviceManager = new StreamAudioDeviceManager(InternalLowLevelClient.RtcSession, this, _logs);
 
             SubscribeTo(InternalLowLevelClient);
         }

Packages/StreamVideo/Runtime/Core/Utils/DebugStopwatchScope.cs

@@ -0,0 +1,55 @@
+using System;
+using System.Diagnostics;
+using StreamVideo.Libs.Logs;
+
+namespace StreamVideo.Core.Utils
+{
+    /// <summary>
+    /// [ONLY IF STREAM_DEBUG_ENABLED] Measure scope execution. 
+    /// </summary>
+    internal class DebugStopwatchScope : IDisposable
+    {
+        public enum Units
+        {
+            MilliSeconds,
+            Seconds,
+            Minutes
+        }
+
+        public DebugStopwatchScope(ILogs logs, string message, Units units = Units.MilliSeconds)
+        {
+#if STREAM_DEBUG_ENABLED
+            _message = !message.IsNullOrEmpty() ? message : throw new ArgumentException("");
+            _units = units;
+            _logs = logs ?? throw new ArgumentNullException(nameof(logs));
+            _sw = new Stopwatch();
+            _sw.Start();
+#endif
+        }
+
+        private readonly ILogs _logs;
+        private readonly Stopwatch _sw;
+        private readonly Units _units;
+        private readonly string _message;
+
+        public void Dispose()
+        {
+#if STREAM_DEBUG_ENABLED
+            _sw.Stop();
+            _logs.Warning($"`{_message}` executed in: {GetTimeLog()}");
+#endif
+        }
+
+        private string GetTimeLog()
+        {
+            switch (_units)
+            {
+                case Units.MilliSeconds: return $"{_sw.Elapsed.TotalMilliseconds} ms";
+                case Units.Seconds: return $"{_sw.Elapsed.TotalSeconds} s";
+                case Units.Minutes: return $"{_sw.Elapsed.TotalMinutes} min";
+                default:
+                    throw new ArgumentOutOfRangeException();
+            }
+        }
+    }
+}

Packages/StreamVideo/Tests/Editor/RepositoryTests.cs

@@ -101,7 +101,6 @@ private async Task Dtos_do_not_contain_json_required_always_flag_Async()
             var internalDtosDirectory = Path.Combine(packageSourcePath, "Runtime", "Core", "InternalDTO");
 
             const string internalDtoNamespace = "namespace StreamVideo.Core.InternalDTO";
-            const string sfuNamespace = "namespace StreamVideo.v1.Sfu";
             var sfuNamespaces = new string[] { "namespace StreamVideo.v1.Sfu", "StreamVideo.Core.Sfu" };
             const string jsonAlwaysRequiredFlag = "Newtonsoft.Json.Required.Always";