Merge pull request #115 from GetStream/PBE-6258-Add-audio-delay-testi…

…ng-mode

Pbe 6258 add audio delay testing mode

Assets/Samples/Stream Video & Audio Chat SDK/0.7.0/Video & Audio Chat Example Project/Scripts/UI/UIManager.cs

@@ -46,8 +46,7 @@ public bool HasUserAuthorizedCameraPermission()
         {
 #if UNITY_STANDALONE
             return true; //StreamTodo: check if this is true for all platforms    
-#endif
-#if UNITY_ANDROID
+#elif UNITY_ANDROID
             return Permission.HasUserAuthorizedPermission(Permission.Camera);
 #else
             Debug.LogError($"Handling permissions not implemented for platform: " + Application.platform);
@@ -81,8 +80,7 @@ public bool HasUserAuthorizedMicrophonePermission()
         {
 #if UNITY_STANDALONE
             return true; //StreamTodo: check if this is true for all platforms    
-#endif
-#if UNITY_ANDROID
+#elif UNITY_ANDROID
             return Permission.HasUserAuthorizedPermission(Permission.Microphone);
 #else
             Debug.LogError($"Handling permissions not implemented for platform: " + Application.platform);

Packages/StreamVideo/Editor/StreamEditorTools.cs

@@ -25,6 +25,10 @@ public static void ToggleStreamDebugModeCompilerFlag()
         [MenuItem(MenuPrefix + "Toggle " + StreamLocalSfuModeEnabledCompilerFlag + " compiler flag")]
         public static void ToggleStreamLocalSfuCompilerFlag()
             => ToggleCompilerFlag(StreamLocalSfuModeEnabledCompilerFlag);
+
+        [MenuItem(MenuPrefix + "Toggle " + StreamAudioBenchmarkEnabledCompilerFlag + " compiler flag")]
+        public static void ToggleStreamAudioBenchmarkCompilerFlag()
+            => ToggleCompilerFlag(StreamAudioBenchmarkEnabledCompilerFlag);
 
         public static void BuildSampleApp()
         {
@@ -160,5 +164,6 @@ public static void SetStreamTestsEnabledCompilerFlag(string flagKeyword, bool en
         private const string StreamTestsEnabledCompilerFlag = "STREAM_TESTS_ENABLED";
         private const string StreamDebugModeEnabledCompilerFlag = "STREAM_DEBUG_ENABLED";
         private const string StreamLocalSfuModeEnabledCompilerFlag = "STREAM_LOCAL_SFU";
+        private const string StreamAudioBenchmarkEnabledCompilerFlag = "STREAM_AUDIO_BENCHMARK_ENABLED";
     }
 }

Packages/StreamVideo/Runtime/Core/LowLevelClient/RtcSession.cs

@@ -166,6 +166,12 @@ public RtcSession(SfuWebSocket sfuWebSocket, Func<IStreamCall, HttpClient> httpC
 
             var statsCollector = new UnityWebRtcStatsCollector(this, _serializer);
             _statsSender = new WebRtcStatsSender(this, statsCollector, _timeService, _logs);
+
+            //StreamTodo: enable this only if a special mode e.g. compiler flag 
+#if STREAM_AUDIO_BENCHMARK_ENABLED
+            _logs.Warning($"Audio benchmark enabled. Waiting for a special video stream to measure audio-video sync. Check {nameof(VideoAudioSyncBenchmark)} summary for more details.");
+            _videoAudioSyncBenchmark = new VideoAudioSyncBenchmark(_timeService, _logs);
+#endif
         }
 
         public void Dispose()
@@ -184,6 +190,7 @@ public void Update()
             _sfuWebSocket.Update();
             Publisher?.Update();
             _statsSender.Update();
+            _videoAudioSyncBenchmark?.Update();
 
             //StreamTodo: we could remove this if we'd maintain a collection of tracks and update them directly
             if (ActiveCall != null)
@@ -270,6 +277,7 @@ public async Task StartAsync(StreamCall call)
 
             //StreamTodo: validate when this state should set
             CallState = CallingState.Joined;
+            _videoAudioSyncBenchmark?.Init(call);
         }
 
         public async Task StopAsync()
@@ -278,6 +286,7 @@ public async Task StopAsync()
             //StreamTodo: check with js definition of "offline" 
             CallState = CallingState.Offline;
             await _sfuWebSocket.DisconnectAsync(WebSocketCloseStatus.NormalClosure, "Video session stopped");
+            _videoAudioSyncBenchmark?.Finish();
         }
 
         //StreamTodo: call by call.reconnectOrSwitchSfu()
@@ -324,6 +333,8 @@ public void TrySetVideoTrackEnabled(bool isEnabled)
         private readonly IStreamClientConfig _config;
         private readonly Func<IStreamCall, HttpClient> _httpClientFactory;
         private readonly WebRtcStatsSender _statsSender;
+        private readonly VideoAudioSyncBenchmark _videoAudioSyncBenchmark;
+        private readonly SdpMungeUtils _sdpMungeUtils = new SdpMungeUtils();
 
         private readonly List<SfuICETrickle> _pendingIceTrickleRequests = new List<SfuICETrickle>();
         private readonly PublisherVideoSettings _publisherVideoSettings = PublisherVideoSettings.Default;
@@ -340,7 +351,6 @@ private readonly Dictionary<string, VideoResolution> _videoResolutionByParticipa
         private bool _trackSubscriptionRequested;
         private bool _trackSubscriptionRequestInProgress;
 
-        private SdpMungeUtils _sdpMungeUtils = new SdpMungeUtils();
         private AudioSource _audioInput;
         private WebCamTexture _videoInput;
         private Camera _videoSceneInput;

Packages/StreamVideo/Runtime/Core/StatefulModels/Tracks/StreamAudioTrack.cs

@@ -1,10 +1,13 @@
-using Unity.WebRTC;
+using System;
+using Unity.WebRTC;
 using UnityEngine;
 
 namespace StreamVideo.Core.StatefulModels.Tracks
 {
     public class StreamAudioTrack : BaseStreamTrack<AudioStreamTrack>
     {
+        internal AudioSource TargetAudioSource;
+
         public StreamAudioTrack(AudioStreamTrack track) 
             : base(track)
         {
@@ -13,9 +16,15 @@ public StreamAudioTrack(AudioStreamTrack track)
 
         public void SetAudioSourceTarget(AudioSource audioSource)
         {
-            audioSource.SetTrack(Track);
-            audioSource.loop = true;
-            audioSource.Play();
+            if (audioSource == null)
+            {
+                throw new ArgumentNullException($"{nameof(audioSource)} cannot be null");
+            }
+
+            TargetAudioSource = audioSource;
+            TargetAudioSource.SetTrack(Track);
+            TargetAudioSource.loop = true;
+            TargetAudioSource.Play();
         }
     }
 }

Packages/StreamVideo/Runtime/Core/StatefulModels/Tracks/StreamVideoTrack.cs

@@ -7,11 +7,13 @@ namespace StreamVideo.Core.StatefulModels.Tracks
 {
     public class StreamVideoTrack : BaseStreamTrack<VideoStreamTrack>
     {
-        public StreamVideoTrack(MediaStreamTrack track) 
+        internal RenderTexture TargetTexture => _targetTexture;
+
+        public StreamVideoTrack(MediaStreamTrack track)
             : base(track)
         {
         }
-        
+
         //StreamTodo: can we remove Unity dependency? 
         public void SetRenderTarget(RawImage targetImage)
         {
@@ -39,7 +41,7 @@ internal override void Update()
                 _targetTexture = new RenderTexture(source.width, source.height, 0, RenderTextureFormat.Default);
                 _targetImage.texture = _targetTexture;
             }
-            
+
             var sizeRatio = (float)source.width / source.height;
 
             var sizeChanged = source.width != _targetTexture.width || source.height != _targetTexture.height;
@@ -55,13 +57,13 @@ internal override void Update()
                 _targetTexture.height = source.height;
                 _targetTexture.Create();
             }
-            
+
             //StreamTodo: debug this size, it can get to negative values
             var rect = _targetImage.GetComponent<RectTransform>();
             var current = rect.sizeDelta;
-            rect.sizeDelta = new Vector2(current.x, current.x * (1/sizeRatio));
-            
-            //StreamTodo: investigate if copying texture is really necessary. Perhaps we can just use the texture from the VideoStreamTrack
+            rect.sizeDelta = new Vector2(current.x, current.x * (1 / sizeRatio));
+
+            //StreamTodo: PERFORMANCE investigate if copying texture is really necessary. Perhaps we can just use the texture from the VideoStreamTrack. Test cross-platform
 
             //StreamTodo: use CopyTexture if available on this GPU
             Graphics.Blit(source, _targetTexture);