From 97a62c1f9ffafa65187e465dd19c10efee87ed60 Mon Sep 17 00:00:00 2001 From: Tomasz Rybakiewicz Date: Thu, 20 Jun 2024 17:00:43 -0400 Subject: [PATCH 1/3] NAVAND-2991 - Optimized TextToSpeech usage Moved language setting and speak calls to a background thread. --- LICENSE.md | 10 ++- changelog/unreleased/bugfixes/NAVAND-2991.md | 1 + .../examples/core/MapboxVoiceActivity.kt | 34 ++++---- libnavui-voice/build.gradle | 1 + .../voice/api/VoiceInstructionsTextPlayer.kt | 85 ++++++++++++------- 5 files changed, 81 insertions(+), 50 deletions(-) create mode 100644 changelog/unreleased/bugfixes/NAVAND-2991.md diff --git a/LICENSE.md b/LICENSE.md index 3beb69c7f20..1bfaa410841 100644 --- a/LICENSE.md +++ b/LICENSE.md @@ -2469,7 +2469,7 @@ License: [The Apache Software License, Version 2.0](http://www.apache.org/licens =========================================================================== Mapbox Navigation uses portions of the Android Support Library compat (The Support Library is a static library that you can add to your Android application in order to use APIs that are either not available for older platform versions or utility APIs that aren't a part of the framework APIs. Compatible on devices running API 14 or later.). -URL: [https://developer.android.com/jetpack/androidx/releases/core#1.5.0](https://developer.android.com/jetpack/androidx/releases/core#1.5.0) +URL: [https://developer.android.com/jetpack/androidx/releases/core#1.6.0](https://developer.android.com/jetpack/androidx/releases/core#1.6.0) License: [The Apache Software License, Version 2.0](http://www.apache.org/licenses/LICENSE-2.0.txt) =========================================================================== @@ -2544,8 +2544,14 @@ License: [Apache 2.0](https://www.apache.org/licenses/LICENSE-2.0.txt) =========================================================================== +Mapbox Navigation uses portions of the Core Kotlin Extensions (Kotlin extensions for 'core' artifact). +URL: [https://developer.android.com/jetpack/androidx/releases/core#1.6.0](https://developer.android.com/jetpack/androidx/releases/core#1.6.0) +License: [The Apache Software License, Version 2.0](http://www.apache.org/licenses/LICENSE-2.0.txt) + +=========================================================================== + Mapbox Navigation uses portions of the Experimental annotation (Java annotation for use on unstable Android API surfaces. When used in conjunction with the Experimental annotation lint checks, this annotation provides functional parity with Kotlin's Experimental annotation.). -URL: [https://developer.android.com/jetpack/androidx](https://developer.android.com/jetpack/androidx) +URL: [https://developer.android.com/jetpack/androidx/releases/annotation#1.1.0](https://developer.android.com/jetpack/androidx/releases/annotation#1.1.0) License: [The Apache Software License, Version 2.0](http://www.apache.org/licenses/LICENSE-2.0.txt) =========================================================================== diff --git a/changelog/unreleased/bugfixes/NAVAND-2991.md b/changelog/unreleased/bugfixes/NAVAND-2991.md new file mode 100644 index 00000000000..4ae5b84cd0d --- /dev/null +++ b/changelog/unreleased/bugfixes/NAVAND-2991.md @@ -0,0 +1 @@ +- Fixed UI jank caused by on-device TextToSpeech player. \ No newline at end of file diff --git a/examples/src/main/java/com/mapbox/navigation/examples/core/MapboxVoiceActivity.kt b/examples/src/main/java/com/mapbox/navigation/examples/core/MapboxVoiceActivity.kt index 8eba9e210bb..8642b717de5 100644 --- a/examples/src/main/java/com/mapbox/navigation/examples/core/MapboxVoiceActivity.kt +++ b/examples/src/main/java/com/mapbox/navigation/examples/core/MapboxVoiceActivity.kt @@ -6,6 +6,7 @@ import android.location.Location import android.os.Bundle import android.view.View import androidx.appcompat.app.AppCompatActivity +import androidx.lifecycle.coroutineScope import androidx.lifecycle.lifecycleScope import com.mapbox.api.directions.v5.models.DirectionsRoute import com.mapbox.api.directions.v5.models.RouteOptions @@ -69,6 +70,7 @@ import com.mapbox.navigation.ui.voice.model.SpeechVolume import com.mapbox.navigation.ui.voice.options.VoiceInstructionsPlayerOptions import com.mapbox.navigation.utils.internal.ifNonNull import com.mapbox.navigation.utils.internal.logD +import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.launch import java.util.Locale @@ -112,7 +114,7 @@ class MapboxVoiceActivity : AppCompatActivity(), OnMapLongClickListener { * has to be played. [MapboxVoiceInstructionsPlayer] should be instantiated in * `Activity#onCreate`. */ - private lateinit var voiceInstructionsPlayer: MapboxVoiceInstructionsPlayer + private var voiceInstructionsPlayer: MapboxVoiceInstructionsPlayer? = null private val routeLineResources: RouteLineResources by lazy { RouteLineResources.Builder().build() @@ -163,7 +165,7 @@ class MapboxVoiceActivity : AppCompatActivity(), OnMapLongClickListener { logD("play(fallback): '${error.fallback.announcement}'", TAG) // The data obtained in the form of an error is played using // voiceInstructionsPlayer. - voiceInstructionsPlayer.play( + voiceInstructionsPlayer?.play( error.fallback, voiceInstructionsPlayerCallback ) @@ -172,7 +174,7 @@ class MapboxVoiceActivity : AppCompatActivity(), OnMapLongClickListener { logD("play: '${value.announcement.announcement}'", TAG) // The data obtained in the form of speech announcement is played using // voiceInstructionsPlayer. - voiceInstructionsPlayer.play( + voiceInstructionsPlayer?.play( value.announcement, voiceInstructionsPlayerCallback ) @@ -220,7 +222,7 @@ class MapboxVoiceActivity : AppCompatActivity(), OnMapLongClickListener { RoutesObserver { result -> // Every time a new route is obtained make sure to cancel the [MapboxSpeechApi] and // clear the [MapboxVoiceInstructionsPlayer] speechApi.cancel() - voiceInstructionsPlayer.clear() + voiceInstructionsPlayer?.clear() if (result.navigationRoutes.isNotEmpty()) { lifecycleScope.launch { routeLineApi.setNavigationRoutes( @@ -300,7 +302,7 @@ class MapboxVoiceActivity : AppCompatActivity(), OnMapLongClickListener { } binding.addPlay.setOnClickListener { - voiceInstructionsPlayer.play( + voiceInstructionsPlayer?.play( SpeechAnnouncement.Builder("Test hybrid speech player.").build(), voiceInstructionsPlayerCallback ) @@ -323,10 +325,10 @@ class MapboxVoiceActivity : AppCompatActivity(), OnMapLongClickListener { private fun handleSoundState(value: Boolean) { if (value) { // This is used to set the speech volume to mute. - voiceInstructionsPlayer.volume(SpeechVolume(0.0f)) + voiceInstructionsPlayer?.volume(SpeechVolume(0.0f)) } else { // This is used to set the speech volume to max - voiceInstructionsPlayer.volume(SpeechVolume(1.0f)) + voiceInstructionsPlayer?.volume(SpeechVolume(1.0f)) } isMuted = value } @@ -387,13 +389,15 @@ class MapboxVoiceActivity : AppCompatActivity(), OnMapLongClickListener { setLocationProvider(navigationLocationProvider) enabled = true } - voiceInstructionsPlayer = MapboxVoiceInstructionsPlayer( - this, - Locale.US.toLanguageTag(), - VoiceInstructionsPlayerOptions.Builder() - .abandonFocusDelay(PLAYER_ABANDON_FOCUS_DELAY) - .build() - ) + lifecycle.coroutineScope.launch(Dispatchers.Default) { + voiceInstructionsPlayer = MapboxVoiceInstructionsPlayer( + applicationContext, + Locale.US.toLanguageTag(), + VoiceInstructionsPlayerOptions.Builder() + .abandonFocusDelay(PLAYER_ABANDON_FOCUS_DELAY) + .build() + ) + } init() voiceInstructionsPrefetcher.onAttached(mapboxNavigation) } @@ -428,7 +432,7 @@ class MapboxVoiceActivity : AppCompatActivity(), OnMapLongClickListener { mapboxNavigation.onDestroy() speechApi.cancel() voiceInstructionsPrefetcher.onDetached(mapboxNavigation) - voiceInstructionsPlayer.shutdown() + voiceInstructionsPlayer?.shutdown() } override fun onMapLongClick(point: Point): Boolean { diff --git a/libnavui-voice/build.gradle b/libnavui-voice/build.gradle index 22bcbe8bbc3..1658735e666 100644 --- a/libnavui-voice/build.gradle +++ b/libnavui-voice/build.gradle @@ -50,6 +50,7 @@ dependencies { // androidX implementation dependenciesList.androidXConstraintLayout implementation dependenciesList.androidXAppCompat + implementation dependenciesList.androidXCoreKtx apply from: "../gradle/unit-testing-dependencies.gradle" testImplementation(project(':libtesting-utils')) diff --git a/libnavui-voice/src/main/java/com/mapbox/navigation/ui/voice/api/VoiceInstructionsTextPlayer.kt b/libnavui-voice/src/main/java/com/mapbox/navigation/ui/voice/api/VoiceInstructionsTextPlayer.kt index 18eb20dbf45..0c128693474 100644 --- a/libnavui-voice/src/main/java/com/mapbox/navigation/ui/voice/api/VoiceInstructionsTextPlayer.kt +++ b/libnavui-voice/src/main/java/com/mapbox/navigation/ui/voice/api/VoiceInstructionsTextPlayer.kt @@ -4,9 +4,14 @@ import android.content.Context import android.speech.tts.TextToSpeech import android.speech.tts.UtteranceProgressListener import androidx.annotation.VisibleForTesting +import androidx.core.os.trace import com.mapbox.navigation.ui.voice.model.SpeechAnnouncement import com.mapbox.navigation.ui.voice.model.SpeechVolume +import com.mapbox.navigation.utils.internal.InternalJobControlFactory.createDefaultScopeJobControl +import com.mapbox.navigation.utils.internal.logD import com.mapbox.navigation.utils.internal.logE +import kotlinx.coroutines.cancelChildren +import kotlinx.coroutines.launch import java.util.Locale /** @@ -25,18 +30,18 @@ internal class VoiceInstructionsTextPlayer( internal var isLanguageSupported: Boolean = false private var textToSpeechInitStatus: Int? = null + private val jobControl = createDefaultScopeJobControl() @VisibleForTesting - internal val textToSpeech = + internal val textToSpeech = trace(TRACE_GET_TTS) { TextToSpeechProvider.getTextToSpeech(context.applicationContext) { status -> textToSpeechInitStatus = status if (status == TextToSpeech.SUCCESS) { initializeWithLanguage(Locale(language)) - if (isLanguageSupported) { - setUpUtteranceProgressListener() - } + setUpUtteranceProgressListener() } } + } @VisibleForTesting internal var volumeLevel: Float = DEFAULT_VOLUME_LEVEL @@ -69,14 +74,11 @@ internal class VoiceInstructionsTextPlayer( "Only one announcement can be played at a time." } currentPlay = announcement - val announcement = announcement.announcement - if (isLanguageSupported && announcement.isNotBlank()) { - play(announcement) + val text = announcement.announcement + if (isLanguageSupported && text.isNotBlank()) { + play(text) } else { - logE( - "$LANGUAGE_NOT_SUPPORTED or announcement from state is blank", - LOG_CATEGORY - ) + logE { "$LANGUAGE_NOT_SUPPORTED or announcement from state is blank" } donePlaying() } } @@ -108,6 +110,7 @@ internal class VoiceInstructionsTextPlayer( * the announcement should end immediately and any announcements queued should be cleared. */ override fun shutdown() { + jobControl.job.cancelChildren() textToSpeech.setOnUtteranceProgressListener(null) textToSpeech.shutdown() currentPlay = null @@ -116,16 +119,20 @@ internal class VoiceInstructionsTextPlayer( @VisibleForTesting internal fun initializeWithLanguage(language: Locale) { - isLanguageSupported = if (playerAttributes.options.checkIsLanguageAvailable) { - textToSpeech.isLanguageAvailable(language) == TextToSpeech.LANG_AVAILABLE - } else { - true - } - if (!isLanguageSupported) { - logE(LANGUAGE_NOT_SUPPORTED, LOG_CATEGORY) - return + jobControl.scope.launch { + trace(TRACE_INIT_LANG) { + isLanguageSupported = if (playerAttributes.options.checkIsLanguageAvailable) { + textToSpeech.isLanguageAvailable(language) == TextToSpeech.LANG_AVAILABLE + } else { + true + } + if (!isLanguageSupported) { + logE { LANGUAGE_NOT_SUPPORTED } + return@trace + } + textToSpeech.language = language + } } - textToSpeech.language = language } private fun setUpUtteranceProgressListener() { @@ -136,12 +143,12 @@ internal class VoiceInstructionsTextPlayer( override fun onError(utteranceId: String?) { // Deprecated, may be called due to https://issuetracker.google.com/issues/138321382 - logE("Unexpected TextToSpeech error", LOG_CATEGORY) + logE { "Unexpected TextToSpeech error" } donePlaying() } override fun onError(utteranceId: String?, errorCode: Int) { - logE("TextToSpeech error: $errorCode", LOG_CATEGORY) + logE { "TextToSpeech error: $errorCode" } donePlaying() } @@ -163,18 +170,23 @@ internal class VoiceInstructionsTextPlayer( } private fun play(announcement: String) { - val currentBundle = BundleProvider.retrieveBundle() - val bundle = currentBundle.apply { - putFloat(TextToSpeech.Engine.KEY_PARAM_VOLUME, volumeLevel) + logD { "play: $announcement" } + jobControl.scope.launch { + trace(TRACE_PLAY) { + val currentBundle = BundleProvider.retrieveBundle() + val bundle = currentBundle.apply { + putFloat(TextToSpeech.Engine.KEY_PARAM_VOLUME, volumeLevel) + } + playerAttributes.applyOn(textToSpeech, bundle) + + textToSpeech.speak( + announcement, + TextToSpeech.QUEUE_FLUSH, + bundle, + DEFAULT_UTTERANCE_ID + ) + } } - playerAttributes.applyOn(textToSpeech, bundle) - - textToSpeech.speak( - announcement, - TextToSpeech.QUEUE_FLUSH, - bundle, - DEFAULT_UTTERANCE_ID - ) } private companion object { @@ -184,5 +196,12 @@ internal class VoiceInstructionsTextPlayer( private const val DEFAULT_UTTERANCE_ID = "default_id" private const val DEFAULT_VOLUME_LEVEL = 1.0f private const val MUTE_VOLUME_LEVEL = 0.0f + + private const val TRACE_GET_TTS = "VoiceInstructionsTextPlayer.getTextToSpeech" + private const val TRACE_INIT_LANG = "VoiceInstructionsTextPlayer.initializeWithLanguage" + private const val TRACE_PLAY = "VoiceInstructionsTextPlayer.play" + + private inline fun logD(msg: () -> String) = logD(LOG_CATEGORY, msg) + private inline fun logE(msg: () -> String) = logE(LOG_CATEGORY, msg) } } From 9d94f8d0de88c9d9af3baae063ae0cfcd4ca1763 Mon Sep 17 00:00:00 2001 From: runner Date: Thu, 20 Jun 2024 21:11:40 +0000 Subject: [PATCH 2/3] Rename changelog files --- changelog/unreleased/bugfixes/{NAVAND-2991.md => 7833.md} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename changelog/unreleased/bugfixes/{NAVAND-2991.md => 7833.md} (100%) diff --git a/changelog/unreleased/bugfixes/NAVAND-2991.md b/changelog/unreleased/bugfixes/7833.md similarity index 100% rename from changelog/unreleased/bugfixes/NAVAND-2991.md rename to changelog/unreleased/bugfixes/7833.md From 1901e79ddbccd1ccbc61cbaa05fe4f80b2d79ad5 Mon Sep 17 00:00:00 2001 From: Tomasz Rybakiewicz Date: Thu, 20 Jun 2024 17:23:31 -0400 Subject: [PATCH 3/3] NAVAND-2991 - Optimized TextToSpeech usage Unit-test fix --- .../voice/api/VoiceInstructionsTextPlayerTest.kt | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/libnavui-voice/src/test/java/com/mapbox/navigation/ui/voice/api/VoiceInstructionsTextPlayerTest.kt b/libnavui-voice/src/test/java/com/mapbox/navigation/ui/voice/api/VoiceInstructionsTextPlayerTest.kt index 24b15f82723..64ad6e9d86b 100644 --- a/libnavui-voice/src/test/java/com/mapbox/navigation/ui/voice/api/VoiceInstructionsTextPlayerTest.kt +++ b/libnavui-voice/src/test/java/com/mapbox/navigation/ui/voice/api/VoiceInstructionsTextPlayerTest.kt @@ -7,15 +7,19 @@ import android.speech.tts.TextToSpeech.LANG_AVAILABLE import android.speech.tts.TextToSpeech.LANG_NOT_SUPPORTED import android.speech.tts.TextToSpeech.OnInitListener import com.mapbox.navigation.testing.LoggingFrontendTestRule +import com.mapbox.navigation.testing.MainCoroutineRule import com.mapbox.navigation.ui.voice.model.SpeechAnnouncement import com.mapbox.navigation.ui.voice.model.SpeechVolume import com.mapbox.navigation.ui.voice.options.VoiceInstructionsPlayerOptions +import com.mapbox.navigation.utils.internal.InternalJobControlFactory +import com.mapbox.navigation.utils.internal.JobControl import io.mockk.clearMocks import io.mockk.every import io.mockk.mockk import io.mockk.mockkObject import io.mockk.unmockkObject import io.mockk.verify +import kotlinx.coroutines.job import org.junit.After import org.junit.Assert.assertEquals import org.junit.Before @@ -25,6 +29,9 @@ import java.util.Locale class VoiceInstructionsTextPlayerTest { + @get:Rule + var coroutineRule = MainCoroutineRule() + @get:Rule val loggerRule = LoggingFrontendTestRule() @@ -33,6 +40,13 @@ class VoiceInstructionsTextPlayerTest { @Before fun setUp() { + mockkObject(InternalJobControlFactory) + every { + InternalJobControlFactory.createDefaultScopeJobControl() + } answers { + val defaultScope = coroutineRule.createTestScope() + JobControl(defaultScope.coroutineContext.job, defaultScope) + } mockkObject(BundleProvider) mockkObject(TextToSpeechProvider) every { BundleProvider.retrieveBundle() } returns mockedBundle @@ -41,6 +55,7 @@ class VoiceInstructionsTextPlayerTest { @After fun tearDown() { + unmockkObject(InternalJobControlFactory) unmockkObject(BundleProvider) unmockkObject(TextToSpeechProvider) }