From 648d1c204ab223721605fe0f1ca3cfccbe3b7bcc Mon Sep 17 00:00:00 2001 From: Quentin Gliosca Date: Mon, 28 Nov 2022 13:24:10 +0100 Subject: [PATCH 01/27] Modify ContentTokenizer --- .../navigator/tts/PublicationSpeechSynthesizer.kt | 3 ++- .../services/content/ContentTokenizer.kt | 15 ++++++++++----- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/tts/PublicationSpeechSynthesizer.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/tts/PublicationSpeechSynthesizer.kt index a04cffe01f..0d32c68956 100644 --- a/readium/navigator/src/main/java/org/readium/r2/navigator/tts/PublicationSpeechSynthesizer.kt +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/tts/PublicationSpeechSynthesizer.kt @@ -96,7 +96,8 @@ class PublicationSpeechSynthesizer private constructor( val defaultTokenizerFactory: (Language?) -> ContentTokenizer = { language -> TextContentTokenizer( unit = TextUnit.Sentence, - defaultLanguage = language + language = language, + overrideContentLanguage = false ) } diff --git a/readium/shared/src/main/java/org/readium/r2/shared/publication/services/content/ContentTokenizer.kt b/readium/shared/src/main/java/org/readium/r2/shared/publication/services/content/ContentTokenizer.kt index 48472caf60..552639aa18 100644 --- a/readium/shared/src/main/java/org/readium/r2/shared/publication/services/content/ContentTokenizer.kt +++ b/readium/shared/src/main/java/org/readium/r2/shared/publication/services/content/ContentTokenizer.kt @@ -26,7 +26,8 @@ fun interface ContentTokenizer : Tokenizer */ @ExperimentalReadiumApi class TextContentTokenizer( - private val defaultLanguage: Language?, + private val language: Language?, + private val overrideContentLanguage: Boolean = false, private val contextSnippetLength: Int = 50, private val textTokenizerFactory: (Language?) -> TextTokenizer ) : ContentTokenizer { @@ -34,9 +35,10 @@ class TextContentTokenizer( /** * A [ContentTokenizer] using the default [TextTokenizer] to split the text of the [Content.Element]. */ - constructor(defaultLanguage: Language?, unit: TextUnit) : this( - defaultLanguage = defaultLanguage, - textTokenizerFactory = { language -> DefaultTextContentTokenizer(unit, language) } + constructor(language: Language?, overrideContentLanguage: Boolean, unit: TextUnit) : this( + language = language, + overrideContentLanguage = overrideContentLanguage, + textTokenizerFactory = { contentLanguage -> DefaultTextContentTokenizer(unit, contentLanguage) }, ) override fun tokenize(data: Content.Element): List = listOf( @@ -50,7 +52,7 @@ class TextContentTokenizer( ) private fun tokenize(segment: Content.TextElement.Segment): List = - textTokenizerFactory(segment.language ?: defaultLanguage).tokenize(segment.text) + textTokenizerFactory(resolveSegmentLanguage(segment)).tokenize(segment.text) .map { range -> segment.copy( locator = segment.locator.copy(text = extractTextContextIn(segment.text, range)), @@ -58,6 +60,9 @@ class TextContentTokenizer( ) } + private fun resolveSegmentLanguage(segment: Content.TextElement.Segment): Language? = + segment.language.takeUnless { overrideContentLanguage } ?: language + private fun extractTextContextIn(string: String, range: IntRange): Locator.Text { val after = string.substring(range.last, (range.last + contextSnippetLength).coerceAtMost(string.length)) val before = string.substring((range.first - contextSnippetLength).coerceAtLeast(0), range.first) From 92cafbf66c3c63701510ef6964df182aad8dbc7d Mon Sep 17 00:00:00 2001 From: Quentin Gliosca Date: Sat, 7 Jan 2023 16:27:22 +0100 Subject: [PATCH 02/27] Add TTS in background --- gradle/libs.versions.toml | 5 + readium/navigator/build.gradle.kts | 1 + .../media3/androidtts/AndroidTtsEngine.kt | 205 ++++++++ .../androidtts/AndroidTtsEngineProvider.kt | 41 ++ .../androidtts/AndroidTtsPreferences.kt | 29 ++ .../androidtts/AndroidTtsPreferencesEditor.kt | 84 ++++ .../AndroidTtsPreferencesFilters.kt | 36 ++ .../AndroidTtsPreferencesSerializer.kt | 24 + .../media3/androidtts/AndroidTtsSettings.kt | 19 + .../androidtts/AndroidTtsSettingsResolver.kt | 26 + .../media3/androidtts/AndroidTtsVoice.kt | 33 ++ .../media3/api/DefaultMetadataFactory.kt | 75 +++ .../media3/api/DefaultMetadataProvider.kt | 16 + .../media3/api/MediaMetadataFactory.kt | 27 ++ .../r2/navigator/media3/api/MediaNavigator.kt | 63 +++ .../media3/api/MediaNavigatorInternal.kt | 77 +++ .../navigator/media3/api/MetadataProvider.kt | 14 + .../media3/api/SynchronizedPlayback.kt | 20 + .../media3/exoplayer/ExoPlayerDataSource.kt | 148 ++++++ .../exoplayer/ExoPlayerEngineProvider.kt | 56 +++ .../media3/exoplayer/ExoPlayerPreferences.kt | 22 + .../exoplayer/ExoPlayerPreferencesEditor.kt | 21 + .../exoplayer/ExoPlayerPreferencesFilters.kt | 30 ++ .../ExoPlayerPreferencesSerializer.kt | 24 + .../media3/exoplayer/ExoPlayerSettings.kt | 15 + .../exoplayer/ExoPlayerSettingsResolver.kt | 23 + .../media3/player/DurationSerializer.kt | 28 ++ .../media3/player/MediaEngineProvider.kt | 35 ++ .../navigator/media3/player/PlayerLocator.kt | 26 + .../media3/player/PlayerNavigator.kt | 125 +++++ .../media3/player/PlayerNavigatorFactory.kt | 59 +++ .../media3/player/PlayerNavigatorInternal.kt | 53 +++ .../navigator/media3/player/PlayerPlayback.kt | 17 + .../SynchronizedNarrationNavigator.kt | 70 +++ .../SynchronizedNarrationNavigatorInternal.kt | 50 ++ .../syncnarr/SynchronizedNarrationPlayback.kt | 19 + .../media3/tts2/TtsContentIterator.kt | 163 +++++++ .../r2/navigator/media3/tts2/TtsEngine.kt | 77 +++ .../navigator/media3/tts2/TtsEngineFacade.kt | 172 +++++++ .../media3/tts2/TtsEngineFacadeListener.kt | 14 + .../media3/tts2/TtsEngineFacadePlayback.kt | 26 + .../media3/tts2/TtsEngineListener.kt | 88 ++++ .../media3/tts2/TtsEngineProvider.kt | 21 + .../r2/navigator/media3/tts2/TtsLocator.kt | 70 +++ .../r2/navigator/media3/tts2/TtsNavigator.kt | 148 ++++++ .../media3/tts2/TtsNavigatorFactory.kt | 80 ++++ .../media3/tts2/TtsNavigatorInternal.kt | 114 +++++ .../media3/tts2/TtsNavigatorListener.kt | 14 + .../r2/navigator/media3/tts2/TtsPlayback.kt | 18 + .../r2/navigator/media3/tts2/TtsPlayer.kt | 448 ++++++++++++++++++ .../navigator/media3/tts2/TtsPreferences.kt | 17 + .../r2/navigator/media3/tts2/TtsSettings.kt | 17 + .../r2/navigator/media3/tts2/TtsTimeline.kt | 56 +++ .../r2/navigator/media3/tts2/TtsUtterance.kt | 17 + test-app/build.gradle.kts | 2 +- test-app/src/main/AndroidManifest.xml | 14 +- .../org/readium/r2/testapp/Application.kt | 48 +- .../r2/testapp/bookshelf/BookshelfFragment.kt | 9 +- .../testapp/bookshelf/BookshelfViewModel.kt | 41 ++ .../r2/testapp/{ => reader}/MediaService.kt | 73 ++- .../r2/testapp/reader/ReaderActivity.kt | 4 +- .../r2/testapp/reader/ReaderInitData.kt | 32 +- .../r2/testapp/reader/ReaderRepository.kt | 82 +++- .../r2/testapp/reader/ReaderViewModel.kt | 42 +- .../r2/testapp/reader/VisualReaderFragment.kt | 36 +- .../reader/preferences/PreferencesManagers.kt | 30 ++ .../r2/testapp/reader/tts/TtsControls.kt | 58 ++- .../r2/testapp/reader/tts/TtsService.kt | 199 ++++++++ .../r2/testapp/reader/tts/TtsViewModel.kt | 238 +++++----- ...ce.kt => LifecycleMedia2SessionService.kt} | 8 +- .../utils/LifecycleMedia3SessionService.kt | 63 +++ 71 files changed, 3893 insertions(+), 262 deletions(-) create mode 100644 readium/navigator/src/main/java/org/readium/r2/navigator/media3/androidtts/AndroidTtsEngine.kt create mode 100644 readium/navigator/src/main/java/org/readium/r2/navigator/media3/androidtts/AndroidTtsEngineProvider.kt create mode 100644 readium/navigator/src/main/java/org/readium/r2/navigator/media3/androidtts/AndroidTtsPreferences.kt create mode 100644 readium/navigator/src/main/java/org/readium/r2/navigator/media3/androidtts/AndroidTtsPreferencesEditor.kt create mode 100644 readium/navigator/src/main/java/org/readium/r2/navigator/media3/androidtts/AndroidTtsPreferencesFilters.kt create mode 100644 readium/navigator/src/main/java/org/readium/r2/navigator/media3/androidtts/AndroidTtsPreferencesSerializer.kt create mode 100644 readium/navigator/src/main/java/org/readium/r2/navigator/media3/androidtts/AndroidTtsSettings.kt create mode 100644 readium/navigator/src/main/java/org/readium/r2/navigator/media3/androidtts/AndroidTtsSettingsResolver.kt create mode 100644 readium/navigator/src/main/java/org/readium/r2/navigator/media3/androidtts/AndroidTtsVoice.kt create mode 100644 readium/navigator/src/main/java/org/readium/r2/navigator/media3/api/DefaultMetadataFactory.kt create mode 100644 readium/navigator/src/main/java/org/readium/r2/navigator/media3/api/DefaultMetadataProvider.kt create mode 100644 readium/navigator/src/main/java/org/readium/r2/navigator/media3/api/MediaMetadataFactory.kt create mode 100644 readium/navigator/src/main/java/org/readium/r2/navigator/media3/api/MediaNavigator.kt create mode 100644 readium/navigator/src/main/java/org/readium/r2/navigator/media3/api/MediaNavigatorInternal.kt create mode 100644 readium/navigator/src/main/java/org/readium/r2/navigator/media3/api/MetadataProvider.kt create mode 100644 readium/navigator/src/main/java/org/readium/r2/navigator/media3/api/SynchronizedPlayback.kt create mode 100644 readium/navigator/src/main/java/org/readium/r2/navigator/media3/exoplayer/ExoPlayerDataSource.kt create mode 100644 readium/navigator/src/main/java/org/readium/r2/navigator/media3/exoplayer/ExoPlayerEngineProvider.kt create mode 100644 readium/navigator/src/main/java/org/readium/r2/navigator/media3/exoplayer/ExoPlayerPreferences.kt create mode 100644 readium/navigator/src/main/java/org/readium/r2/navigator/media3/exoplayer/ExoPlayerPreferencesEditor.kt create mode 100644 readium/navigator/src/main/java/org/readium/r2/navigator/media3/exoplayer/ExoPlayerPreferencesFilters.kt create mode 100644 readium/navigator/src/main/java/org/readium/r2/navigator/media3/exoplayer/ExoPlayerPreferencesSerializer.kt create mode 100644 readium/navigator/src/main/java/org/readium/r2/navigator/media3/exoplayer/ExoPlayerSettings.kt create mode 100644 readium/navigator/src/main/java/org/readium/r2/navigator/media3/exoplayer/ExoPlayerSettingsResolver.kt create mode 100644 readium/navigator/src/main/java/org/readium/r2/navigator/media3/player/DurationSerializer.kt create mode 100644 readium/navigator/src/main/java/org/readium/r2/navigator/media3/player/MediaEngineProvider.kt create mode 100644 readium/navigator/src/main/java/org/readium/r2/navigator/media3/player/PlayerLocator.kt create mode 100644 readium/navigator/src/main/java/org/readium/r2/navigator/media3/player/PlayerNavigator.kt create mode 100644 readium/navigator/src/main/java/org/readium/r2/navigator/media3/player/PlayerNavigatorFactory.kt create mode 100644 readium/navigator/src/main/java/org/readium/r2/navigator/media3/player/PlayerNavigatorInternal.kt create mode 100644 readium/navigator/src/main/java/org/readium/r2/navigator/media3/player/PlayerPlayback.kt create mode 100644 readium/navigator/src/main/java/org/readium/r2/navigator/media3/syncnarr/SynchronizedNarrationNavigator.kt create mode 100644 readium/navigator/src/main/java/org/readium/r2/navigator/media3/syncnarr/SynchronizedNarrationNavigatorInternal.kt create mode 100644 readium/navigator/src/main/java/org/readium/r2/navigator/media3/syncnarr/SynchronizedNarrationPlayback.kt create mode 100644 readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsContentIterator.kt create mode 100644 readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsEngine.kt create mode 100644 readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsEngineFacade.kt create mode 100644 readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsEngineFacadeListener.kt create mode 100644 readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsEngineFacadePlayback.kt create mode 100644 readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsEngineListener.kt create mode 100644 readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsEngineProvider.kt create mode 100644 readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsLocator.kt create mode 100644 readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsNavigator.kt create mode 100644 readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsNavigatorFactory.kt create mode 100644 readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsNavigatorInternal.kt create mode 100644 readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsNavigatorListener.kt create mode 100644 readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsPlayback.kt create mode 100644 readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsPlayer.kt create mode 100644 readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsPreferences.kt create mode 100644 readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsSettings.kt create mode 100644 readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsTimeline.kt create mode 100644 readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsUtterance.kt rename test-app/src/main/java/org/readium/r2/testapp/{ => reader}/MediaService.kt (60%) create mode 100644 test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsService.kt rename test-app/src/main/java/org/readium/r2/testapp/utils/{LifecycleMediaSessionService.kt => LifecycleMedia2SessionService.kt} (86%) create mode 100644 test-app/src/main/java/org/readium/r2/testapp/utils/LifecycleMedia3SessionService.kt diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 7a19bb7765..f601553cf2 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -23,6 +23,7 @@ androidx-lifecycle = "2.5.1" androidx-lifecycle-extensions = "2.2.0" androidx-media = "1.6.0" androidx-media2 = "1.2.1" +androidx-media3 = "1.0.0-beta02" androidx-navigation = "2.5.2" androidx-paging = "3.1.1" androidx-recyclerview = "1.2.1" @@ -87,6 +88,9 @@ androidx-lifecycle-vmsavedstate = { group = "androidx.lifecycle", name = "lifecy androidx-media = { group = "androidx.media", name = "media", version.ref = "androidx-media" } androidx-media2-session = { group = "androidx.media2", name = "media2-session", version.ref = "androidx-media2" } androidx-media2-player = { group = "androidx.media2", name = "media2-player", version.ref = "androidx-media2" } +androidx-media3-common = { group = "androidx.media3", name = "media3-common", version.ref = "androidx-media3" } +androidx-media3-session = { group = "androidx.media3", name = "media3-session", version.ref = "androidx-media3" } +androidx-media3-exoplayer = { group = "androidx.media3", name = "media3-exoplayer", version.ref = "androidx-media3" } androidx-navigation-fragment = { group = "androidx.navigation", name = "navigation-fragment-ktx", version.ref = "androidx-navigation" } androidx-navigation-ui = { group = "androidx.navigation", name = "navigation-ui-ktx", version.ref = "androidx-navigation" } androidx-paging = { group = "androidx.paging", name = "paging-runtime-ktx", version.ref = "androidx-paging" } @@ -138,6 +142,7 @@ coroutines = ["kotlinx-coroutines-core", "kotlinx-coroutines-android"] exoplayer = ["google-exoplayer-core", "google-exoplayer-ui", "google-exoplayer-mediasession", "google-exoplayer-workmanager", "google-exoplayer-extension-media2"] lifecycle = ["androidx-lifecycle-common", "androidx-lifecycle-extensions", "androidx-lifecycle-livedata", "androidx-lifecycle-runtime", "androidx-lifecycle-viewmodel", "androidx-lifecycle-vmsavedstate", "androidx-lifecycle-viewmodel-compose"] media2 = ["androidx-media2-session", "androidx-media2-player"] +media3 = ["androidx-media3-session", "androidx-media3-common", "androidx-media3-exoplayer"] navigation = ["androidx-navigation-fragment", "androidx-navigation-ui"] room = ["androidx-room-runtime", "androidx-room-ktx"] test-frameworks = ["junit", "androidx-ext-junit", "androidx-expresso-core", "robolectric", "kotlin-junit", "assertj", "kotlinx-coroutines-test"] diff --git a/readium/navigator/build.gradle.kts b/readium/navigator/build.gradle.kts index d21d98abfb..193a2ba1f9 100644 --- a/readium/navigator/build.gradle.kts +++ b/readium/navigator/build.gradle.kts @@ -81,6 +81,7 @@ dependencies { implementation(libs.bundles.lifecycle) implementation(libs.androidx.recyclerview) implementation(libs.androidx.media) + implementation(libs.bundles.media3) implementation(libs.androidx.viewpager2) implementation(libs.androidx.webkit) // Needed to avoid a crash with API 31, see https://stackoverflow.com/a/69152986/1474476 diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/androidtts/AndroidTtsEngine.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/androidtts/AndroidTtsEngine.kt new file mode 100644 index 0000000000..1aad39735a --- /dev/null +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/androidtts/AndroidTtsEngine.kt @@ -0,0 +1,205 @@ +/* + * Copyright 2022 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + +package org.readium.r2.navigator.media3.androidtts + +import android.content.Context +import android.content.Intent +import android.speech.tts.TextToSpeech +import android.speech.tts.TextToSpeech.QUEUE_ADD +import android.speech.tts.UtteranceProgressListener +import kotlinx.coroutines.* +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.StateFlow +import kotlinx.coroutines.flow.asStateFlow +import org.readium.r2.navigator.media3.tts2.TtsEngine +import org.readium.r2.navigator.media3.tts2.TtsUtterance +import org.readium.r2.shared.ExperimentalReadiumApi +import org.readium.r2.shared.publication.Metadata + +/** + * Default [TtsEngine] implementation using Android's native text to speech engine. + */ +@ExperimentalReadiumApi +class AndroidTtsEngine( + private val engine: TextToSpeech, + metadata: Metadata, + initialPreferences: AndroidTtsPreferences +) : TtsEngine { + + companion object { + + suspend operator fun invoke( + context: Context, + metadata: Metadata, + initialPreferences: AndroidTtsPreferences + ): AndroidTtsEngine? { + + val init = CompletableDeferred() + + val listener = TextToSpeech.OnInitListener { status -> + init.complete(status == TextToSpeech.SUCCESS) + } + val engine = TextToSpeech(context, listener) + + return if (init.await()) + AndroidTtsEngine(engine, metadata, initialPreferences) + else + null + } + } + + /** + * Android's TTS error code. + * See https://developer.android.com/reference/android/speech/tts/TextToSpeech#ERROR + */ + enum class EngineError(val code: Int) { + /** Denotes a generic operation failure. */ + Unknown(-1), + /** Denotes a failure caused by an invalid request. */ + InvalidRequest(-8), + /** Denotes a failure caused by a network connectivity problems. */ + Network(-6), + /** Denotes a failure caused by network timeout. */ + NetworkTimeout(-7), + /** Denotes a failure caused by an unfinished download of the voice data. */ + NotInstalledYet(-9), + /** Denotes a failure related to the output (audio device or a file). */ + Output(-5), + /** Denotes a failure of a TTS service. */ + Service(-4), + /** Denotes a failure of a TTS engine to synthesize the given input. */ + Synthesis(-3); + + companion object { + fun getOrDefault(key: Int): EngineError = + values() + .firstOrNull { it.code == key } + ?: Unknown + } + } + + class EngineException(code: Int) : Exception("Android TTS engine error: $code") { + val error: EngineError = + EngineError.getOrDefault(code) + } + + init { + engine.setOnUtteranceProgressListener(Listener()) + } + + private val scope: CoroutineScope = + CoroutineScope(SupervisorJob() + Dispatchers.Default) + + private var listener: TtsEngine.Listener? = + null + + private val settingsResolver: AndroidTtsSettingsResolver = + AndroidTtsSettingsResolver(metadata) + + private val _settings: MutableStateFlow = + MutableStateFlow(settingsResolver.settings(initialPreferences)) + + override fun close() { + scope.cancel() + engine.shutdown() + } + + override fun speak( + utterance: TtsUtterance, + requestId: String, + ) { + engine.speak(utterance.text, QUEUE_ADD, null, requestId) + } + + /** + * Start the activity to install additional language data. + * To be called if you receive a [TtsEngine.Exception.LanguageSupportIncomplete] error. + * + * Returns whether the request was successful. + * + * See https://android-developers.googleblog.com/2009/09/introduction-to-text-to-speech-in.html + */ + fun requestInstallMissingVoice( + context: Context, + intentFlags: Int = Intent.FLAG_ACTIVITY_NEW_TASK + ): Boolean { + val intent = Intent() + .setAction(TextToSpeech.Engine.ACTION_INSTALL_TTS_DATA) + .setFlags(intentFlags) + + if (context.packageManager.queryIntentActivities(intent, 0).isEmpty()) { + return false + } + + context.startActivity(intent) + return true + } + + // Engine + + override fun stop() { + engine.stop() + } + + override fun setListener(listener: TtsEngine.Listener?) { + this.listener = listener + } + + override val settings: StateFlow = + _settings.asStateFlow() + + override fun submitPreferences(preferences: AndroidTtsPreferences) { + val newSettings = settingsResolver.settings(preferences) + engine.setup(newSettings) + _settings.value = newSettings + } + + private fun TextToSpeech.setup(settings: AndroidTtsSettings) { + setSpeechRate(settings.speedRate.toFloat()) + + val localeResult = engine.setLanguage(settings.language?.locale) + if (localeResult < TextToSpeech.LANG_AVAILABLE) { + if (localeResult == TextToSpeech.LANG_MISSING_DATA) + throw org.readium.r2.navigator.tts.TtsEngine.Exception.LanguageSupportIncomplete(settings.language!!) + else + throw org.readium.r2.navigator.tts.TtsEngine.Exception.LanguageNotSupported(settings.language!!) + } + } + + inner class Listener : UtteranceProgressListener() { + override fun onStart(utteranceId: String?) { + listener?.onStart(utteranceId!!) + } + + override fun onStop(utteranceId: String?, interrupted: Boolean) { + listener?.let { + if (interrupted) { + it.onInterrupted(utteranceId!!) + } else { + it.onFlushed(utteranceId!!) + } + } + } + + override fun onDone(utteranceId: String?) { + listener?.onDone(utteranceId!!) + } + + @Deprecated("Deprecated in the interface", ReplaceWith("onError(utteranceId, -1)")) + override fun onError(utteranceId: String?) { + onError(utteranceId, -1) + } + + override fun onError(utteranceId: String?, errorCode: Int) { + // listener?.onError(utteranceId!!, EngineException(errorCode)) + } + + override fun onRangeStart(utteranceId: String?, start: Int, end: Int, frame: Int) { + listener?.onRange(utteranceId!!, start until end) + } + } +} diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/androidtts/AndroidTtsEngineProvider.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/androidtts/AndroidTtsEngineProvider.kt new file mode 100644 index 0000000000..ac0720eb6e --- /dev/null +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/androidtts/AndroidTtsEngineProvider.kt @@ -0,0 +1,41 @@ +/* + * Copyright 2022 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + +package org.readium.r2.navigator.media3.androidtts + +import android.content.Context +import org.readium.r2.navigator.media3.tts2.TtsEngineProvider +import org.readium.r2.shared.ExperimentalReadiumApi +import org.readium.r2.shared.publication.Metadata +import org.readium.r2.shared.publication.Publication + +@ExperimentalReadiumApi +class AndroidTtsEngineProvider( + private val context: Context, +) : TtsEngineProvider { + + override suspend fun createEngine( + publication: Publication, + initialPreferences: AndroidTtsPreferences + ): AndroidTtsEngine? { + return AndroidTtsEngine(context, publication.metadata, initialPreferences) + } + + fun computeSettings( + metadata: Metadata, + preferences: AndroidTtsPreferences + ): AndroidTtsSettings = + AndroidTtsSettingsResolver(metadata).settings(preferences) + + override fun createPreferencesEditor( + publication: Publication, + initialPreferences: AndroidTtsPreferences + ): AndroidTtsPreferencesEditor = + AndroidTtsPreferencesEditor(initialPreferences, publication.metadata) + + override fun createEmptyPreferences(): AndroidTtsPreferences = + AndroidTtsPreferences() +} diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/androidtts/AndroidTtsPreferences.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/androidtts/AndroidTtsPreferences.kt new file mode 100644 index 0000000000..f4da59498a --- /dev/null +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/androidtts/AndroidTtsPreferences.kt @@ -0,0 +1,29 @@ +/* + * Copyright 2022 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + +package org.readium.r2.navigator.media3.androidtts + +import kotlinx.serialization.Serializable +import org.readium.r2.navigator.media3.tts2.TtsPreferences +import org.readium.r2.shared.ExperimentalReadiumApi +import org.readium.r2.shared.util.Language + +@ExperimentalReadiumApi +@Serializable +data class AndroidTtsPreferences( + override val language: Language? = null, + val voiceId: String? = null, + val pitchRate: Double? = null, + val speedRate: Double? = null, +) : TtsPreferences { + + override fun plus(other: AndroidTtsPreferences): AndroidTtsPreferences = + AndroidTtsPreferences( + language = other.language ?: language, + voiceId = other.voiceId ?: voiceId, + speedRate = other.speedRate ?: speedRate, + ) +} diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/androidtts/AndroidTtsPreferencesEditor.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/androidtts/AndroidTtsPreferencesEditor.kt new file mode 100644 index 0000000000..6176a70d74 --- /dev/null +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/androidtts/AndroidTtsPreferencesEditor.kt @@ -0,0 +1,84 @@ +/* + * Copyright 2022 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + +package org.readium.r2.navigator.media3.androidtts + +import org.readium.r2.navigator.extensions.format +import org.readium.r2.navigator.preferences.* +import org.readium.r2.shared.ExperimentalReadiumApi +import org.readium.r2.shared.publication.Metadata +import org.readium.r2.shared.util.Language + +@ExperimentalReadiumApi +class AndroidTtsPreferencesEditor( + initialPreferences: AndroidTtsPreferences, + publicationMetadata: Metadata, +) : PreferencesEditor { + + private data class State( + val preferences: AndroidTtsPreferences, + val settings: AndroidTtsSettings + ) + + private val settingsResolver: AndroidTtsSettingsResolver = + AndroidTtsSettingsResolver(publicationMetadata) + + private var state: State = + initialPreferences.toState() + + override val preferences: AndroidTtsPreferences + get() = state.preferences + + override fun clear() { + updateValues { AndroidTtsPreferences() } + } + + val language: Preference = + PreferenceDelegate( + getValue = { preferences.language }, + getEffectiveValue = { state.settings.language }, + getIsEffective = { true }, + updateValue = { value -> updateValues { it.copy(language = value) } }, + ) + + val pitchRate: RangePreference = + RangePreferenceDelegate( + getValue = { preferences.pitchRate }, + getEffectiveValue = { state.settings.pitchRate }, + getIsEffective = { true }, + updateValue = { value -> updateValues { it.copy(pitchRate = value) } }, + supportedRange = 0.0..Double.MAX_VALUE, + progressionStrategy = DoubleIncrement(0.1), + valueFormatter = { it.format(1) }, + ) + + val speedRate: RangePreference = + RangePreferenceDelegate( + getValue = { preferences.speedRate }, + getEffectiveValue = { state.settings.speedRate }, + getIsEffective = { true }, + updateValue = { value -> updateValues { it.copy(speedRate = value) } }, + supportedRange = 0.0..Double.MAX_VALUE, + progressionStrategy = DoubleIncrement(0.1), + valueFormatter = { "${it.format(1)}x" }, + ) + + val voiceId: Preference = + PreferenceDelegate( + getValue = { preferences.voiceId }, + getEffectiveValue = { state.settings.voiceId }, + getIsEffective = { true }, + updateValue = { value -> updateValues { it.copy(voiceId = value) } }, + ) + + private fun updateValues(updater: (AndroidTtsPreferences) -> AndroidTtsPreferences) { + val newPreferences = updater(preferences) + state = newPreferences.toState() + } + + private fun AndroidTtsPreferences.toState() = + State(preferences = this, settings = settingsResolver.settings(this)) +} diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/androidtts/AndroidTtsPreferencesFilters.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/androidtts/AndroidTtsPreferencesFilters.kt new file mode 100644 index 0000000000..41849741cb --- /dev/null +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/androidtts/AndroidTtsPreferencesFilters.kt @@ -0,0 +1,36 @@ +/* + * Copyright 2022 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + +package org.readium.r2.navigator.media3.androidtts + +import org.readium.r2.navigator.preferences.PreferencesFilter +import org.readium.r2.shared.ExperimentalReadiumApi + +/** + * Suggested filter to keep only shared [AndroidTtsPreferences]. + */ +@ExperimentalReadiumApi +object AndroidTtsSharedPreferencesFilter : PreferencesFilter { + + override fun filter(preferences: AndroidTtsPreferences): AndroidTtsPreferences = + preferences.copy( + language = null, + voiceId = null + ) +} + +/** + * Suggested filter to keep only publication-specific [AndroidTtsPreferences]. + */ +@ExperimentalReadiumApi +object AndroidTtsPublicationPreferencesFilter : PreferencesFilter { + + override fun filter(preferences: AndroidTtsPreferences): AndroidTtsPreferences = + AndroidTtsPreferences( + pitchRate = preferences.pitchRate, + speedRate = preferences.speedRate + ) +} diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/androidtts/AndroidTtsPreferencesSerializer.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/androidtts/AndroidTtsPreferencesSerializer.kt new file mode 100644 index 0000000000..40acbc5b2a --- /dev/null +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/androidtts/AndroidTtsPreferencesSerializer.kt @@ -0,0 +1,24 @@ +/* + * Copyright 2022 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + +package org.readium.r2.navigator.media3.androidtts + +import kotlinx.serialization.json.Json +import org.readium.r2.navigator.preferences.PreferencesSerializer +import org.readium.r2.shared.ExperimentalReadiumApi + +/** + * JSON serializer of [AndroidTtsPreferences]. + */ +@ExperimentalReadiumApi +class AndroidTtsPreferencesSerializer : PreferencesSerializer { + + override fun serialize(preferences: AndroidTtsPreferences): String = + Json.encodeToString(AndroidTtsPreferences.serializer(), preferences) + + override fun deserialize(preferences: String): AndroidTtsPreferences = + Json.decodeFromString(AndroidTtsPreferences.serializer(), preferences) +} diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/androidtts/AndroidTtsSettings.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/androidtts/AndroidTtsSettings.kt new file mode 100644 index 0000000000..6ea70aa422 --- /dev/null +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/androidtts/AndroidTtsSettings.kt @@ -0,0 +1,19 @@ +/* + * Copyright 2022 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + +package org.readium.r2.navigator.media3.androidtts + +import org.readium.r2.navigator.media3.tts2.TtsSettings +import org.readium.r2.shared.ExperimentalReadiumApi +import org.readium.r2.shared.util.Language + +@ExperimentalReadiumApi +data class AndroidTtsSettings( + override val language: Language?, + val voiceId: String?, + val pitchRate: Double, + val speedRate: Double, +) : TtsSettings diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/androidtts/AndroidTtsSettingsResolver.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/androidtts/AndroidTtsSettingsResolver.kt new file mode 100644 index 0000000000..ae96baf296 --- /dev/null +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/androidtts/AndroidTtsSettingsResolver.kt @@ -0,0 +1,26 @@ +/* + * Copyright 2022 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + +package org.readium.r2.navigator.media3.androidtts + +import org.readium.r2.shared.ExperimentalReadiumApi +import org.readium.r2.shared.publication.Metadata + +@ExperimentalReadiumApi +internal class AndroidTtsSettingsResolver( + private val metadata: Metadata, +) { + + fun settings(preferences: AndroidTtsPreferences): AndroidTtsSettings { + + return AndroidTtsSettings( + language = preferences.language ?: metadata.language, + voiceId = preferences.voiceId, + pitchRate = preferences.pitchRate ?: 1.0, + speedRate = preferences.speedRate ?: 1.0, + ) + } +} diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/androidtts/AndroidTtsVoice.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/androidtts/AndroidTtsVoice.kt new file mode 100644 index 0000000000..a1970b3533 --- /dev/null +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/androidtts/AndroidTtsVoice.kt @@ -0,0 +1,33 @@ +/* + * Copyright 2022 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + +package org.readium.r2.navigator.media3.androidtts + +import org.readium.r2.shared.ExperimentalReadiumApi +import org.readium.r2.shared.util.Language + +/** + * Represents a voice provided by the TTS engine which can speak an utterance. + * + * @param id Unique and stable identifier for this voice. Can be used to store and retrieve the + * voice from the user preferences. + * @param name Human-friendly name for this voice, when available. + * @param language Language (and region) this voice belongs to. + * @param quality Voice quality. + * @param requiresNetwork Indicates whether using this voice requires an Internet connection. + */ +@ExperimentalReadiumApi +data class AndroidTtsVoice( + val id: String, + val name: String? = null, + val language: Language, + val quality: Quality = Quality.Normal, + val requiresNetwork: Boolean = false, +) { + enum class Quality { + Lowest, Low, Normal, High, Highest + } +} diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/api/DefaultMetadataFactory.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/api/DefaultMetadataFactory.kt new file mode 100644 index 0000000000..5f96847149 --- /dev/null +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/api/DefaultMetadataFactory.kt @@ -0,0 +1,75 @@ +/* + * Copyright 2022 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + +package org.readium.r2.navigator.media3.api + +import androidx.media3.common.MediaMetadata +import androidx.media3.common.MediaMetadata.PICTURE_TYPE_FRONT_COVER +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Deferred +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.async +import org.readium.r2.shared.publication.Publication + +internal class DefaultMetadataFactory(private val publication: Publication) : MediaMetadataFactory { + + private val coroutineScope = + CoroutineScope(Dispatchers.Default) + + private val authors: String? + get() = publication.metadata.authors + .joinToString(", ") { it.name }.takeIf { it.isNotBlank() } + + private val cover: Deferred = coroutineScope.async { + publication.linkWithRel("cover") + ?.let { publication.get(it) } + ?.read() + ?.getOrNull() + } + + @androidx.annotation.OptIn(androidx.media3.common.util.UnstableApi::class) + override suspend fun publicationMetadata(): MediaMetadata { + val builder = MediaMetadata.Builder() + .setTitle(publication.metadata.title) + .setAlbumTitle(publication.metadata.title) + .setTotalTrackCount(publication.readingOrder.size) + + authors + ?.let { + builder.setArtist(it) + builder.setAlbumArtist(it) + } + + cover.await() + ?.let { builder.maybeSetArtworkData(it, PICTURE_TYPE_FRONT_COVER) } + + return builder.build() + } + + override suspend fun resourceMetadata(index: Int): MediaMetadata { + + val builder = MediaMetadata.Builder() + val link = publication.readingOrder[index] + + builder.setTrackNumber(index) + // builder.setMediaUri(link.href) + builder.setTitle(link.title) + builder.setTitle(publication.metadata.title) + builder.setAlbumTitle(publication.metadata.title) + // builder.setDuration(MediaMetadata.METADATA_KEY_DURATION, (link.duration?.toLong() ?: 0) * 1000) + + authors + ?.let { + builder.setArtist(it) + builder.setAlbumArtist(it) + } + + cover.await() + ?.let { builder.maybeSetArtworkData(it, PICTURE_TYPE_FRONT_COVER) } + + return builder.build() + } +} diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/api/DefaultMetadataProvider.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/api/DefaultMetadataProvider.kt new file mode 100644 index 0000000000..702c9cec1d --- /dev/null +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/api/DefaultMetadataProvider.kt @@ -0,0 +1,16 @@ +/* + * Copyright 2022 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + +package org.readium.r2.navigator.media3.api + +import org.readium.r2.shared.publication.Publication + +class DefaultMetadataProvider : MetadataProvider { + + override fun createMetadataFactory(publication: Publication): MediaMetadataFactory { + return DefaultMetadataFactory(publication) + } +} diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/api/MediaMetadataFactory.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/api/MediaMetadataFactory.kt new file mode 100644 index 0000000000..0e705f354b --- /dev/null +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/api/MediaMetadataFactory.kt @@ -0,0 +1,27 @@ +/* + * Copyright 2022 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + +package org.readium.r2.navigator.media3.api + +import androidx.media3.common.MediaMetadata + +/** + * Factory for the [MediaMetadata] associated with the publication and its resources. + * + * The metadata are used for example in the media-style Android notification. + */ +interface MediaMetadataFactory { + + /** + * Creates the [MediaMetadata] for the whole publication. + */ + suspend fun publicationMetadata(): MediaMetadata + + /** + * Creates the [MediaMetadata] for the reading order resource at the given [index]. + */ + suspend fun resourceMetadata(index: Int): MediaMetadata +} diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/api/MediaNavigator.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/api/MediaNavigator.kt new file mode 100644 index 0000000000..80bc2092e1 --- /dev/null +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/api/MediaNavigator.kt @@ -0,0 +1,63 @@ +/* + * Copyright 2022 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + +package org.readium.r2.navigator.media3.api + +import androidx.media3.common.Player +import kotlin.time.Duration +import kotlinx.coroutines.flow.StateFlow +import org.readium.r2.navigator.Navigator +import org.readium.r2.shared.ExperimentalReadiumApi +import org.readium.r2.shared.publication.Locator +import org.readium.r2.shared.util.Closeable + +@ExperimentalReadiumApi +interface MediaNavigator

: Navigator, Closeable { + + enum class State { + Playing, + Paused, + Ended; + } + + data class Buffer( + val isPlayable: Boolean, + val position: Duration + ) + + interface Playback { + + val state: State + val locator: Locator + } + + interface TextSynchronization { + + val token: Locator? + } + + interface BufferProvider { + + val buffer: Buffer + } + + val playback: StateFlow

+ + /** + * Resumes the playback at the current location or start it again from the beginning if it has finished. + */ + fun play() + + /** + * Pauses the playback. + */ + fun pause() + + /** + * Adapts this navigator to the media3 [Player] interface. + */ + fun asPlayer(): Player +} diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/api/MediaNavigatorInternal.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/api/MediaNavigatorInternal.kt new file mode 100644 index 0000000000..f728eea2aa --- /dev/null +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/api/MediaNavigatorInternal.kt @@ -0,0 +1,77 @@ +/* + * Copyright 2022 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + +package org.readium.r2.navigator.media3.api + +import androidx.media3.common.Player +import kotlin.time.Duration +import kotlinx.coroutines.flow.StateFlow +import org.readium.r2.shared.InternalReadiumApi + +@InternalReadiumApi +interface MediaNavigatorInternal> { + + interface Locator + + enum class State { + Playing, + Paused, + Ended; + } + + data class Buffer( + val isPlayable: Boolean, + val position: Duration + ) + + interface Playback { + + val state: State + val locator: L + } + + interface TextSynchronization { + + val token: Locator? + } + + interface BufferProvider { + + val buffer: Buffer + } + + val playback: StateFlow

+ + /** + * Resumes the playback at the current location or start it again from the beginning if it has finished. + */ + fun play() + + /** + * Pauses the playback. + */ + fun pause() + + /** + * Seeks to the given locator. + */ + fun go(locator: L) + + /** + * Skips forward + */ + fun goForward() + + /** + * Skips backward. + */ + fun goBackward() + + /** + * Adapts this navigator to the media3 [Player] interface. + */ + fun asPlayer(): Player +} diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/api/MetadataProvider.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/api/MetadataProvider.kt new file mode 100644 index 0000000000..377ca88e53 --- /dev/null +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/api/MetadataProvider.kt @@ -0,0 +1,14 @@ +/* + * Copyright 2022 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + +package org.readium.r2.navigator.media3.api + +import org.readium.r2.shared.publication.Publication + +interface MetadataProvider { + + fun createMetadataFactory(publication: Publication): MediaMetadataFactory +} diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/api/SynchronizedPlayback.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/api/SynchronizedPlayback.kt new file mode 100644 index 0000000000..5dc470d7f3 --- /dev/null +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/api/SynchronizedPlayback.kt @@ -0,0 +1,20 @@ +/* + * Copyright 2022 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + +package org.readium.r2.navigator.media3.api + +import org.readium.r2.shared.ExperimentalReadiumApi + +@ExperimentalReadiumApi +interface SynchronizedPlayback : + MediaNavigatorInternal.Playback, MediaNavigatorInternal.TextSynchronization { + + override val state: MediaNavigatorInternal.State + + override val locator: L + + override val token: L? +} diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/exoplayer/ExoPlayerDataSource.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/exoplayer/ExoPlayerDataSource.kt new file mode 100644 index 0000000000..a587e4c266 --- /dev/null +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/exoplayer/ExoPlayerDataSource.kt @@ -0,0 +1,148 @@ +/* + * Copyright 2022 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + +package org.readium.r2.navigator.media3.player + +import android.net.Uri +import androidx.media3.common.C.LENGTH_UNSET +import androidx.media3.common.C.RESULT_END_OF_INPUT +import androidx.media3.datasource.BaseDataSource +import androidx.media3.datasource.DataSource +import androidx.media3.datasource.DataSpec +import androidx.media3.datasource.TransferListener +import java.io.IOException +import kotlinx.coroutines.runBlocking +import org.readium.r2.shared.fetcher.Resource +import org.readium.r2.shared.fetcher.buffered +import org.readium.r2.shared.publication.Publication + +sealed class ExoPlayerDataSourceException(message: String, cause: Throwable?) : IOException(message, cause) { + class NotOpened(message: String) : ExoPlayerDataSourceException(message, null) + class NotFound(message: String) : ExoPlayerDataSourceException(message, null) + class ReadFailed(uri: Uri, offset: Int, readLength: Int, cause: Throwable) : ExoPlayerDataSourceException("Failed to read $readLength bytes of URI $uri at offset $offset.", cause) +} + +/** + * An ExoPlayer's [DataSource] which retrieves resources from a [Publication]. + */ +@androidx.annotation.OptIn(androidx.media3.common.util.UnstableApi::class) +internal class ExoPlayerDataSource internal constructor(private val publication: Publication) : BaseDataSource(/* isNetwork = */ true) { + + class Factory( + private val publication: Publication, + private val transferListener: TransferListener? = null + ) : DataSource.Factory { + + override fun createDataSource(): DataSource = + ExoPlayerDataSource(publication).apply { + if (transferListener != null) { + addTransferListener(transferListener) + } + } + } + + private data class OpenedResource( + val resource: Resource, + val uri: Uri, + var position: Long, + ) + + private var openedResource: OpenedResource? = null + + override fun open(dataSpec: DataSpec): Long { + val link = publication.linkWithHref(dataSpec.uri.toString()) + ?: throw ExoPlayerDataSourceException.NotFound("Can't find a [Link] for URI: ${dataSpec.uri}. Make sure you only request resources declared in the manifest.") + + val resource = publication.get(link) + // Significantly improves performances, in particular with deflated ZIP entries. + .buffered(resourceLength = cachedLengths[dataSpec.uri.toString()]) + + openedResource = OpenedResource( + resource = resource, + uri = dataSpec.uri, + position = dataSpec.position, + ) + + val bytesToRead = + if (dataSpec.length != LENGTH_UNSET.toLong()) { + dataSpec.length + } else { + val contentLength = contentLengthOf(dataSpec.uri, resource) + ?: return dataSpec.length + contentLength - dataSpec.position + } + + return bytesToRead + } + + /** Cached content lengths indexed by their URL. */ + private var cachedLengths: MutableMap = mutableMapOf() + + private fun contentLengthOf(uri: Uri, resource: Resource): Long? { + cachedLengths[uri.toString()]?.let { return it } + + val length = runBlocking { resource.length() }.getOrNull() + ?: return null + + cachedLengths[uri.toString()] = length + return length + } + + override fun read(target: ByteArray, offset: Int, length: Int): Int { + if (length <= 0) { + return 0 + } + + val openedResource = openedResource ?: throw ExoPlayerDataSourceException.NotOpened("No opened resource to read from. Did you call open()?") + + try { + val data = runBlocking { + openedResource.resource + .read(range = openedResource.position until (openedResource.position + length)) + .getOrThrow() + } + + if (data.isEmpty()) { + return RESULT_END_OF_INPUT + } + + data.copyInto( + destination = target, + destinationOffset = offset, + startIndex = 0, + endIndex = data.size + ) + + openedResource.position += data.count() + return data.count() + } catch (e: Exception) { + if (e is InterruptedException) { + return 0 + } + throw ExoPlayerDataSourceException.ReadFailed( + uri = openedResource.uri, + offset = offset, + readLength = length, + cause = e + ) + } + } + + override fun getUri(): Uri? = openedResource?.uri + + override fun close() { + openedResource?.run { + try { + runBlocking { resource.close() } + } catch (e: Exception) { + if (e !is InterruptedException) { + throw e + } + } + } + openedResource = null + } +} diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/exoplayer/ExoPlayerEngineProvider.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/exoplayer/ExoPlayerEngineProvider.kt new file mode 100644 index 0000000000..08609c2b64 --- /dev/null +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/exoplayer/ExoPlayerEngineProvider.kt @@ -0,0 +1,56 @@ +/* + * Copyright 2022 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + +package org.readium.r2.navigator.media3.exoplayer + +import android.content.Context +import androidx.media3.common.AudioAttributes +import androidx.media3.common.C +import androidx.media3.datasource.DataSource +import androidx.media3.exoplayer.ExoPlayer +import androidx.media3.exoplayer.source.DefaultMediaSourceFactory +import org.readium.r2.navigator.media3.player.ExoPlayerDataSource +import org.readium.r2.navigator.media3.player.MediaEngineProvider +import org.readium.r2.shared.ExperimentalReadiumApi +import org.readium.r2.shared.publication.Metadata +import org.readium.r2.shared.publication.Publication + +@ExperimentalReadiumApi +@androidx.annotation.OptIn(androidx.media3.common.util.UnstableApi::class) +class ExoPlayerEngineProvider( + private val context: Context, +) : MediaEngineProvider { + + override suspend fun createPlayer(publication: Publication): ExoPlayer { + val dataSourceFactory: DataSource.Factory = ExoPlayerDataSource.Factory(publication) + return ExoPlayer.Builder(context) + .setMediaSourceFactory(DefaultMediaSourceFactory(dataSourceFactory)) + .setAudioAttributes( + AudioAttributes.Builder() + .setContentType(C.AUDIO_CONTENT_TYPE_MUSIC) + .setUsage(C.USAGE_MEDIA) + .build(), + true + ) + .setHandleAudioBecomingNoisy(true) + .build() + } + + override fun computeSettings( + metadata: Metadata, + preferences: ExoPlayerPreferences + ): ExoPlayerSettings = + ExoPlayerSettingsResolver(metadata).settings(preferences) + + override fun createPreferenceEditor( + publication: Publication, + initialPreferences: ExoPlayerPreferences + ): ExoPlayerPreferencesEditor = + ExoPlayerPreferencesEditor() + + override fun createEmptyPreferences(): ExoPlayerPreferences = + ExoPlayerPreferences() +} diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/exoplayer/ExoPlayerPreferences.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/exoplayer/ExoPlayerPreferences.kt new file mode 100644 index 0000000000..c77c551475 --- /dev/null +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/exoplayer/ExoPlayerPreferences.kt @@ -0,0 +1,22 @@ +/* + * Copyright 2022 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + +package org.readium.r2.navigator.media3.exoplayer + +import org.readium.r2.navigator.preferences.Configurable +import org.readium.r2.shared.ExperimentalReadiumApi + +@ExperimentalReadiumApi +@kotlinx.serialization.Serializable +data class ExoPlayerPreferences( + val rateMultiplier: Double? = null, +) : Configurable.Preferences { + + override fun plus(other: ExoPlayerPreferences): ExoPlayerPreferences = + ExoPlayerPreferences( + rateMultiplier = other.rateMultiplier ?: rateMultiplier, + ) +} diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/exoplayer/ExoPlayerPreferencesEditor.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/exoplayer/ExoPlayerPreferencesEditor.kt new file mode 100644 index 0000000000..6cc4efe150 --- /dev/null +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/exoplayer/ExoPlayerPreferencesEditor.kt @@ -0,0 +1,21 @@ +/* + * Copyright 2022 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + +package org.readium.r2.navigator.media3.exoplayer + +import org.readium.r2.navigator.preferences.PreferencesEditor +import org.readium.r2.shared.ExperimentalReadiumApi + +@ExperimentalReadiumApi +class ExoPlayerPreferencesEditor : PreferencesEditor { + + override val preferences: ExoPlayerPreferences + get() = TODO("Not yet implemented") + + override fun clear() { + TODO("Not yet implemented") + } +} diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/exoplayer/ExoPlayerPreferencesFilters.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/exoplayer/ExoPlayerPreferencesFilters.kt new file mode 100644 index 0000000000..db3f3b7aad --- /dev/null +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/exoplayer/ExoPlayerPreferencesFilters.kt @@ -0,0 +1,30 @@ +/* + * Copyright 2022 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + +package org.readium.r2.navigator.media3.exoplayer + +import org.readium.r2.navigator.preferences.PreferencesFilter +import org.readium.r2.shared.ExperimentalReadiumApi + +/** + * Suggested filter to keep only shared [ExoPlayerPreferences]. + */ +@ExperimentalReadiumApi +object ExoPlayerSharedPreferencesFilter : PreferencesFilter { + + override fun filter(preferences: ExoPlayerPreferences): ExoPlayerPreferences = + preferences.copy() +} + +/** + * Suggested filter to keep only publication-specific [ExoPlayerPreferences]. + */ +@ExperimentalReadiumApi +object ExoPlayerPublicationPreferencesFilter : PreferencesFilter { + + override fun filter(preferences: ExoPlayerPreferences): ExoPlayerPreferences = + ExoPlayerPreferences() +} diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/exoplayer/ExoPlayerPreferencesSerializer.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/exoplayer/ExoPlayerPreferencesSerializer.kt new file mode 100644 index 0000000000..aada34c11e --- /dev/null +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/exoplayer/ExoPlayerPreferencesSerializer.kt @@ -0,0 +1,24 @@ +/* + * Copyright 2022 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + +package org.readium.r2.navigator.media3.exoplayer + +import kotlinx.serialization.json.Json +import org.readium.r2.navigator.preferences.PreferencesSerializer +import org.readium.r2.shared.ExperimentalReadiumApi + +/** + * JSON serializer of [ExoPlayerPreferences]. + */ +@ExperimentalReadiumApi +class ExoPlayerPreferencesSerializer : PreferencesSerializer { + + override fun serialize(preferences: ExoPlayerPreferences): String = + Json.encodeToString(ExoPlayerPreferences.serializer(), preferences) + + override fun deserialize(preferences: String): ExoPlayerPreferences = + Json.decodeFromString(ExoPlayerPreferences.serializer(), preferences) +} diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/exoplayer/ExoPlayerSettings.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/exoplayer/ExoPlayerSettings.kt new file mode 100644 index 0000000000..58bc54b1e5 --- /dev/null +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/exoplayer/ExoPlayerSettings.kt @@ -0,0 +1,15 @@ +/* + * Copyright 2022 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + +package org.readium.r2.navigator.media3.exoplayer + +import org.readium.r2.navigator.preferences.Configurable +import org.readium.r2.shared.ExperimentalReadiumApi + +@ExperimentalReadiumApi +data class ExoPlayerSettings( + val rateMultiplier: Double +) : Configurable.Settings diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/exoplayer/ExoPlayerSettingsResolver.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/exoplayer/ExoPlayerSettingsResolver.kt new file mode 100644 index 0000000000..b940630a3e --- /dev/null +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/exoplayer/ExoPlayerSettingsResolver.kt @@ -0,0 +1,23 @@ +/* + * Copyright 2022 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + +package org.readium.r2.navigator.media3.exoplayer + +import org.readium.r2.shared.ExperimentalReadiumApi +import org.readium.r2.shared.publication.Metadata + +@ExperimentalReadiumApi +internal class ExoPlayerSettingsResolver( + private val metadata: Metadata, +) { + + fun settings(preferences: ExoPlayerPreferences): ExoPlayerSettings { + + return ExoPlayerSettings( + rateMultiplier = preferences.rateMultiplier ?: 1.0, + ) + } +} diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/player/DurationSerializer.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/player/DurationSerializer.kt new file mode 100644 index 0000000000..9a717708a3 --- /dev/null +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/player/DurationSerializer.kt @@ -0,0 +1,28 @@ +/* + * Copyright 2022 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + +package org.readium.r2.navigator.media3.player + +import kotlin.time.Duration +import kotlinx.serialization.KSerializer +import kotlinx.serialization.builtins.serializer +import kotlinx.serialization.descriptors.SerialDescriptor +import kotlinx.serialization.encoding.Decoder +import kotlinx.serialization.encoding.Encoder + +object DurationSerializer : KSerializer { + + private val serializer = Duration.serializer() + + override val descriptor: SerialDescriptor = serializer.descriptor + + override fun deserialize(decoder: Decoder): Duration = + decoder.decodeSerializableValue(serializer) + + override fun serialize(encoder: Encoder, value: Duration) { + encoder.encodeSerializableValue(serializer, value) + } +} diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/player/MediaEngineProvider.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/player/MediaEngineProvider.kt new file mode 100644 index 0000000000..e6e5b4e3f3 --- /dev/null +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/player/MediaEngineProvider.kt @@ -0,0 +1,35 @@ +/* + * Copyright 2022 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + +package org.readium.r2.navigator.media3.player + +import androidx.media3.common.Player +import org.readium.r2.navigator.preferences.Configurable +import org.readium.r2.navigator.preferences.PreferencesEditor +import org.readium.r2.shared.ExperimentalReadiumApi +import org.readium.r2.shared.publication.Metadata +import org.readium.r2.shared.publication.Publication + +@ExperimentalReadiumApi +interface MediaEngineProvider, E : PreferencesEditor

> { + + suspend fun createPlayer(publication: Publication): Player + + /** + * Creates settings for [metadata] and [preferences]. + */ + fun computeSettings(metadata: Metadata, preferences: P): S + + /** + * Creates a preferences editor for [publication] and [initialPreferences]. + */ + fun createPreferenceEditor(publication: Publication, initialPreferences: P): E + + /** + * Creates an empty set of preferences of this TTS engine provider. + */ + fun createEmptyPreferences(): P +} diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/player/PlayerLocator.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/player/PlayerLocator.kt new file mode 100644 index 0000000000..545fd25302 --- /dev/null +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/player/PlayerLocator.kt @@ -0,0 +1,26 @@ +/* + * Copyright 2022 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + +package org.readium.r2.navigator.media3.player + +import kotlin.time.Duration +import kotlin.time.Duration.Companion.seconds +import kotlinx.serialization.Serializable +import org.readium.r2.navigator.extensions.fragmentParameters +import org.readium.r2.navigator.media3.api.MediaNavigatorInternal +import org.readium.r2.shared.ExperimentalReadiumApi +import org.readium.r2.shared.publication.Locator + +@ExperimentalReadiumApi +@Serializable +data class PlayerLocator( + val index: Int, + @Serializable(with = DurationSerializer::class) + val position: Duration +) : MediaNavigatorInternal.Locator + +internal val Locator.Locations.time: Duration? get() = + fragmentParameters["t"]?.toIntOrNull()?.seconds diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/player/PlayerNavigator.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/player/PlayerNavigator.kt new file mode 100644 index 0000000000..9bbb40df00 --- /dev/null +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/player/PlayerNavigator.kt @@ -0,0 +1,125 @@ +/* + * Copyright 2022 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + +package org.readium.r2.navigator.media3.player + +import androidx.media3.common.Player +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.SupervisorJob +import kotlinx.coroutines.flow.StateFlow +import org.readium.r2.navigator.Navigator +import org.readium.r2.navigator.media3.api.MediaMetadataFactory +import org.readium.r2.navigator.media3.api.MediaNavigator +import org.readium.r2.navigator.preferences.Configurable +import org.readium.r2.shared.ExperimentalReadiumApi +import org.readium.r2.shared.extensions.mapStateIn +import org.readium.r2.shared.publication.Link +import org.readium.r2.shared.publication.Locator +import org.readium.r2.shared.publication.Publication + +@ExperimentalReadiumApi +class PlayerNavigator> private constructor( + override val publication: Publication, + private val playerNavigator: PlayerNavigatorInternal, +) : MediaNavigator, Navigator, Configurable { + + companion object { + + suspend operator fun > invoke( + publication: Publication, + mediaEngineProvider: MediaEngineProvider, + metadataFactory: MediaMetadataFactory, + initialPreferences: P = mediaEngineProvider.createEmptyPreferences(), + initialLocator: Locator = publication.startLocator, + ): PlayerNavigator { + TODO("Not yet implemented") + /*val player = mediaEngineProvider.createPlayer(publication) + + val playlist = publication.readingOrder.indices.map { index -> + val metadata = metadataFactory.resourceMetadata(index) + MediaItem.Builder() + .setMediaMetadata(metadata) + .build() + } + + val publicationMetadata = metadataFactory.publicationMetadata() + + val settingsResolver = ExoPlayerSettingsResolver(publication.metadata) + + return PlayerNavigator( + + )*/ + } + + private val Publication.startLocator: Locator + get() = locatorFromLink(readingOrder.first())!! + } + + data class Playback( + override val state: MediaNavigator.State, + override val locator: Locator, + override val buffer: MediaNavigator.Buffer + ) : MediaNavigator.Playback, MediaNavigator.BufferProvider + + private val coroutineScope = + CoroutineScope(SupervisorJob() + Dispatchers.Main.immediate) + + // Configurable + + override val settings: StateFlow = + TODO("Not yet implemented") + + override fun submitPreferences(preferences: P) { + TODO("Not yet implemented") + } + + // MediaNavigator + + override val playback: StateFlow + get() = TODO("Not yet implemented") + + override fun play() { + TODO("Not yet implemented") + } + + override fun pause() { + TODO("Not yet implemented") + } + + override fun asPlayer(): Player { + return playerNavigator.asPlayer() + } + + // Navigator + + override val currentLocator: StateFlow = + playback.mapStateIn(coroutineScope) { it.locator } + + override fun go(locator: Locator, animated: Boolean, completion: () -> Unit): Boolean { + TODO("Not yet implemented") + } + + override fun go(link: Link, animated: Boolean, completion: () -> Unit): Boolean { + val locator = publication.locatorFromLink(link) + ?: return false + go(locator) + return true + } + + override fun goForward(animated: Boolean, completion: () -> Unit): Boolean { + playerNavigator.goForward() + return true + } + + override fun goBackward(animated: Boolean, completion: () -> Unit): Boolean { + playerNavigator.goBackward() + return true + } + + override fun close() { + } +} diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/player/PlayerNavigatorFactory.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/player/PlayerNavigatorFactory.kt new file mode 100644 index 0000000000..b49a437f42 --- /dev/null +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/player/PlayerNavigatorFactory.kt @@ -0,0 +1,59 @@ +/* + * Copyright 2022 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + +package org.readium.r2.navigator.media3.player + +import org.readium.r2.navigator.media3.api.MetadataProvider +import org.readium.r2.navigator.preferences.Configurable +import org.readium.r2.navigator.preferences.PreferencesEditor +import org.readium.r2.shared.ExperimentalReadiumApi +import org.readium.r2.shared.publication.Locator +import org.readium.r2.shared.publication.Publication + +@ExperimentalReadiumApi +class PlayerNavigatorFactory, E : PreferencesEditor

>( + private val publication: Publication, + private val mediaEngineProvider: MediaEngineProvider, + private val playerNavigator: PlayerNavigator, +) { + + companion object { + + suspend operator fun , E : PreferencesEditor

> invoke( + publication: Publication, + mediaEngineProvider: MediaEngineProvider, + metadataProvider: MetadataProvider, + initialPreferences: P, + initialLocator: Locator + ): PlayerNavigatorFactory { + + val navigator = PlayerNavigator( + publication, + mediaEngineProvider, + metadataProvider.createMetadataFactory(publication), + initialPreferences, + initialLocator, + ) + + return PlayerNavigatorFactory( + publication, + mediaEngineProvider, + navigator + ) + } + } + + fun getMediaNavigator(): PlayerNavigator = + playerNavigator + + fun createPreferencesEditor( + initialPreferences: P + ): E = + mediaEngineProvider.createPreferenceEditor( + publication, + initialPreferences + ) +} diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/player/PlayerNavigatorInternal.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/player/PlayerNavigatorInternal.kt new file mode 100644 index 0000000000..7de5ee2056 --- /dev/null +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/player/PlayerNavigatorInternal.kt @@ -0,0 +1,53 @@ +/* + * Copyright 2022 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + +package org.readium.r2.navigator.media3.player + +import androidx.media3.common.Player +import kotlinx.coroutines.flow.StateFlow +import org.readium.r2.navigator.media3.api.MediaNavigatorInternal +import org.readium.r2.navigator.preferences.Configurable +import org.readium.r2.shared.ExperimentalReadiumApi + +@ExperimentalReadiumApi +class PlayerNavigatorInternal>( + private val player: Player +) : MediaNavigatorInternal, Configurable { + + override val playback: StateFlow + get() = TODO("Not yet implemented") + + override fun play() { + TODO("Not yet implemented") + } + + override fun pause() { + TODO("Not yet implemented") + } + + override fun go(locator: PlayerLocator) { + TODO("Not yet implemented") + } + + override fun goForward() { + TODO("Not yet implemented") + } + + override fun goBackward() { + TODO("Not yet implemented") + } + + override fun asPlayer(): Player { + return player + } + + override val settings: StateFlow + get() = TODO("Not yet implemented") + + override fun submitPreferences(preferences: P) { + TODO("Not yet implemented") + } +} diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/player/PlayerPlayback.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/player/PlayerPlayback.kt new file mode 100644 index 0000000000..09629bb96c --- /dev/null +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/player/PlayerPlayback.kt @@ -0,0 +1,17 @@ +/* + * Copyright 2022 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + +package org.readium.r2.navigator.media3.player + +import org.readium.r2.navigator.media3.api.MediaNavigatorInternal +import org.readium.r2.shared.ExperimentalReadiumApi + +@ExperimentalReadiumApi +data class PlayerPlayback( + override val state: MediaNavigatorInternal.State, + override val locator: PlayerLocator, + override val buffer: MediaNavigatorInternal.Buffer +) : MediaNavigatorInternal.Playback, MediaNavigatorInternal.BufferProvider diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/syncnarr/SynchronizedNarrationNavigator.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/syncnarr/SynchronizedNarrationNavigator.kt new file mode 100644 index 0000000000..a898a9614d --- /dev/null +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/syncnarr/SynchronizedNarrationNavigator.kt @@ -0,0 +1,70 @@ +/* + * Copyright 2022 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + +package org.readium.r2.navigator.media3.syncnarr + +import androidx.media3.common.Player +import kotlinx.coroutines.flow.StateFlow +import org.readium.r2.navigator.media3.api.MediaNavigator +import org.readium.r2.navigator.preferences.Configurable +import org.readium.r2.shared.ExperimentalReadiumApi +import org.readium.r2.shared.publication.Link +import org.readium.r2.shared.publication.Locator +import org.readium.r2.shared.publication.Publication + +@ExperimentalReadiumApi +class SynchronizedNarrationNavigator>( + private val internalNavigator: SynchronizedNarrationNavigatorInternal +) : MediaNavigator { + + data class Playback( + override val state: MediaNavigator.State, + override val locator: Locator, + override val token: Locator?, + override val buffer: MediaNavigator.Buffer + ) : MediaNavigator.Playback, MediaNavigator.BufferProvider, MediaNavigator.TextSynchronization + + override val publication: Publication + get() = TODO("Not yet implemented") + + override val currentLocator: StateFlow + get() = TODO("Not yet implemented") + + override fun go(locator: Locator, animated: Boolean, completion: () -> Unit): Boolean { + TODO("Not yet implemented") + } + + override fun go(link: Link, animated: Boolean, completion: () -> Unit): Boolean { + TODO("Not yet implemented") + } + + override fun goForward(animated: Boolean, completion: () -> Unit): Boolean { + TODO("Not yet implemented") + } + + override fun goBackward(animated: Boolean, completion: () -> Unit): Boolean { + TODO("Not yet implemented") + } + + override fun close() { + TODO("Not yet implemented") + } + + override val playback: StateFlow + get() = TODO("Not yet implemented") + + override fun play() { + TODO("Not yet implemented") + } + + override fun pause() { + TODO("Not yet implemented") + } + + override fun asPlayer(): Player { + TODO("Not yet implemented") + } +} diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/syncnarr/SynchronizedNarrationNavigatorInternal.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/syncnarr/SynchronizedNarrationNavigatorInternal.kt new file mode 100644 index 0000000000..21251f1298 --- /dev/null +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/syncnarr/SynchronizedNarrationNavigatorInternal.kt @@ -0,0 +1,50 @@ +/* + * Copyright 2022 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + +package org.readium.r2.navigator.media3.syncnarr + +import androidx.media3.common.Player +import kotlinx.coroutines.flow.StateFlow +import org.readium.r2.navigator.media3.api.MediaNavigatorInternal +import org.readium.r2.navigator.media3.api.SynchronizedPlayback +import org.readium.r2.navigator.media3.player.PlayerLocator +import org.readium.r2.navigator.media3.player.PlayerNavigatorInternal +import org.readium.r2.navigator.preferences.Configurable +import org.readium.r2.shared.ExperimentalReadiumApi + +@ExperimentalReadiumApi +class SynchronizedNarrationNavigatorInternal>( + private val playerNavigator: PlayerNavigatorInternal + // media overlays data +) : MediaNavigatorInternal> { + + override val playback: StateFlow> + get() = TODO("Not yet implemented") + + override fun play() { + TODO("Not yet implemented") + } + + override fun pause() { + TODO("Not yet implemented") + } + + override fun go(locator: PlayerLocator) { + TODO("Not yet implemented") + } + + override fun goForward() { + TODO("Not yet implemented") + } + + override fun goBackward() { + TODO("Not yet implemented") + } + + override fun asPlayer(): Player { + TODO("Not yet implemented") + } +} diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/syncnarr/SynchronizedNarrationPlayback.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/syncnarr/SynchronizedNarrationPlayback.kt new file mode 100644 index 0000000000..a5ea2df3c9 --- /dev/null +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/syncnarr/SynchronizedNarrationPlayback.kt @@ -0,0 +1,19 @@ +/* + * Copyright 2022 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + +package org.readium.r2.navigator.media3.syncnarr + +import org.readium.r2.navigator.media3.api.MediaNavigatorInternal +import org.readium.r2.navigator.media3.api.SynchronizedPlayback +import org.readium.r2.navigator.media3.player.PlayerLocator +import org.readium.r2.shared.ExperimentalReadiumApi + +@ExperimentalReadiumApi +data class SynchronizedNarrationPlayback( + override val state: MediaNavigatorInternal.State, + override val locator: PlayerLocator, + override val token: PlayerLocator?, +) : SynchronizedPlayback diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsContentIterator.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsContentIterator.kt new file mode 100644 index 0000000000..74d8ef6910 --- /dev/null +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsContentIterator.kt @@ -0,0 +1,163 @@ +/* + * Copyright 2022 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + +package org.readium.r2.navigator.media3.tts2 + +import org.readium.r2.shared.ExperimentalReadiumApi +import org.readium.r2.shared.publication.Locator +import org.readium.r2.shared.publication.Publication +import org.readium.r2.shared.publication.services.content.Content +import org.readium.r2.shared.publication.services.content.ContentTokenizer +import org.readium.r2.shared.publication.services.content.content +import org.readium.r2.shared.util.CursorList +import org.readium.r2.shared.util.Language + +@ExperimentalReadiumApi +internal class TtsContentIterator( + private val publication: Publication, + private val tokenizerFactory: (language: Language?) -> ContentTokenizer, + initialLocator: TtsLocator? +) { + enum class Direction { + Forward, Backward; + } + + private var utterances: CursorList = + CursorList() + + /** + * [Content.Iterator] used to iterate through the [publication]. + */ + private var publicationIterator: Content.Iterator = createIterator(initialLocator) + set(value) { + field = value + utterances = CursorList() + } + + private var language: Language? = + null + + /** + * Sets the tokenizer language. + * + * The change is not immediate, it will be applied as soon as possible. + */ + fun setLanguage(language: Language?) { + this.language = language + } + + /** + * Gets the next utterance in the given [direction], or null when reaching the beginning or the + * end. + */ + suspend fun nextUtterance(direction: Direction): TtsUtterance? { + val utterance = utterances.nextIn(direction) + if (utterance == null && loadNextUtterances(direction)) { + return nextUtterance(direction) + } + return utterance + } + + /** + * Moves the iterator to the position provided in [locator]. + */ + fun seek(locator: TtsLocator) { + publicationIterator = createIterator(locator) + } + + fun restart() { + publicationIterator = createIterator(null) + } + + private fun createIterator(locator: TtsLocator?) = + publication.content(locator?.toLocator(publication)) + ?.iterator() + ?: throw IllegalStateException("No ContentService.") + + /** + * Loads the utterances for the next publication [Content.Element] item in the given [direction]. + */ + private suspend fun loadNextUtterances(direction: Direction): Boolean { + val content = publicationIterator.nextIn(direction) + ?: return false + + val nextUtterances = content + .tokenize() + .flatMap { it.utterances() } + + if (nextUtterances.isEmpty()) { + return loadNextUtterances(direction) + } + + utterances = CursorList( + list = nextUtterances, + startIndex = when (direction) { + Direction.Forward -> 0 + Direction.Backward -> nextUtterances.size - 1 + } + ) + + return true + } + + /** + * Splits a publication [Content.Element] item into smaller chunks using the provided tokenizer. + * + * This is used to split a paragraph into sentences, for example. + */ + private fun Content.Element.tokenize(): List = + tokenizerFactory(language).tokenize(this) + + /** + * Splits a publication [Content.Element] item into the utterances to be spoken. + */ + private fun Content.Element.utterances(): List { + fun utterance(text: String, locator: Locator, language: Language? = null): TtsUtterance? { + if (!text.any { it.isLetterOrDigit() }) + return null + + return TtsUtterance( + text = text, + locator = checkNotNull(locator.toTtsLocator(publication)) { "Missing data in locator." }, + language = language + ) + } + + return when (this) { + is Content.TextElement -> { + segments.mapNotNull { segment -> + utterance( + text = segment.text, + locator = segment.locator, + language = segment.language + ) + } + } + + is Content.TextualElement -> { + listOfNotNull( + text + ?.takeIf { it.isNotBlank() } + ?.let { utterance(text = it, locator = locator) } + ) + } + + else -> emptyList() + } + } + + private fun CursorList.nextIn(direction: Direction): E? = + when (direction) { + Direction.Forward -> next() + Direction.Backward -> previous() + } + + private suspend fun Content.Iterator.nextIn(direction: Direction): Content.Element? = + when (direction) { + Direction.Forward -> nextOrNull() + Direction.Backward -> previousOrNull() + } +} diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsEngine.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsEngine.kt new file mode 100644 index 0000000000..d2d4cdded7 --- /dev/null +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsEngine.kt @@ -0,0 +1,77 @@ +/* + * Copyright 2022 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + +package org.readium.r2.navigator.media3.tts2 + +import org.readium.r2.navigator.preferences.Configurable +import org.readium.r2.shared.ExperimentalReadiumApi +import org.readium.r2.shared.util.Closeable +import org.readium.r2.shared.util.Language + +/** + * A text-to-speech engine synthesizes text utterances (e.g. sentence). + */ +@ExperimentalReadiumApi +interface TtsEngine> : Configurable, Closeable { + + @ExperimentalReadiumApi + sealed class Exception private constructor( + override val message: String, + cause: Throwable? = null + ) : kotlin.Exception(message, cause) { + /** Failed to initialize the TTS engine. */ + class InitializationFailed(cause: Throwable? = null) : + Exception("The TTS engine failed to initialize", cause) + + /** Tried to synthesize an utterance with an unsupported language. */ + class LanguageNotSupported(val language: Language, cause: Throwable? = null) : + Exception("The language ${language.code} is not supported by the TTS engine", cause) + + /** The selected language is missing downloadable data. */ + class LanguageSupportIncomplete(val language: Language, cause: Throwable? = null) : + Exception("The language ${language.code} requires additional files by the TTS engine", cause) + + /** Error during network calls. */ + class Network(cause: Throwable? = null) : + Exception("A network error occurred", cause) + + /** Other engine-specific errors. */ + class Other(override val cause: Throwable) : + Exception(cause.message ?: "An unknown error occurred", cause) + + companion object { + fun wrap(e: Throwable): Exception = when (e) { + is Exception -> e + else -> Other(e) + } + } + } + + /** + * TTS engine callbacks. + */ + @ExperimentalReadiumApi + interface Listener { + + fun onStart(id: String) + + fun onRange(id: String, range: IntRange) + + fun onInterrupted(id: String) + + fun onFlushed(id: String) + + fun onDone(id: String) + + fun onError(id: String, error: Exception) + } + + fun speak(utterance: TtsUtterance, requestId: String) + + fun stop() + + fun setListener(listener: Listener?) +} diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsEngineFacade.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsEngineFacade.kt new file mode 100644 index 0000000000..8e1fd61883 --- /dev/null +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsEngineFacade.kt @@ -0,0 +1,172 @@ +/* + * Copyright 2022 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + +package org.readium.r2.navigator.media3.tts2 + +import java.util.* +import kotlinx.coroutines.* +import kotlinx.coroutines.flow.* +import org.readium.r2.navigator.preferences.Configurable +import org.readium.r2.shared.ExperimentalReadiumApi + +@ExperimentalReadiumApi +internal class TtsEngineFacade>( + private val ttsEngine: TtsEngine, + private val ttsContentIterator: TtsContentIterator, + private val listener: TtsEngineFacadeListener, + firstUtterance: TtsUtterance, +) : Configurable by ttsEngine { + + companion object { + + suspend operator fun > invoke( + ttsEngine: TtsEngine, + ttsContentIterator: TtsContentIterator, + listener: TtsEngineFacadeListener + ): TtsEngineFacade? { + + val firstUtterance = ttsContentIterator.nextUtterance(TtsContentIterator.Direction.Forward) + ?: run { + ttsContentIterator.restart() + ttsContentIterator.nextUtterance(TtsContentIterator.Direction.Forward) + } ?: return null + + return TtsEngineFacade(ttsEngine, ttsContentIterator, listener, firstUtterance) + } + } + + private val coroutineScope: CoroutineScope = + MainScope() + + private var pendingUtterance: TtsUtterance? = + firstUtterance + + private var playbackJob: Job? = null + + private val ttsEngineListener = TtsEngineListener() + + private val playbackMutable: MutableStateFlow = + MutableStateFlow( + TtsEngineFacadePlayback( + index = firstUtterance.locator.resourceIndex, + state = TtsEngineFacadePlayback.State.READY, + isPlaying = false, + playWhenReady = false, + locator = firstUtterance.locator, + range = null + ) + ) + + init { + ttsEngine.setListener(ttsEngineListener) + ttsContentIterator.setLanguage(ttsEngine.settings.value.language) + + ttsEngineListener.state + .onEach { engineState -> onEngineStateChanged(engineState) } + .launchIn(coroutineScope) + } + + private fun onEngineStateChanged(state: TtsEngineListener.EngineState) { + val newPlayback = playbackMutable.value.copy( + isPlaying = state.utteranceId != null, + range = state.range + ) + + playbackMutable.value = newPlayback + } + + val playback: StateFlow = + playbackMutable.asStateFlow() + + fun play() { + replacePlaybackJob { + playbackMutable.value = + playbackMutable.value.copy( + state = TtsEngineFacadePlayback.State.READY, + playWhenReady = true, + isPlaying = true + ) + playContinuous() + } + } + + fun pause() { + replacePlaybackJob { + playbackMutable.value = + playbackMutable.value.copy( + playWhenReady = false, + isPlaying = false + ) + } + } + + fun go(locator: TtsLocator) { + replacePlaybackJob { + pendingUtterance = null + ttsContentIterator.seek(locator) + playContinuous() + } + } + + fun nextUtterance() { + replacePlaybackJob { + pendingUtterance = null + ttsContentIterator.nextUtterance(TtsContentIterator.Direction.Forward) + playContinuous() + } + } + + fun previousUtterance() { + replacePlaybackJob { + pendingUtterance = null + ttsContentIterator.nextUtterance(TtsContentIterator.Direction.Backward) + playContinuous() + } + } + + private fun replacePlaybackJob(block: suspend CoroutineScope.() -> Unit) { + coroutineScope.launch { + ttsEngine.stop() + playbackJob?.cancelAndJoin() + ttsEngineListener.removeAllCallbacks() + playbackJob = launch { + block() + } + } + } + + private suspend fun playContinuous() { + if (pendingUtterance == null) { + pendingUtterance = ttsContentIterator.nextUtterance(TtsContentIterator.Direction.Forward) + } + pendingUtterance?.let { + playbackMutable.value = playbackMutable.value.copy(range = null, locator = it.locator) + speakUtterance(it) + pendingUtterance = null + playContinuous() + } ?: run { + playbackMutable.value = playbackMutable.value.copy( + isPlaying = false, playWhenReady = false, state = TtsEngineFacadePlayback.State.ENDED + ) + } + } + + private suspend fun speakUtterance(utterance: TtsUtterance) = + suspendCancellableCoroutine { continuation -> + val id = UUID.randomUUID().toString() + ttsEngineListener.addCallback(id, continuation) + ttsEngine.speak(utterance, id) + } + + fun stop() { + listener.onNavigatorStopped() + pause() + } + + fun close() { + ttsEngine.close() + } +} diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsEngineFacadeListener.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsEngineFacadeListener.kt new file mode 100644 index 0000000000..85ba373b3f --- /dev/null +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsEngineFacadeListener.kt @@ -0,0 +1,14 @@ +/* + * Copyright 2022 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + +package org.readium.r2.navigator.media3.tts2 + +interface TtsEngineFacadeListener { + + fun onNavigatorStopped() + + fun onPlaybackException() +} diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsEngineFacadePlayback.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsEngineFacadePlayback.kt new file mode 100644 index 0000000000..6d03875864 --- /dev/null +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsEngineFacadePlayback.kt @@ -0,0 +1,26 @@ +/* + * Copyright 2022 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + +package org.readium.r2.navigator.media3.tts2 + +import androidx.media3.common.Player +import org.readium.r2.shared.ExperimentalReadiumApi + +@ExperimentalReadiumApi +internal data class TtsEngineFacadePlayback( + val state: State, + val isPlaying: Boolean, + val playWhenReady: Boolean, + val index: Int, + val locator: TtsLocator, + val range: IntRange? +) { + + enum class State(val value: Int) { + READY(Player.STATE_READY), + ENDED(Player.STATE_ENDED); + } +} diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsEngineListener.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsEngineListener.kt new file mode 100644 index 0000000000..5e81c67ed3 --- /dev/null +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsEngineListener.kt @@ -0,0 +1,88 @@ +/* + * Copyright 2022 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + +package org.readium.r2.navigator.media3.tts2 + +import kotlinx.coroutines.CancellableContinuation +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.StateFlow +import kotlinx.coroutines.flow.asStateFlow +import org.readium.r2.shared.ExperimentalReadiumApi +import timber.log.Timber + +@ExperimentalReadiumApi +internal class TtsEngineListener : TtsEngine.Listener { + + private data class UtteranceTask( + val id: String, + val continuation: CancellableContinuation + ) + + data class EngineState( + val utteranceId: String?, + val range: IntRange?, + ) + + private val waitingTasks: MutableList = + mutableListOf() + + private val stateMutable: MutableStateFlow = + MutableStateFlow( + EngineState(null, null) + ) + + val state: StateFlow = + stateMutable.asStateFlow() + + fun addCallback(id: String, continuation: CancellableContinuation) { + val task = UtteranceTask(id, continuation) + waitingTasks.add(task) + } + + fun removeAllCallbacks() { + waitingTasks.clear() + } + + override fun onStart(id: String) { + Timber.d("onStart") + stateMutable.value = EngineState(id, null) + } + + override fun onRange(id: String, range: IntRange) { + Timber.d("onRange") + stateMutable.value = EngineState(id, range) + } + + override fun onInterrupted(id: String) { + Timber.d("onInterrupted") + stateMutable.value = EngineState(null, null) + waitingTasks.forEach { + it.continuation.resume(null) {} + waitingTasks.remove(it) + } + } + + override fun onFlushed(id: String) { + Timber.d("onFlushed") + waitingTasks.forEach { + it.continuation.resume(null) {} + waitingTasks.remove(it) + } + } + + override fun onDone(id: String) { + Timber.d("onDone") + stateMutable.value = EngineState(null, null) + waitingTasks.forEach { + it.continuation.resume(null) {} + waitingTasks.remove(it) + } + } + + override fun onError(id: String, error: TtsEngine.Exception) { + Timber.e("onError $error") + } +} diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsEngineProvider.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsEngineProvider.kt new file mode 100644 index 0000000000..6274a10bcd --- /dev/null +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsEngineProvider.kt @@ -0,0 +1,21 @@ +/* + * Copyright 2022 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + +package org.readium.r2.navigator.media3.tts2 + +import org.readium.r2.navigator.preferences.PreferencesEditor +import org.readium.r2.shared.ExperimentalReadiumApi +import org.readium.r2.shared.publication.Publication + +@ExperimentalReadiumApi +interface TtsEngineProvider, E : PreferencesEditor

> { + + suspend fun createEngine(publication: Publication, initialPreferences: P): TtsEngine? + + fun createPreferencesEditor(publication: Publication, initialPreferences: P): E + + fun createEmptyPreferences(): P +} diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsLocator.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsLocator.kt new file mode 100644 index 0000000000..80d524be8a --- /dev/null +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsLocator.kt @@ -0,0 +1,70 @@ +/* + * Copyright 2022 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + +package org.readium.r2.navigator.media3.tts2 + +import kotlinx.serialization.Serializable +import org.readium.r2.navigator.media3.api.MediaNavigatorInternal +import org.readium.r2.shared.ExperimentalReadiumApi +import org.readium.r2.shared.publication.Locator +import org.readium.r2.shared.publication.Publication +import org.readium.r2.shared.publication.html.cssSelector +import org.readium.r2.shared.publication.indexOfFirstWithHref + +@ExperimentalReadiumApi +@Serializable +data class TtsLocator( + val resourceIndex: Int, + val text: String, + val textBefore: String?, + val textAfter: String?, + val cssSelector: String?, +) : MediaNavigatorInternal.Locator + +@ExperimentalReadiumApi +internal fun TtsLocator.toLocator(publication: Publication): Locator { + return publication + .locatorFromLink(publication.readingOrder[resourceIndex])!! + .copyWithLocations( + progression = null, + otherLocations = buildMap { + cssSelector?.let { put("cssSelector", it) } + } + ).copy( + text = + Locator.Text( + highlight = text, + before = textBefore, + after = textAfter + ) + ) +} + +@ExperimentalReadiumApi +internal fun Locator.toTtsLocator(publication: Publication): TtsLocator? { + val resourceIndex = publication.readingOrder.indexOfFirstWithHref(href) + ?: return null + + val contentText = text.highlight + ?: return null + + return TtsLocator( + resourceIndex = resourceIndex, + text = contentText, + textBefore = text.before, + textAfter = text.after, + cssSelector = locations.cssSelector, + ) +} + +@ExperimentalReadiumApi +internal fun TtsLocator.substring(range: IntRange): TtsLocator { + return copy( + textBefore = textBefore.orEmpty() + text.substring(0, range.first), + text = text.substring(range), + textAfter = text.substring(range.last) + textAfter.orEmpty() + ) +} diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsNavigator.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsNavigator.kt new file mode 100644 index 0000000000..508acf3237 --- /dev/null +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsNavigator.kt @@ -0,0 +1,148 @@ +/* + * Copyright 2022 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + +package org.readium.r2.navigator.media3.tts2 + +import android.app.Application +import androidx.media3.common.MediaItem +import androidx.media3.common.Player +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.MainScope +import kotlinx.coroutines.flow.StateFlow +import org.readium.r2.navigator.Navigator +import org.readium.r2.navigator.media3.api.MediaNavigator +import org.readium.r2.navigator.media3.api.MediaNavigatorInternal +import org.readium.r2.navigator.media3.api.MetadataProvider +import org.readium.r2.navigator.preferences.Configurable +import org.readium.r2.shared.ExperimentalReadiumApi +import org.readium.r2.shared.extensions.mapStateIn +import org.readium.r2.shared.publication.Link +import org.readium.r2.shared.publication.Locator +import org.readium.r2.shared.publication.Publication +import org.readium.r2.shared.publication.services.content.ContentService +import org.readium.r2.shared.publication.services.content.ContentTokenizer +import org.readium.r2.shared.util.Language + +@ExperimentalReadiumApi +class TtsNavigator> private constructor( + override val publication: Publication, + private val ttsNavigator: TtsNavigatorInternal +) : MediaNavigator, Navigator, Configurable by ttsNavigator { + + companion object { + + suspend operator fun > invoke( + application: Application, + publication: Publication, + ttsEngineProvider: TtsEngineProvider, + tokenizerFactory: (defaultLanguage: Language?) -> ContentTokenizer, + metadataProvider: MetadataProvider, + listener: TtsNavigatorListener, + initialPreferences: P? = null, + initialLocator: Locator? = null, + ): TtsNavigator? { + + if (publication.findService(ContentService::class) == null) { + return null + } + + val actualInitialLocator = initialLocator + ?.toTtsLocator(publication) + + val actualInitialPreferences = initialPreferences + ?: ttsEngineProvider.createEmptyPreferences() + + val contentIterator = + TtsContentIterator(publication, tokenizerFactory, actualInitialLocator) + + val ttsEngine = + ttsEngineProvider.createEngine(publication, actualInitialPreferences) + ?: return null + + val metadataFactory = + metadataProvider.createMetadataFactory(publication) + + val playlistMetadata = + metadataFactory.publicationMetadata() + + val mediaItems = + publication.readingOrder.indices.map { index -> + val metadata = metadataFactory.resourceMetadata(index) + MediaItem.Builder() + .setMediaMetadata(metadata) + .build() + } + + val internalNavigator = + TtsNavigatorInternal(application, ttsEngine, contentIterator, playlistMetadata, mediaItems, listener) + ?: return null + + return TtsNavigator(publication, internalNavigator) + } + } + + data class Playback( + override val state: MediaNavigator.State, + override val locator: Locator, + override val token: Locator? + ) : MediaNavigator.Playback, MediaNavigator.TextSynchronization + + private val coroutineScope: CoroutineScope = + MainScope() + + override val playback: StateFlow = + ttsNavigator.playback.mapStateIn(coroutineScope) { it.toPlayback() } + + private fun TtsPlayback.toPlayback() = + Playback( + state = when (state) { + MediaNavigatorInternal.State.Playing -> MediaNavigator.State.Playing + MediaNavigatorInternal.State.Paused -> MediaNavigator.State.Paused + MediaNavigatorInternal.State.Ended -> MediaNavigator.State.Ended + }, + locator = locator.toLocator(publication), + token = token?.toLocator(publication) + ) + + override val currentLocator: StateFlow = + playback.mapStateIn(coroutineScope) { it.locator } + + override fun go(locator: Locator, animated: Boolean, completion: () -> Unit): Boolean { + val ttsLocator = locator.toTtsLocator(publication) ?: return false + ttsNavigator.go(ttsLocator) + return true + } + + override fun go(link: Link, animated: Boolean, completion: () -> Unit): Boolean { + val locator = publication.locatorFromLink(link) ?: return false + return go(locator) + } + + override fun goForward(animated: Boolean, completion: () -> Unit): Boolean { + ttsNavigator.goForward() + return true + } + + override fun goBackward(animated: Boolean, completion: () -> Unit): Boolean { + ttsNavigator.goBackward() + return true + } + + override fun close() { + ttsNavigator.close() + } + + override fun play() { + ttsNavigator.play() + } + + override fun pause() { + ttsNavigator.pause() + } + + override fun asPlayer(): Player = + ttsNavigator.asPlayer() +} diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsNavigatorFactory.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsNavigatorFactory.kt new file mode 100644 index 0000000000..3fca996067 --- /dev/null +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsNavigatorFactory.kt @@ -0,0 +1,80 @@ +/* + * Copyright 2022 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + +package org.readium.r2.navigator.media3.tts2 + +import android.app.Application +import org.readium.r2.navigator.media3.api.DefaultMetadataProvider +import org.readium.r2.navigator.media3.api.MetadataProvider +import org.readium.r2.navigator.preferences.PreferencesEditor +import org.readium.r2.shared.ExperimentalReadiumApi +import org.readium.r2.shared.publication.Locator +import org.readium.r2.shared.publication.Publication +import org.readium.r2.shared.publication.services.content.* +import org.readium.r2.shared.util.Language +import org.readium.r2.shared.util.tokenizer.TextUnit + +@ExperimentalReadiumApi +class TtsNavigatorFactory, E : PreferencesEditor

>( + private val application: Application, + private val publication: Publication, + private val ttsEngineProvider: TtsEngineProvider, + private val tokenizerFactory: (defaultLanguage: Language?) -> ContentTokenizer, + private val metadataProvider: MetadataProvider +) { + companion object { + + suspend operator fun , E : PreferencesEditor

> invoke( + application: Application, + publication: Publication, + ttsEngineProvider: TtsEngineProvider, + tokenizerFactory: (defaultLanguage: Language?) -> ContentTokenizer = defaultTokenizerFactory, + metadataProvider: MetadataProvider = defaultMetadataProvider + ): TtsNavigatorFactory? { + + publication.content() + ?.iterator() + ?.takeIf { it.hasNext() } + ?: return null + + return TtsNavigatorFactory(application, publication, ttsEngineProvider, tokenizerFactory, metadataProvider) + } + + /** + * The default content tokenizer will split the [Content.Element] items into individual sentences. + */ + val defaultTokenizerFactory: (Language?) -> ContentTokenizer = { language -> + TextContentTokenizer( + unit = TextUnit.Sentence, + language = language, + overrideContentLanguage = false + ) + } + + val defaultMetadataProvider: MetadataProvider = DefaultMetadataProvider() + } + + suspend fun createNavigator( + listener: TtsNavigatorListener, + initialPreferences: P? = null, + initialLocator: Locator? = null + ): TtsNavigator { + return TtsNavigator( + application, + publication, + ttsEngineProvider, + tokenizerFactory, + metadataProvider, + listener, + initialPreferences, + initialLocator + )!! + } + + fun createTtsPreferencesEditor( + currentPreferences: P, + ): E = ttsEngineProvider.createPreferencesEditor(publication, currentPreferences) +} diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsNavigatorInternal.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsNavigatorInternal.kt new file mode 100644 index 0000000000..ad02a9e5c7 --- /dev/null +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsNavigatorInternal.kt @@ -0,0 +1,114 @@ +/* + * Copyright 2022 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + +package org.readium.r2.navigator.media3.tts2 + +import android.app.Application +import androidx.media3.common.MediaItem +import androidx.media3.common.MediaMetadata +import androidx.media3.common.Player +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.ExperimentalCoroutinesApi +import kotlinx.coroutines.MainScope +import kotlinx.coroutines.flow.StateFlow +import org.readium.r2.navigator.media3.api.MediaNavigatorInternal +import org.readium.r2.navigator.preferences.Configurable +import org.readium.r2.shared.ExperimentalReadiumApi +import org.readium.r2.shared.extensions.mapStateIn + +@OptIn(ExperimentalCoroutinesApi::class) +@ExperimentalReadiumApi +internal class TtsNavigatorInternal>( + application: Application, + private val ttsEngineFacade: TtsEngineFacade, + private val player: TtsPlayer, +) : MediaNavigatorInternal, Configurable by ttsEngineFacade { + + companion object { + + suspend operator fun > invoke( + application: Application, + ttsEngine: TtsEngine, + ttsContentIterator: TtsContentIterator, + playlistMetadata: MediaMetadata, + mediaItems: List, + listener: TtsNavigatorListener + ): TtsNavigatorInternal? { + + val facadeListener = object : TtsEngineFacadeListener { + + override fun onNavigatorStopped() { + listener.onStopRequested() + } + + override fun onPlaybackException() { + listener.onPlaybackException() + } + } + + val ttsEngineFacade = + TtsEngineFacade(ttsEngine, ttsContentIterator, facadeListener) + ?: return null + + val player = + TtsPlayer(application, ttsEngineFacade, playlistMetadata, mediaItems) + + return TtsNavigatorInternal(application, ttsEngineFacade, player) + } + } + + private val coroutineScope: CoroutineScope = + MainScope() + + override val playback: StateFlow = + ttsEngineFacade.playback + .mapStateIn(coroutineScope) { playback -> + val state = when (playback.state) { + TtsEngineFacadePlayback.State.READY -> + if (playback.playWhenReady) MediaNavigatorInternal.State.Playing + else MediaNavigatorInternal.State.Paused + TtsEngineFacadePlayback.State.ENDED -> + MediaNavigatorInternal.State.Ended + } + + val token = playback.range + ?.let { playback.locator.substring(playback.range) } + + TtsPlayback( + state = state, + locator = playback.locator, + token = token + ) + } + + override fun play() { + ttsEngineFacade.play() + } + + override fun pause() { + ttsEngineFacade.pause() + } + + override fun go(locator: TtsLocator) { + ttsEngineFacade.go(locator) + } + + override fun goForward() { + ttsEngineFacade.nextUtterance() + } + + override fun goBackward() { + ttsEngineFacade.previousUtterance() + } + + override fun asPlayer(): Player { + return player + } + + fun close() { + ttsEngineFacade.close() + } +} diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsNavigatorListener.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsNavigatorListener.kt new file mode 100644 index 0000000000..af67a340f0 --- /dev/null +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsNavigatorListener.kt @@ -0,0 +1,14 @@ +/* + * Copyright 2022 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + +package org.readium.r2.navigator.media3.tts2 + +interface TtsNavigatorListener { + + fun onStopRequested() + + fun onPlaybackException() +} diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsPlayback.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsPlayback.kt new file mode 100644 index 0000000000..e853533b45 --- /dev/null +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsPlayback.kt @@ -0,0 +1,18 @@ +/* + * Copyright 2022 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + +package org.readium.r2.navigator.media3.tts2 + +import org.readium.r2.navigator.media3.api.MediaNavigatorInternal +import org.readium.r2.navigator.media3.api.SynchronizedPlayback +import org.readium.r2.shared.ExperimentalReadiumApi + +@ExperimentalReadiumApi +internal data class TtsPlayback( + override val state: MediaNavigatorInternal.State, + override val locator: TtsLocator, + override val token: TtsLocator? +) : SynchronizedPlayback diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsPlayer.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsPlayer.kt new file mode 100644 index 0000000000..729c265118 --- /dev/null +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsPlayer.kt @@ -0,0 +1,448 @@ +/* + * Copyright 2022 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + +package org.readium.r2.navigator.media3.tts2 + +import android.app.Application +import android.content.Context +import android.media.AudioManager +import android.os.Looper +import android.view.Surface +import android.view.SurfaceHolder +import android.view.SurfaceView +import android.view.TextureView +import androidx.media3.common.* +import androidx.media3.common.C.* +import androidx.media3.common.text.CueGroup +import androidx.media3.common.util.Clock +import androidx.media3.common.util.ListenerSet +import androidx.media3.common.util.Util +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.MainScope +import kotlinx.coroutines.flow.launchIn +import kotlinx.coroutines.flow.onEach +import org.readium.r2.shared.ExperimentalReadiumApi +import timber.log.Timber + +@ExperimentalReadiumApi +@androidx.annotation.OptIn(androidx.media3.common.util.UnstableApi::class) +internal class TtsPlayer( + private val application: Application, + private val ttsEngineFacade: TtsEngineFacade<*, *>, + private val playlistMetadata: MediaMetadata, + private val mediaItems: List +) : BasePlayer() { + + private val coroutineScope: CoroutineScope = + MainScope() + + private var lastPlayback: TtsEngineFacadePlayback = + ttsEngineFacade.playback.value + + init { + ttsEngineFacade.playback + .onEach { playback -> + notifyListeners(lastPlayback, playback) + lastPlayback = playback + }.launchIn(coroutineScope) + } + + private var listeners: ListenerSet = + ListenerSet( + applicationLooper, + Clock.DEFAULT, + ) { listener: Player.Listener, flags: FlagSet? -> + listener.onEvents(this, Player.Events(flags!!)) + } + + private val permanentAvailableCommands = + Player.Commands.Builder() + .addAll( + COMMAND_PLAY_PAUSE, + COMMAND_STOP, + COMMAND_SEEK_TO_NEXT_MEDIA_ITEM, + COMMAND_SEEK_TO_PREVIOUS_MEDIA_ITEM, + COMMAND_SEEK_TO_NEXT, + COMMAND_SEEK_TO_PREVIOUS + // COMMAND_GET_AUDIO_ATTRIBUTES, + // COMMAND_GET_CURRENT_MEDIA_ITEM, + // COMMAND_GET_MEDIA_ITEMS_METADATA, + // COMMAND_GET_TEXT + ).build() + + private val audioManager: AudioManager = + application.getSystemService(Context.AUDIO_SERVICE) as AudioManager + + override fun getApplicationLooper(): Looper { + return Looper.getMainLooper() + } + + override fun addListener(listener: Player.Listener) { + Timber.d("addListener") + listeners.add(listener) + } + + override fun removeListener(listener: Player.Listener) { + Timber.d("removeListener") + listeners.remove(listener) + } + + override fun setMediaItems(mediaItems: MutableList, resetPosition: Boolean) { + throw NotImplementedError() + } + + override fun setMediaItems( + mediaItems: MutableList, + startIndex: Int, + startPositionMs: Long + ) { + throw NotImplementedError() + } + + override fun addMediaItems(index: Int, mediaItems: MutableList) { + throw NotImplementedError() + } + + override fun moveMediaItems(fromIndex: Int, toIndex: Int, newIndex: Int) { + throw NotImplementedError() + } + + override fun removeMediaItems(fromIndex: Int, toIndex: Int) { + throw NotImplementedError() + } + + override fun getAvailableCommands(): Player.Commands { + return Player.Commands.Builder() + .addAll(permanentAvailableCommands) + .build() + } + + override fun prepare() { + throw NotImplementedError() + } + + override fun getPlaybackState(): Int { + return ttsEngineFacade.playback.value.state.value + } + + override fun getPlaybackSuppressionReason(): Int { + return PLAYBACK_SUPPRESSION_REASON_NONE // TODO + } + + override fun getPlayerError(): PlaybackException? { + return null // TODO + } + + override fun setPlayWhenReady(playWhenReady: Boolean) { + if (playWhenReady) { + ttsEngineFacade.play() + } else { + ttsEngineFacade.pause() + } + } + + override fun getPlayWhenReady(): Boolean { + return ttsEngineFacade.playback.value.playWhenReady + } + + override fun setRepeatMode(repeatMode: Int) { + throw NotImplementedError() + } + + override fun getRepeatMode(): Int { + return REPEAT_MODE_OFF + } + + override fun setShuffleModeEnabled(shuffleModeEnabled: Boolean) { + throw NotImplementedError() + } + + override fun getShuffleModeEnabled(): Boolean { + return false + } + + override fun isLoading(): Boolean { + return false + } + + override fun seekTo(mediaItemIndex: Int, positionMs: Long) { + throw NotImplementedError() + } + + override fun getSeekBackIncrement(): Long { + return 0 + } + + override fun getSeekForwardIncrement(): Long { + return 0 + } + + override fun getMaxSeekToPreviousPosition(): Long { + return 0 + } + + override fun setPlaybackParameters(playbackParameters: PlaybackParameters) { + throw NotImplementedError() // TODO + } + + override fun getPlaybackParameters(): PlaybackParameters { + return PlaybackParameters.DEFAULT + } + + override fun stop() { + ttsEngineFacade.stop() + } + + @Deprecated("Deprecated in Java") + override fun stop(reset: Boolean) {} + + override fun release() { + ttsEngineFacade.close() + } + + override fun getCurrentTracks(): Tracks { + throw NotImplementedError() + } + + override fun getTrackSelectionParameters(): TrackSelectionParameters { + return TrackSelectionParameters.Builder(application) + .build() + } + + override fun setTrackSelectionParameters(parameters: TrackSelectionParameters) { + throw NotImplementedError() + } + + override fun getMediaMetadata(): MediaMetadata { + return currentTimeline.getWindow(currentMediaItemIndex, window).mediaItem.mediaMetadata + } + + override fun getPlaylistMetadata(): MediaMetadata { + return playlistMetadata + } + + override fun setPlaylistMetadata(mediaMetadata: MediaMetadata) { + throw NotImplementedError() + } + + override fun getCurrentTimeline(): Timeline { + // MediaNotificationManager requires a non-empty timeline to start foreground playing. + return TtsTimeline(mediaItems) + /*return SinglePeriodTimeline( + TIME_UNSET, false, false, false, null, mediaItem)*/ + } + + override fun getCurrentPeriodIndex(): Int { + return lastPlayback.index + } + + override fun getCurrentMediaItemIndex(): Int { + return lastPlayback.index + } + + override fun getDuration(): Long { + return TIME_UNSET + } + + override fun getCurrentPosition(): Long { + return 0 + } + + override fun getBufferedPosition(): Long { + return 0 + } + + override fun getTotalBufferedDuration(): Long { + return 0 + } + + override fun isPlayingAd(): Boolean { + return false + } + + override fun getCurrentAdGroupIndex(): Int { + return INDEX_UNSET + } + + override fun getCurrentAdIndexInAdGroup(): Int { + return INDEX_UNSET + } + + override fun getContentPosition(): Long { + return 0 + } + + override fun getContentBufferedPosition(): Long { + return 0 + } + + override fun getAudioAttributes(): AudioAttributes { + return AudioAttributes.Builder() + .setUsage(USAGE_MEDIA) + .setContentType(AUDIO_CONTENT_TYPE_SPEECH) + .build() + } + + override fun setVolume(volume: Float) { + throw NotImplementedError() + } + + override fun getVolume(): Float { + return 1.0f + } + + override fun clearVideoSurface() { + throw NotImplementedError() + } + + override fun clearVideoSurface(surface: Surface?) { + throw NotImplementedError() + } + + override fun setVideoSurface(surface: Surface?) { + throw NotImplementedError() + } + + override fun setVideoSurfaceHolder(surfaceHolder: SurfaceHolder?) { + throw NotImplementedError() + } + + override fun clearVideoSurfaceHolder(surfaceHolder: SurfaceHolder?) { + throw NotImplementedError() + } + + override fun setVideoSurfaceView(surfaceView: SurfaceView?) { + throw NotImplementedError() + } + + override fun clearVideoSurfaceView(surfaceView: SurfaceView?) { + throw NotImplementedError() + } + + override fun setVideoTextureView(textureView: TextureView?) { + throw NotImplementedError() + } + + override fun clearVideoTextureView(textureView: TextureView?) { + throw NotImplementedError() + } + + override fun getVideoSize(): VideoSize { + return VideoSize.UNKNOWN + } + + override fun getCurrentCues(): CueGroup { + return CueGroup(emptyList()) + } + + override fun getDeviceInfo(): DeviceInfo { + val minVolume = if (Util.SDK_INT >= 28) audioManager.getStreamMinVolume(STREAM_TYPE_MUSIC) else 0 + val maxVolume = audioManager.getStreamMaxVolume(STREAM_TYPE_MUSIC) + return DeviceInfo(DeviceInfo.PLAYBACK_TYPE_LOCAL, minVolume, maxVolume) + } + + override fun getDeviceVolume(): Int { + return audioManager.getStreamVolume(STREAM_TYPE_MUSIC) + } + + override fun isDeviceMuted(): Boolean { + return if (Util.SDK_INT >= 23) { + audioManager.isStreamMute(STREAM_TYPE_MUSIC) + } else { + deviceVolume == 0 + } + } + + override fun setDeviceVolume(volume: Int) { + throw NotImplementedError() + } + + override fun increaseDeviceVolume() { + throw NotImplementedError() + } + + override fun decreaseDeviceVolume() { + throw NotImplementedError() + } + + override fun setDeviceMuted(muted: Boolean) { + throw NotImplementedError() + } + + private fun notifyListeners( + previousPlaybackInfo: TtsEngineFacadePlayback, + playbackInfo: TtsEngineFacadePlayback, + // playWhenReadyChangeReason: @Player.PlayWhenReadyChangeReason Int, + ) { + /*if (previousPlaybackInfo.playbackError != playbackInfo.playbackError) { + listeners.queueEvent( + EVENT_PLAYER_ERROR + ) { listener: Player.Listener -> + listener.onPlayerErrorChanged( + playbackInfo.playbackError + ) + } + if (playbackInfo.playbackError != null) { + listeners.queueEvent( + EVENT_PLAYER_ERROR + ) { listener: Player.Listener -> + listener.onPlayerError( + playbackInfo.playbackError!! + ) + } + } + }*/ + + if (previousPlaybackInfo.isPlaying != playbackInfo.isPlaying) { + listeners.queueEvent( + EVENT_PLAYBACK_STATE_CHANGED + ) { listener: Player.Listener -> + listener.onPlaybackStateChanged( + playbackInfo.state.value + ) + } + } + + if (previousPlaybackInfo.playWhenReady != playbackInfo.playWhenReady) { + listeners.queueEvent( + EVENT_PLAY_WHEN_READY_CHANGED + ) { listener: Player.Listener -> + listener.onPlayWhenReadyChanged( + playbackInfo.playWhenReady, + if (playbackInfo.state == TtsEngineFacadePlayback.State.ENDED) + PLAY_WHEN_READY_CHANGE_REASON_END_OF_MEDIA_ITEM + else + PLAY_WHEN_READY_CHANGE_REASON_USER_REQUEST + // PLAYBACK_SUPPRESSION_REASON_NONE + // playWhenReadyChangeReason + ) + } + } + + if (isPlaying(previousPlaybackInfo) != isPlaying(playbackInfo)) { + listeners.queueEvent( + EVENT_IS_PLAYING_CHANGED + ) { listener: Player.Listener -> + listener.onIsPlayingChanged(isPlaying(playbackInfo)) + } + } + /*if (previousPlaybackInfo.playbackParameters != playbackInfo.playbackParameters) { + listeners.queueEvent( + EVENT_PLAYBACK_PARAMETERS_CHANGED + ) { listener: Player.Listener -> + listener.onPlaybackParametersChanged( + playbackInfo.playbackParameters + ) + } + }*/ + + listeners.flushEvents() + } + + private fun isPlaying(playbackInfo: TtsEngineFacadePlayback): Boolean { + return (playbackInfo.state == TtsEngineFacadePlayback.State.READY && playbackInfo.playWhenReady) + } +} diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsPreferences.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsPreferences.kt new file mode 100644 index 0000000000..7e91b3b018 --- /dev/null +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsPreferences.kt @@ -0,0 +1,17 @@ +/* + * Copyright 2022 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + +package org.readium.r2.navigator.media3.tts2 + +import org.readium.r2.navigator.preferences.Configurable +import org.readium.r2.shared.ExperimentalReadiumApi +import org.readium.r2.shared.util.Language + +@ExperimentalReadiumApi +interface TtsPreferences

> : Configurable.Preferences

{ + + val language: Language? +} diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsSettings.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsSettings.kt new file mode 100644 index 0000000000..ebe4731983 --- /dev/null +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsSettings.kt @@ -0,0 +1,17 @@ +/* + * Copyright 2022 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + +package org.readium.r2.navigator.media3.tts2 + +import org.readium.r2.navigator.preferences.Configurable +import org.readium.r2.shared.ExperimentalReadiumApi +import org.readium.r2.shared.util.Language + +@ExperimentalReadiumApi +interface TtsSettings : Configurable.Settings { + + val language: Language? +} diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsTimeline.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsTimeline.kt new file mode 100644 index 0000000000..cd3104c81e --- /dev/null +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsTimeline.kt @@ -0,0 +1,56 @@ +/* + * Copyright 2022 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + +package org.readium.r2.navigator.media3.tts2 + +import androidx.media3.common.MediaItem +import androidx.media3.common.Timeline +import java.util.* + +@androidx.annotation.OptIn(androidx.media3.common.util.UnstableApi::class) +internal class TtsTimeline( + private val mediaItems: List, +) : Timeline() { + + private val uuids = (0 until windowCount) + .map { UUID.randomUUID() } + + override fun getWindowCount(): Int { + return mediaItems.size + } + + override fun getWindow( + windowIndex: Int, + window: Window, + defaultPositionProjectionUs: Long + ): Window { + window.uid = uuids[windowIndex] + window.firstPeriodIndex = windowIndex + window.lastPeriodIndex = windowIndex + window.mediaItem = mediaItems[windowIndex] + return window + } + + override fun getPeriodCount(): Int { + return mediaItems.size + } + + override fun getPeriod(periodIndex: Int, period: Period, setIds: Boolean): Period { + period.windowIndex += periodIndex + if (setIds) { + period.uid = uuids[periodIndex] + } + return period + } + + override fun getIndexOfPeriod(uid: Any): Int { + return uuids.indexOfFirst { it == uid } + } + + override fun getUidOfPeriod(periodIndex: Int): Any { + return uuids[periodIndex] + } +} diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsUtterance.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsUtterance.kt new file mode 100644 index 0000000000..8da9a53b36 --- /dev/null +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsUtterance.kt @@ -0,0 +1,17 @@ +/* + * Copyright 2022 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + +package org.readium.r2.navigator.media3.tts2 + +import org.readium.r2.shared.ExperimentalReadiumApi +import org.readium.r2.shared.util.Language + +@ExperimentalReadiumApi +data class TtsUtterance( + val text: String, + val locator: TtsLocator, + val language: Language? +) diff --git a/test-app/build.gradle.kts b/test-app/build.gradle.kts index 53670c41b8..018fa7d8cd 100644 --- a/test-app/build.gradle.kts +++ b/test-app/build.gradle.kts @@ -102,7 +102,7 @@ dependencies { implementation(libs.kotlinx.coroutines.core) implementation(libs.jsoup) - implementation(libs.bundles.media2) + implementation(libs.bundles.media3) // Room database implementation(libs.bundles.room) diff --git a/test-app/src/main/AndroidManifest.xml b/test-app/src/main/AndroidManifest.xml index 63344a09e2..1e9738b983 100644 --- a/test-app/src/main/AndroidManifest.xml +++ b/test-app/src/main/AndroidManifest.xml @@ -189,16 +189,26 @@ android:name=".reader.ReaderActivity" android:label="Reader" /> - - + + + + + + + diff --git a/test-app/src/main/java/org/readium/r2/testapp/Application.kt b/test-app/src/main/java/org/readium/r2/testapp/Application.kt index 576ee32e52..6a4b65a0c5 100755 --- a/test-app/src/main/java/org/readium/r2/testapp/Application.kt +++ b/test-app/src/main/java/org/readium/r2/testapp/Application.kt @@ -6,11 +6,7 @@ package org.readium.r2.testapp -import android.content.ComponentName -import android.content.Context -import android.content.Intent -import android.content.ServiceConnection -import android.os.IBinder +import android.content.* import androidx.datastore.core.DataStore import androidx.datastore.preferences.core.Preferences import androidx.datastore.preferences.preferencesDataStore @@ -43,27 +39,6 @@ class Application : android.app.Application() { private val Context.navigatorPreferences: DataStore by preferencesDataStore(name = "navigator-preferences") - private val mediaServiceBinder: CompletableDeferred = - CompletableDeferred() - - private val mediaServiceConnection = object : ServiceConnection { - - override fun onServiceConnected(name: ComponentName?, service: IBinder) { - Timber.d("MediaService bound.") - mediaServiceBinder.complete(service as MediaService.Binder) - } - - override fun onServiceDisconnected(name: ComponentName) { - Timber.d("MediaService disconnected.") - // Should not happen, do nothing. - } - - override fun onNullBinding(name: ComponentName) { - Timber.d("Failed to bind to MediaService.") - // Should not happen, do nothing. - } - } - override fun onCreate() { super.onCreate() DynamicColors.applyToActivitiesIfAvailable(this) @@ -73,29 +48,26 @@ class Application : android.app.Application() { storageDir = computeStorageDir() - /* - * Starting media service. - */ - - // MediaSessionService.onBind requires the intent to have a non-null action. - val intent = Intent(MediaService.SERVICE_INTERFACE) - .apply { setClass(applicationContext, MediaService::class.java) } - startService(intent) - bindService(intent, mediaServiceConnection, 0) - /* * Initializing repositories */ bookRepository = BookDatabase.getDatabase(this).booksDao() - .let { BookRepository(this, it, storageDir, readium.lcpService, readium.streamer) } + .let { dao -> + BookRepository( + applicationContext, + dao, + storageDir, + readium.lcpService, + readium.streamer + ) + } readerRepository = coroutineScope.async { ReaderRepository( this@Application, readium, - mediaServiceBinder.await(), bookRepository, navigatorPreferences ) diff --git a/test-app/src/main/java/org/readium/r2/testapp/bookshelf/BookshelfFragment.kt b/test-app/src/main/java/org/readium/r2/testapp/bookshelf/BookshelfFragment.kt index 6e136d6dbe..8c0374e9f6 100644 --- a/test-app/src/main/java/org/readium/r2/testapp/bookshelf/BookshelfFragment.kt +++ b/test-app/src/main/java/org/readium/r2/testapp/bookshelf/BookshelfFragment.kt @@ -23,7 +23,6 @@ import androidx.fragment.app.activityViewModels import androidx.recyclerview.widget.RecyclerView import com.google.android.material.dialog.MaterialAlertDialogBuilder import com.google.android.material.snackbar.Snackbar -import org.readium.r2.shared.extensions.tryOrLog import org.readium.r2.testapp.R import org.readium.r2.testapp.databinding.FragmentBookshelfBinding import org.readium.r2.testapp.domain.model.Book @@ -66,11 +65,6 @@ class BookshelfFragment : Fragment() { } } - readerLauncher = - registerForActivityResult(ReaderActivityContract()) { input -> - input?.let { tryOrLog { bookshelfViewModel.closePublication(input.bookId) } } - } - binding.bookshelfBookList.apply { setHasFixedSize(true) layoutManager = GridAutoFitLayoutManager(requireContext(), 120) @@ -147,7 +141,8 @@ class BookshelfFragment : Fragment() { } is BookshelfViewModel.Event.LaunchReader -> { - readerLauncher.launch(event.arguments) + val intent = ReaderActivityContract().createIntent(requireContext(), event.arguments) + startActivity(intent) null } } diff --git a/test-app/src/main/java/org/readium/r2/testapp/bookshelf/BookshelfViewModel.kt b/test-app/src/main/java/org/readium/r2/testapp/bookshelf/BookshelfViewModel.kt index 676869ab44..a79d414a3e 100644 --- a/test-app/src/main/java/org/readium/r2/testapp/bookshelf/BookshelfViewModel.kt +++ b/test-app/src/main/java/org/readium/r2/testapp/bookshelf/BookshelfViewModel.kt @@ -9,14 +9,23 @@ package org.readium.r2.testapp.bookshelf import android.app.Activity import android.app.Application import android.content.Context +import android.graphics.Bitmap +import android.graphics.BitmapFactory import android.net.Uri import androidx.lifecycle.AndroidViewModel import androidx.lifecycle.viewModelScope +import java.io.File +import java.io.FileOutputStream +import java.io.IOException +import java.net.HttpURLConnection +import java.net.URL import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.channels.Channel import kotlinx.coroutines.launch import kotlinx.coroutines.withContext import org.readium.r2.shared.UserException +import org.readium.r2.shared.publication.Publication +import org.readium.r2.shared.publication.services.cover import org.readium.r2.testapp.BuildConfig import org.readium.r2.testapp.R import org.readium.r2.testapp.domain.model.Book @@ -115,6 +124,38 @@ class BookshelfViewModel(application: Application) : AndroidViewModel(applicatio readerRepository.close(bookId) } + private fun storeCoverImage(publication: Publication, imageName: String) = + viewModelScope.launch(Dispatchers.IO) { + // TODO Figure out where to store these cover images + val coverImageDir = File(app.storageDir, "covers/") + if (!coverImageDir.exists()) { + coverImageDir.mkdirs() + } + val coverImageFile = File(app.storageDir, "covers/$imageName.png") + + val bitmap: Bitmap? = publication.cover() + + val resized = bitmap?.let { Bitmap.createScaledBitmap(it, 120, 200, true) } + val fos = FileOutputStream(coverImageFile) + resized?.compress(Bitmap.CompressFormat.PNG, 80, fos) + fos.flush() + fos.close() + } + + private fun getBitmapFromURL(src: String): Bitmap? { + return try { + val url = URL(src) + val connection = url.openConnection() as HttpURLConnection + connection.doInput = true + connection.connect() + val input = connection.inputStream + BitmapFactory.decodeStream(input) + } catch (e: IOException) { + e.printStackTrace() + null + } + } + sealed class Event { object ImportPublicationSuccess : Event() diff --git a/test-app/src/main/java/org/readium/r2/testapp/MediaService.kt b/test-app/src/main/java/org/readium/r2/testapp/reader/MediaService.kt similarity index 60% rename from test-app/src/main/java/org/readium/r2/testapp/MediaService.kt rename to test-app/src/main/java/org/readium/r2/testapp/reader/MediaService.kt index ba121a94c1..e8469aae1e 100644 --- a/test-app/src/main/java/org/readium/r2/testapp/MediaService.kt +++ b/test-app/src/main/java/org/readium/r2/testapp/reader/MediaService.kt @@ -4,16 +4,18 @@ * available in the top-level LICENSE file of the project. */ -package org.readium.r2.testapp +package org.readium.r2.testapp.reader +import android.app.Application import android.app.PendingIntent +import android.content.ComponentName import android.content.Intent +import android.content.ServiceConnection import android.os.Build import android.os.IBinder import androidx.lifecycle.lifecycleScope import androidx.media2.session.MediaSession -import kotlin.time.ExperimentalTime -import kotlinx.coroutines.ExperimentalCoroutinesApi +import kotlinx.coroutines.CompletableDeferred import kotlinx.coroutines.FlowPreview import kotlinx.coroutines.Job import kotlinx.coroutines.flow.launchIn @@ -21,20 +23,19 @@ import kotlinx.coroutines.flow.onEach import kotlinx.coroutines.flow.sample import org.readium.navigator.media2.ExperimentalMedia2 import org.readium.navigator.media2.MediaNavigator -import org.readium.r2.testapp.reader.ReaderActivityContract -import org.readium.r2.testapp.utils.LifecycleMediaSessionService +import org.readium.r2.testapp.utils.LifecycleMedia2SessionService import timber.log.Timber -@OptIn(ExperimentalTime::class, ExperimentalMedia2::class, ExperimentalCoroutinesApi::class) -class MediaService : LifecycleMediaSessionService() { +@OptIn(ExperimentalMedia2::class) +class MediaService : LifecycleMedia2SessionService() { /** * The service interface to be used by the app. */ inner class Binder : android.os.Binder() { - private val app: Application - get() = application as Application + private val app: org.readium.r2.testapp.Application + get() = application as org.readium.r2.testapp.Application private var saveLocationJob: Job? = null @@ -82,8 +83,8 @@ class MediaService : LifecycleMediaSessionService() { applicationContext, ReaderActivityContract.Arguments(bookId) ) - intent.addFlags(Intent.FLAG_ACTIVITY_CLEAR_TOP) - + /* intent.addFlags(Intent.FLAG_ACTIVITY_CLEAR_TOP) + .addFlags(Intent.FLAG_ACTIVITY_SINGLE_TOP) */ return PendingIntent.getActivity(applicationContext, 0, intent, flags) } } @@ -130,6 +131,54 @@ class MediaService : LifecycleMediaSessionService() { } companion object { - const val SERVICE_INTERFACE = "org.readium.r2.testapp.MediaService" + + const val SERVICE_INTERFACE = "org.readium.r2.testapp.reader.MediaService" + + fun start(application: Application) { + val intent = intent(application) + application.startService(intent) + } + + suspend fun bind(application: Application): MediaService.Binder { + val mediaServiceBinder: CompletableDeferred = + CompletableDeferred() + + val mediaServiceConnection = object : ServiceConnection { + + override fun onServiceConnected(name: ComponentName?, service: IBinder) { + Timber.d("MediaService bound.") + mediaServiceBinder.complete(service as MediaService.Binder) + } + + override fun onServiceDisconnected(name: ComponentName) { + Timber.e("MediaService disconnected.") + + // Should not happen, do nothing. + } + + override fun onNullBinding(name: ComponentName) { + val errorMessage = "Failed to bind to MediaService." + Timber.e(errorMessage) + val exception = IllegalStateException(errorMessage) + mediaServiceBinder.completeExceptionally(exception) + // Should not happen, do nothing. + } + } + + val intent = intent(application) + application.bindService(intent, mediaServiceConnection, 0) + + return mediaServiceBinder.await() + } + + fun stop(application: Application) { + val intent = intent(application) + application.stopService(intent) + } + + private fun intent(application: Application) = + Intent(SERVICE_INTERFACE) + // MediaSessionService.onBind requires the intent to have a non-null action + .apply { setClass(application, MediaService::class.java) } } } diff --git a/test-app/src/main/java/org/readium/r2/testapp/reader/ReaderActivity.kt b/test-app/src/main/java/org/readium/r2/testapp/reader/ReaderActivity.kt index cc8867dc7b..42354a09fe 100644 --- a/test-app/src/main/java/org/readium/r2/testapp/reader/ReaderActivity.kt +++ b/test-app/src/main/java/org/readium/r2/testapp/reader/ReaderActivity.kt @@ -6,8 +6,6 @@ package org.readium.r2.testapp.reader -import android.app.Activity -import android.content.Intent import android.os.Build import android.os.Bundle import android.view.MenuItem @@ -150,7 +148,7 @@ open class ReaderActivity : AppCompatActivity() { } override fun finish() { - setResult(Activity.RESULT_OK, Intent().putExtras(intent)) + model.close() super.finish() } diff --git a/test-app/src/main/java/org/readium/r2/testapp/reader/ReaderInitData.kt b/test-app/src/main/java/org/readium/r2/testapp/reader/ReaderInitData.kt index 21086934e1..10884f7fd2 100644 --- a/test-app/src/main/java/org/readium/r2/testapp/reader/ReaderInitData.kt +++ b/test-app/src/main/java/org/readium/r2/testapp/reader/ReaderInitData.kt @@ -15,10 +15,15 @@ import org.readium.navigator.media2.ExperimentalMedia2 import org.readium.navigator.media2.MediaNavigator import org.readium.r2.navigator.epub.EpubNavigatorFactory import org.readium.r2.navigator.epub.EpubPreferences +import org.readium.r2.navigator.media3.androidtts.AndroidTtsPreferences +import org.readium.r2.navigator.media3.androidtts.AndroidTtsPreferencesEditor +import org.readium.r2.navigator.media3.androidtts.AndroidTtsSettings +import org.readium.r2.navigator.media3.tts2.TtsNavigatorFactory import org.readium.r2.navigator.pdf.PdfNavigatorFactory import org.readium.r2.shared.ExperimentalReadiumApi import org.readium.r2.shared.publication.* import org.readium.r2.testapp.reader.preferences.PreferencesManager +import org.readium.r2.testapp.reader.tts.TtsService sealed class ReaderInitData { abstract val bookId: Long @@ -29,35 +34,42 @@ sealed class VisualReaderInitData( override val bookId: Long, override val publication: Publication, val initialLocation: Locator?, + val ttsInitData: TtsInitData?, ) : ReaderInitData() class ImageReaderInitData( bookId: Long, publication: Publication, - initialLocation: Locator? -) : VisualReaderInitData(bookId, publication, initialLocation) + initialLocation: Locator?, + ttsInitData: TtsInitData?, +) : VisualReaderInitData(bookId, publication, initialLocation, ttsInitData) class EpubReaderInitData( bookId: Long, publication: Publication, initialLocation: Locator?, val preferencesManager: PreferencesManager, - val navigatorFactory: EpubNavigatorFactory -) : VisualReaderInitData(bookId, publication, initialLocation) + val navigatorFactory: EpubNavigatorFactory, + ttsInitData: TtsInitData?, +) : VisualReaderInitData(bookId, publication, initialLocation, ttsInitData) class PdfReaderInitData( bookId: Long, publication: Publication, initialLocation: Locator?, val preferencesManager: PreferencesManager, - val navigatorFactory: PdfNavigatorFactory -) : VisualReaderInitData(bookId, publication, initialLocation) + val navigatorFactory: PdfNavigatorFactory, + ttsInitData: TtsInitData?, +) : VisualReaderInitData(bookId, publication, initialLocation, ttsInitData) -@ExperimentalMedia2 +@OptIn(ExperimentalMedia2::class) class MediaReaderInitData( override val bookId: Long, override val publication: Publication, val mediaNavigator: MediaNavigator, + val sessionBinder: MediaService.Binder + // val preferencesManager: PreferencesManager, + // val navigatorFactory: PlayerNavigatorFactory ) : ReaderInitData() class DummyReaderInitData( @@ -69,3 +81,9 @@ class DummyReaderInitData( ) ) } + +class TtsInitData( + val sessionBinder: TtsService.Binder, + val ttsNavigatorFactory: TtsNavigatorFactory, + val preferencesManager: PreferencesManager, +) diff --git a/test-app/src/main/java/org/readium/r2/testapp/reader/ReaderRepository.kt b/test-app/src/main/java/org/readium/r2/testapp/reader/ReaderRepository.kt index 5af2a13c6c..43f16de7b5 100644 --- a/test-app/src/main/java/org/readium/r2/testapp/reader/ReaderRepository.kt +++ b/test-app/src/main/java/org/readium/r2/testapp/reader/ReaderRepository.kt @@ -16,6 +16,8 @@ import org.readium.adapters.pdfium.navigator.PdfiumEngineProvider import org.readium.navigator.media2.ExperimentalMedia2 import org.readium.navigator.media2.MediaNavigator import org.readium.r2.navigator.epub.EpubNavigatorFactory +import org.readium.r2.navigator.media3.androidtts.AndroidTtsEngineProvider +import org.readium.r2.navigator.media3.tts2.TtsNavigatorFactory import org.readium.r2.navigator.pdf.PdfNavigatorFactory import org.readium.r2.shared.ExperimentalReadiumApi import org.readium.r2.shared.publication.Locator @@ -25,24 +27,25 @@ import org.readium.r2.shared.publication.services.isRestricted import org.readium.r2.shared.publication.services.protectionError import org.readium.r2.shared.util.Try import org.readium.r2.shared.util.getOrElse -import org.readium.r2.testapp.MediaService import org.readium.r2.testapp.Readium import org.readium.r2.testapp.bookshelf.BookRepository +import org.readium.r2.testapp.reader.preferences.AndroidTtsPreferencesManagerFactory import org.readium.r2.testapp.reader.preferences.EpubPreferencesManagerFactory import org.readium.r2.testapp.reader.preferences.PdfiumPreferencesManagerFactory +import org.readium.r2.testapp.reader.tts.TtsService +import timber.log.Timber /** * Open and store publications in order for them to be listened or read. * * Ensure you call [open] before any attempt to start a [ReaderActivity]. * Pass the method result to the activity to enable it to know which current publication it must - * retrieve from this rep+ository - media or visual. + * retrieve from this repository - media or visual. */ -@OptIn(ExperimentalMedia2::class, ExperimentalReadiumApi::class) +@OptIn(ExperimentalReadiumApi::class) class ReaderRepository( private val application: Application, private val readium: Readium, - private val mediaBinder: MediaService.Binder, private val bookRepository: BookRepository, private val preferencesDataStore: DataStore, ) { @@ -108,16 +111,53 @@ class ReaderRepository( publication: Publication, initialLocator: Locator? ): MediaReaderInitData { + val navigator = MediaNavigator.create( application, publication, initialLocator ).getOrElse { throw Exception("Cannot open audiobook.") } + MediaService.start(application) + val mediaBinder = MediaService.bind(application) mediaBinder.bindNavigator(navigator, bookId) - return MediaReaderInitData(bookId, publication, navigator) + return MediaReaderInitData(bookId, publication, navigator, mediaBinder) } + /* private suspend fun openAudio( + bookId: Long, + publication: Publication, + initialLocator: Locator? + ): MediaReaderInitData { + + val preferencesManager = ExoPlayerPreferencesManagerFactory(preferencesDataStore) + .createPreferenceManager(bookId) + val mediaEngine = ExoPlayerEngineProvider(application) + val initialPreferences = preferencesManager.preferences.value + val actualInitialLocator = initialLocator + ?: publication.locatorFromLink(publication.readingOrder[0])!! + + val navigatorFactory = PlayerNavigatorFactory( + publication, + mediaEngine, + DefaultMetadataProvider(), + initialPreferences, + actualInitialLocator, + ) + + val navigator = navigatorFactory.getMediaNavigator() + .getOrElse { throw Exception("Cannot open audiobook.") } + + val navigator = MediaNavigator.create( + application, + publication, + initialLocator + ).getOrElse { throw Exception("Cannot open audiobook.") } + + mediaBinder.bindNavigator(navigator, bookId) + return MediaReaderInitData(bookId, publication,, preferencesManager, navigatorFactory) + } */ + private suspend fun openEpub( bookId: Long, publication: Publication, @@ -127,10 +167,11 @@ class ReaderRepository( val preferencesManager = EpubPreferencesManagerFactory(preferencesDataStore) .createPreferenceManager(bookId) val navigatorFactory = EpubNavigatorFactory(publication) + val ttsInitData = getTtsInitData(bookId, publication) return EpubReaderInitData( bookId, publication, initialLocator, - preferencesManager, navigatorFactory + preferencesManager, navigatorFactory, ttsInitData ) } @@ -144,14 +185,16 @@ class ReaderRepository( .createPreferenceManager(bookId) val pdfEngine = PdfiumEngineProvider() val navigatorFactory = PdfNavigatorFactory(publication, pdfEngine) + val ttsInitData = getTtsInitData(bookId, publication) return PdfReaderInitData( bookId, publication, initialLocator, - preferencesManager, navigatorFactory + preferencesManager, navigatorFactory, + ttsInitData ) } - private fun openImage( + private suspend fun openImage( bookId: Long, publication: Publication, initialLocator: Locator? @@ -159,17 +202,36 @@ class ReaderRepository( return ImageReaderInitData( bookId = bookId, publication = publication, - initialLocation = initialLocator + initialLocation = initialLocator, + ttsInitData = getTtsInitData(bookId, publication) ) } + private suspend fun getTtsInitData( + bookId: Long, + publication: Publication, + ): TtsInitData? { + TtsService.start(application) + val sessionBinder = TtsService.bind(application) + val preferencesManager = AndroidTtsPreferencesManagerFactory(preferencesDataStore) + .createPreferenceManager(bookId) + val ttsEngine = AndroidTtsEngineProvider(application) + val navigatorFactory = TtsNavigatorFactory(application, publication, ttsEngine) ?: return null + return TtsInitData(sessionBinder, navigatorFactory, preferencesManager) + } + fun close(bookId: Long) { + Timber.d("Closing Publication") when (val initData = repository.remove(bookId)) { is MediaReaderInitData -> { - mediaBinder.closeNavigator() + initData.publication.close() + initData.sessionBinder.closeNavigator() + MediaService.stop(application) } is VisualReaderInitData -> { initData.publication.close() + initData.ttsInitData?.sessionBinder?.closeNavigator() + TtsService.stop(application) } null, is DummyReaderInitData -> { // Do nothing diff --git a/test-app/src/main/java/org/readium/r2/testapp/reader/ReaderViewModel.kt b/test-app/src/main/java/org/readium/r2/testapp/reader/ReaderViewModel.kt index b42551c0b9..0be6cb2e81 100644 --- a/test-app/src/main/java/org/readium/r2/testapp/reader/ReaderViewModel.kt +++ b/test-app/src/main/java/org/readium/r2/testapp/reader/ReaderViewModel.kt @@ -37,20 +37,26 @@ import org.readium.r2.testapp.reader.tts.TtsViewModel import org.readium.r2.testapp.search.SearchPagingSource import org.readium.r2.testapp.utils.EventChannel import org.readium.r2.testapp.utils.createViewModelFactory +import timber.log.Timber @OptIn(Search::class, ExperimentalDecorator::class, ExperimentalCoroutinesApi::class) class ReaderViewModel( - application: Application, - val readerInitData: ReaderInitData, + private val bookId: Long, + private val readerRepository: ReaderRepository, private val bookRepository: BookRepository, ) : ViewModel() { + val readerInitData = + try { + checkNotNull(readerRepository[bookId]) + } catch (e: Exception) { + // Fallbacks on a dummy Publication to avoid crashing the app until the Activity finishes. + DummyReaderInitData(bookId) + } + val publication: Publication = readerInitData.publication - val bookId: Long = - readerInitData.bookId - val activityChannel: EventChannel = EventChannel(Channel(Channel.BUFFERED), viewModelScope) @@ -58,9 +64,8 @@ class ReaderViewModel( EventChannel(Channel(Channel.BUFFERED), viewModelScope) val tts: TtsViewModel? = TtsViewModel( - context = application, - publication = readerInitData.publication, - scope = viewModelScope + viewModelScope = viewModelScope, + readerInitData = readerInitData ) val settings: UserPreferencesViewModel<*, *>? = UserPreferencesViewModel( @@ -68,12 +73,13 @@ class ReaderViewModel( readerInitData = readerInitData ) - override fun onCleared() { - super.onCleared() - tts?.onCleared() + fun close() { + tts?.stop() + readerRepository.close(bookId) } fun saveProgression(locator: Locator) = viewModelScope.launch { + Timber.v("Saving locator for book $bookId: $locator.") bookRepository.saveProgression(locator, bookId) } @@ -255,16 +261,10 @@ class ReaderViewModel( companion object { fun createFactory(application: Application, arguments: ReaderActivityContract.Arguments) = createViewModelFactory { - val readerInitData = - try { - val readerRepository = application.readerRepository.getCompleted() - checkNotNull(readerRepository[arguments.bookId]) - } catch (e: Exception) { - // Fallbacks on a dummy Publication to avoid crashing the app until the Activity finishes. - DummyReaderInitData(arguments.bookId) - } - - ReaderViewModel(application, readerInitData, application.bookRepository) + val readerRepository = + application.readerRepository.getCompleted() + + ReaderViewModel(arguments.bookId, readerRepository, application.bookRepository) } } } diff --git a/test-app/src/main/java/org/readium/r2/testapp/reader/VisualReaderFragment.kt b/test-app/src/main/java/org/readium/r2/testapp/reader/VisualReaderFragment.kt index 9bfbdaddb6..691eb4444b 100644 --- a/test-app/src/main/java/org/readium/r2/testapp/reader/VisualReaderFragment.kt +++ b/test-app/src/main/java/org/readium/r2/testapp/reader/VisualReaderFragment.kt @@ -35,7 +35,6 @@ import androidx.fragment.app.Fragment import androidx.lifecycle.Lifecycle import androidx.lifecycle.lifecycleScope import androidx.lifecycle.repeatOnLifecycle -import kotlin.time.Duration.Companion.milliseconds import kotlin.time.Duration.Companion.seconds import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.flow.* @@ -80,6 +79,8 @@ abstract class VisualReaderFragment : BaseReaderFragment(), VisualNavigator.List */ private var disableTouches by mutableStateOf(false) + private var preventProgressionSaving: Boolean = false + override fun onViewCreated(view: View, savedInstanceState: Bundle?) { super.onViewCreated(view, savedInstanceState) @@ -134,7 +135,11 @@ abstract class VisualReaderFragment : BaseReaderFragment(), VisualNavigator.List viewLifecycleOwner.lifecycleScope.launch { viewLifecycleOwner.repeatOnLifecycle(Lifecycle.State.STARTED) { navigator.currentLocator - .onEach { model.saveProgression(it) } + .onEach { + if (!preventProgressionSaving) { + model.saveProgression(it) + } + } .launchIn(this) setupHighlights(this) @@ -179,16 +184,6 @@ abstract class VisualReaderFragment : BaseReaderFragment(), VisualNavigator.List } .launchIn(scope) - // Navigate to the currently spoken utterance. - state.map { it.playingUtterance } - .filterNotNull() - // Prevent jumping to many locations when the user skips repeatedly forward/backward. - .throttleLatest(500.milliseconds) - .onEach { locator -> - navigator.go(locator, animated = false) - } - .launchIn(scope) - // Navigate to the currently spoken word. // This will automatically turn pages when needed. state.map { it.playingWordRange } @@ -225,6 +220,13 @@ abstract class VisualReaderFragment : BaseReaderFragment(), VisualNavigator.List } .launchIn(scope) } + + state.map { it.showControls } + .distinctUntilChanged() + .onEach { showControls -> + preventProgressionSaving = showControls + } + .launchIn(scope) } } @@ -249,12 +251,6 @@ abstract class VisualReaderFragment : BaseReaderFragment(), VisualNavigator.List super.onDestroyView() } - override fun onStop() { - super.onStop() - - model.tts?.pause() - } - override fun onHiddenChanged(hidden: Boolean) { super.onHiddenChanged(hidden) setMenuVisibility(!hidden) @@ -269,6 +265,10 @@ abstract class VisualReaderFragment : BaseReaderFragment(), VisualNavigator.List override fun onOptionsItemSelected(item: MenuItem): Boolean { when (item.itemId) { R.id.tts -> checkNotNull(model.tts).start(navigator) + R.id.toc -> { + model.tts?.stop() + super.onOptionsItemSelected(item) + } else -> return super.onOptionsItemSelected(item) } return true diff --git a/test-app/src/main/java/org/readium/r2/testapp/reader/preferences/PreferencesManagers.kt b/test-app/src/main/java/org/readium/r2/testapp/reader/preferences/PreferencesManagers.kt index e9b575e304..2452c6e45e 100644 --- a/test-app/src/main/java/org/readium/r2/testapp/reader/preferences/PreferencesManagers.kt +++ b/test-app/src/main/java/org/readium/r2/testapp/reader/preferences/PreferencesManagers.kt @@ -27,6 +27,14 @@ import org.readium.r2.navigator.epub.EpubPreferences import org.readium.r2.navigator.epub.EpubPreferencesSerializer import org.readium.r2.navigator.epub.EpubPublicationPreferencesFilter import org.readium.r2.navigator.epub.EpubSharedPreferencesFilter +import org.readium.r2.navigator.media3.androidtts.AndroidTtsPreferences +import org.readium.r2.navigator.media3.androidtts.AndroidTtsPreferencesSerializer +import org.readium.r2.navigator.media3.androidtts.AndroidTtsPublicationPreferencesFilter +import org.readium.r2.navigator.media3.androidtts.AndroidTtsSharedPreferencesFilter +import org.readium.r2.navigator.media3.exoplayer.ExoPlayerPreferences +import org.readium.r2.navigator.media3.exoplayer.ExoPlayerPreferencesSerializer +import org.readium.r2.navigator.media3.exoplayer.ExoPlayerPublicationPreferencesFilter +import org.readium.r2.navigator.media3.exoplayer.ExoPlayerSharedPreferencesFilter import org.readium.r2.navigator.preferences.Configurable import org.readium.r2.navigator.preferences.PreferencesFilter import org.readium.r2.navigator.preferences.PreferencesSerializer @@ -130,3 +138,25 @@ class PdfiumPreferencesManagerFactory( preferencesSerializer = PdfiumPreferencesSerializer(), emptyPreferences = PdfiumPreferences() ) + +class ExoPlayerPreferencesManagerFactory( + dataStore: DataStore +) : PreferencesManagerFactory( + dataStore = dataStore, + klass = ExoPlayerPreferences::class, + sharedPreferencesFilter = ExoPlayerSharedPreferencesFilter, + publicationPreferencesFilter = ExoPlayerPublicationPreferencesFilter, + preferencesSerializer = ExoPlayerPreferencesSerializer(), + emptyPreferences = ExoPlayerPreferences() +) + +class AndroidTtsPreferencesManagerFactory( + dataStore: DataStore +) : PreferencesManagerFactory( + dataStore = dataStore, + klass = AndroidTtsPreferences::class, + sharedPreferencesFilter = AndroidTtsSharedPreferencesFilter, + publicationPreferencesFilter = AndroidTtsPublicationPreferencesFilter, + preferencesSerializer = AndroidTtsPreferencesSerializer(), + emptyPreferences = AndroidTtsPreferences() +) diff --git a/test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsControls.kt b/test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsControls.kt index 423e1dcd63..66743f3adb 100644 --- a/test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsControls.kt +++ b/test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsControls.kt @@ -16,7 +16,7 @@ import androidx.compose.ui.Modifier import androidx.compose.ui.res.stringResource import androidx.compose.ui.unit.dp import java.text.DecimalFormat -import org.readium.r2.navigator.tts.PublicationSpeechSynthesizer.Configuration +import org.readium.r2.navigator.media3.androidtts.AndroidTtsPreferencesEditor import org.readium.r2.navigator.tts.TtsEngine.Voice import org.readium.r2.shared.ExperimentalReadiumApi import org.readium.r2.shared.util.Language @@ -32,18 +32,21 @@ import org.readium.r2.testapp.utils.extensions.asStateWhenStarted fun TtsControls(model: TtsViewModel, modifier: Modifier = Modifier) { val showControls by model.state.asStateWhenStarted { it.showControls } val isPlaying by model.state.asStateWhenStarted { it.isPlaying } - val settings by model.state.asStateWhenStarted { it.settings } + // val settings by model.state.asStateWhenStarted { it.settings } + + val editor = remember { mutableStateOf(model.preferencesEditor, policy = neverEqualPolicy()) } + val commit: () -> Unit = { editor.value = editor.value ; model.commitPreferences() } if (showControls) { TtsControls( playing = isPlaying, - availableRates = listOf(0.25, 0.5, 0.75, 1.0, 1.25, 1.5, 2.0) - .filter { it in settings.rateRange }, - availableLanguages = settings.availableLanguages, - availableVoices = settings.availableVoices, - config = settings.config, - onConfigChange = model::setConfig, - onPlayPause = model::pauseOrResume, + availableRates = listOf(0.25, 0.5, 0.75, 1.0, 1.25, 1.5, 2.0), + // .filter { it in settings.rateRange }, + availableLanguages = emptyList(), // settings.availableLanguages, + availableVoices = emptyList(), // settings.availableVoices, + editor = editor.value, + commit = commit, + onPlayPause = { if (isPlaying) model.pause() else model.play() }, onStop = model::stop, onPrevious = model::previous, onNext = model::next, @@ -59,8 +62,8 @@ fun TtsControls( availableRates: List, availableLanguages: List, availableVoices: List, - config: Configuration?, - onConfigChange: (Configuration) -> Unit, + editor: AndroidTtsPreferencesEditor, + commit: () -> Unit, onPlayPause: () -> Unit, onStop: () -> Unit, onPrevious: () -> Unit, @@ -69,13 +72,13 @@ fun TtsControls( ) { var showSettings by remember { mutableStateOf(false) } - if (config != null && showSettings) { - TtsSettingsDialog( + if (showSettings) { + TtsPreferencesDialog( availableRates = availableRates, availableLanguages = availableLanguages, availableVoices = availableVoices, - config = config, - onConfigChange = onConfigChange, + editor = editor, + commit = commit, onDismiss = { showSettings = false } ) } @@ -139,12 +142,12 @@ fun TtsControls( @OptIn(ExperimentalReadiumApi::class) @Composable -private fun TtsSettingsDialog( +private fun TtsPreferencesDialog( availableRates: List, availableLanguages: List, availableVoices: List, - config: Configuration, - onConfigChange: (Configuration) -> Unit, + editor: AndroidTtsPreferencesEditor, + commit: () -> Unit, onDismiss: () -> Unit ) { AlertDialog( @@ -161,12 +164,13 @@ private fun TtsSettingsDialog( SelectorListItem( label = stringResource(R.string.tts_rate), values = availableRates, - selection = config.rateMultiplier, + selection = editor.speedRate.value, titleForValue = { rate -> DecimalFormat("x#.##").format(rate) }, onSelected = { - onConfigChange(config.copy(rateMultiplier = it)) + editor.speedRate.set(it) + commit() } ) } @@ -174,20 +178,26 @@ private fun TtsSettingsDialog( SelectorListItem( label = stringResource(R.string.language), values = availableLanguages, - selection = config.defaultLanguage, + selection = editor.language.value, titleForValue = { language -> language?.locale?.displayName ?: stringResource(R.string.auto) }, - onSelected = { onConfigChange(config.copy(defaultLanguage = it, voiceId = null)) } + onSelected = { + editor.language.set(it) + commit() + } ) SelectorListItem( label = stringResource(R.string.tts_voice), values = availableVoices, - selection = availableVoices.firstOrNull { it.id == config.voiceId }, + selection = availableVoices.firstOrNull { it.id == editor.voiceId.value }, titleForValue = { it?.name ?: it?.id ?: stringResource(R.string.auto) }, - onSelected = { onConfigChange(config.copy(voiceId = it?.id)) } + onSelected = { + editor.voiceId.set(it?.id) + commit() + } ) } } diff --git a/test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsService.kt b/test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsService.kt new file mode 100644 index 0000000000..3ded1984c5 --- /dev/null +++ b/test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsService.kt @@ -0,0 +1,199 @@ +/* + * Copyright 2022 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + +package org.readium.r2.testapp.reader.tts + +import android.app.Application +import android.app.PendingIntent +import android.content.ComponentName +import android.content.Intent +import android.content.ServiceConnection +import android.os.Build +import android.os.IBinder +import androidx.lifecycle.lifecycleScope +import androidx.media3.session.MediaSession +import kotlinx.coroutines.CompletableDeferred +import kotlinx.coroutines.Job +import kotlinx.coroutines.flow.launchIn +import kotlinx.coroutines.flow.onEach +import kotlinx.coroutines.flow.sample +import org.readium.r2.navigator.media3.androidtts.AndroidTtsPreferences +import org.readium.r2.navigator.media3.androidtts.AndroidTtsSettings +import org.readium.r2.navigator.media3.tts2.TtsNavigator +import org.readium.r2.shared.ExperimentalReadiumApi +import org.readium.r2.testapp.reader.ReaderActivityContract +import org.readium.r2.testapp.utils.LifecycleMedia3SessionService +import timber.log.Timber + +@OptIn(ExperimentalReadiumApi::class) +@androidx.annotation.OptIn(androidx.media3.common.util.UnstableApi::class) +class TtsService : LifecycleMedia3SessionService() { + + /** + * The service interface to be used by the app. + */ + inner class Binder : android.os.Binder() { + + private val app: org.readium.r2.testapp.Application + get() = application as org.readium.r2.testapp.Application + + private var saveLocationJob: Job? = null + + var mediaNavigator: TtsNavigator? = null + + var mediaSession: MediaSession? = null + + fun closeNavigator() { + stopForeground(true) + mediaSession?.release() + mediaSession = null + saveLocationJob?.cancel() + saveLocationJob = null + mediaNavigator?.close() + mediaNavigator = null + } + + fun bindNavigator( + navigator: TtsNavigator, + bookId: Long + ) { + val activityIntent = createSessionActivityIntent(bookId) + mediaNavigator = navigator + val session = MediaSession.Builder(applicationContext, navigator.asPlayer()) + .setSessionActivity(activityIntent) + .setId(bookId.toString()) + .build() + + mediaSession = session + addSession(session) + + /* + * Launch a job for saving progression even when playback is going on in the background + * with no ReaderActivity opened. + */ + saveLocationJob = navigator.currentLocator + .sample(3000) + .onEach { locator -> + Timber.d("Saving TTS progression $locator") + app.bookRepository.saveProgression(locator, bookId) + }.launchIn(lifecycleScope) + } + + private fun createSessionActivityIntent(bookId: Long): PendingIntent { + // This intent will be triggered when the notification is clicked. + var flags = PendingIntent.FLAG_UPDATE_CURRENT + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) { + flags = flags or PendingIntent.FLAG_IMMUTABLE + } + + val intent = + ReaderActivityContract().createIntent( + applicationContext, + ReaderActivityContract.Arguments(bookId) + ) + /*intent.addFlags(Intent.FLAG_ACTIVITY_CLEAR_TOP) + intent.addFlags(Intent.FLAG_ACTIVITY_SINGLE_TOP)*/ + + return PendingIntent.getActivity(applicationContext, 0, intent, flags) + } + } + + private val binder by lazy { + Binder() + } + + override fun onCreate() { + super.onCreate() + Timber.d("TtsService created.") + } + + override fun onBind(intent: Intent?): IBinder? { + Timber.d("onBind called with $intent") + + return if (intent?.action == SERVICE_INTERFACE) { + super.onBind(intent) + // Readium-aware client. + Timber.d("Returning custom binder.") + binder + } else { + // External controller. + Timber.d("Returning MediaSessionService binder.") + super.onBind(intent) + } + } + + override fun onGetSession(controllerInfo: MediaSession.ControllerInfo): MediaSession? { + return binder.mediaSession + } + + override fun onUpdateNotification(session: MediaSession) { + Timber.d("onUpdateNotification") + super.onUpdateNotification(session) + } + + override fun onDestroy() { + super.onDestroy() + Timber.d("MediaService destroyed.") + } + + override fun onTaskRemoved(rootIntent: Intent) { + super.onTaskRemoved(rootIntent) + Timber.d("Task removed. Stopping session and service.") + // Close the navigator to allow the service to be stopped. + binder.closeNavigator() + stopSelf() + } + + companion object { + + const val SERVICE_INTERFACE = "org.readium.r2.testapp.reader.tts.TtsService" + + fun start(application: Application) { + val intent = intent(application) + application.startService(intent) + } + + suspend fun bind(application: Application): TtsService.Binder { + val mediaServiceBinder: CompletableDeferred = + CompletableDeferred() + + val mediaServiceConnection = object : ServiceConnection { + + override fun onServiceConnected(name: ComponentName?, service: IBinder) { + Timber.d("MediaService bound.") + mediaServiceBinder.complete(service as Binder) + } + + override fun onServiceDisconnected(name: ComponentName) { + Timber.d("MediaService disconnected.") + // Should not happen, do nothing. + } + + override fun onNullBinding(name: ComponentName) { + val errorMessage = "Failed to bind to MediaService." + Timber.e(errorMessage) + val exception = IllegalStateException(errorMessage) + mediaServiceBinder.completeExceptionally(exception) + } + } + + val intent = intent(application) + application.bindService(intent, mediaServiceConnection, 0) + + return mediaServiceBinder.await() + } + + fun stop(application: Application) { + val intent = intent(application) + application.stopService(intent) + } + + private fun intent(application: Application) = + Intent(SERVICE_INTERFACE) + // MediaSessionService.onBind requires the intent to have a non-null action + .apply { setClass(application, TtsService::class.java) } + } +} diff --git a/test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsViewModel.kt b/test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsViewModel.kt index fca7ea5845..e6395c124e 100644 --- a/test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsViewModel.kt +++ b/test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsViewModel.kt @@ -11,32 +11,41 @@ import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.channels.Channel import kotlinx.coroutines.flow.* import kotlinx.coroutines.launch -import kotlinx.coroutines.runBlocking import org.readium.r2.navigator.Navigator import org.readium.r2.navigator.VisualNavigator -import org.readium.r2.navigator.tts.AndroidTtsEngine +import org.readium.r2.navigator.media3.androidtts.AndroidTtsPreferences +import org.readium.r2.navigator.media3.androidtts.AndroidTtsPreferencesEditor +import org.readium.r2.navigator.media3.androidtts.AndroidTtsSettings +import org.readium.r2.navigator.media3.api.MediaNavigator +import org.readium.r2.navigator.media3.tts2.TtsNavigator +import org.readium.r2.navigator.media3.tts2.TtsNavigatorFactory +import org.readium.r2.navigator.media3.tts2.TtsNavigatorListener import org.readium.r2.navigator.tts.PublicationSpeechSynthesizer -import org.readium.r2.navigator.tts.PublicationSpeechSynthesizer.Configuration -import org.readium.r2.navigator.tts.PublicationSpeechSynthesizer.State as TtsState -import org.readium.r2.navigator.tts.TtsEngine -import org.readium.r2.navigator.tts.TtsEngine.Voice import org.readium.r2.shared.DelicateReadiumApi import org.readium.r2.shared.ExperimentalReadiumApi import org.readium.r2.shared.UserException import org.readium.r2.shared.publication.Locator import org.readium.r2.shared.publication.Publication import org.readium.r2.shared.util.Language -import org.readium.r2.testapp.R +import org.readium.r2.testapp.reader.ReaderInitData +import org.readium.r2.testapp.reader.VisualReaderInitData +import org.readium.r2.testapp.reader.preferences.PreferencesManager +import timber.log.Timber /** - * View model controlling a [PublicationSpeechSynthesizer] to read a publication aloud. + * View model controlling a [TtsNavigator] to read a publication aloud. * * Note: This is not an Android [ViewModel], but it is a component of [ReaderViewModel]. */ @OptIn(ExperimentalReadiumApi::class) class TtsViewModel private constructor( - private val synthesizer: PublicationSpeechSynthesizer, - private val scope: CoroutineScope + private val viewModelScope: CoroutineScope, + private val bookId: Long, + private val publication: Publication, + private val ttsNavigatorFactory: TtsNavigatorFactory, + private val ttsSessionBinder: TtsService.Binder, + private val preferencesManager: PreferencesManager, + val preferencesEditor: AndroidTtsPreferencesEditor ) { companion object { @@ -45,12 +54,28 @@ class TtsViewModel private constructor( * TTS engine. */ operator fun invoke( - context: Context, - publication: Publication, - scope: CoroutineScope - ): TtsViewModel? = - PublicationSpeechSynthesizer(context, publication) - ?.let { TtsViewModel(it, scope) } + viewModelScope: CoroutineScope, + readerInitData: ReaderInitData + ): TtsViewModel? { + if (readerInitData !is VisualReaderInitData || readerInitData.ttsInitData == null) { + return null + } + + val preferencesEditor = + readerInitData.ttsInitData.ttsNavigatorFactory.createTtsPreferencesEditor( + readerInitData.ttsInitData.preferencesManager.preferences.value + ) + + return TtsViewModel( + viewModelScope = viewModelScope, + bookId = readerInitData.bookId, + publication = readerInitData.publication, + ttsNavigatorFactory = readerInitData.ttsInitData.ttsNavigatorFactory, + ttsSessionBinder = readerInitData.ttsInitData.sessionBinder, + preferencesManager = readerInitData.ttsInitData.preferencesManager, + preferencesEditor = preferencesEditor + ) + } } /** @@ -58,27 +83,12 @@ class TtsViewModel private constructor( * @param isPlaying Whether the TTS is currently speaking. * @param playingWordRange Locator to the currently spoken word. * @param playingUtterance Locator for the currently spoken utterance (e.g. sentence). - * @param settings Current user settings and their constraints. */ data class State( val showControls: Boolean = false, val isPlaying: Boolean = false, val playingWordRange: Locator? = null, val playingUtterance: Locator? = null, - val settings: Settings = Settings() - ) - - /** - * @param config Currently selected user settings. - * @param rateRange Supported range for the rate setting. - * @param availableLanguages Languages supported by the TTS engine. - * @param availableVoices Voices supported by the TTS engine, for the selected language. - */ - data class Settings( - val config: Configuration = Configuration(), - val rateRange: ClosedRange = 1.0..1.0, - val availableLanguages: List = emptyList(), - val availableVoices: List = emptyList(), ) sealed class Event { @@ -93,124 +103,116 @@ class TtsViewModel private constructor( class OnMissingVoiceData(val language: Language) : Event() } + private val ttsNavigator: TtsNavigator? get() = + ttsSessionBinder.mediaNavigator + /** * Current state of the view model. */ - val state: StateFlow + private val _state: MutableStateFlow = MutableStateFlow( + stateFromPlayback(ttsNavigator?.playback?.value) + ) + + init { + ttsNavigator?.let { bindStateToNavigator(it) } + } + val state: StateFlow = _state.asStateFlow() private val _events: Channel = Channel(Channel.BUFFERED) val events: Flow = _events.receiveAsFlow() /** - * Indicates whether the TTS is in the Stopped state. + * Starts the TTS using the first visible locator in the given [navigator]. */ - private val isStopped: StateFlow + fun start(navigator: Navigator) { + if (ttsNavigator != null) return - init { - synthesizer.listener = SynthesizerListener() - - // Automatically close the TTS when reaching the Stopped state. - isStopped = synthesizer.state - .map { it == PublicationSpeechSynthesizer.State.Stopped } - .stateIn(scope, SharingStarted.Lazily, initialValue = true) - - // Supported voices grouped by their language. - val voicesByLanguage: Flow>> = - synthesizer.availableVoices - .map { voices -> voices.groupBy { it.language } } - - // All supported languages. - val languages: Flow> = voicesByLanguage - .map { voices -> - voices.keys.sortedBy { it.locale.displayName } - } + viewModelScope.launch { + val ttsNavigator = createTtsNavigator(navigator) + bindStateToNavigator(ttsNavigator) + ttsNavigator.play() + } + } + + private suspend fun createTtsNavigator(navigator: Navigator): TtsNavigator { + val start = (navigator as? VisualNavigator)?.firstVisibleElementLocator() + + val listener = object : TtsNavigatorListener { - // Supported voices for the language selected in the synthesizer configuration. - val voicesForSelectedLanguage: Flow> = - combine( - synthesizer.config.map { it.defaultLanguage }, - voicesByLanguage, - ) { language, voices -> - language - ?.let { voices[it] } - ?.sortedBy { it.name ?: it.id } - ?: emptyList() + override fun onStopRequested() { + stop() } - // Settings model for the current configuration. - val settings: Flow = combine( - synthesizer.config, - languages, - voicesForSelectedLanguage, - ) { config, langs, voices -> - Settings( - config = config, - rateRange = synthesizer.rateMultiplierRange, - availableLanguages = langs, - availableVoices = voices - ) + override fun onPlaybackException() { + TODO("Not yet implemented") + } } - // Current view model state. - state = combine( - isStopped, - synthesizer.state, - settings - ) { isStopped, state, currentSettings -> - val playing = (state as? TtsState.Playing) - val paused = (state as? TtsState.Paused) - - State( - showControls = !isStopped, - isPlaying = (playing != null), - playingWordRange = playing?.range, - playingUtterance = (playing?.utterance ?: paused?.utterance)?.locator, - settings = currentSettings - ) - }.stateIn(scope, SharingStarted.Eagerly, initialValue = State()) + val ttsNavigator = ttsNavigatorFactory.createNavigator( + listener, + preferencesManager.preferences.value, + start + ) + + ttsSessionBinder.bindNavigator(ttsNavigator, bookId) + return ttsNavigator } - fun onCleared() { - runBlocking { - synthesizer.close() - } + private fun bindStateToNavigator( + ttsNavigator: TtsNavigator + ) { + ttsNavigator.playback + .onEach { playback -> + Timber.d("new TTS playback $playback") + _state.value = stateFromPlayback(playback) + Timber.d("new TTS state ${_state.value}") + }.launchIn(viewModelScope) } - /** - * Starts the TTS using the first visible locator in the given [navigator]. - */ - fun start(navigator: Navigator) { - if (!isStopped.value) return + private fun stateFromPlayback(playback: TtsNavigator.Playback?): State { + if (playback == null) + return State() - scope.launch { - val start = (navigator as? VisualNavigator)?.firstVisibleElementLocator() - synthesizer.start(fromLocator = start) - } + return State( + showControls = playback.state != MediaNavigator.State.Ended, + isPlaying = playback.state == MediaNavigator.State.Playing, + playingWordRange = playback.token, + playingUtterance = playback.locator + ) } fun stop() { - if (isStopped.value) return - synthesizer.stop() + if (ttsNavigator == null) return + + _state.value = State( + showControls = false, + isPlaying = false, + playingWordRange = null, + playingUtterance = null, + ) + ttsSessionBinder.closeNavigator() } - fun pauseOrResume() { - synthesizer.pauseOrResume() + fun play() { + ttsNavigator?.play() } fun pause() { - synthesizer.pause() + ttsNavigator?.pause() } fun previous() { - synthesizer.previous() + ttsNavigator?.goBackward() } fun next() { - synthesizer.next() + ttsNavigator?.goForward() } - fun setConfig(config: Configuration) { - synthesizer.setConfig(config) + fun commitPreferences() { + viewModelScope.launch { + preferencesManager.setPreferences(preferencesEditor.preferences) + } } /** @@ -218,15 +220,15 @@ class TtsViewModel private constructor( */ @OptIn(DelicateReadiumApi::class) fun requestInstallVoice(context: Context) { - synthesizer.engine.requestInstallMissingVoice(context) + // synthesizer.engine.requestInstallMissingVoice(context) } - private inner class SynthesizerListener : PublicationSpeechSynthesizer.Listener { + /*private inner class SynthesizerListener : PublicationSpeechSynthesizer.Listener { override fun onUtteranceError( utterance: PublicationSpeechSynthesizer.Utterance, error: PublicationSpeechSynthesizer.Exception ) { - scope.launch { + viewModelScope.launch { // The synthesizer is paused when encountering an error while playing an // utterance. Here we will skip to the next utterance unless the exception is // recoverable. @@ -238,7 +240,7 @@ class TtsViewModel private constructor( } override fun onError(error: PublicationSpeechSynthesizer.Exception) { - scope.launch { + viewModelScope.launch { handleTtsException(error) } } @@ -282,5 +284,5 @@ class TtsViewModel private constructor( is TtsEngine.Exception.Other -> UserException(R.string.tts_error_other) } - } + }*/ } diff --git a/test-app/src/main/java/org/readium/r2/testapp/utils/LifecycleMediaSessionService.kt b/test-app/src/main/java/org/readium/r2/testapp/utils/LifecycleMedia2SessionService.kt similarity index 86% rename from test-app/src/main/java/org/readium/r2/testapp/utils/LifecycleMediaSessionService.kt rename to test-app/src/main/java/org/readium/r2/testapp/utils/LifecycleMedia2SessionService.kt index 5dacff318b..7a26c0c4dc 100644 --- a/test-app/src/main/java/org/readium/r2/testapp/utils/LifecycleMediaSessionService.kt +++ b/test-app/src/main/java/org/readium/r2/testapp/utils/LifecycleMedia2SessionService.kt @@ -1,3 +1,9 @@ +/* + * Copyright 2022 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + package org.readium.r2.testapp.utils import android.content.Intent @@ -13,7 +19,7 @@ import androidx.media2.session.MediaSessionService * https://android.googlesource.com/platform/frameworks/support/+/refs/heads/androidx-main/lifecycle/lifecycle-service/src/main/java/androidx/lifecycle/LifecycleService.java */ -abstract class LifecycleMediaSessionService : MediaSessionService(), LifecycleOwner { +abstract class LifecycleMedia2SessionService : MediaSessionService(), LifecycleOwner { @Suppress("LeakingThis") private val lifecycleDispatcher = ServiceLifecycleDispatcher(this) diff --git a/test-app/src/main/java/org/readium/r2/testapp/utils/LifecycleMedia3SessionService.kt b/test-app/src/main/java/org/readium/r2/testapp/utils/LifecycleMedia3SessionService.kt new file mode 100644 index 0000000000..dda4958b3e --- /dev/null +++ b/test-app/src/main/java/org/readium/r2/testapp/utils/LifecycleMedia3SessionService.kt @@ -0,0 +1,63 @@ +/* + * Copyright 2022 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + +package org.readium.r2.testapp.utils + +import android.content.Intent +import android.os.IBinder +import androidx.annotation.CallSuper +import androidx.lifecycle.Lifecycle +import androidx.lifecycle.LifecycleOwner +import androidx.lifecycle.ServiceLifecycleDispatcher +import androidx.media3.session.MediaSessionService + +/* + * Borrowed from + * https://android.googlesource.com/platform/frameworks/support/+/refs/heads/androidx-main/lifecycle/lifecycle-service/src/main/java/androidx/lifecycle/LifecycleService.java + */ + +abstract class LifecycleMedia3SessionService : MediaSessionService(), LifecycleOwner { + + @Suppress("LeakingThis") + private val lifecycleDispatcher = ServiceLifecycleDispatcher(this) + + @CallSuper + override fun onCreate() { + lifecycleDispatcher.onServicePreSuperOnCreate() + super.onCreate() + } + + @CallSuper + override fun onBind(intent: Intent?): IBinder? { + lifecycleDispatcher.onServicePreSuperOnBind() + return super.onBind(intent) + } + + @CallSuper + override fun onStart(intent: Intent?, startId: Int) { + lifecycleDispatcher.onServicePreSuperOnStart() + super.onStart(intent, startId) + } + + // this method is added only to annotate it with @CallSuper. + // In usual service super.onStartCommand is no-op, but in LifecycleService + // it results in mDispatcher.onServicePreSuperOnStart() call, because + // super.onStartCommand calls onStart(). + @CallSuper + override fun onStartCommand(intent: Intent?, flags: Int, startId: Int): Int { + return super.onStartCommand(intent, flags, startId) + } + + @CallSuper + override fun onDestroy() { + lifecycleDispatcher.onServicePreSuperOnDestroy() + super.onDestroy() + } + + override fun getLifecycle(): Lifecycle { + return lifecycleDispatcher.lifecycle + } +} From 0de1cfc9f8150c8de1ab1badb5ad3fce4acd80a2 Mon Sep 17 00:00:00 2001 From: Quentin Gliosca Date: Mon, 9 Jan 2023 12:15:20 +0100 Subject: [PATCH 03/27] Work in progress --- .../navigator/epub/EpubNavigatorFragment.kt | 4 + .../media3/androidtts/AndroidTtsEngine.kt | 37 +- .../androidtts/AndroidTtsPreferences.kt | 11 +- .../androidtts/AndroidTtsPreferencesEditor.kt | 28 +- .../AndroidTtsPreferencesFilters.kt | 6 +- .../media3/androidtts/AndroidTtsSettings.kt | 6 +- .../androidtts/AndroidTtsSettingsResolver.kt | 6 +- .../media3/tts2/TtsContentIterator.kt | 68 ++- .../r2/navigator/media3/tts2/TtsEngine.kt | 16 +- .../navigator/media3/tts2/TtsEngineFacade.kt | 191 ++----- .../media3/tts2/TtsEngineFacadeListener.kt | 14 - .../media3/tts2/TtsEngineFacadePlayback.kt | 26 - .../media3/tts2/TtsEngineListener.kt | 88 --- .../media3/tts2/TtsEngineProvider.kt | 3 + .../media3/tts2/TtsNavigatorInternal.kt | 43 +- .../r2/navigator/media3/tts2/TtsPlayer.kt | 519 ++++-------------- .../media3/tts2/TtsSessionAdapter.kt | 451 +++++++++++++++ .../{TtsTimeline.kt => TtsSessionTimeline.kt} | 2 +- .../r2/testapp/reader/ReaderInitData.kt | 2 +- .../r2/testapp/reader/VisualReaderFragment.kt | 2 + .../reader/preferences/UserPreferences.kt | 392 +------------ .../r2/testapp/reader/tts/TtsControls.kt | 124 ++--- .../r2/testapp/reader/tts/TtsService.kt | 95 +++- .../r2/testapp/reader/tts/TtsViewModel.kt | 75 ++- .../readium/r2/testapp/shared/views/List.kt | 63 --- .../r2/testapp/shared/views/Preferences.kt | 472 ++++++++++++++++ test-app/src/main/res/values/strings.xml | 3 +- 27 files changed, 1417 insertions(+), 1330 deletions(-) delete mode 100644 readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsEngineFacadeListener.kt delete mode 100644 readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsEngineFacadePlayback.kt delete mode 100644 readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsEngineListener.kt create mode 100644 readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsSessionAdapter.kt rename readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/{TtsTimeline.kt => TtsSessionTimeline.kt} (97%) delete mode 100644 test-app/src/main/java/org/readium/r2/testapp/shared/views/List.kt create mode 100644 test-app/src/main/java/org/readium/r2/testapp/shared/views/Preferences.kt diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/epub/EpubNavigatorFragment.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/epub/EpubNavigatorFragment.kt index 5ebca0739f..43f298df56 100644 --- a/readium/navigator/src/main/java/org/readium/r2/navigator/epub/EpubNavigatorFragment.kt +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/epub/EpubNavigatorFragment.kt @@ -930,6 +930,10 @@ class EpubNavigatorFragment internal constructor( } private fun notifyCurrentLocation() { + if (view == null) { + return + } + val navigator = this debounceLocationNotificationJob?.cancel() debounceLocationNotificationJob = viewLifecycleOwner.lifecycleScope.launch { diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/androidtts/AndroidTtsEngine.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/androidtts/AndroidTtsEngine.kt index 1aad39735a..2709b66203 100644 --- a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/androidtts/AndroidTtsEngine.kt +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/androidtts/AndroidTtsEngine.kt @@ -16,9 +16,9 @@ import kotlinx.coroutines.flow.MutableStateFlow import kotlinx.coroutines.flow.StateFlow import kotlinx.coroutines.flow.asStateFlow import org.readium.r2.navigator.media3.tts2.TtsEngine -import org.readium.r2.navigator.media3.tts2.TtsUtterance import org.readium.r2.shared.ExperimentalReadiumApi import org.readium.r2.shared.publication.Metadata +import org.readium.r2.shared.util.Language /** * Default [TtsEngine] implementation using Android's native text to speech engine. @@ -91,9 +91,6 @@ class AndroidTtsEngine( engine.setOnUtteranceProgressListener(Listener()) } - private val scope: CoroutineScope = - CoroutineScope(SupervisorJob() + Dispatchers.Default) - private var listener: TtsEngine.Listener? = null @@ -104,15 +101,16 @@ class AndroidTtsEngine( MutableStateFlow(settingsResolver.settings(initialPreferences)) override fun close() { - scope.cancel() engine.shutdown() } override fun speak( - utterance: TtsUtterance, requestId: String, + text: String, + language: Language? ) { - engine.speak(utterance.text, QUEUE_ADD, null, requestId) + engine.language = language?.locale ?: settings.value.language?.locale + engine.speak(text, QUEUE_ADD, null, requestId) } /** @@ -159,7 +157,8 @@ class AndroidTtsEngine( } private fun TextToSpeech.setup(settings: AndroidTtsSettings) { - setSpeechRate(settings.speedRate.toFloat()) + setSpeechRate(settings.speed.toFloat()) + setPitch(settings.pitch.toFloat()) val localeResult = engine.setLanguage(settings.language?.locale) if (localeResult < TextToSpeech.LANG_AVAILABLE) { @@ -171,35 +170,35 @@ class AndroidTtsEngine( } inner class Listener : UtteranceProgressListener() { - override fun onStart(utteranceId: String?) { - listener?.onStart(utteranceId!!) + override fun onStart(utteranceId: String) { + listener?.onStart(utteranceId) } - override fun onStop(utteranceId: String?, interrupted: Boolean) { + override fun onStop(utteranceId: String, interrupted: Boolean) { listener?.let { if (interrupted) { - it.onInterrupted(utteranceId!!) + it.onInterrupted(utteranceId) } else { - it.onFlushed(utteranceId!!) + it.onFlushed(utteranceId) } } } - override fun onDone(utteranceId: String?) { - listener?.onDone(utteranceId!!) + override fun onDone(utteranceId: String) { + listener?.onDone(utteranceId) } @Deprecated("Deprecated in the interface", ReplaceWith("onError(utteranceId, -1)")) - override fun onError(utteranceId: String?) { + override fun onError(utteranceId: String) { onError(utteranceId, -1) } - override fun onError(utteranceId: String?, errorCode: Int) { + override fun onError(utteranceId: String, errorCode: Int) { // listener?.onError(utteranceId!!, EngineException(errorCode)) } - override fun onRangeStart(utteranceId: String?, start: Int, end: Int, frame: Int) { - listener?.onRange(utteranceId!!, start until end) + override fun onRangeStart(utteranceId: String, start: Int, end: Int, frame: Int) { + listener?.onRange(utteranceId, start until end) } } } diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/androidtts/AndroidTtsPreferences.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/androidtts/AndroidTtsPreferences.kt index f4da59498a..47433d0c93 100644 --- a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/androidtts/AndroidTtsPreferences.kt +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/androidtts/AndroidTtsPreferences.kt @@ -15,15 +15,16 @@ import org.readium.r2.shared.util.Language @Serializable data class AndroidTtsPreferences( override val language: Language? = null, - val voiceId: String? = null, - val pitchRate: Double? = null, - val speedRate: Double? = null, + val voices: Map? = null, + val pitch: Double? = null, + val speed: Double? = null, ) : TtsPreferences { override fun plus(other: AndroidTtsPreferences): AndroidTtsPreferences = AndroidTtsPreferences( language = other.language ?: language, - voiceId = other.voiceId ?: voiceId, - speedRate = other.speedRate ?: speedRate, + voices = other.voices ?: voices, + speed = other.speed ?: speed, + pitch = other.pitch ?: pitch ) } diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/androidtts/AndroidTtsPreferencesEditor.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/androidtts/AndroidTtsPreferencesEditor.kt index 6176a70d74..944d570bdf 100644 --- a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/androidtts/AndroidTtsPreferencesEditor.kt +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/androidtts/AndroidTtsPreferencesEditor.kt @@ -44,34 +44,34 @@ class AndroidTtsPreferencesEditor( updateValue = { value -> updateValues { it.copy(language = value) } }, ) - val pitchRate: RangePreference = + val pitch: RangePreference = RangePreferenceDelegate( - getValue = { preferences.pitchRate }, - getEffectiveValue = { state.settings.pitchRate }, + getValue = { preferences.pitch }, + getEffectiveValue = { state.settings.pitch }, getIsEffective = { true }, - updateValue = { value -> updateValues { it.copy(pitchRate = value) } }, + updateValue = { value -> updateValues { it.copy(pitch = value) } }, supportedRange = 0.0..Double.MAX_VALUE, progressionStrategy = DoubleIncrement(0.1), - valueFormatter = { it.format(1) }, + valueFormatter = { "${it.format(2)}x" }, ) - val speedRate: RangePreference = + val speed: RangePreference = RangePreferenceDelegate( - getValue = { preferences.speedRate }, - getEffectiveValue = { state.settings.speedRate }, + getValue = { preferences.speed }, + getEffectiveValue = { state.settings.speed }, getIsEffective = { true }, - updateValue = { value -> updateValues { it.copy(speedRate = value) } }, + updateValue = { value -> updateValues { it.copy(speed = value) } }, supportedRange = 0.0..Double.MAX_VALUE, progressionStrategy = DoubleIncrement(0.1), - valueFormatter = { "${it.format(1)}x" }, + valueFormatter = { "${it.format(2)}x" }, ) - val voiceId: Preference = + val voices: Preference> = PreferenceDelegate( - getValue = { preferences.voiceId }, - getEffectiveValue = { state.settings.voiceId }, + getValue = { preferences.voices }, + getEffectiveValue = { state.settings.voices }, getIsEffective = { true }, - updateValue = { value -> updateValues { it.copy(voiceId = value) } }, + updateValue = { value -> updateValues { it.copy(voices = value) } }, ) private fun updateValues(updater: (AndroidTtsPreferences) -> AndroidTtsPreferences) { diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/androidtts/AndroidTtsPreferencesFilters.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/androidtts/AndroidTtsPreferencesFilters.kt index 41849741cb..5807c6d665 100644 --- a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/androidtts/AndroidTtsPreferencesFilters.kt +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/androidtts/AndroidTtsPreferencesFilters.kt @@ -18,7 +18,7 @@ object AndroidTtsSharedPreferencesFilter : PreferencesFilter, + val pitch: Double, + val speed: Double, ) : TtsSettings diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/androidtts/AndroidTtsSettingsResolver.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/androidtts/AndroidTtsSettingsResolver.kt index ae96baf296..01d790ec70 100644 --- a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/androidtts/AndroidTtsSettingsResolver.kt +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/androidtts/AndroidTtsSettingsResolver.kt @@ -18,9 +18,9 @@ internal class AndroidTtsSettingsResolver( return AndroidTtsSettings( language = preferences.language ?: metadata.language, - voiceId = preferences.voiceId, - pitchRate = preferences.pitchRate ?: 1.0, - speedRate = preferences.speedRate ?: 1.0, + voices = preferences.voices ?: emptyMap(), + pitch = preferences.pitch ?: 1.0, + speed = preferences.speed ?: 1.0, ) } } diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsContentIterator.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsContentIterator.kt index 74d8ef6910..6b3f06a67f 100644 --- a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsContentIterator.kt +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsContentIterator.kt @@ -21,10 +21,10 @@ internal class TtsContentIterator( private val tokenizerFactory: (language: Language?) -> ContentTokenizer, initialLocator: TtsLocator? ) { - enum class Direction { - Forward, Backward; - } + /** + * Current subset of utterances with a cursor. + */ private var utterances: CursorList = CursorList() @@ -37,29 +37,13 @@ internal class TtsContentIterator( utterances = CursorList() } - private var language: Language? = - null - /** - * Sets the tokenizer language. + * The tokenizer language. * - * The change is not immediate, it will be applied as soon as possible. - */ - fun setLanguage(language: Language?) { - this.language = language - } - - /** - * Gets the next utterance in the given [direction], or null when reaching the beginning or the - * end. + * Modifying this property is not immediate, the new value will be applied as soon as possible. */ - suspend fun nextUtterance(direction: Direction): TtsUtterance? { - val utterance = utterances.nextIn(direction) - if (utterance == null && loadNextUtterances(direction)) { - return nextUtterance(direction) - } - return utterance - } + var language: Language? = + null /** * Moves the iterator to the position provided in [locator]. @@ -68,15 +52,49 @@ internal class TtsContentIterator( publicationIterator = createIterator(locator) } - fun restart() { + /** + * Moves the iterator to the beginning of the publication. + */ + fun seekToBeginning() { publicationIterator = createIterator(null) } - private fun createIterator(locator: TtsLocator?) = + /** + * Creates a fresh content iterator for the publication starting from [locator]. + */ + private fun createIterator(locator: TtsLocator?): Content.Iterator = publication.content(locator?.toLocator(publication)) ?.iterator() ?: throw IllegalStateException("No ContentService.") + /** + * Advances to the previous item and returns it, or null if we reached the beginning. + */ + suspend fun previousUtterance(): TtsUtterance? = + nextUtterance(Direction.Backward) + + /** + * Advances to the next item and returns it, or null if we reached the end. + */ + suspend fun nextUtterance(): TtsUtterance? = + nextUtterance(Direction.Forward) + + private enum class Direction { + Forward, Backward; + } + + /** + * Gets the next utterance in the given [direction], or null when reaching the beginning or the + * end. + */ + private suspend fun nextUtterance(direction: Direction): TtsUtterance? { + val utterance = utterances.nextIn(direction) + if (utterance == null && loadNextUtterances(direction)) { + return nextUtterance(direction) + } + return utterance + } + /** * Loads the utterances for the next publication [Content.Element] item in the given [direction]. */ diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsEngine.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsEngine.kt index d2d4cdded7..68ce4e3765 100644 --- a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsEngine.kt +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsEngine.kt @@ -17,7 +17,6 @@ import org.readium.r2.shared.util.Language @ExperimentalReadiumApi interface TtsEngine> : Configurable, Closeable { - @ExperimentalReadiumApi sealed class Exception private constructor( override val message: String, cause: Throwable? = null @@ -53,23 +52,22 @@ interface TtsEngine> : Configurable /** * TTS engine callbacks. */ - @ExperimentalReadiumApi interface Listener { - fun onStart(id: String) + fun onStart(requestId: String) - fun onRange(id: String, range: IntRange) + fun onRange(requestId: String, range: IntRange) - fun onInterrupted(id: String) + fun onInterrupted(requestId: String) - fun onFlushed(id: String) + fun onFlushed(requestId: String) - fun onDone(id: String) + fun onDone(requestId: String) - fun onError(id: String, error: Exception) + fun onError(requestId: String, error: Exception) } - fun speak(utterance: TtsUtterance, requestId: String) + fun speak(requestId: String, text: String, language: Language?) fun stop() diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsEngineFacade.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsEngineFacade.kt index 8e1fd61883..19923b1dbe 100644 --- a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsEngineFacade.kt +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsEngineFacade.kt @@ -7,166 +7,89 @@ package org.readium.r2.navigator.media3.tts2 import java.util.* -import kotlinx.coroutines.* -import kotlinx.coroutines.flow.* +import kotlinx.coroutines.CancellableContinuation +import kotlinx.coroutines.ExperimentalCoroutinesApi +import kotlinx.coroutines.suspendCancellableCoroutine import org.readium.r2.navigator.preferences.Configurable import org.readium.r2.shared.ExperimentalReadiumApi +import org.readium.r2.shared.util.Language @ExperimentalReadiumApi +@OptIn(ExperimentalCoroutinesApi::class) internal class TtsEngineFacade>( - private val ttsEngine: TtsEngine, - private val ttsContentIterator: TtsContentIterator, - private val listener: TtsEngineFacadeListener, - firstUtterance: TtsUtterance, + private val ttsEngine: TtsEngine ) : Configurable by ttsEngine { - companion object { - - suspend operator fun > invoke( - ttsEngine: TtsEngine, - ttsContentIterator: TtsContentIterator, - listener: TtsEngineFacadeListener - ): TtsEngineFacade? { - - val firstUtterance = ttsContentIterator.nextUtterance(TtsContentIterator.Direction.Forward) - ?: run { - ttsContentIterator.restart() - ttsContentIterator.nextUtterance(TtsContentIterator.Direction.Forward) - } ?: return null - - return TtsEngineFacade(ttsEngine, ttsContentIterator, listener, firstUtterance) - } - } - - private val coroutineScope: CoroutineScope = - MainScope() - - private var pendingUtterance: TtsUtterance? = - firstUtterance - - private var playbackJob: Job? = null - - private val ttsEngineListener = TtsEngineListener() - - private val playbackMutable: MutableStateFlow = - MutableStateFlow( - TtsEngineFacadePlayback( - index = firstUtterance.locator.resourceIndex, - state = TtsEngineFacadePlayback.State.READY, - isPlaying = false, - playWhenReady = false, - locator = firstUtterance.locator, - range = null - ) - ) - init { - ttsEngine.setListener(ttsEngineListener) - ttsContentIterator.setLanguage(ttsEngine.settings.value.language) - - ttsEngineListener.state - .onEach { engineState -> onEngineStateChanged(engineState) } - .launchIn(coroutineScope) + val listener = TtsEngineListener() + ttsEngine.setListener(listener) } - private fun onEngineStateChanged(state: TtsEngineListener.EngineState) { - val newPlayback = playbackMutable.value.copy( - isPlaying = state.utteranceId != null, - range = state.range - ) - - playbackMutable.value = newPlayback - } + private var currentTask: UtteranceTask? = null - val playback: StateFlow = - playbackMutable.asStateFlow() - - fun play() { - replacePlaybackJob { - playbackMutable.value = - playbackMutable.value.copy( - state = TtsEngineFacadePlayback.State.READY, - playWhenReady = true, - isPlaying = true - ) - playContinuous() + suspend fun speak(text: String, language: Language?, onRange: (IntRange) -> Unit) { + suspendCancellableCoroutine { continuation -> + continuation.invokeOnCancellation { ttsEngine.stop() } + val id = UUID.randomUUID().toString() + currentTask?.continuation?.cancel() + currentTask = UtteranceTask(id, continuation, onRange) + ttsEngine.speak(id, text, language) } } - fun pause() { - replacePlaybackJob { - playbackMutable.value = - playbackMutable.value.copy( - playWhenReady = false, - isPlaying = false - ) - } + fun close() { + currentTask?.continuation?.cancel() + ttsEngine.close() } - fun go(locator: TtsLocator) { - replacePlaybackJob { - pendingUtterance = null - ttsContentIterator.seek(locator) - playContinuous() - } - } + private data class UtteranceTask( + val requestId: String, + val continuation: CancellableContinuation, + val onRange: (IntRange) -> Unit + ) - fun nextUtterance() { - replacePlaybackJob { - pendingUtterance = null - ttsContentIterator.nextUtterance(TtsContentIterator.Direction.Forward) - playContinuous() - } - } + private inner class TtsEngineListener : TtsEngine.Listener { - fun previousUtterance() { - replacePlaybackJob { - pendingUtterance = null - ttsContentIterator.nextUtterance(TtsContentIterator.Direction.Backward) - playContinuous() + override fun onStart(requestId: String) { } - } - private fun replacePlaybackJob(block: suspend CoroutineScope.() -> Unit) { - coroutineScope.launch { - ttsEngine.stop() - playbackJob?.cancelAndJoin() - ttsEngineListener.removeAllCallbacks() - playbackJob = launch { - block() - } + override fun onRange(requestId: String, range: IntRange) { + currentTask + ?.takeIf { it.requestId == requestId } + ?.onRange + ?.invoke(range) } - } - private suspend fun playContinuous() { - if (pendingUtterance == null) { - pendingUtterance = ttsContentIterator.nextUtterance(TtsContentIterator.Direction.Forward) - } - pendingUtterance?.let { - playbackMutable.value = playbackMutable.value.copy(range = null, locator = it.locator) - speakUtterance(it) - pendingUtterance = null - playContinuous() - } ?: run { - playbackMutable.value = playbackMutable.value.copy( - isPlaying = false, playWhenReady = false, state = TtsEngineFacadePlayback.State.ENDED - ) + override fun onInterrupted(requestId: String) { + currentTask + ?.takeIf { it.requestId == requestId } + ?.continuation + ?.cancel() + currentTask = null } - } - private suspend fun speakUtterance(utterance: TtsUtterance) = - suspendCancellableCoroutine { continuation -> - val id = UUID.randomUUID().toString() - ttsEngineListener.addCallback(id, continuation) - ttsEngine.speak(utterance, id) + override fun onFlushed(requestId: String) { + currentTask + ?.takeIf { it.requestId == requestId } + ?.continuation + ?.cancel() + currentTask = null } - fun stop() { - listener.onNavigatorStopped() - pause() - } + override fun onDone(requestId: String) { + currentTask + ?.takeIf { it.requestId == requestId } + ?.continuation + ?.resume(null) {} + currentTask = null + } - fun close() { - ttsEngine.close() + override fun onError(requestId: String, error: TtsEngine.Exception) { + currentTask + ?.takeIf { it.requestId == requestId } + ?.continuation + ?.resume(error) {} + currentTask = null + } } } diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsEngineFacadeListener.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsEngineFacadeListener.kt deleted file mode 100644 index 85ba373b3f..0000000000 --- a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsEngineFacadeListener.kt +++ /dev/null @@ -1,14 +0,0 @@ -/* - * Copyright 2022 Readium Foundation. All rights reserved. - * Use of this source code is governed by the BSD-style license - * available in the top-level LICENSE file of the project. - */ - -package org.readium.r2.navigator.media3.tts2 - -interface TtsEngineFacadeListener { - - fun onNavigatorStopped() - - fun onPlaybackException() -} diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsEngineFacadePlayback.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsEngineFacadePlayback.kt deleted file mode 100644 index 6d03875864..0000000000 --- a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsEngineFacadePlayback.kt +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Copyright 2022 Readium Foundation. All rights reserved. - * Use of this source code is governed by the BSD-style license - * available in the top-level LICENSE file of the project. - */ - -package org.readium.r2.navigator.media3.tts2 - -import androidx.media3.common.Player -import org.readium.r2.shared.ExperimentalReadiumApi - -@ExperimentalReadiumApi -internal data class TtsEngineFacadePlayback( - val state: State, - val isPlaying: Boolean, - val playWhenReady: Boolean, - val index: Int, - val locator: TtsLocator, - val range: IntRange? -) { - - enum class State(val value: Int) { - READY(Player.STATE_READY), - ENDED(Player.STATE_ENDED); - } -} diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsEngineListener.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsEngineListener.kt deleted file mode 100644 index 5e81c67ed3..0000000000 --- a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsEngineListener.kt +++ /dev/null @@ -1,88 +0,0 @@ -/* - * Copyright 2022 Readium Foundation. All rights reserved. - * Use of this source code is governed by the BSD-style license - * available in the top-level LICENSE file of the project. - */ - -package org.readium.r2.navigator.media3.tts2 - -import kotlinx.coroutines.CancellableContinuation -import kotlinx.coroutines.flow.MutableStateFlow -import kotlinx.coroutines.flow.StateFlow -import kotlinx.coroutines.flow.asStateFlow -import org.readium.r2.shared.ExperimentalReadiumApi -import timber.log.Timber - -@ExperimentalReadiumApi -internal class TtsEngineListener : TtsEngine.Listener { - - private data class UtteranceTask( - val id: String, - val continuation: CancellableContinuation - ) - - data class EngineState( - val utteranceId: String?, - val range: IntRange?, - ) - - private val waitingTasks: MutableList = - mutableListOf() - - private val stateMutable: MutableStateFlow = - MutableStateFlow( - EngineState(null, null) - ) - - val state: StateFlow = - stateMutable.asStateFlow() - - fun addCallback(id: String, continuation: CancellableContinuation) { - val task = UtteranceTask(id, continuation) - waitingTasks.add(task) - } - - fun removeAllCallbacks() { - waitingTasks.clear() - } - - override fun onStart(id: String) { - Timber.d("onStart") - stateMutable.value = EngineState(id, null) - } - - override fun onRange(id: String, range: IntRange) { - Timber.d("onRange") - stateMutable.value = EngineState(id, range) - } - - override fun onInterrupted(id: String) { - Timber.d("onInterrupted") - stateMutable.value = EngineState(null, null) - waitingTasks.forEach { - it.continuation.resume(null) {} - waitingTasks.remove(it) - } - } - - override fun onFlushed(id: String) { - Timber.d("onFlushed") - waitingTasks.forEach { - it.continuation.resume(null) {} - waitingTasks.remove(it) - } - } - - override fun onDone(id: String) { - Timber.d("onDone") - stateMutable.value = EngineState(null, null) - waitingTasks.forEach { - it.continuation.resume(null) {} - waitingTasks.remove(it) - } - } - - override fun onError(id: String, error: TtsEngine.Exception) { - Timber.e("onError $error") - } -} diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsEngineProvider.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsEngineProvider.kt index 6274a10bcd..c0d6cc69fc 100644 --- a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsEngineProvider.kt +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsEngineProvider.kt @@ -10,6 +10,9 @@ import org.readium.r2.navigator.preferences.PreferencesEditor import org.readium.r2.shared.ExperimentalReadiumApi import org.readium.r2.shared.publication.Publication +/** + * To be implemented by adapters for third-party TTS engines which can be used with [TtsNavigator]. + */ @ExperimentalReadiumApi interface TtsEngineProvider, E : PreferencesEditor

> { diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsNavigatorInternal.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsNavigatorInternal.kt index ad02a9e5c7..bcd0ee080d 100644 --- a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsNavigatorInternal.kt +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsNavigatorInternal.kt @@ -22,10 +22,9 @@ import org.readium.r2.shared.extensions.mapStateIn @OptIn(ExperimentalCoroutinesApi::class) @ExperimentalReadiumApi internal class TtsNavigatorInternal>( - application: Application, - private val ttsEngineFacade: TtsEngineFacade, - private val player: TtsPlayer, -) : MediaNavigatorInternal, Configurable by ttsEngineFacade { + private val ttsPlayer: TtsPlayer, + private val sessionAdapter: TtsSessionAdapter, +) : MediaNavigatorInternal, Configurable by ttsPlayer { companion object { @@ -38,25 +37,21 @@ internal class TtsNavigatorInternal>( listener: TtsNavigatorListener ): TtsNavigatorInternal? { - val facadeListener = object : TtsEngineFacadeListener { - - override fun onNavigatorStopped() { - listener.onStopRequested() - } + val playerListener = object : TtsPlayer.Listener { override fun onPlaybackException() { listener.onPlaybackException() } } - val ttsEngineFacade = - TtsEngineFacade(ttsEngine, ttsContentIterator, facadeListener) + val ttsPlayer = + TtsPlayer(ttsEngine, ttsContentIterator, playerListener) ?: return null - val player = - TtsPlayer(application, ttsEngineFacade, playlistMetadata, mediaItems) + val sessionAdapter = + TtsSessionAdapter(application, ttsPlayer, playlistMetadata, mediaItems, listener::onStopRequested) - return TtsNavigatorInternal(application, ttsEngineFacade, player) + return TtsNavigatorInternal(ttsPlayer, sessionAdapter) } } @@ -64,13 +59,13 @@ internal class TtsNavigatorInternal>( MainScope() override val playback: StateFlow = - ttsEngineFacade.playback + ttsPlayer.playback .mapStateIn(coroutineScope) { playback -> val state = when (playback.state) { - TtsEngineFacadePlayback.State.READY -> + TtsPlayer.Playback.State.READY -> if (playback.playWhenReady) MediaNavigatorInternal.State.Playing else MediaNavigatorInternal.State.Paused - TtsEngineFacadePlayback.State.ENDED -> + TtsPlayer.Playback.State.ENDED -> MediaNavigatorInternal.State.Ended } @@ -85,30 +80,30 @@ internal class TtsNavigatorInternal>( } override fun play() { - ttsEngineFacade.play() + ttsPlayer.play() } override fun pause() { - ttsEngineFacade.pause() + ttsPlayer.pause() } override fun go(locator: TtsLocator) { - ttsEngineFacade.go(locator) + ttsPlayer.go(locator) } override fun goForward() { - ttsEngineFacade.nextUtterance() + ttsPlayer.nextUtterance() } override fun goBackward() { - ttsEngineFacade.previousUtterance() + ttsPlayer.previousUtterance() } override fun asPlayer(): Player { - return player + return sessionAdapter } fun close() { - ttsEngineFacade.close() + ttsPlayer.close() } } diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsPlayer.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsPlayer.kt index 729c265118..dcc0d0e9f6 100644 --- a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsPlayer.kt +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsPlayer.kt @@ -6,443 +6,168 @@ package org.readium.r2.navigator.media3.tts2 -import android.app.Application -import android.content.Context -import android.media.AudioManager -import android.os.Looper -import android.view.Surface -import android.view.SurfaceHolder -import android.view.SurfaceView -import android.view.TextureView -import androidx.media3.common.* -import androidx.media3.common.C.* -import androidx.media3.common.text.CueGroup -import androidx.media3.common.util.Clock -import androidx.media3.common.util.ListenerSet -import androidx.media3.common.util.Util -import kotlinx.coroutines.CoroutineScope -import kotlinx.coroutines.MainScope -import kotlinx.coroutines.flow.launchIn -import kotlinx.coroutines.flow.onEach +import androidx.media3.common.Player +import kotlinx.coroutines.* +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.StateFlow +import kotlinx.coroutines.flow.asStateFlow +import org.readium.r2.navigator.preferences.Configurable import org.readium.r2.shared.ExperimentalReadiumApi import timber.log.Timber @ExperimentalReadiumApi -@androidx.annotation.OptIn(androidx.media3.common.util.UnstableApi::class) -internal class TtsPlayer( - private val application: Application, - private val ttsEngineFacade: TtsEngineFacade<*, *>, - private val playlistMetadata: MediaMetadata, - private val mediaItems: List -) : BasePlayer() { - - private val coroutineScope: CoroutineScope = - MainScope() - - private var lastPlayback: TtsEngineFacadePlayback = - ttsEngineFacade.playback.value - - init { - ttsEngineFacade.playback - .onEach { playback -> - notifyListeners(lastPlayback, playback) - lastPlayback = playback - }.launchIn(coroutineScope) - } - - private var listeners: ListenerSet = - ListenerSet( - applicationLooper, - Clock.DEFAULT, - ) { listener: Player.Listener, flags: FlagSet? -> - listener.onEvents(this, Player.Events(flags!!)) +internal class TtsPlayer>( + private val engineFacade: TtsEngineFacade, + private val contentIterator: TtsContentIterator, + private val listener: Listener, + firstUtterance: TtsUtterance, +) : Configurable by engineFacade { + + companion object { + + suspend operator fun > invoke( + engine: TtsEngine, + contentIterator: TtsContentIterator, + listener: Listener + ): TtsPlayer? { + + val firstUtterance = contentIterator.nextUtterance() + ?: run { + contentIterator.seekToBeginning() + contentIterator.nextUtterance() + } ?: return null + + val ttsEngineFacade = TtsEngineFacade(engine) + + return TtsPlayer(ttsEngineFacade, contentIterator, listener, firstUtterance) } - - private val permanentAvailableCommands = - Player.Commands.Builder() - .addAll( - COMMAND_PLAY_PAUSE, - COMMAND_STOP, - COMMAND_SEEK_TO_NEXT_MEDIA_ITEM, - COMMAND_SEEK_TO_PREVIOUS_MEDIA_ITEM, - COMMAND_SEEK_TO_NEXT, - COMMAND_SEEK_TO_PREVIOUS - // COMMAND_GET_AUDIO_ATTRIBUTES, - // COMMAND_GET_CURRENT_MEDIA_ITEM, - // COMMAND_GET_MEDIA_ITEMS_METADATA, - // COMMAND_GET_TEXT - ).build() - - private val audioManager: AudioManager = - application.getSystemService(Context.AUDIO_SERVICE) as AudioManager - - override fun getApplicationLooper(): Looper { - return Looper.getMainLooper() } - override fun addListener(listener: Player.Listener) { - Timber.d("addListener") - listeners.add(listener) - } - - override fun removeListener(listener: Player.Listener) { - Timber.d("removeListener") - listeners.remove(listener) - } + interface Listener { - override fun setMediaItems(mediaItems: MutableList, resetPosition: Boolean) { - throw NotImplementedError() + fun onPlaybackException() } - override fun setMediaItems( - mediaItems: MutableList, - startIndex: Int, - startPositionMs: Long + @ExperimentalReadiumApi + data class Playback( + val state: State, + val isPlaying: Boolean, + val playWhenReady: Boolean, + val index: Int, + val locator: TtsLocator, + val range: IntRange? ) { - throw NotImplementedError() - } - - override fun addMediaItems(index: Int, mediaItems: MutableList) { - throw NotImplementedError() - } - - override fun moveMediaItems(fromIndex: Int, toIndex: Int, newIndex: Int) { - throw NotImplementedError() - } - - override fun removeMediaItems(fromIndex: Int, toIndex: Int) { - throw NotImplementedError() - } - - override fun getAvailableCommands(): Player.Commands { - return Player.Commands.Builder() - .addAll(permanentAvailableCommands) - .build() - } - - override fun prepare() { - throw NotImplementedError() - } - - override fun getPlaybackState(): Int { - return ttsEngineFacade.playback.value.state.value - } - override fun getPlaybackSuppressionReason(): Int { - return PLAYBACK_SUPPRESSION_REASON_NONE // TODO - } - - override fun getPlayerError(): PlaybackException? { - return null // TODO - } - - override fun setPlayWhenReady(playWhenReady: Boolean) { - if (playWhenReady) { - ttsEngineFacade.play() - } else { - ttsEngineFacade.pause() + enum class State(val value: Int) { + READY(Player.STATE_READY), + ENDED(Player.STATE_ENDED); } } - override fun getPlayWhenReady(): Boolean { - return ttsEngineFacade.playback.value.playWhenReady - } - - override fun setRepeatMode(repeatMode: Int) { - throw NotImplementedError() - } - - override fun getRepeatMode(): Int { - return REPEAT_MODE_OFF - } - - override fun setShuffleModeEnabled(shuffleModeEnabled: Boolean) { - throw NotImplementedError() - } - - override fun getShuffleModeEnabled(): Boolean { - return false - } - - override fun isLoading(): Boolean { - return false - } - - override fun seekTo(mediaItemIndex: Int, positionMs: Long) { - throw NotImplementedError() - } - - override fun getSeekBackIncrement(): Long { - return 0 - } - - override fun getSeekForwardIncrement(): Long { - return 0 - } - - override fun getMaxSeekToPreviousPosition(): Long { - return 0 - } - - override fun setPlaybackParameters(playbackParameters: PlaybackParameters) { - throw NotImplementedError() // TODO - } - - override fun getPlaybackParameters(): PlaybackParameters { - return PlaybackParameters.DEFAULT - } - - override fun stop() { - ttsEngineFacade.stop() - } - - @Deprecated("Deprecated in Java") - override fun stop(reset: Boolean) {} - - override fun release() { - ttsEngineFacade.close() - } - - override fun getCurrentTracks(): Tracks { - throw NotImplementedError() - } - - override fun getTrackSelectionParameters(): TrackSelectionParameters { - return TrackSelectionParameters.Builder(application) - .build() - } - - override fun setTrackSelectionParameters(parameters: TrackSelectionParameters) { - throw NotImplementedError() - } - - override fun getMediaMetadata(): MediaMetadata { - return currentTimeline.getWindow(currentMediaItemIndex, window).mediaItem.mediaMetadata - } - - override fun getPlaylistMetadata(): MediaMetadata { - return playlistMetadata - } - - override fun setPlaylistMetadata(mediaMetadata: MediaMetadata) { - throw NotImplementedError() - } - - override fun getCurrentTimeline(): Timeline { - // MediaNotificationManager requires a non-empty timeline to start foreground playing. - return TtsTimeline(mediaItems) - /*return SinglePeriodTimeline( - TIME_UNSET, false, false, false, null, mediaItem)*/ - } - - override fun getCurrentPeriodIndex(): Int { - return lastPlayback.index - } - - override fun getCurrentMediaItemIndex(): Int { - return lastPlayback.index - } - - override fun getDuration(): Long { - return TIME_UNSET - } - - override fun getCurrentPosition(): Long { - return 0 - } - - override fun getBufferedPosition(): Long { - return 0 - } - - override fun getTotalBufferedDuration(): Long { - return 0 - } - - override fun isPlayingAd(): Boolean { - return false - } - - override fun getCurrentAdGroupIndex(): Int { - return INDEX_UNSET - } - - override fun getCurrentAdIndexInAdGroup(): Int { - return INDEX_UNSET - } - - override fun getContentPosition(): Long { - return 0 - } - - override fun getContentBufferedPosition(): Long { - return 0 - } - - override fun getAudioAttributes(): AudioAttributes { - return AudioAttributes.Builder() - .setUsage(USAGE_MEDIA) - .setContentType(AUDIO_CONTENT_TYPE_SPEECH) - .build() - } - - override fun setVolume(volume: Float) { - throw NotImplementedError() - } - - override fun getVolume(): Float { - return 1.0f - } - - override fun clearVideoSurface() { - throw NotImplementedError() - } - - override fun clearVideoSurface(surface: Surface?) { - throw NotImplementedError() - } - - override fun setVideoSurface(surface: Surface?) { - throw NotImplementedError() - } - - override fun setVideoSurfaceHolder(surfaceHolder: SurfaceHolder?) { - throw NotImplementedError() - } - - override fun clearVideoSurfaceHolder(surfaceHolder: SurfaceHolder?) { - throw NotImplementedError() - } - - override fun setVideoSurfaceView(surfaceView: SurfaceView?) { - throw NotImplementedError() - } - - override fun clearVideoSurfaceView(surfaceView: SurfaceView?) { - throw NotImplementedError() - } - - override fun setVideoTextureView(textureView: TextureView?) { - throw NotImplementedError() - } + private val coroutineScope: CoroutineScope = + MainScope() - override fun clearVideoTextureView(textureView: TextureView?) { - throw NotImplementedError() - } + private val playbackMutable: MutableStateFlow = + MutableStateFlow( + Playback( + index = firstUtterance.locator.resourceIndex, + state = Playback.State.READY, + isPlaying = false, + playWhenReady = false, + locator = firstUtterance.locator, + range = null + ) + ) - override fun getVideoSize(): VideoSize { - return VideoSize.UNKNOWN - } + private var pendingUtterance: TtsUtterance? = + firstUtterance - override fun getCurrentCues(): CueGroup { - return CueGroup(emptyList()) - } + private var playbackJob: Job? = null - override fun getDeviceInfo(): DeviceInfo { - val minVolume = if (Util.SDK_INT >= 28) audioManager.getStreamMinVolume(STREAM_TYPE_MUSIC) else 0 - val maxVolume = audioManager.getStreamMaxVolume(STREAM_TYPE_MUSIC) - return DeviceInfo(DeviceInfo.PLAYBACK_TYPE_LOCAL, minVolume, maxVolume) + init { + contentIterator.language = engineFacade.settings.value.language } - override fun getDeviceVolume(): Int { - return audioManager.getStreamVolume(STREAM_TYPE_MUSIC) - } + val playback: StateFlow = + playbackMutable.asStateFlow() - override fun isDeviceMuted(): Boolean { - return if (Util.SDK_INT >= 23) { - audioManager.isStreamMute(STREAM_TYPE_MUSIC) - } else { - deviceVolume == 0 + fun play() { + replacePlaybackJob { + playbackMutable.value = + playbackMutable.value.copy( + state = Playback.State.READY, + playWhenReady = true, + isPlaying = true + ) + playContinuous() } } - override fun setDeviceVolume(volume: Int) { - throw NotImplementedError() + fun pause() { + replacePlaybackJob { + playbackMutable.value = + playbackMutable.value.copy( + playWhenReady = false, + isPlaying = false + ) + } } - override fun increaseDeviceVolume() { - throw NotImplementedError() + fun go(locator: TtsLocator) { + replacePlaybackJob { + pendingUtterance = null + contentIterator.seek(locator) + playContinuous() + } } - override fun decreaseDeviceVolume() { - throw NotImplementedError() + fun nextUtterance() { + replacePlaybackJob { + pendingUtterance = null + playContinuous() + } } - override fun setDeviceMuted(muted: Boolean) { - throw NotImplementedError() + fun previousUtterance() { + replacePlaybackJob { + pendingUtterance = null + pendingUtterance = contentIterator.previousUtterance() + playContinuous() + } } - private fun notifyListeners( - previousPlaybackInfo: TtsEngineFacadePlayback, - playbackInfo: TtsEngineFacadePlayback, - // playWhenReadyChangeReason: @Player.PlayWhenReadyChangeReason Int, - ) { - /*if (previousPlaybackInfo.playbackError != playbackInfo.playbackError) { - listeners.queueEvent( - EVENT_PLAYER_ERROR - ) { listener: Player.Listener -> - listener.onPlayerErrorChanged( - playbackInfo.playbackError - ) - } - if (playbackInfo.playbackError != null) { - listeners.queueEvent( - EVENT_PLAYER_ERROR - ) { listener: Player.Listener -> - listener.onPlayerError( - playbackInfo.playbackError!! - ) - } - } - }*/ - - if (previousPlaybackInfo.isPlaying != playbackInfo.isPlaying) { - listeners.queueEvent( - EVENT_PLAYBACK_STATE_CHANGED - ) { listener: Player.Listener -> - listener.onPlaybackStateChanged( - playbackInfo.state.value - ) + private fun replacePlaybackJob(block: suspend CoroutineScope.() -> Unit) { + coroutineScope.launch { + playbackJob?.cancelAndJoin() + playbackJob = launch { + block() } } + } - if (previousPlaybackInfo.playWhenReady != playbackInfo.playWhenReady) { - listeners.queueEvent( - EVENT_PLAY_WHEN_READY_CHANGED - ) { listener: Player.Listener -> - listener.onPlayWhenReadyChanged( - playbackInfo.playWhenReady, - if (playbackInfo.state == TtsEngineFacadePlayback.State.ENDED) - PLAY_WHEN_READY_CHANGE_REASON_END_OF_MEDIA_ITEM - else - PLAY_WHEN_READY_CHANGE_REASON_USER_REQUEST - // PLAYBACK_SUPPRESSION_REASON_NONE - // playWhenReadyChangeReason - ) - } + private suspend fun playContinuous() { + if (pendingUtterance == null) { + pendingUtterance = contentIterator.nextUtterance() } - - if (isPlaying(previousPlaybackInfo) != isPlaying(playbackInfo)) { - listeners.queueEvent( - EVENT_IS_PLAYING_CHANGED - ) { listener: Player.Listener -> - listener.onIsPlayingChanged(isPlaying(playbackInfo)) - } + pendingUtterance?.let { + Timber.d("Setting playback to locator ${it.locator}") + playbackMutable.value = playbackMutable.value.copy(range = null, locator = it.locator, isPlaying = true) + engineFacade.speak(it.text, it.language, ::onRangeChanged) + pendingUtterance = null + playContinuous() + } ?: run { + playbackMutable.value = playbackMutable.value.copy( + isPlaying = false, playWhenReady = false, state = Playback.State.ENDED + ) } - /*if (previousPlaybackInfo.playbackParameters != playbackInfo.playbackParameters) { - listeners.queueEvent( - EVENT_PLAYBACK_PARAMETERS_CHANGED - ) { listener: Player.Listener -> - listener.onPlaybackParametersChanged( - playbackInfo.playbackParameters - ) - } - }*/ + } - listeners.flushEvents() + private fun onRangeChanged(range: IntRange) { + val newPlayback = playbackMutable.value.copy(range = range) + playbackMutable.value = newPlayback } - private fun isPlaying(playbackInfo: TtsEngineFacadePlayback): Boolean { - return (playbackInfo.state == TtsEngineFacadePlayback.State.READY && playbackInfo.playWhenReady) + fun close() { + engineFacade.close() } } diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsSessionAdapter.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsSessionAdapter.kt new file mode 100644 index 0000000000..64851f707d --- /dev/null +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsSessionAdapter.kt @@ -0,0 +1,451 @@ +/* + * Copyright 2022 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + +package org.readium.r2.navigator.media3.tts2 + +import android.app.Application +import android.content.Context +import android.media.AudioManager +import android.os.Looper +import android.view.Surface +import android.view.SurfaceHolder +import android.view.SurfaceView +import android.view.TextureView +import androidx.media3.common.* +import androidx.media3.common.C.* +import androidx.media3.common.text.CueGroup +import androidx.media3.common.util.Clock +import androidx.media3.common.util.ListenerSet +import androidx.media3.common.util.Util +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.MainScope +import kotlinx.coroutines.flow.launchIn +import kotlinx.coroutines.flow.onEach +import org.readium.r2.shared.ExperimentalReadiumApi +import timber.log.Timber + +@ExperimentalReadiumApi +@androidx.annotation.OptIn(androidx.media3.common.util.UnstableApi::class) +internal class TtsSessionAdapter( + private val application: Application, + private val ttsPlayer: TtsPlayer<*, *>, + private val playlistMetadata: MediaMetadata, + private val mediaItems: List, + private val onStop: () -> Unit +) : BasePlayer() { + + private val coroutineScope: CoroutineScope = + MainScope() + + private var lastPlayback: TtsPlayer.Playback = + ttsPlayer.playback.value + + init { + ttsPlayer.playback + .onEach { playback -> + notifyListeners(lastPlayback, playback) + lastPlayback = playback + }.launchIn(coroutineScope) + } + + private var listeners: ListenerSet = + ListenerSet( + applicationLooper, + Clock.DEFAULT, + ) { listener: Player.Listener, flags: FlagSet? -> + listener.onEvents(this, Player.Events(flags!!)) + } + + private val permanentAvailableCommands = + Player.Commands.Builder() + .addAll( + COMMAND_PLAY_PAUSE, + COMMAND_STOP, + + // COMMAND_SEEK_TO_NEXT_MEDIA_ITEM, + // COMMAND_SEEK_TO_PREVIOUS_MEDIA_ITEM, + // COMMAND_SEEK_TO_NEXT, + // COMMAND_SEEK_TO_PREVIOUS + + // COMMAND_GET_AUDIO_ATTRIBUTES, + // COMMAND_GET_CURRENT_MEDIA_ITEM, + // COMMAND_GET_MEDIA_ITEMS_METADATA, + // COMMAND_GET_TEXT + ).build() + + private val audioManager: AudioManager = + application.getSystemService(Context.AUDIO_SERVICE) as AudioManager + + override fun getApplicationLooper(): Looper { + return Looper.getMainLooper() + } + + override fun addListener(listener: Player.Listener) { + Timber.d("addListener") + listeners.add(listener) + } + + override fun removeListener(listener: Player.Listener) { + Timber.d("removeListener") + listeners.remove(listener) + } + + override fun setMediaItems(mediaItems: MutableList, resetPosition: Boolean) { + throw NotImplementedError() + } + + override fun setMediaItems( + mediaItems: MutableList, + startIndex: Int, + startPositionMs: Long + ) { + throw NotImplementedError() + } + + override fun addMediaItems(index: Int, mediaItems: MutableList) { + throw NotImplementedError() + } + + override fun moveMediaItems(fromIndex: Int, toIndex: Int, newIndex: Int) { + throw NotImplementedError() + } + + override fun removeMediaItems(fromIndex: Int, toIndex: Int) { + throw NotImplementedError() + } + + override fun getAvailableCommands(): Player.Commands { + return Player.Commands.Builder() + .addAll(permanentAvailableCommands) + .build() + } + + override fun prepare() { + throw NotImplementedError() + } + + override fun getPlaybackState(): Int { + return ttsPlayer.playback.value.state.value + } + + override fun getPlaybackSuppressionReason(): Int { + return PLAYBACK_SUPPRESSION_REASON_NONE // TODO + } + + override fun getPlayerError(): PlaybackException? { + return null // TODO + } + + override fun setPlayWhenReady(playWhenReady: Boolean) { + if (playWhenReady) { + ttsPlayer.play() + } else { + ttsPlayer.pause() + } + } + + override fun getPlayWhenReady(): Boolean { + return ttsPlayer.playback.value.playWhenReady + } + + override fun setRepeatMode(repeatMode: Int) { + throw NotImplementedError() + } + + override fun getRepeatMode(): Int { + return REPEAT_MODE_OFF + } + + override fun setShuffleModeEnabled(shuffleModeEnabled: Boolean) { + throw NotImplementedError() + } + + override fun getShuffleModeEnabled(): Boolean { + return false + } + + override fun isLoading(): Boolean { + return false + } + + override fun seekTo(mediaItemIndex: Int, positionMs: Long) { + throw NotImplementedError() + } + + override fun getSeekBackIncrement(): Long { + return 0 + } + + override fun getSeekForwardIncrement(): Long { + return 0 + } + + override fun getMaxSeekToPreviousPosition(): Long { + return 0 + } + + override fun setPlaybackParameters(playbackParameters: PlaybackParameters) { + throw NotImplementedError() // TODO + } + + override fun getPlaybackParameters(): PlaybackParameters { + return PlaybackParameters.DEFAULT + } + + override fun stop() { + onStop() + } + + @Deprecated("Deprecated in Java") + override fun stop(reset: Boolean) {} + + override fun release() { + ttsPlayer.close() + } + + override fun getCurrentTracks(): Tracks { + throw NotImplementedError() + } + + override fun getTrackSelectionParameters(): TrackSelectionParameters { + return TrackSelectionParameters.Builder(application) + .build() + } + + override fun setTrackSelectionParameters(parameters: TrackSelectionParameters) { + throw NotImplementedError() + } + + override fun getMediaMetadata(): MediaMetadata { + return currentTimeline.getWindow(currentMediaItemIndex, window).mediaItem.mediaMetadata + } + + override fun getPlaylistMetadata(): MediaMetadata { + return playlistMetadata + } + + override fun setPlaylistMetadata(mediaMetadata: MediaMetadata) { + throw NotImplementedError() + } + + override fun getCurrentTimeline(): Timeline { + // MediaNotificationManager requires a non-empty timeline to start foreground playing. + return TtsSessionTimeline(mediaItems) + /*return SinglePeriodTimeline( + TIME_UNSET, false, false, false, null, mediaItem)*/ + } + + override fun getCurrentPeriodIndex(): Int { + return lastPlayback.index + } + + override fun getCurrentMediaItemIndex(): Int { + return lastPlayback.index + } + + override fun getDuration(): Long { + return TIME_UNSET + } + + override fun getCurrentPosition(): Long { + return 0 + } + + override fun getBufferedPosition(): Long { + return 0 + } + + override fun getTotalBufferedDuration(): Long { + return 0 + } + + override fun isPlayingAd(): Boolean { + return false + } + + override fun getCurrentAdGroupIndex(): Int { + return INDEX_UNSET + } + + override fun getCurrentAdIndexInAdGroup(): Int { + return INDEX_UNSET + } + + override fun getContentPosition(): Long { + return 0 + } + + override fun getContentBufferedPosition(): Long { + return 0 + } + + override fun getAudioAttributes(): AudioAttributes { + return AudioAttributes.Builder() + .setUsage(USAGE_MEDIA) + .setContentType(AUDIO_CONTENT_TYPE_SPEECH) + .build() + } + + override fun setVolume(volume: Float) { + throw NotImplementedError() + } + + override fun getVolume(): Float { + return 1.0f + } + + override fun clearVideoSurface() { + throw NotImplementedError() + } + + override fun clearVideoSurface(surface: Surface?) { + throw NotImplementedError() + } + + override fun setVideoSurface(surface: Surface?) { + throw NotImplementedError() + } + + override fun setVideoSurfaceHolder(surfaceHolder: SurfaceHolder?) { + throw NotImplementedError() + } + + override fun clearVideoSurfaceHolder(surfaceHolder: SurfaceHolder?) { + throw NotImplementedError() + } + + override fun setVideoSurfaceView(surfaceView: SurfaceView?) { + throw NotImplementedError() + } + + override fun clearVideoSurfaceView(surfaceView: SurfaceView?) { + throw NotImplementedError() + } + + override fun setVideoTextureView(textureView: TextureView?) { + throw NotImplementedError() + } + + override fun clearVideoTextureView(textureView: TextureView?) { + throw NotImplementedError() + } + + override fun getVideoSize(): VideoSize { + return VideoSize.UNKNOWN + } + + override fun getCurrentCues(): CueGroup { + return CueGroup(emptyList()) + } + + override fun getDeviceInfo(): DeviceInfo { + val minVolume = if (Util.SDK_INT >= 28) audioManager.getStreamMinVolume(STREAM_TYPE_MUSIC) else 0 + val maxVolume = audioManager.getStreamMaxVolume(STREAM_TYPE_MUSIC) + return DeviceInfo(DeviceInfo.PLAYBACK_TYPE_LOCAL, minVolume, maxVolume) + } + + override fun getDeviceVolume(): Int { + return audioManager.getStreamVolume(STREAM_TYPE_MUSIC) + } + + override fun isDeviceMuted(): Boolean { + return if (Util.SDK_INT >= 23) { + audioManager.isStreamMute(STREAM_TYPE_MUSIC) + } else { + deviceVolume == 0 + } + } + + override fun setDeviceVolume(volume: Int) { + throw NotImplementedError() + } + + override fun increaseDeviceVolume() { + throw NotImplementedError() + } + + override fun decreaseDeviceVolume() { + throw NotImplementedError() + } + + override fun setDeviceMuted(muted: Boolean) { + throw NotImplementedError() + } + + private fun notifyListeners( + previousPlaybackInfo: TtsPlayer.Playback, + playbackInfo: TtsPlayer.Playback, + // playWhenReadyChangeReason: @Player.PlayWhenReadyChangeReason Int, + ) { + /*if (previousPlaybackInfo.playbackError != playbackInfo.playbackError) { + listeners.queueEvent( + EVENT_PLAYER_ERROR + ) { listener: Player.Listener -> + listener.onPlayerErrorChanged( + playbackInfo.playbackError + ) + } + if (playbackInfo.playbackError != null) { + listeners.queueEvent( + EVENT_PLAYER_ERROR + ) { listener: Player.Listener -> + listener.onPlayerError( + playbackInfo.playbackError!! + ) + } + } + }*/ + + if (previousPlaybackInfo.isPlaying != playbackInfo.isPlaying) { + listeners.queueEvent( + EVENT_PLAYBACK_STATE_CHANGED + ) { listener: Player.Listener -> + listener.onPlaybackStateChanged( + playbackInfo.state.value + ) + } + } + + if (previousPlaybackInfo.playWhenReady != playbackInfo.playWhenReady) { + listeners.queueEvent( + EVENT_PLAY_WHEN_READY_CHANGED + ) { listener: Player.Listener -> + listener.onPlayWhenReadyChanged( + playbackInfo.playWhenReady, + if (playbackInfo.state == TtsPlayer.Playback.State.ENDED) + PLAY_WHEN_READY_CHANGE_REASON_END_OF_MEDIA_ITEM + else + PLAY_WHEN_READY_CHANGE_REASON_USER_REQUEST + // PLAYBACK_SUPPRESSION_REASON_NONE + // playWhenReadyChangeReason + ) + } + } + + if (isPlaying(previousPlaybackInfo) != isPlaying(playbackInfo)) { + listeners.queueEvent( + EVENT_IS_PLAYING_CHANGED + ) { listener: Player.Listener -> + listener.onIsPlayingChanged(isPlaying(playbackInfo)) + } + } + /*if (previousPlaybackInfo.playbackParameters != playbackInfo.playbackParameters) { + listeners.queueEvent( + EVENT_PLAYBACK_PARAMETERS_CHANGED + ) { listener: Player.Listener -> + listener.onPlaybackParametersChanged( + playbackInfo.playbackParameters + ) + } + }*/ + + listeners.flushEvents() + } + + private fun isPlaying(playbackInfo: TtsPlayer.Playback): Boolean { + return (playbackInfo.state == TtsPlayer.Playback.State.READY && playbackInfo.playWhenReady) + } +} diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsTimeline.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsSessionTimeline.kt similarity index 97% rename from readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsTimeline.kt rename to readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsSessionTimeline.kt index cd3104c81e..85f232d9ee 100644 --- a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsTimeline.kt +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsSessionTimeline.kt @@ -11,7 +11,7 @@ import androidx.media3.common.Timeline import java.util.* @androidx.annotation.OptIn(androidx.media3.common.util.UnstableApi::class) -internal class TtsTimeline( +internal class TtsSessionTimeline( private val mediaItems: List, ) : Timeline() { diff --git a/test-app/src/main/java/org/readium/r2/testapp/reader/ReaderInitData.kt b/test-app/src/main/java/org/readium/r2/testapp/reader/ReaderInitData.kt index 10884f7fd2..296bbfcfcf 100644 --- a/test-app/src/main/java/org/readium/r2/testapp/reader/ReaderInitData.kt +++ b/test-app/src/main/java/org/readium/r2/testapp/reader/ReaderInitData.kt @@ -33,7 +33,7 @@ sealed class ReaderInitData { sealed class VisualReaderInitData( override val bookId: Long, override val publication: Publication, - val initialLocation: Locator?, + var initialLocation: Locator?, val ttsInitData: TtsInitData?, ) : ReaderInitData() diff --git a/test-app/src/main/java/org/readium/r2/testapp/reader/VisualReaderFragment.kt b/test-app/src/main/java/org/readium/r2/testapp/reader/VisualReaderFragment.kt index 691eb4444b..6ef8169725 100644 --- a/test-app/src/main/java/org/readium/r2/testapp/reader/VisualReaderFragment.kt +++ b/test-app/src/main/java/org/readium/r2/testapp/reader/VisualReaderFragment.kt @@ -52,6 +52,7 @@ import org.readium.r2.testapp.reader.tts.TtsViewModel import org.readium.r2.testapp.utils.* import org.readium.r2.testapp.utils.extensions.confirmDialog import org.readium.r2.testapp.utils.extensions.throttleLatest +import timber.log.Timber /* * Base reader fragment class @@ -209,6 +210,7 @@ abstract class VisualReaderFragment : BaseReaderFragment(), VisualNavigator.List state.map { it.playingUtterance } .distinctUntilChanged() .onEach { locator -> + Timber.d("Highlighting $locator") val decoration = locator?.let { Decoration( id = "tts", diff --git a/test-app/src/main/java/org/readium/r2/testapp/reader/preferences/UserPreferences.kt b/test-app/src/main/java/org/readium/r2/testapp/reader/preferences/UserPreferences.kt index 6cc167d785..aea474426e 100644 --- a/test-app/src/main/java/org/readium/r2/testapp/reader/preferences/UserPreferences.kt +++ b/test-app/src/main/java/org/readium/r2/testapp/reader/preferences/UserPreferences.kt @@ -8,33 +8,26 @@ package org.readium.r2.testapp.reader.preferences -import androidx.compose.foundation.clickable import androidx.compose.foundation.layout.* import androidx.compose.material.* -import androidx.compose.material.icons.Icons -import androidx.compose.material.icons.filled.* -import androidx.compose.runtime.* +import androidx.compose.runtime.Composable +import androidx.compose.runtime.collectAsState +import androidx.compose.runtime.getValue import androidx.compose.ui.Alignment import androidx.compose.ui.Modifier -import androidx.compose.ui.graphics.Color -import androidx.compose.ui.graphics.luminance import androidx.compose.ui.text.style.TextAlign import androidx.compose.ui.unit.dp -import androidx.compose.ui.window.Dialog -import java.util.* import org.readium.adapters.pdfium.navigator.PdfiumPreferencesEditor import org.readium.r2.navigator.epub.EpubPreferencesEditor import org.readium.r2.navigator.preferences.* -import org.readium.r2.navigator.preferences.Color as ReadiumColor import org.readium.r2.navigator.preferences.TextAlign as ReadiumTextAlign import org.readium.r2.shared.ExperimentalReadiumApi import org.readium.r2.shared.publication.epub.EpubLayout import org.readium.r2.shared.util.Language import org.readium.r2.testapp.LITERATA import org.readium.r2.testapp.reader.ReaderViewModel -import org.readium.r2.testapp.utils.compose.ColorPicker +import org.readium.r2.testapp.shared.views.* import org.readium.r2.testapp.utils.compose.DropdownMenuButton -import org.readium.r2.testapp.utils.compose.ToggleButtonGroup /** * Stateful user settings component paired with a [ReaderViewModel]. @@ -249,7 +242,7 @@ private fun ColumnScope.FixedLayoutUserPreferences( @Composable private fun ColumnScope.ReflowableUserPreferences( commit: () -> Unit, - backgroundColor: Preference? = null, + backgroundColor: Preference? = null, columnCount: EnumPreference? = null, fontFamily: Preference? = null, fontSize: RangePreference? = null, @@ -267,7 +260,7 @@ private fun ColumnScope.ReflowableUserPreferences( readingProgression: EnumPreference? = null, scroll: Preference? = null, textAlign: EnumPreference? = null, - textColor: Preference? = null, + textColor: Preference? = null, textNormalization: Preference? = null, theme: EnumPreference? = null, typeScale: RangePreference? = null, @@ -399,6 +392,7 @@ private fun ColumnScope.ReflowableUserPreferences( title = "Typeface", preference = fontFamily .withSupportedValues( + null, FontFamily.LITERATA, FontFamily.SANS_SERIF, FontFamily.IA_WRITER_DUOSPACE, @@ -535,378 +529,6 @@ private fun ColumnScope.ReflowableUserPreferences( } } -/** - * Component for an [EnumPreference] displayed as a group of mutually exclusive buttons. - * This works best with a small number of enum values. - */ -@Composable -private fun ButtonGroupItem( - title: String, - preference: EnumPreference, - commit: () -> Unit, - formatValue: (T) -> String -) { - ButtonGroupItem( - title = title, - options = preference.supportedValues, - isActive = preference.isEffective, - activeOption = preference.effectiveValue, - selectedOption = preference.value, - formatValue = formatValue, - onClear = { preference.clear(); commit() } - .takeIf { preference.value != null }, - onSelectedOptionChanged = { newValue -> - if (newValue == preference.value) { - preference.clear() - } else { - preference.set(newValue) - } - commit() - } - ) -} - -/** - * Group of mutually exclusive buttons. - */ -@Composable -private fun ButtonGroupItem( - title: String, - options: List, - isActive: Boolean, - activeOption: T, - selectedOption: T?, - formatValue: (T) -> String, - onClear: (() -> Unit)?, - onSelectedOptionChanged: (T) -> Unit, -) { - Item(title, isActive = isActive, onClear = onClear) { - ToggleButtonGroup( - options = options, - activeOption = activeOption, - selectedOption = selectedOption, - onSelectOption = { option -> onSelectedOptionChanged(option) } - ) { option -> - Text( - text = formatValue(option), - style = MaterialTheme.typography.caption - ) - } - } -} - -/** - * Component for an [EnumPreference] displayed as a dropdown menu. - */ -@Composable -private fun MenuItem( - title: String, - preference: EnumPreference, - commit: () -> Unit, - formatValue: (T?) -> String -) { - MenuItem( - title = title, - value = preference.value ?: preference.effectiveValue, - values = listOf(null) + preference.supportedValues, - isActive = preference.isEffective, - formatValue = formatValue, - onClear = { preference.clear(); commit() } - .takeIf { preference.value != null }, - onValueChanged = { value -> - preference.set(value) - commit() - } - ) -} - -/** - * Dropdown menu. - */ -@Composable -private fun MenuItem( - title: String, - value: T, - values: List, - isActive: Boolean, - formatValue: (T) -> String, - onValueChanged: (T) -> Unit, - onClear: (() -> Unit)? -) { - Item(title, isActive = isActive, onClear = onClear) { - DropdownMenuButton( - text = { - Text( - text = formatValue(value), - style = MaterialTheme.typography.caption - ) - } - ) { dismiss -> - for (value in values) { - DropdownMenuItem( - onClick = { - dismiss() - onValueChanged(value) - } - ) { - Text(formatValue(value)) - } - } - } - } -} - -/** - * Component for a [RangePreference] with decrement and increment buttons. - */ -@Composable -private fun > StepperItem( - title: String, - preference: RangePreference, - commit: () -> Unit -) { - StepperItem( - title = title, - isActive = preference.isEffective, - value = preference.value ?: preference.effectiveValue, - formatValue = preference::formatValue, - onDecrement = { preference.decrement(); commit() }, - onIncrement = { preference.increment(); commit() }, - onClear = { preference.clear(); commit() } - .takeIf { preference.value != null }, - ) -} - -/** - * Component for a [RangePreference] with decrement and increment buttons. - */ -@Composable -private fun StepperItem( - title: String, - isActive: Boolean, - value: T, - formatValue: (T) -> String, - onDecrement: () -> Unit, - onIncrement: () -> Unit, - onClear: (() -> Unit)? -) { - Item(title, isActive = isActive, onClear = onClear) { - Row( - verticalAlignment = Alignment.CenterVertically, - horizontalArrangement = Arrangement.spacedBy(16.dp) - ) { - IconButton( - onClick = onDecrement, - content = { - Icon(Icons.Default.Remove, contentDescription = "Less") - } - ) - - Text( - text = formatValue(value), - modifier = Modifier.widthIn(min = 30.dp), - textAlign = TextAlign.Center - ) - - IconButton( - onClick = onIncrement, - content = { - Icon(Icons.Default.Add, contentDescription = "More") - } - ) - } - } -} - -/** - * Component for a boolean [Preference]. - */ -@Composable -private fun SwitchItem( - title: String, - preference: Preference, - commit: () -> Unit -) { - SwitchItem( - title = title, - value = preference.value ?: preference.effectiveValue, - isActive = preference.isEffective, - onCheckedChange = { preference.set(it); commit() }, - onToggle = { preference.toggle(); commit() }, - onClear = { preference.clear(); commit() } - .takeIf { preference.value != null }, - ) -} - -/** - * Switch - */ -@Composable -private fun SwitchItem( - title: String, - value: Boolean, - isActive: Boolean, - onCheckedChange: (Boolean) -> Unit, - onToggle: () -> Unit, - onClear: (() -> Unit)? -) { - Item( - title = title, - isActive = isActive, - onClick = onToggle, - onClear = onClear - ) { - Switch( - checked = value, - onCheckedChange = onCheckedChange - ) - } -} - -/** - * Component for a [Preference]. - */ -@Composable -private fun ColorItem( - title: String, - preference: Preference, - commit: () -> Unit -) { - ColorItem( - title = title, - isActive = preference.isEffective, - value = preference.value ?: preference.effectiveValue, - noValueSelected = preference.value == null, - onColorChanged = { preference.set(it); commit() }, - onClear = { preference.clear(); commit() } - .takeIf { preference.value != null } - ) -} - -/** - * Color picker - */ -@Composable -private fun ColorItem( - title: String, - isActive: Boolean, - value: ReadiumColor, - noValueSelected: Boolean, - onColorChanged: (ReadiumColor?) -> Unit, - onClear: (() -> Unit)? -) { - var isPicking by remember { mutableStateOf(false) } - - Item( - title = title, - isActive = isActive, - onClick = { isPicking = true }, - onClear = onClear - ) { - val color = Color(value.int) - - OutlinedButton( - onClick = { isPicking = true }, - colors = ButtonDefaults.buttonColors(backgroundColor = color) - ) { - if (noValueSelected) { - Icon( - imageVector = Icons.Default.Palette, - contentDescription = "Change color", - tint = if (color.luminance() > 0.5) Color.Black else Color.White - ) - } - } - - if (isPicking) { - Dialog( - onDismissRequest = { isPicking = false } - ) { - Column( - horizontalAlignment = Alignment.End - ) { - ColorPicker { color -> - isPicking = false - onColorChanged(ReadiumColor(color)) - } - Button( - onClick = { - isPicking = false - onColorChanged(null) - } - ) { - Text("Clear") - } - } - } - } - } -} - -/** - * Component for a [Preference]`. - */ -@Composable -fun LanguageItem( - preference: Preference, - commit: () -> Unit -) { - val languages = remember { - Locale.getAvailableLocales() - .map { Language(it).removeRegion() } - .distinct() - .sortedBy { it.locale.displayName } - } - - MenuItem( - title = "Language", - isActive = preference.isEffective, - value = preference.value ?: preference.effectiveValue, - values = languages, - formatValue = { it?.locale?.displayName ?: "Unknown" }, - onClear = { preference.clear(); commit() } - .takeIf { preference.value != null }, - onValueChanged = { value -> - preference.set(value) - commit() - } - ) -} - -@OptIn(ExperimentalMaterialApi::class) -@Composable -private fun Item( - title: String, - isActive: Boolean = true, - onClick: (() -> Unit)? = null, - onClear: (() -> Unit)? = null, - content: @Composable () -> Unit -) { - ListItem( - modifier = - if (onClick != null) Modifier.clickable(onClick = onClick) - else Modifier, - text = { - val alpha = if (isActive) 1.0f else ContentAlpha.disabled - CompositionLocalProvider(LocalContentAlpha provides alpha) { - Text(title) - } - }, - trailing = { - Row { - content() - - IconButton(onClick = onClear ?: {}, enabled = onClear != null) { - Icon( - Icons.Default.Backspace, - contentDescription = "Clear" - ) - } - } - } - ) -} - @Composable private fun Divider() { Divider(modifier = Modifier.padding(vertical = 16.dp)) diff --git a/test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsControls.kt b/test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsControls.kt index 66743f3adb..450a3fa9f4 100644 --- a/test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsControls.kt +++ b/test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsControls.kt @@ -4,6 +4,8 @@ * available in the top-level LICENSE file of the project. */ +@file:OptIn(ExperimentalReadiumApi::class) + package org.readium.r2.testapp.reader.tts import androidx.compose.foundation.layout.* @@ -14,38 +16,31 @@ import androidx.compose.runtime.* import androidx.compose.ui.Alignment import androidx.compose.ui.Modifier import androidx.compose.ui.res.stringResource +import androidx.compose.ui.text.style.TextAlign import androidx.compose.ui.unit.dp -import java.text.DecimalFormat import org.readium.r2.navigator.media3.androidtts.AndroidTtsPreferencesEditor -import org.readium.r2.navigator.tts.TtsEngine.Voice +import org.readium.r2.navigator.preferences.* import org.readium.r2.shared.ExperimentalReadiumApi import org.readium.r2.shared.util.Language import org.readium.r2.testapp.R -import org.readium.r2.testapp.shared.views.SelectorListItem +import org.readium.r2.testapp.shared.views.LanguageItem +import org.readium.r2.testapp.shared.views.MenuItem import org.readium.r2.testapp.utils.extensions.asStateWhenStarted /** * TTS controls bar displayed at the bottom of the screen when speaking a publication. */ -@OptIn(ExperimentalReadiumApi::class) @Composable fun TtsControls(model: TtsViewModel, modifier: Modifier = Modifier) { val showControls by model.state.asStateWhenStarted { it.showControls } val isPlaying by model.state.asStateWhenStarted { it.isPlaying } - // val settings by model.state.asStateWhenStarted { it.settings } - - val editor = remember { mutableStateOf(model.preferencesEditor, policy = neverEqualPolicy()) } - val commit: () -> Unit = { editor.value = editor.value ; model.commitPreferences() } + val editor by model.editor.collectAsState() if (showControls) { TtsControls( playing = isPlaying, - availableRates = listOf(0.25, 0.5, 0.75, 1.0, 1.25, 1.5, 2.0), - // .filter { it in settings.rateRange }, - availableLanguages = emptyList(), // settings.availableLanguages, - availableVoices = emptyList(), // settings.availableVoices, - editor = editor.value, - commit = commit, + editor = editor, + commit = model::commit, onPlayPause = { if (isPlaying) model.pause() else model.play() }, onStop = model::stop, onPrevious = model::previous, @@ -59,9 +54,6 @@ fun TtsControls(model: TtsViewModel, modifier: Modifier = Modifier) { @Composable fun TtsControls( playing: Boolean, - availableRates: List, - availableLanguages: List, - availableVoices: List, editor: AndroidTtsPreferencesEditor, commit: () -> Unit, onPlayPause: () -> Unit, @@ -74,10 +66,10 @@ fun TtsControls( if (showSettings) { TtsPreferencesDialog( - availableRates = availableRates, - availableLanguages = availableLanguages, - availableVoices = availableVoices, - editor = editor, + speed = editor.speed, + pitch = editor.pitch, + language = editor.language, + voices = editor.voices, commit = commit, onDismiss = { showSettings = false } ) @@ -140,13 +132,12 @@ fun TtsControls( } } -@OptIn(ExperimentalReadiumApi::class) @Composable private fun TtsPreferencesDialog( - availableRates: List, - availableLanguages: List, - availableVoices: List, - editor: AndroidTtsPreferencesEditor, + speed: RangePreference, + pitch: RangePreference, + language: Preference, + voices: Preference>, commit: () -> Unit, onDismiss: () -> Unit ) { @@ -157,47 +148,54 @@ private fun TtsPreferencesDialog( Text(text = stringResource(R.string.close)) } }, - title = { Text(stringResource(R.string.tts_settings)) }, + title = { + Text( + text = stringResource(R.string.tts_settings), + textAlign = TextAlign.Center, + style = MaterialTheme.typography.h6, + modifier = Modifier + .fillMaxWidth() + ) + }, text = { Column { - if (availableRates.size > 1) { - SelectorListItem( - label = stringResource(R.string.tts_rate), - values = availableRates, - selection = editor.speedRate.value, - titleForValue = { rate -> - DecimalFormat("x#.##").format(rate) + MenuItem( + title = stringResource(R.string.speed_rate), + preference = speed.withSupportedValues(0.25, 0.5, 0.75, 1.0, 1.25, 1.5, 2.0), + formatValue = speed::formatValue, + commit = commit + ) + MenuItem( + title = stringResource(R.string.pitch_rate), + preference = pitch.withSupportedValues(0.25, 0.5, 0.75, 1.0, 1.25, 1.5, 2.0), + formatValue = pitch::formatValue, + commit = commit + ) + LanguageItem( + preference = language, + commit = commit + ) + MenuItem( + title = stringResource(R.string.tts_voice), + preference = voices.map( + from = { voices -> + language.effectiveValue?.let { voices[it] } }, - onSelected = { - editor.speedRate.set(it) - commit() + to = { voice -> + buildMap { + voices.value?.let { putAll(it) } + language.effectiveValue?.let { + if (voice == null) { + remove(it) + } else { + put(it, voice) + } + } + } } - ) - } - - SelectorListItem( - label = stringResource(R.string.language), - values = availableLanguages, - selection = editor.language.value, - titleForValue = { language -> - language?.locale?.displayName - ?: stringResource(R.string.auto) - }, - onSelected = { - editor.language.set(it) - commit() - } - ) - - SelectorListItem( - label = stringResource(R.string.tts_voice), - values = availableVoices, - selection = availableVoices.firstOrNull { it.id == editor.voiceId.value }, - titleForValue = { it?.name ?: it?.id ?: stringResource(R.string.auto) }, - onSelected = { - editor.voiceId.set(it?.id) - commit() - } + ).withSupportedValues("a", "b", "c"), + formatValue = { it ?: "Default" }, // it?.name ?: it?.id ?: stringResource(R.string.auto) }, + commit = commit ) } } diff --git a/test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsService.kt b/test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsService.kt index 3ded1984c5..3e2e8c379b 100644 --- a/test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsService.kt +++ b/test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsService.kt @@ -6,17 +6,19 @@ package org.readium.r2.testapp.reader.tts -import android.app.Application -import android.app.PendingIntent +import android.app.* import android.content.ComponentName +import android.content.Context import android.content.Intent import android.content.ServiceConnection +import android.graphics.Color import android.os.Build import android.os.IBinder -import androidx.lifecycle.lifecycleScope +import androidx.annotation.RequiresApi +import androidx.core.app.NotificationCompat +import androidx.core.content.ContextCompat import androidx.media3.session.MediaSession -import kotlinx.coroutines.CompletableDeferred -import kotlinx.coroutines.Job +import kotlinx.coroutines.* import kotlinx.coroutines.flow.launchIn import kotlinx.coroutines.flow.onEach import kotlinx.coroutines.flow.sample @@ -32,6 +34,14 @@ import timber.log.Timber @androidx.annotation.OptIn(androidx.media3.common.util.UnstableApi::class) class TtsService : LifecycleMedia3SessionService() { + class Session( + val bookId: Long, + val navigator: TtsNavigator, + val mediaSession: MediaSession, + ) { + val coroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.Main) + } + /** * The service interface to be used by the app. */ @@ -40,46 +50,48 @@ class TtsService : LifecycleMedia3SessionService() { private val app: org.readium.r2.testapp.Application get() = application as org.readium.r2.testapp.Application - private var saveLocationJob: Job? = null - - var mediaNavigator: TtsNavigator? = null - - var mediaSession: MediaSession? = null + var session: Session? = null fun closeNavigator() { stopForeground(true) - mediaSession?.release() - mediaSession = null - saveLocationJob?.cancel() - saveLocationJob = null - mediaNavigator?.close() - mediaNavigator = null + session?.mediaSession?.release() + session?.navigator?.close() + session?.coroutineScope?.cancel() + session = null } fun bindNavigator( navigator: TtsNavigator, bookId: Long - ) { + ): Session { val activityIntent = createSessionActivityIntent(bookId) - mediaNavigator = navigator - val session = MediaSession.Builder(applicationContext, navigator.asPlayer()) + val mediaSession = MediaSession.Builder(applicationContext, navigator.asPlayer()) .setSessionActivity(activityIntent) .setId(bookId.toString()) .build() - mediaSession = session - addSession(session) + addSession(mediaSession) + + val session = Session( + bookId, + navigator, + mediaSession + ) + + this@Binder.session = session /* * Launch a job for saving progression even when playback is going on in the background * with no ReaderActivity opened. */ - saveLocationJob = navigator.currentLocator + navigator.currentLocator .sample(3000) .onEach { locator -> Timber.d("Saving TTS progression $locator") app.bookRepository.saveProgression(locator, bookId) - }.launchIn(lifecycleScope) + }.launchIn(session.coroutineScope) + + return session } private fun createSessionActivityIntent(bookId: Long): PendingIntent { @@ -108,6 +120,39 @@ class TtsService : LifecycleMedia3SessionService() { override fun onCreate() { super.onCreate() Timber.d("TtsService created.") + // val initialNotification = createInitialNotification() + // startForeground(1, initialNotification) + } + + private fun createInitialNotification(): Notification { + return if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) { + val notificationChannelId = createNotificationChannel() + Notification.Builder(this, notificationChannelId) + .setContentTitle("R2 testapp") + .setContentText("rgergergergg") + .setAutoCancel(true) + .build() + } else { + NotificationCompat.Builder(this) + .setContentTitle("R2 testapp") + .setContentText("grgrgrgrg") + .setPriority(NotificationCompat.PRIORITY_DEFAULT) + .setAutoCancel(true) + .build() + } + } + + @RequiresApi(Build.VERSION_CODES.O) + private fun createNotificationChannel(): String { + val notificationChannelId = "example.permanence" + val channelName = "Background Service" + val channel = NotificationChannel(notificationChannelId, channelName, NotificationManager.IMPORTANCE_NONE) + channel.lightColor = Color.BLUE + channel.lockscreenVisibility = Notification.VISIBILITY_PRIVATE + + val manager = getSystemService(Context.NOTIFICATION_SERVICE) as NotificationManager + manager.createNotificationChannel(channel) + return notificationChannelId } override fun onBind(intent: Intent?): IBinder? { @@ -126,7 +171,7 @@ class TtsService : LifecycleMedia3SessionService() { } override fun onGetSession(controllerInfo: MediaSession.ControllerInfo): MediaSession? { - return binder.mediaSession + return binder.session?.mediaSession } override fun onUpdateNotification(session: MediaSession) { @@ -153,7 +198,7 @@ class TtsService : LifecycleMedia3SessionService() { fun start(application: Application) { val intent = intent(application) - application.startService(intent) + ContextCompat.startForegroundService(application, intent) } suspend fun bind(application: Application): TtsService.Binder { diff --git a/test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsViewModel.kt b/test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsViewModel.kt index e6395c124e..b2c3177b32 100644 --- a/test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsViewModel.kt +++ b/test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsViewModel.kt @@ -8,6 +8,7 @@ package org.readium.r2.testapp.reader.tts import android.content.Context import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Job import kotlinx.coroutines.channels.Channel import kotlinx.coroutines.flow.* import kotlinx.coroutines.launch @@ -30,6 +31,7 @@ import org.readium.r2.shared.util.Language import org.readium.r2.testapp.reader.ReaderInitData import org.readium.r2.testapp.reader.VisualReaderInitData import org.readium.r2.testapp.reader.preferences.PreferencesManager +import org.readium.r2.testapp.utils.extensions.mapStateIn import timber.log.Timber /** @@ -45,7 +47,7 @@ class TtsViewModel private constructor( private val ttsNavigatorFactory: TtsNavigatorFactory, private val ttsSessionBinder: TtsService.Binder, private val preferencesManager: PreferencesManager, - val preferencesEditor: AndroidTtsPreferencesEditor + private val createPreferencesEditor: (AndroidTtsPreferences) -> AndroidTtsPreferencesEditor ) { companion object { @@ -55,17 +57,12 @@ class TtsViewModel private constructor( */ operator fun invoke( viewModelScope: CoroutineScope, - readerInitData: ReaderInitData + readerInitData: ReaderInitData, ): TtsViewModel? { if (readerInitData !is VisualReaderInitData || readerInitData.ttsInitData == null) { return null } - val preferencesEditor = - readerInitData.ttsInitData.ttsNavigatorFactory.createTtsPreferencesEditor( - readerInitData.ttsInitData.preferencesManager.preferences.value - ) - return TtsViewModel( viewModelScope = viewModelScope, bookId = readerInitData.bookId, @@ -73,7 +70,7 @@ class TtsViewModel private constructor( ttsNavigatorFactory = readerInitData.ttsInitData.ttsNavigatorFactory, ttsSessionBinder = readerInitData.ttsInitData.sessionBinder, preferencesManager = readerInitData.ttsInitData.preferencesManager, - preferencesEditor = preferencesEditor + createPreferencesEditor = readerInitData.ttsInitData.ttsNavigatorFactory::createTtsPreferencesEditor ) } } @@ -91,6 +88,11 @@ class TtsViewModel private constructor( val playingUtterance: Locator? = null, ) + data class Binding( + val playbackJob: Job, + val submitSettingsJob: Job + ) + sealed class Event { /** * Emitted when the [PublicationSpeechSynthesizer] fails with an error. @@ -103,8 +105,8 @@ class TtsViewModel private constructor( class OnMissingVoiceData(val language: Language) : Event() } - private val ttsNavigator: TtsNavigator? get() = - ttsSessionBinder.mediaNavigator + val editor: StateFlow = preferencesManager.preferences + .mapStateIn(viewModelScope, createPreferencesEditor) /** * Current state of the view model. @@ -113,14 +115,20 @@ class TtsViewModel private constructor( stateFromPlayback(ttsNavigator?.playback?.value) ) - init { - ttsNavigator?.let { bindStateToNavigator(it) } - } val state: StateFlow = _state.asStateFlow() private val _events: Channel = Channel(Channel.BUFFERED) val events: Flow = _events.receiveAsFlow() + private val ttsSession: TtsService.Session? get() = + ttsSessionBinder.session + + private val ttsNavigator: TtsNavigator? get() = + ttsSession?.navigator + + private var binding: Binding? = + ttsSession?.let { bindSession(it) } + /** * Starts the TTS using the first visible locator in the given [navigator]. */ @@ -128,13 +136,12 @@ class TtsViewModel private constructor( if (ttsNavigator != null) return viewModelScope.launch { - val ttsNavigator = createTtsNavigator(navigator) - bindStateToNavigator(ttsNavigator) - ttsNavigator.play() + val session = openSession(navigator) + binding = bindSession(session) } } - private suspend fun createTtsNavigator(navigator: Navigator): TtsNavigator { + private suspend fun openSession(navigator: Navigator): TtsService.Session { val start = (navigator as? VisualNavigator)?.firstVisibleElementLocator() val listener = object : TtsNavigatorListener { @@ -154,19 +161,27 @@ class TtsViewModel private constructor( start ) - ttsSessionBinder.bindNavigator(ttsNavigator, bookId) - return ttsNavigator + // playWhenReady must be true for the MediaSessionService to call Service.startForeground + // and prevent crashing + ttsNavigator.play() + return ttsSessionBinder.bindNavigator(ttsNavigator, bookId) } - private fun bindStateToNavigator( - ttsNavigator: TtsNavigator - ) { - ttsNavigator.playback + private fun bindSession( + ttsSession: TtsService.Session + ): Binding { + val playbackJob = ttsSession.navigator.playback .onEach { playback -> Timber.d("new TTS playback $playback") _state.value = stateFromPlayback(playback) Timber.d("new TTS state ${_state.value}") }.launchIn(viewModelScope) + + val preferencesJob = preferencesManager.preferences + .onEach { ttsSession.navigator.submitPreferences(it) } + .launchIn(viewModelScope) + + return Binding(playbackJob, preferencesJob) } private fun stateFromPlayback(playback: TtsNavigator.Playback?): State { @@ -182,7 +197,7 @@ class TtsViewModel private constructor( } fun stop() { - if (ttsNavigator == null) return + ttsSession ?: return _state.value = State( showControls = false, @@ -190,6 +205,13 @@ class TtsViewModel private constructor( playingWordRange = null, playingUtterance = null, ) + + binding?.apply { + playbackJob.cancel() + submitSettingsJob.cancel() + } + binding = null + ttsSessionBinder.closeNavigator() } @@ -209,12 +231,11 @@ class TtsViewModel private constructor( ttsNavigator?.goForward() } - fun commitPreferences() { + fun commit() { viewModelScope.launch { - preferencesManager.setPreferences(preferencesEditor.preferences) + preferencesManager.setPreferences(editor.value.preferences) } } - /** * Starts the activity to install additional voice data. */ diff --git a/test-app/src/main/java/org/readium/r2/testapp/shared/views/List.kt b/test-app/src/main/java/org/readium/r2/testapp/shared/views/List.kt deleted file mode 100644 index b2ad6b3ddd..0000000000 --- a/test-app/src/main/java/org/readium/r2/testapp/shared/views/List.kt +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Copyright 2022 Readium Foundation. All rights reserved. - * Use of this source code is governed by the BSD-style license - * available in the top-level LICENSE file of the project. - */ - -package org.readium.r2.testapp.shared.views - -import androidx.compose.foundation.clickable -import androidx.compose.material.* -import androidx.compose.runtime.* -import androidx.compose.ui.Modifier - -/** - * A Material [ListItem] displaying a dropdown menu to select a value. The current value is - * displayed on the right. - */ -@OptIn(ExperimentalMaterialApi::class) -@Composable -fun SelectorListItem( - label: String, - values: List, - selection: T, - titleForValue: @Composable (T) -> String, - onSelected: (T) -> Unit, - enabled: Boolean = values.isNotEmpty(), -) { - var isExpanded by remember { mutableStateOf(false) } - fun dismiss() { isExpanded = false } - - ListItem( - modifier = Modifier - .clickable(enabled = enabled) { - isExpanded = true - }, - text = { - Group(enabled = enabled) { - Text(label) - } - }, - trailing = { - Group(enabled = enabled) { - Text(titleForValue(selection)) - } - - DropdownMenu( - expanded = isExpanded, - onDismissRequest = { dismiss() } - ) { - for (value in values) { - DropdownMenuItem( - onClick = { - onSelected(value) - dismiss() - } - ) { - Text(titleForValue(value)) - } - } - } - } - ) -} diff --git a/test-app/src/main/java/org/readium/r2/testapp/shared/views/Preferences.kt b/test-app/src/main/java/org/readium/r2/testapp/shared/views/Preferences.kt new file mode 100644 index 0000000000..38bf618988 --- /dev/null +++ b/test-app/src/main/java/org/readium/r2/testapp/shared/views/Preferences.kt @@ -0,0 +1,472 @@ +/* + * Copyright 2022 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + +@file:OptIn(ExperimentalReadiumApi::class) + +package org.readium.r2.testapp.shared.views + +import androidx.compose.foundation.clickable +import androidx.compose.foundation.layout.Arrangement +import androidx.compose.foundation.layout.Column +import androidx.compose.foundation.layout.Row +import androidx.compose.foundation.layout.widthIn +import androidx.compose.material.* +import androidx.compose.material.icons.Icons +import androidx.compose.material.icons.filled.Add +import androidx.compose.material.icons.filled.Backspace +import androidx.compose.material.icons.filled.Palette +import androidx.compose.material.icons.filled.Remove +import androidx.compose.runtime.* +import androidx.compose.ui.Alignment +import androidx.compose.ui.Modifier +import androidx.compose.ui.graphics.Color +import androidx.compose.ui.graphics.luminance +import androidx.compose.ui.text.style.TextAlign +import androidx.compose.ui.unit.dp +import androidx.compose.ui.window.Dialog +import java.util.* +import org.readium.r2.navigator.preferences.* +import org.readium.r2.navigator.preferences.Color as ReadiumColor +import org.readium.r2.shared.ExperimentalReadiumApi +import org.readium.r2.shared.util.Language +import org.readium.r2.testapp.utils.compose.ColorPicker +import org.readium.r2.testapp.utils.compose.DropdownMenuButton +import org.readium.r2.testapp.utils.compose.ToggleButtonGroup + +/** + * Component for an [EnumPreference] displayed as a group of mutually exclusive buttons. + * This works best with a small number of enum values. + */ +@Composable +fun ButtonGroupItem( + title: String, + preference: EnumPreference, + commit: () -> Unit, + formatValue: (T) -> String +) { + ButtonGroupItem( + title = title, + options = preference.supportedValues, + isActive = preference.isEffective, + activeOption = preference.effectiveValue, + selectedOption = preference.value, + formatValue = formatValue, + onClear = { preference.clear(); commit() } + .takeIf { preference.value != null }, + onSelectedOptionChanged = { newValue -> + if (newValue == preference.value) { + preference.clear() + } else { + preference.set(newValue) + } + commit() + } + ) +} + +/** + * Group of mutually exclusive buttons. + */ +@Composable +private fun ButtonGroupItem( + title: String, + options: List, + isActive: Boolean, + activeOption: T, + selectedOption: T?, + formatValue: (T) -> String, + onClear: (() -> Unit)?, + onSelectedOptionChanged: (T) -> Unit, +) { + Item(title, isActive = isActive, onClear = onClear) { + ToggleButtonGroup( + options = options, + activeOption = activeOption, + selectedOption = selectedOption, + onSelectOption = { option -> onSelectedOptionChanged(option) } + ) { option -> + Text( + text = formatValue(option), + style = MaterialTheme.typography.caption + ) + } + } +} + +/** + * Component for an [EnumPreference] displayed as a dropdown menu. + */ +@Composable +fun MenuItem( + title: String, + preference: EnumPreference, + commit: () -> Unit, + formatValue: (T) -> String +) { + MenuItem( + title = title, + value = preference.value ?: preference.effectiveValue, + values = preference.supportedValues, + isActive = preference.isEffective, + formatValue = formatValue, + onClear = { preference.clear(); commit() } + .takeIf { preference.value != null }, + onValueChanged = { value -> + preference.set(value) + commit() + } + ) +} + +/** + * Dropdown menu. + */ +@Composable +private fun MenuItem( + title: String, + value: T, + values: List, + isActive: Boolean, + formatValue: (T) -> String, + onValueChanged: (T) -> Unit, + onClear: (() -> Unit)? +) { + Item(title, isActive = isActive, onClear = onClear) { + DropdownMenuButton( + text = { + Text( + text = formatValue(value), + style = MaterialTheme.typography.caption + ) + } + ) { dismiss -> + for (value in values) { + DropdownMenuItem( + onClick = { + dismiss() + onValueChanged(value) + } + ) { + Text(formatValue(value)) + } + } + } + } +} + +/** + * Component for a [RangePreference] with decrement and increment buttons. + */ +@Composable +fun > StepperItem( + title: String, + preference: RangePreference, + commit: () -> Unit +) { + StepperItem( + title = title, + isActive = preference.isEffective, + value = preference.value ?: preference.effectiveValue, + formatValue = preference::formatValue, + onDecrement = { preference.decrement(); commit() }, + onIncrement = { preference.increment(); commit() }, + onClear = { preference.clear(); commit() } + .takeIf { preference.value != null }, + ) +} + +/** + * Component for a [RangePreference] with decrement and increment buttons. + */ +@Composable +private fun StepperItem( + title: String, + isActive: Boolean, + value: T, + formatValue: (T) -> String, + onDecrement: () -> Unit, + onIncrement: () -> Unit, + onClear: (() -> Unit)? +) { + Item(title, isActive = isActive, onClear = onClear) { + Row( + verticalAlignment = Alignment.CenterVertically, + horizontalArrangement = Arrangement.spacedBy(16.dp) + ) { + IconButton( + onClick = onDecrement, + content = { + Icon(Icons.Default.Remove, contentDescription = "Less") + } + ) + + Text( + text = formatValue(value), + modifier = Modifier.widthIn(min = 30.dp), + textAlign = TextAlign.Center + ) + + IconButton( + onClick = onIncrement, + content = { + Icon(Icons.Default.Add, contentDescription = "More") + } + ) + } + } +} + +/** + * Component for a boolean [Preference]. + */ +@Composable +fun SwitchItem( + title: String, + preference: Preference, + commit: () -> Unit +) { + SwitchItem( + title = title, + value = preference.value ?: preference.effectiveValue, + isActive = preference.isEffective, + onCheckedChange = { preference.set(it); commit() }, + onToggle = { preference.toggle(); commit() }, + onClear = { preference.clear(); commit() } + .takeIf { preference.value != null }, + ) +} + +/** + * Switch + */ +@Composable +private fun SwitchItem( + title: String, + value: Boolean, + isActive: Boolean, + onCheckedChange: (Boolean) -> Unit, + onToggle: () -> Unit, + onClear: (() -> Unit)? +) { + Item( + title = title, + isActive = isActive, + onClick = onToggle, + onClear = onClear + ) { + Switch( + checked = value, + onCheckedChange = onCheckedChange + ) + } +} + +/** + * Component for a [Preference]. + */ +@Composable +fun ColorItem( + title: String, + preference: Preference, + commit: () -> Unit +) { + ColorItem( + title = title, + isActive = preference.isEffective, + value = preference.value ?: preference.effectiveValue, + noValueSelected = preference.value == null, + onColorChanged = { preference.set(it); commit() }, + onClear = { preference.clear(); commit() } + .takeIf { preference.value != null } + ) +} + +/** + * Color picker + */ +@Composable +private fun ColorItem( + title: String, + isActive: Boolean, + value: ReadiumColor, + noValueSelected: Boolean, + onColorChanged: (ReadiumColor?) -> Unit, + onClear: (() -> Unit)? +) { + var isPicking by remember { mutableStateOf(false) } + + Item( + title = title, + isActive = isActive, + onClick = { isPicking = true }, + onClear = onClear + ) { + val color = Color(value.int) + + OutlinedButton( + onClick = { isPicking = true }, + colors = ButtonDefaults.buttonColors(backgroundColor = color) + ) { + if (noValueSelected) { + Icon( + imageVector = Icons.Default.Palette, + contentDescription = "Change color", + tint = if (color.luminance() > 0.5) Color.Black else Color.White + ) + } + } + + if (isPicking) { + Dialog( + onDismissRequest = { isPicking = false } + ) { + Column( + horizontalAlignment = Alignment.End + ) { + ColorPicker { color -> + isPicking = false + onColorChanged(ReadiumColor(color)) + } + Button( + onClick = { + isPicking = false + onColorChanged(null) + } + ) { + Text("Clear") + } + } + } + } + } +} + +/** + * Component for a [Preference]`. + */ +@Composable +fun LanguageItem( + preference: Preference, + commit: () -> Unit +) { + val languages = remember { + Locale.getAvailableLocales() + .map { Language(it).removeRegion() } + .distinct() + .sortedBy { it.locale.displayName } + } + + MenuItem( + title = "Language", + preference = preference.withSupportedValues(languages + null), + formatValue = { it?.locale?.displayName ?: "Unknown" }, + commit = commit + ) +} + +@OptIn(ExperimentalMaterialApi::class) +@Composable +private fun Item( + title: String, + isActive: Boolean = true, + onClick: (() -> Unit)? = null, + onClear: (() -> Unit)? = null, + content: @Composable () -> Unit +) { + ListItem( + modifier = + if (onClick != null) Modifier.clickable(onClick = onClick) + else Modifier, + text = { + val alpha = if (isActive) 1.0f else ContentAlpha.disabled + CompositionLocalProvider(LocalContentAlpha provides alpha) { + Text(title) + } + }, + trailing = { + Row { + content() + + IconButton(onClick = onClear ?: {}, enabled = onClear != null) { + Icon( + Icons.Default.Backspace, + contentDescription = "Clear" + ) + } + } + } + ) +} + +@Composable +fun SelectorListItem( + title: String, + preference: EnumPreference, + formatValue: (T) -> String, + commit: () -> Unit +) { + SelectorListItem( + title = title, + values = preference.supportedValues, + selection = preference.value ?: preference.effectiveValue, + formatValue = formatValue, + onSelected = { + preference.set(it) + commit() + } + ) +} + +/** + * A Material [ListItem] displaying a dropdown menu to select a value. The current value is + * displayed on the right. + */ +@OptIn(ExperimentalMaterialApi::class) +@Composable +private fun SelectorListItem( + title: String, + values: List, + selection: T, + formatValue: (T) -> String, + onSelected: (T) -> Unit, + enabled: Boolean = values.isNotEmpty(), +) { + var isExpanded by remember { mutableStateOf(false) } + fun dismiss() { isExpanded = false } + + ListItem( + modifier = Modifier + .clickable(enabled = enabled) { + isExpanded = true + }, + text = { + Group(enabled = enabled) { + Text(title) + } + }, + trailing = { + Group(enabled = enabled) { + Text(formatValue(selection)) + } + + DropdownMenu( + expanded = isExpanded, + onDismissRequest = { dismiss() } + ) { + for (value in values) { + DropdownMenuItem( + onClick = { + onSelected(value) + dismiss() + } + ) { + Text(formatValue(value)) + } + } + } + } + ) +} diff --git a/test-app/src/main/res/values/strings.xml b/test-app/src/main/res/values/strings.xml index 6f18a55ec8..592869d255 100644 --- a/test-app/src/main/res/values/strings.xml +++ b/test-app/src/main/res/values/strings.xml @@ -208,7 +208,8 @@ Pause Play Go backward - Rate + Speed + Pitch Speech settings Stop Voice From de80567c2e87b774e0a7f6f0fd15e6f2fe3e1ebf Mon Sep 17 00:00:00 2001 From: Quentin Gliosca Date: Thu, 12 Jan 2023 14:00:44 +0100 Subject: [PATCH 04/27] Refactor Playback --- .../media3/androidtts/AndroidTtsEngine.kt | 142 +++-- .../androidtts/AndroidTtsEngineProvider.kt | 51 +- .../androidtts/AndroidTtsPreferences.kt | 4 +- .../AndroidTtsPreferencesFilters.kt | 6 +- .../media3/androidtts/AndroidTtsSettings.kt | 4 +- .../media3/androidtts/AndroidTtsVoice.kt | 33 -- ...tory.kt => DefaultMediaMetadataFactory.kt} | 4 +- .../media3/api/DefaultMetadataProvider.kt | 16 - ...taProvider.kt => MediaMetadataProvider.kt} | 2 +- .../r2/navigator/media3/api/MediaNavigator.kt | 38 +- .../media3/api/MediaNavigatorInternal.kt | 46 +- .../media3/api/SynchronizedMediaNavigator.kt | 26 + .../api/SynchronizedMediaNavigatorInternal.kt | 22 + .../media3/api/SynchronizedPlayback.kt | 20 - .../media3/exoplayer/ExoPlayerDataSource.kt | 2 +- .../exoplayer/ExoPlayerEngineProvider.kt | 1 - .../navigator/media3/player/PlayerLocator.kt | 2 +- .../media3/player/PlayerNavigator.kt | 125 ---- .../media3/player/PlayerNavigatorFactory.kt | 6 +- .../media3/player/PlayerNavigatorInternal.kt | 53 -- .../navigator/media3/player/PlayerPlayback.kt | 17 - .../SynchronizedNarrationNavigator.kt | 13 +- .../SynchronizedNarrationNavigatorInternal.kt | 50 -- .../syncnarr/SynchronizedNarrationPlayback.kt | 19 - .../media3/tts2/TtsContentIterator.kt | 68 ++- .../r2/navigator/media3/tts2/TtsEngine.kt | 58 +- .../navigator/media3/tts2/TtsEngineFacade.kt | 31 +- .../media3/tts2/TtsEngineProvider.kt | 13 +- .../r2/navigator/media3/tts2/TtsLocator.kt | 70 --- .../r2/navigator/media3/tts2/TtsNavigator.kt | 114 ++-- .../media3/tts2/TtsNavigatorFactory.kt | 30 +- .../media3/tts2/TtsNavigatorInternal.kt | 142 +++-- .../media3/tts2/TtsNavigatorListener.kt | 2 - .../r2/navigator/media3/tts2/TtsPlayback.kt | 18 - .../r2/navigator/media3/tts2/TtsPlayer.kt | 396 ++++++++++--- .../navigator/media3/tts2/TtsPreferences.kt | 17 - .../media3/tts2/TtsSessionAdapter.kt | 538 +++++++++++++++--- .../media3/tts2/TtsSessionTimeline.kt | 1 + .../r2/navigator/media3/tts2/TtsSettings.kt | 17 - .../media3/tts2/TtsStreamVolumeManager.kt | 219 +++++++ .../r2/navigator/media3/tts2/TtsUtterance.kt | 17 - .../r2/navigator/tts/AndroidTtsEngine.kt | 2 +- .../org/readium/r2/testapp/Application.kt | 2 +- .../r2/testapp/reader/ReaderInitData.kt | 10 +- .../r2/testapp/reader/ReaderRepository.kt | 16 +- .../r2/testapp/reader/ReaderViewModel.kt | 6 +- .../r2/testapp/reader/tts/TtsEngine.kt | 21 + .../r2/testapp/reader/tts/TtsService.kt | 13 +- .../r2/testapp/reader/tts/TtsServiceFacade.kt | 53 ++ .../r2/testapp/reader/tts/TtsViewModel.kt | 95 ++-- 50 files changed, 1673 insertions(+), 998 deletions(-) delete mode 100644 readium/navigator/src/main/java/org/readium/r2/navigator/media3/androidtts/AndroidTtsVoice.kt rename readium/navigator/src/main/java/org/readium/r2/navigator/media3/api/{DefaultMetadataFactory.kt => DefaultMediaMetadataFactory.kt} (92%) delete mode 100644 readium/navigator/src/main/java/org/readium/r2/navigator/media3/api/DefaultMetadataProvider.kt rename readium/navigator/src/main/java/org/readium/r2/navigator/media3/api/{MetadataProvider.kt => MediaMetadataProvider.kt} (91%) create mode 100644 readium/navigator/src/main/java/org/readium/r2/navigator/media3/api/SynchronizedMediaNavigator.kt create mode 100644 readium/navigator/src/main/java/org/readium/r2/navigator/media3/api/SynchronizedMediaNavigatorInternal.kt delete mode 100644 readium/navigator/src/main/java/org/readium/r2/navigator/media3/api/SynchronizedPlayback.kt delete mode 100644 readium/navigator/src/main/java/org/readium/r2/navigator/media3/player/PlayerNavigator.kt delete mode 100644 readium/navigator/src/main/java/org/readium/r2/navigator/media3/player/PlayerNavigatorInternal.kt delete mode 100644 readium/navigator/src/main/java/org/readium/r2/navigator/media3/player/PlayerPlayback.kt delete mode 100644 readium/navigator/src/main/java/org/readium/r2/navigator/media3/syncnarr/SynchronizedNarrationNavigatorInternal.kt delete mode 100644 readium/navigator/src/main/java/org/readium/r2/navigator/media3/syncnarr/SynchronizedNarrationPlayback.kt delete mode 100644 readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsLocator.kt delete mode 100644 readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsPlayback.kt delete mode 100644 readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsPreferences.kt delete mode 100644 readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsSettings.kt create mode 100644 readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsStreamVolumeManager.kt delete mode 100644 readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsUtterance.kt create mode 100644 test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsEngine.kt create mode 100644 test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsServiceFacade.kt diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/androidtts/AndroidTtsEngine.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/androidtts/AndroidTtsEngine.kt index 2709b66203..cbe60aeaf9 100644 --- a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/androidtts/AndroidTtsEngine.kt +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/androidtts/AndroidTtsEngine.kt @@ -7,10 +7,12 @@ package org.readium.r2.navigator.media3.androidtts import android.content.Context -import android.content.Intent import android.speech.tts.TextToSpeech import android.speech.tts.TextToSpeech.QUEUE_ADD import android.speech.tts.UtteranceProgressListener +import android.speech.tts.Voice as AndroidVoice +import android.speech.tts.Voice.* +import java.util.* import kotlinx.coroutines.* import kotlinx.coroutines.flow.MutableStateFlow import kotlinx.coroutines.flow.StateFlow @@ -27,14 +29,17 @@ import org.readium.r2.shared.util.Language class AndroidTtsEngine( private val engine: TextToSpeech, metadata: Metadata, + private val defaultVoiceProvider: DefaultVoiceProvider?, initialPreferences: AndroidTtsPreferences -) : TtsEngine { +) : TtsEngine { companion object { suspend operator fun invoke( context: Context, metadata: Metadata, + defaultVoiceProvider: DefaultVoiceProvider?, initialPreferences: AndroidTtsPreferences ): AndroidTtsEngine? { @@ -46,12 +51,17 @@ class AndroidTtsEngine( val engine = TextToSpeech(context, listener) return if (init.await()) - AndroidTtsEngine(engine, metadata, initialPreferences) + AndroidTtsEngine(engine, metadata, defaultVoiceProvider, initialPreferences) else null } } + fun interface DefaultVoiceProvider { + + fun chooseVoice(language: Language?, availableVoices: Set): Voice? + } + /** * Android's TTS error code. * See https://developer.android.com/reference/android/speech/tts/TextToSpeech#ERROR @@ -82,16 +92,38 @@ class AndroidTtsEngine( } } - class EngineException(code: Int) : Exception("Android TTS engine error: $code") { + class Exception(code: Int) : + kotlin.Exception("Android TTS engine error: $code"), TtsEngine.Error { + val error: EngineError = EngineError.getOrDefault(code) } + /** + * Represents a voice provided by the TTS engine which can speak an utterance. + * + * @param name Unique and stable identifier for this voice + * @param language Language (and region) this voice belongs to. + * @param quality Voice quality. + * @param requiresNetwork Indicates whether using this voice requires an Internet connection. + */ + data class Voice( + val name: String, + override val language: Language, + val quality: Quality = Quality.Normal, + val requiresNetwork: Boolean = false, + ) : TtsEngine.Voice { + + enum class Quality { + Lowest, Low, Normal, High, Highest + } + } + init { engine.setOnUtteranceProgressListener(Listener()) } - private var listener: TtsEngine.Listener? = + private var listener: TtsEngine.Listener? = null private val settingsResolver: AndroidTtsSettingsResolver = @@ -100,8 +132,13 @@ class AndroidTtsEngine( private val _settings: MutableStateFlow = MutableStateFlow(settingsResolver.settings(initialPreferences)) - override fun close() { - engine.shutdown() + override val voices: Set get() = + engine.voices + .map { it.toTtsEngineVoice() } + .toSet() + + override fun setListener(listener: TtsEngine.Listener?) { + this.listener = listener } override fun speak( @@ -109,42 +146,16 @@ class AndroidTtsEngine( text: String, language: Language? ) { - engine.language = language?.locale ?: settings.value.language?.locale + engine.setupVoice(settings.value, language, voices) engine.speak(text, QUEUE_ADD, null, requestId) } - /** - * Start the activity to install additional language data. - * To be called if you receive a [TtsEngine.Exception.LanguageSupportIncomplete] error. - * - * Returns whether the request was successful. - * - * See https://android-developers.googleblog.com/2009/09/introduction-to-text-to-speech-in.html - */ - fun requestInstallMissingVoice( - context: Context, - intentFlags: Int = Intent.FLAG_ACTIVITY_NEW_TASK - ): Boolean { - val intent = Intent() - .setAction(TextToSpeech.Engine.ACTION_INSTALL_TTS_DATA) - .setFlags(intentFlags) - - if (context.packageManager.queryIntentActivities(intent, 0).isEmpty()) { - return false - } - - context.startActivity(intent) - return true - } - - // Engine - override fun stop() { engine.stop() } - override fun setListener(listener: TtsEngine.Listener?) { - this.listener = listener + override fun close() { + engine.shutdown() } override val settings: StateFlow = @@ -152,23 +163,59 @@ class AndroidTtsEngine( override fun submitPreferences(preferences: AndroidTtsPreferences) { val newSettings = settingsResolver.settings(preferences) - engine.setup(newSettings) + engine.setupPitchAndSpeed(newSettings) _settings.value = newSettings } - private fun TextToSpeech.setup(settings: AndroidTtsSettings) { + private fun TextToSpeech.setupPitchAndSpeed(settings: AndroidTtsSettings) { setSpeechRate(settings.speed.toFloat()) setPitch(settings.pitch.toFloat()) + } - val localeResult = engine.setLanguage(settings.language?.locale) - if (localeResult < TextToSpeech.LANG_AVAILABLE) { - if (localeResult == TextToSpeech.LANG_MISSING_DATA) - throw org.readium.r2.navigator.tts.TtsEngine.Exception.LanguageSupportIncomplete(settings.language!!) - else - throw org.readium.r2.navigator.tts.TtsEngine.Exception.LanguageNotSupported(settings.language!!) - } + private fun TextToSpeech.setupVoice( + settings: AndroidTtsSettings, + utteranceLanguage: Language?, + voices: Set + ) { + val language = utteranceLanguage + ?: settings.language + + val preferredVoice = language + ?.let { settings.voices[it] } + ?.let { voiceForName(it) } + + val voice = preferredVoice + ?: defaultVoice(language, voices) + + voice + ?.let { engine.voice = it } + ?: run { engine.language = language?.locale ?: Locale.getDefault() } } + private fun defaultVoice(language: Language?, voices: Set): AndroidVoice? = + defaultVoiceProvider + ?.chooseVoice(language, voices) + ?.let { voiceForName(it.name) } + + private fun voiceForName(name: String) = + engine.voices + .firstOrNull { it.name == name } + + private fun AndroidVoice.toTtsEngineVoice() = + Voice( + name = name, + language = Language(locale), + quality = when (quality) { + QUALITY_VERY_HIGH -> Voice.Quality.Highest + QUALITY_HIGH -> Voice.Quality.High + QUALITY_NORMAL -> Voice.Quality.Normal + QUALITY_LOW -> Voice.Quality.Low + QUALITY_VERY_LOW -> Voice.Quality.Lowest + else -> throw IllegalStateException("Unexpected voice quality.") + }, + requiresNetwork = isNetworkConnectionRequired + ) + inner class Listener : UtteranceProgressListener() { override fun onStart(utteranceId: String) { listener?.onStart(utteranceId) @@ -194,7 +241,10 @@ class AndroidTtsEngine( } override fun onError(utteranceId: String, errorCode: Int) { - // listener?.onError(utteranceId!!, EngineException(errorCode)) + listener?.onError( + utteranceId, + Exception(errorCode) + ) } override fun onRangeStart(utteranceId: String, start: Int, end: Int, frame: Int) { diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/androidtts/AndroidTtsEngineProvider.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/androidtts/AndroidTtsEngineProvider.kt index ac0720eb6e..48e37b3bc8 100644 --- a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/androidtts/AndroidTtsEngineProvider.kt +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/androidtts/AndroidTtsEngineProvider.kt @@ -7,21 +7,32 @@ package org.readium.r2.navigator.media3.androidtts import android.content.Context +import androidx.media3.common.PlaybackException +import androidx.media3.common.PlaybackException.* +import androidx.media3.common.PlaybackParameters import org.readium.r2.navigator.media3.tts2.TtsEngineProvider import org.readium.r2.shared.ExperimentalReadiumApi import org.readium.r2.shared.publication.Metadata import org.readium.r2.shared.publication.Publication @ExperimentalReadiumApi +@androidx.annotation.OptIn(androidx.media3.common.util.UnstableApi::class) class AndroidTtsEngineProvider( private val context: Context, -) : TtsEngineProvider { + private val defaultVoiceProvider: AndroidTtsEngine.DefaultVoiceProvider? = null +) : TtsEngineProvider { override suspend fun createEngine( publication: Publication, initialPreferences: AndroidTtsPreferences ): AndroidTtsEngine? { - return AndroidTtsEngine(context, publication.metadata, initialPreferences) + return AndroidTtsEngine( + context, + publication.metadata, + defaultVoiceProvider, + initialPreferences + ) } fun computeSettings( @@ -38,4 +49,40 @@ class AndroidTtsEngineProvider( override fun createEmptyPreferences(): AndroidTtsPreferences = AndroidTtsPreferences() + + override fun getPlaybackParameters( + settings: AndroidTtsSettings + ): PlaybackParameters { + return PlaybackParameters(settings.speed.toFloat(), settings.pitch.toFloat()) + } + + override fun updatePlaybackParameters( + previousPreferences: AndroidTtsPreferences, + playbackParameters: PlaybackParameters + ): AndroidTtsPreferences { + return previousPreferences.copy( + speed = playbackParameters.speed.toDouble(), + pitch = playbackParameters.pitch.toDouble() + ) + } + + override fun mapEngineError(error: AndroidTtsEngine.Exception): PlaybackException = + when (error.error) { + AndroidTtsEngine.EngineError.Unknown -> + PlaybackException(error.message, error.cause, ERROR_CODE_UNSPECIFIED) + AndroidTtsEngine.EngineError.InvalidRequest -> + PlaybackException(error.message, error.cause, ERROR_CODE_IO_BAD_HTTP_STATUS) + AndroidTtsEngine.EngineError.Network -> + PlaybackException(error.message, error.cause, ERROR_CODE_IO_NETWORK_CONNECTION_FAILED) + AndroidTtsEngine.EngineError.NetworkTimeout -> + PlaybackException(error.message, error.cause, ERROR_CODE_IO_NETWORK_CONNECTION_TIMEOUT) + AndroidTtsEngine.EngineError.NotInstalledYet -> + PlaybackException(error.message, error.cause, ERROR_CODE_UNSPECIFIED) + AndroidTtsEngine.EngineError.Output -> + PlaybackException(error.message, error.cause, ERROR_CODE_UNSPECIFIED) + AndroidTtsEngine.EngineError.Service -> + PlaybackException(error.message, error.cause, ERROR_CODE_UNSPECIFIED) + AndroidTtsEngine.EngineError.Synthesis -> + PlaybackException(error.message, error.cause, ERROR_CODE_UNSPECIFIED) + } } diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/androidtts/AndroidTtsPreferences.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/androidtts/AndroidTtsPreferences.kt index 47433d0c93..cb6d81fc68 100644 --- a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/androidtts/AndroidTtsPreferences.kt +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/androidtts/AndroidTtsPreferences.kt @@ -7,7 +7,7 @@ package org.readium.r2.navigator.media3.androidtts import kotlinx.serialization.Serializable -import org.readium.r2.navigator.media3.tts2.TtsPreferences +import org.readium.r2.navigator.media3.tts2.TtsEngine import org.readium.r2.shared.ExperimentalReadiumApi import org.readium.r2.shared.util.Language @@ -18,7 +18,7 @@ data class AndroidTtsPreferences( val voices: Map? = null, val pitch: Double? = null, val speed: Double? = null, -) : TtsPreferences { +) : TtsEngine.Preferences { override fun plus(other: AndroidTtsPreferences): AndroidTtsPreferences = AndroidTtsPreferences( diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/androidtts/AndroidTtsPreferencesFilters.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/androidtts/AndroidTtsPreferencesFilters.kt index 5807c6d665..cebe76957b 100644 --- a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/androidtts/AndroidTtsPreferencesFilters.kt +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/androidtts/AndroidTtsPreferencesFilters.kt @@ -17,8 +17,7 @@ object AndroidTtsSharedPreferencesFilter : PreferencesFilter, val pitch: Double, val speed: Double, -) : TtsSettings +) : TtsEngine.Settings diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/androidtts/AndroidTtsVoice.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/androidtts/AndroidTtsVoice.kt deleted file mode 100644 index a1970b3533..0000000000 --- a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/androidtts/AndroidTtsVoice.kt +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright 2022 Readium Foundation. All rights reserved. - * Use of this source code is governed by the BSD-style license - * available in the top-level LICENSE file of the project. - */ - -package org.readium.r2.navigator.media3.androidtts - -import org.readium.r2.shared.ExperimentalReadiumApi -import org.readium.r2.shared.util.Language - -/** - * Represents a voice provided by the TTS engine which can speak an utterance. - * - * @param id Unique and stable identifier for this voice. Can be used to store and retrieve the - * voice from the user preferences. - * @param name Human-friendly name for this voice, when available. - * @param language Language (and region) this voice belongs to. - * @param quality Voice quality. - * @param requiresNetwork Indicates whether using this voice requires an Internet connection. - */ -@ExperimentalReadiumApi -data class AndroidTtsVoice( - val id: String, - val name: String? = null, - val language: Language, - val quality: Quality = Quality.Normal, - val requiresNetwork: Boolean = false, -) { - enum class Quality { - Lowest, Low, Normal, High, Highest - } -} diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/api/DefaultMetadataFactory.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/api/DefaultMediaMetadataFactory.kt similarity index 92% rename from readium/navigator/src/main/java/org/readium/r2/navigator/media3/api/DefaultMetadataFactory.kt rename to readium/navigator/src/main/java/org/readium/r2/navigator/media3/api/DefaultMediaMetadataFactory.kt index 5f96847149..bfc39f6f78 100644 --- a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/api/DefaultMetadataFactory.kt +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/api/DefaultMediaMetadataFactory.kt @@ -14,7 +14,8 @@ import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.async import org.readium.r2.shared.publication.Publication -internal class DefaultMetadataFactory(private val publication: Publication) : MediaMetadataFactory { +@androidx.annotation.OptIn(androidx.media3.common.util.UnstableApi::class) +internal class DefaultMediaMetadataFactory(private val publication: Publication) : MediaMetadataFactory { private val coroutineScope = CoroutineScope(Dispatchers.Default) @@ -30,7 +31,6 @@ internal class DefaultMetadataFactory(private val publication: Publication) : Me ?.getOrNull() } - @androidx.annotation.OptIn(androidx.media3.common.util.UnstableApi::class) override suspend fun publicationMetadata(): MediaMetadata { val builder = MediaMetadata.Builder() .setTitle(publication.metadata.title) diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/api/DefaultMetadataProvider.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/api/DefaultMetadataProvider.kt deleted file mode 100644 index 702c9cec1d..0000000000 --- a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/api/DefaultMetadataProvider.kt +++ /dev/null @@ -1,16 +0,0 @@ -/* - * Copyright 2022 Readium Foundation. All rights reserved. - * Use of this source code is governed by the BSD-style license - * available in the top-level LICENSE file of the project. - */ - -package org.readium.r2.navigator.media3.api - -import org.readium.r2.shared.publication.Publication - -class DefaultMetadataProvider : MetadataProvider { - - override fun createMetadataFactory(publication: Publication): MediaMetadataFactory { - return DefaultMetadataFactory(publication) - } -} diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/api/MetadataProvider.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/api/MediaMetadataProvider.kt similarity index 91% rename from readium/navigator/src/main/java/org/readium/r2/navigator/media3/api/MetadataProvider.kt rename to readium/navigator/src/main/java/org/readium/r2/navigator/media3/api/MediaMetadataProvider.kt index 377ca88e53..0f2acd9170 100644 --- a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/api/MetadataProvider.kt +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/api/MediaMetadataProvider.kt @@ -8,7 +8,7 @@ package org.readium.r2.navigator.media3.api import org.readium.r2.shared.publication.Publication -interface MetadataProvider { +interface MediaMetadataProvider { fun createMetadataFactory(publication: Publication): MediaMetadataFactory } diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/api/MediaNavigator.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/api/MediaNavigator.kt index 80bc2092e1..979befbd29 100644 --- a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/api/MediaNavigator.kt +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/api/MediaNavigator.kt @@ -7,44 +7,30 @@ package org.readium.r2.navigator.media3.api import androidx.media3.common.Player -import kotlin.time.Duration import kotlinx.coroutines.flow.StateFlow import org.readium.r2.navigator.Navigator import org.readium.r2.shared.ExperimentalReadiumApi -import org.readium.r2.shared.publication.Locator import org.readium.r2.shared.util.Closeable @ExperimentalReadiumApi -interface MediaNavigator

: Navigator, Closeable { +interface MediaNavigator : Navigator, Closeable { + + interface Error enum class State { - Playing, - Paused, - Ended; + Ready, + Buffering, + Ended, + Error; } - data class Buffer( - val isPlayable: Boolean, - val position: Duration + data class Playback( + val state: State, + val playWhenReady: Boolean, + val error: E? ) - interface Playback { - - val state: State - val locator: Locator - } - - interface TextSynchronization { - - val token: Locator? - } - - interface BufferProvider { - - val buffer: Buffer - } - - val playback: StateFlow

+ val playback: StateFlow> /** * Resumes the playback at the current location or start it again from the beginning if it has finished. diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/api/MediaNavigatorInternal.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/api/MediaNavigatorInternal.kt index f728eea2aa..19a22d2da3 100644 --- a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/api/MediaNavigatorInternal.kt +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/api/MediaNavigatorInternal.kt @@ -7,46 +7,38 @@ package org.readium.r2.navigator.media3.api import androidx.media3.common.Player -import kotlin.time.Duration import kotlinx.coroutines.flow.StateFlow import org.readium.r2.shared.InternalReadiumApi @InternalReadiumApi -interface MediaNavigatorInternal> { +interface MediaNavigatorInternal

{ - interface Locator + interface Position - enum class State { - Playing, - Paused, - Ended; - } - - data class Buffer( - val isPlayable: Boolean, - val position: Duration - ) - - interface Playback { + interface RelaxedPosition - val state: State - val locator: L - } - - interface TextSynchronization { + interface Error - val token: Locator? + enum class State { + Ready, + Buffering, + Ended, + Error; } - interface BufferProvider { + data class Playback( + val state: State, + val playWhenReady: Boolean, + val error: E? + ) - val buffer: Buffer - } + val playback: StateFlow> - val playback: StateFlow

+ val progression: StateFlow

/** - * Resumes the playback at the current location or start it again from the beginning if it has finished. + * Resumes the playback at the current location. */ fun play() @@ -58,7 +50,7 @@ interface MediaNavigatorInternal : MediaNavigator { + + data class Utterance( + val locator: Locator, + val range: IntRange? + ) { + + val rangeLocator: Locator? = range + ?.let { locator.copy(text = locator.text.substring(it)) } + } + + val utterance: StateFlow +} diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/api/SynchronizedMediaNavigatorInternal.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/api/SynchronizedMediaNavigatorInternal.kt new file mode 100644 index 0000000000..c45d0f8a6b --- /dev/null +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/api/SynchronizedMediaNavigatorInternal.kt @@ -0,0 +1,22 @@ +/* + * Copyright 2022 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + +package org.readium.r2.navigator.media3.api + +import kotlinx.coroutines.flow.StateFlow + +interface SynchronizedMediaNavigatorInternal

: + MediaNavigatorInternal { + + data class Utterance

( + val text: String, + val position: P, + val range: IntRange? + ) + + val utterance: StateFlow> +} diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/api/SynchronizedPlayback.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/api/SynchronizedPlayback.kt deleted file mode 100644 index 5dc470d7f3..0000000000 --- a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/api/SynchronizedPlayback.kt +++ /dev/null @@ -1,20 +0,0 @@ -/* - * Copyright 2022 Readium Foundation. All rights reserved. - * Use of this source code is governed by the BSD-style license - * available in the top-level LICENSE file of the project. - */ - -package org.readium.r2.navigator.media3.api - -import org.readium.r2.shared.ExperimentalReadiumApi - -@ExperimentalReadiumApi -interface SynchronizedPlayback : - MediaNavigatorInternal.Playback, MediaNavigatorInternal.TextSynchronization { - - override val state: MediaNavigatorInternal.State - - override val locator: L - - override val token: L? -} diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/exoplayer/ExoPlayerDataSource.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/exoplayer/ExoPlayerDataSource.kt index a587e4c266..c3411199fd 100644 --- a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/exoplayer/ExoPlayerDataSource.kt +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/exoplayer/ExoPlayerDataSource.kt @@ -4,7 +4,7 @@ * available in the top-level LICENSE file of the project. */ -package org.readium.r2.navigator.media3.player +package org.readium.r2.navigator.media3.exoplayer import android.net.Uri import androidx.media3.common.C.LENGTH_UNSET diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/exoplayer/ExoPlayerEngineProvider.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/exoplayer/ExoPlayerEngineProvider.kt index 08609c2b64..3bae8b85c1 100644 --- a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/exoplayer/ExoPlayerEngineProvider.kt +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/exoplayer/ExoPlayerEngineProvider.kt @@ -12,7 +12,6 @@ import androidx.media3.common.C import androidx.media3.datasource.DataSource import androidx.media3.exoplayer.ExoPlayer import androidx.media3.exoplayer.source.DefaultMediaSourceFactory -import org.readium.r2.navigator.media3.player.ExoPlayerDataSource import org.readium.r2.navigator.media3.player.MediaEngineProvider import org.readium.r2.shared.ExperimentalReadiumApi import org.readium.r2.shared.publication.Metadata diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/player/PlayerLocator.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/player/PlayerLocator.kt index 545fd25302..b78d5fb3e4 100644 --- a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/player/PlayerLocator.kt +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/player/PlayerLocator.kt @@ -20,7 +20,7 @@ data class PlayerLocator( val index: Int, @Serializable(with = DurationSerializer::class) val position: Duration -) : MediaNavigatorInternal.Locator +) : MediaNavigatorInternal.Position internal val Locator.Locations.time: Duration? get() = fragmentParameters["t"]?.toIntOrNull()?.seconds diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/player/PlayerNavigator.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/player/PlayerNavigator.kt deleted file mode 100644 index 9bbb40df00..0000000000 --- a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/player/PlayerNavigator.kt +++ /dev/null @@ -1,125 +0,0 @@ -/* - * Copyright 2022 Readium Foundation. All rights reserved. - * Use of this source code is governed by the BSD-style license - * available in the top-level LICENSE file of the project. - */ - -package org.readium.r2.navigator.media3.player - -import androidx.media3.common.Player -import kotlinx.coroutines.CoroutineScope -import kotlinx.coroutines.Dispatchers -import kotlinx.coroutines.SupervisorJob -import kotlinx.coroutines.flow.StateFlow -import org.readium.r2.navigator.Navigator -import org.readium.r2.navigator.media3.api.MediaMetadataFactory -import org.readium.r2.navigator.media3.api.MediaNavigator -import org.readium.r2.navigator.preferences.Configurable -import org.readium.r2.shared.ExperimentalReadiumApi -import org.readium.r2.shared.extensions.mapStateIn -import org.readium.r2.shared.publication.Link -import org.readium.r2.shared.publication.Locator -import org.readium.r2.shared.publication.Publication - -@ExperimentalReadiumApi -class PlayerNavigator> private constructor( - override val publication: Publication, - private val playerNavigator: PlayerNavigatorInternal, -) : MediaNavigator, Navigator, Configurable { - - companion object { - - suspend operator fun > invoke( - publication: Publication, - mediaEngineProvider: MediaEngineProvider, - metadataFactory: MediaMetadataFactory, - initialPreferences: P = mediaEngineProvider.createEmptyPreferences(), - initialLocator: Locator = publication.startLocator, - ): PlayerNavigator { - TODO("Not yet implemented") - /*val player = mediaEngineProvider.createPlayer(publication) - - val playlist = publication.readingOrder.indices.map { index -> - val metadata = metadataFactory.resourceMetadata(index) - MediaItem.Builder() - .setMediaMetadata(metadata) - .build() - } - - val publicationMetadata = metadataFactory.publicationMetadata() - - val settingsResolver = ExoPlayerSettingsResolver(publication.metadata) - - return PlayerNavigator( - - )*/ - } - - private val Publication.startLocator: Locator - get() = locatorFromLink(readingOrder.first())!! - } - - data class Playback( - override val state: MediaNavigator.State, - override val locator: Locator, - override val buffer: MediaNavigator.Buffer - ) : MediaNavigator.Playback, MediaNavigator.BufferProvider - - private val coroutineScope = - CoroutineScope(SupervisorJob() + Dispatchers.Main.immediate) - - // Configurable - - override val settings: StateFlow = - TODO("Not yet implemented") - - override fun submitPreferences(preferences: P) { - TODO("Not yet implemented") - } - - // MediaNavigator - - override val playback: StateFlow - get() = TODO("Not yet implemented") - - override fun play() { - TODO("Not yet implemented") - } - - override fun pause() { - TODO("Not yet implemented") - } - - override fun asPlayer(): Player { - return playerNavigator.asPlayer() - } - - // Navigator - - override val currentLocator: StateFlow = - playback.mapStateIn(coroutineScope) { it.locator } - - override fun go(locator: Locator, animated: Boolean, completion: () -> Unit): Boolean { - TODO("Not yet implemented") - } - - override fun go(link: Link, animated: Boolean, completion: () -> Unit): Boolean { - val locator = publication.locatorFromLink(link) - ?: return false - go(locator) - return true - } - - override fun goForward(animated: Boolean, completion: () -> Unit): Boolean { - playerNavigator.goForward() - return true - } - - override fun goBackward(animated: Boolean, completion: () -> Unit): Boolean { - playerNavigator.goBackward() - return true - } - - override fun close() { - } -} diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/player/PlayerNavigatorFactory.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/player/PlayerNavigatorFactory.kt index b49a437f42..b1002b22b8 100644 --- a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/player/PlayerNavigatorFactory.kt +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/player/PlayerNavigatorFactory.kt @@ -6,7 +6,7 @@ package org.readium.r2.navigator.media3.player -import org.readium.r2.navigator.media3.api.MetadataProvider +/*import org.readium.r2.navigator.media3.api.MediaMetadataProvider import org.readium.r2.navigator.preferences.Configurable import org.readium.r2.navigator.preferences.PreferencesEditor import org.readium.r2.shared.ExperimentalReadiumApi @@ -25,7 +25,7 @@ class PlayerNavigatorFactory, E : PreferencesEditor

> invoke( publication: Publication, mediaEngineProvider: MediaEngineProvider, - metadataProvider: MetadataProvider, + metadataProvider: MediaMetadataProvider, initialPreferences: P, initialLocator: Locator ): PlayerNavigatorFactory { @@ -56,4 +56,4 @@ class PlayerNavigatorFactory>( - private val player: Player -) : MediaNavigatorInternal, Configurable { - - override val playback: StateFlow - get() = TODO("Not yet implemented") - - override fun play() { - TODO("Not yet implemented") - } - - override fun pause() { - TODO("Not yet implemented") - } - - override fun go(locator: PlayerLocator) { - TODO("Not yet implemented") - } - - override fun goForward() { - TODO("Not yet implemented") - } - - override fun goBackward() { - TODO("Not yet implemented") - } - - override fun asPlayer(): Player { - return player - } - - override val settings: StateFlow - get() = TODO("Not yet implemented") - - override fun submitPreferences(preferences: P) { - TODO("Not yet implemented") - } -} diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/player/PlayerPlayback.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/player/PlayerPlayback.kt deleted file mode 100644 index 09629bb96c..0000000000 --- a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/player/PlayerPlayback.kt +++ /dev/null @@ -1,17 +0,0 @@ -/* - * Copyright 2022 Readium Foundation. All rights reserved. - * Use of this source code is governed by the BSD-style license - * available in the top-level LICENSE file of the project. - */ - -package org.readium.r2.navigator.media3.player - -import org.readium.r2.navigator.media3.api.MediaNavigatorInternal -import org.readium.r2.shared.ExperimentalReadiumApi - -@ExperimentalReadiumApi -data class PlayerPlayback( - override val state: MediaNavigatorInternal.State, - override val locator: PlayerLocator, - override val buffer: MediaNavigatorInternal.Buffer -) : MediaNavigatorInternal.Playback, MediaNavigatorInternal.BufferProvider diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/syncnarr/SynchronizedNarrationNavigator.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/syncnarr/SynchronizedNarrationNavigator.kt index a898a9614d..64158df9dc 100644 --- a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/syncnarr/SynchronizedNarrationNavigator.kt +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/syncnarr/SynchronizedNarrationNavigator.kt @@ -6,16 +6,7 @@ package org.readium.r2.navigator.media3.syncnarr -import androidx.media3.common.Player -import kotlinx.coroutines.flow.StateFlow -import org.readium.r2.navigator.media3.api.MediaNavigator -import org.readium.r2.navigator.preferences.Configurable -import org.readium.r2.shared.ExperimentalReadiumApi -import org.readium.r2.shared.publication.Link -import org.readium.r2.shared.publication.Locator -import org.readium.r2.shared.publication.Publication - -@ExperimentalReadiumApi +/*@ExperimentalReadiumApi class SynchronizedNarrationNavigator>( private val internalNavigator: SynchronizedNarrationNavigatorInternal ) : MediaNavigator { @@ -67,4 +58,4 @@ class SynchronizedNarrationNavigator>( - private val playerNavigator: PlayerNavigatorInternal - // media overlays data -) : MediaNavigatorInternal> { - - override val playback: StateFlow> - get() = TODO("Not yet implemented") - - override fun play() { - TODO("Not yet implemented") - } - - override fun pause() { - TODO("Not yet implemented") - } - - override fun go(locator: PlayerLocator) { - TODO("Not yet implemented") - } - - override fun goForward() { - TODO("Not yet implemented") - } - - override fun goBackward() { - TODO("Not yet implemented") - } - - override fun asPlayer(): Player { - TODO("Not yet implemented") - } -} diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/syncnarr/SynchronizedNarrationPlayback.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/syncnarr/SynchronizedNarrationPlayback.kt deleted file mode 100644 index a5ea2df3c9..0000000000 --- a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/syncnarr/SynchronizedNarrationPlayback.kt +++ /dev/null @@ -1,19 +0,0 @@ -/* - * Copyright 2022 Readium Foundation. All rights reserved. - * Use of this source code is governed by the BSD-style license - * available in the top-level LICENSE file of the project. - */ - -package org.readium.r2.navigator.media3.syncnarr - -import org.readium.r2.navigator.media3.api.MediaNavigatorInternal -import org.readium.r2.navigator.media3.api.SynchronizedPlayback -import org.readium.r2.navigator.media3.player.PlayerLocator -import org.readium.r2.shared.ExperimentalReadiumApi - -@ExperimentalReadiumApi -data class SynchronizedNarrationPlayback( - override val state: MediaNavigatorInternal.State, - override val locator: PlayerLocator, - override val token: PlayerLocator?, -) : SynchronizedPlayback diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsContentIterator.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsContentIterator.kt index 6b3f06a67f..70f9b800fa 100644 --- a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsContentIterator.kt +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsContentIterator.kt @@ -9,6 +9,8 @@ package org.readium.r2.navigator.media3.tts2 import org.readium.r2.shared.ExperimentalReadiumApi import org.readium.r2.shared.publication.Locator import org.readium.r2.shared.publication.Publication +import org.readium.r2.shared.publication.html.cssSelector +import org.readium.r2.shared.publication.indexOfFirstWithHref import org.readium.r2.shared.publication.services.content.Content import org.readium.r2.shared.publication.services.content.ContentTokenizer import org.readium.r2.shared.publication.services.content.content @@ -19,13 +21,21 @@ import org.readium.r2.shared.util.Language internal class TtsContentIterator( private val publication: Publication, private val tokenizerFactory: (language: Language?) -> ContentTokenizer, - initialLocator: TtsLocator? + initialLocator: Locator? ) { + data class Utterance( + val resourceIndex: Int, + val cssSelector: String, + val text: String, + val textBefore: String?, + val textAfter: String?, + val language: Language? + ) /** * Current subset of utterances with a cursor. */ - private var utterances: CursorList = + private var utterances: CursorList = CursorList() /** @@ -45,10 +55,13 @@ internal class TtsContentIterator( var language: Language? = null + val resourceCount: Int = + publication.readingOrder.size + /** * Moves the iterator to the position provided in [locator]. */ - fun seek(locator: TtsLocator) { + fun seek(locator: Locator) { publicationIterator = createIterator(locator) } @@ -56,27 +69,37 @@ internal class TtsContentIterator( * Moves the iterator to the beginning of the publication. */ fun seekToBeginning() { - publicationIterator = createIterator(null) + publicationIterator = createIterator(locator = null) + } + + /** + * Moves the iterator to the resource with the given [index] in the publication reading order. + */ + fun seekToResource(index: Int) { + val link = publication.readingOrder.getOrNull(index) ?: return + val locator = publication.locatorFromLink(link) + publicationIterator = createIterator(locator) } /** - * Creates a fresh content iterator for the publication starting from [locator]. + * Creates a fresh content iterator for the publication starting from [Locator]. */ - private fun createIterator(locator: TtsLocator?): Content.Iterator = - publication.content(locator?.toLocator(publication)) + + private fun createIterator(locator: Locator?): Content.Iterator = + publication.content(locator) ?.iterator() ?: throw IllegalStateException("No ContentService.") /** * Advances to the previous item and returns it, or null if we reached the beginning. */ - suspend fun previousUtterance(): TtsUtterance? = + suspend fun previousUtterance(): Utterance? = nextUtterance(Direction.Backward) /** * Advances to the next item and returns it, or null if we reached the end. */ - suspend fun nextUtterance(): TtsUtterance? = + suspend fun nextUtterance(): Utterance? = nextUtterance(Direction.Forward) private enum class Direction { @@ -87,7 +110,7 @@ internal class TtsContentIterator( * Gets the next utterance in the given [direction], or null when reaching the beginning or the * end. */ - private suspend fun nextUtterance(direction: Direction): TtsUtterance? { + private suspend fun nextUtterance(direction: Direction): Utterance? { val utterance = utterances.nextIn(direction) if (utterance == null && loadNextUtterances(direction)) { return nextUtterance(direction) @@ -126,21 +149,32 @@ internal class TtsContentIterator( * * This is used to split a paragraph into sentences, for example. */ - private fun Content.Element.tokenize(): List = - tokenizerFactory(language).tokenize(this) + private fun Content.Element.tokenize(): List { + val language = this@tokenize.language ?: this@TtsContentIterator.language + return tokenizerFactory(language).tokenize(this) + } /** * Splits a publication [Content.Element] item into the utterances to be spoken. */ - private fun Content.Element.utterances(): List { - fun utterance(text: String, locator: Locator, language: Language? = null): TtsUtterance? { + private fun Content.Element.utterances(): List { + fun utterance(text: String, locator: Locator, language: Language? = null): Utterance? { if (!text.any { it.isLetterOrDigit() }) return null - return TtsUtterance( + val resourceIndex = publication.readingOrder.indexOfFirstWithHref(locator.href) + ?: throw IllegalStateException("Content Element cannot be found in readingOrder.") + + val cssSelector = locator.locations.cssSelector + ?: throw IllegalStateException("Css selectors are expected in iterator locators.") + + return Utterance( text = text, - locator = checkNotNull(locator.toTtsLocator(publication)) { "Missing data in locator." }, - language = language + language = language, + resourceIndex = resourceIndex, + textBefore = locator.text.before, + textAfter = locator.text.after, + cssSelector = cssSelector, ) } diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsEngine.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsEngine.kt index 68ce4e3765..f491d3268c 100644 --- a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsEngine.kt +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsEngine.kt @@ -15,44 +15,30 @@ import org.readium.r2.shared.util.Language * A text-to-speech engine synthesizes text utterances (e.g. sentence). */ @ExperimentalReadiumApi -interface TtsEngine> : Configurable, Closeable { - - sealed class Exception private constructor( - override val message: String, - cause: Throwable? = null - ) : kotlin.Exception(message, cause) { - /** Failed to initialize the TTS engine. */ - class InitializationFailed(cause: Throwable? = null) : - Exception("The TTS engine failed to initialize", cause) - - /** Tried to synthesize an utterance with an unsupported language. */ - class LanguageNotSupported(val language: Language, cause: Throwable? = null) : - Exception("The language ${language.code} is not supported by the TTS engine", cause) - - /** The selected language is missing downloadable data. */ - class LanguageSupportIncomplete(val language: Language, cause: Throwable? = null) : - Exception("The language ${language.code} requires additional files by the TTS engine", cause) - - /** Error during network calls. */ - class Network(cause: Throwable? = null) : - Exception("A network error occurred", cause) - - /** Other engine-specific errors. */ - class Other(override val cause: Throwable) : - Exception(cause.message ?: "An unknown error occurred", cause) - - companion object { - fun wrap(e: Throwable): Exception = when (e) { - is Exception -> e - else -> Other(e) - } - } +interface TtsEngine, + E : TtsEngine.Error, V : TtsEngine.Voice> : Configurable, Closeable { + + interface Preferences

> : Configurable.Preferences

{ + + val language: Language? + } + + interface Settings : Configurable.Settings { + + val language: Language? } + interface Voice { + + val language: Language + } + + interface Error + /** * TTS engine callbacks. */ - interface Listener { + interface Listener { fun onStart(requestId: String) @@ -64,12 +50,14 @@ interface TtsEngine> : Configurable fun onDone(requestId: String) - fun onError(requestId: String, error: Exception) + fun onError(requestId: String, error: E) } + val voices: Set + fun speak(requestId: String, text: String, language: Language?) fun stop() - fun setListener(listener: Listener?) + fun setListener(listener: Listener?) } diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsEngineFacade.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsEngineFacade.kt index 19923b1dbe..402672d62c 100644 --- a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsEngineFacade.kt +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsEngineFacade.kt @@ -16,39 +16,42 @@ import org.readium.r2.shared.util.Language @ExperimentalReadiumApi @OptIn(ExperimentalCoroutinesApi::class) -internal class TtsEngineFacade>( - private val ttsEngine: TtsEngine -) : Configurable by ttsEngine { +internal class TtsEngineFacade, + E : TtsEngine.Error, V : TtsEngine.Voice>( + private val engine: TtsEngine +) : Configurable by engine { init { val listener = TtsEngineListener() - ttsEngine.setListener(listener) + engine.setListener(listener) } - private var currentTask: UtteranceTask? = null + private var currentTask: UtteranceTask? = null - suspend fun speak(text: String, language: Language?, onRange: (IntRange) -> Unit) { + val voices: Set + get() = engine.voices + + suspend fun speak(text: String, language: Language?, onRange: (IntRange) -> Unit): E? = suspendCancellableCoroutine { continuation -> - continuation.invokeOnCancellation { ttsEngine.stop() } + continuation.invokeOnCancellation { engine.stop() } val id = UUID.randomUUID().toString() currentTask?.continuation?.cancel() currentTask = UtteranceTask(id, continuation, onRange) - ttsEngine.speak(id, text, language) + engine.speak(id, text, language) } - } fun close() { currentTask?.continuation?.cancel() - ttsEngine.close() + engine.close() } - private data class UtteranceTask( + private data class UtteranceTask( val requestId: String, - val continuation: CancellableContinuation, + val continuation: CancellableContinuation, val onRange: (IntRange) -> Unit ) - private inner class TtsEngineListener : TtsEngine.Listener { + private inner class TtsEngineListener : TtsEngine.Listener { override fun onStart(requestId: String) { } @@ -84,7 +87,7 @@ internal class TtsEngineFacade>( currentTask = null } - override fun onError(requestId: String, error: TtsEngine.Exception) { + override fun onError(requestId: String, error: E) { currentTask ?.takeIf { it.requestId == requestId } ?.continuation diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsEngineProvider.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsEngineProvider.kt index c0d6cc69fc..b0716c0e97 100644 --- a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsEngineProvider.kt +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsEngineProvider.kt @@ -6,6 +6,8 @@ package org.readium.r2.navigator.media3.tts2 +import androidx.media3.common.PlaybackException +import androidx.media3.common.PlaybackParameters import org.readium.r2.navigator.preferences.PreferencesEditor import org.readium.r2.shared.ExperimentalReadiumApi import org.readium.r2.shared.publication.Publication @@ -14,11 +16,18 @@ import org.readium.r2.shared.publication.Publication * To be implemented by adapters for third-party TTS engines which can be used with [TtsNavigator]. */ @ExperimentalReadiumApi -interface TtsEngineProvider, E : PreferencesEditor

> { +interface TtsEngineProvider, E : PreferencesEditor

, + F : TtsEngine.Error, V : TtsEngine.Voice> { - suspend fun createEngine(publication: Publication, initialPreferences: P): TtsEngine? + suspend fun createEngine(publication: Publication, initialPreferences: P): TtsEngine? fun createPreferencesEditor(publication: Publication, initialPreferences: P): E fun createEmptyPreferences(): P + + fun getPlaybackParameters(settings: S): PlaybackParameters + + fun updatePlaybackParameters(previousPreferences: P, playbackParameters: PlaybackParameters): P + + fun mapEngineError(error: F): PlaybackException } diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsLocator.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsLocator.kt deleted file mode 100644 index 80d524be8a..0000000000 --- a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsLocator.kt +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Copyright 2022 Readium Foundation. All rights reserved. - * Use of this source code is governed by the BSD-style license - * available in the top-level LICENSE file of the project. - */ - -package org.readium.r2.navigator.media3.tts2 - -import kotlinx.serialization.Serializable -import org.readium.r2.navigator.media3.api.MediaNavigatorInternal -import org.readium.r2.shared.ExperimentalReadiumApi -import org.readium.r2.shared.publication.Locator -import org.readium.r2.shared.publication.Publication -import org.readium.r2.shared.publication.html.cssSelector -import org.readium.r2.shared.publication.indexOfFirstWithHref - -@ExperimentalReadiumApi -@Serializable -data class TtsLocator( - val resourceIndex: Int, - val text: String, - val textBefore: String?, - val textAfter: String?, - val cssSelector: String?, -) : MediaNavigatorInternal.Locator - -@ExperimentalReadiumApi -internal fun TtsLocator.toLocator(publication: Publication): Locator { - return publication - .locatorFromLink(publication.readingOrder[resourceIndex])!! - .copyWithLocations( - progression = null, - otherLocations = buildMap { - cssSelector?.let { put("cssSelector", it) } - } - ).copy( - text = - Locator.Text( - highlight = text, - before = textBefore, - after = textAfter - ) - ) -} - -@ExperimentalReadiumApi -internal fun Locator.toTtsLocator(publication: Publication): TtsLocator? { - val resourceIndex = publication.readingOrder.indexOfFirstWithHref(href) - ?: return null - - val contentText = text.highlight - ?: return null - - return TtsLocator( - resourceIndex = resourceIndex, - text = contentText, - textBefore = text.before, - textAfter = text.after, - cssSelector = locations.cssSelector, - ) -} - -@ExperimentalReadiumApi -internal fun TtsLocator.substring(range: IntRange): TtsLocator { - return copy( - textBefore = textBefore.orEmpty() + text.substring(0, range.first), - text = text.substring(range), - textAfter = text.substring(range.last) + textAfter.orEmpty() - ) -} diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsNavigator.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsNavigator.kt index 508acf3237..ab4a81d942 100644 --- a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsNavigator.kt +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsNavigator.kt @@ -13,9 +13,7 @@ import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.MainScope import kotlinx.coroutines.flow.StateFlow import org.readium.r2.navigator.Navigator -import org.readium.r2.navigator.media3.api.MediaNavigator -import org.readium.r2.navigator.media3.api.MediaNavigatorInternal -import org.readium.r2.navigator.media3.api.MetadataProvider +import org.readium.r2.navigator.media3.api.* import org.readium.r2.navigator.preferences.Configurable import org.readium.r2.shared.ExperimentalReadiumApi import org.readium.r2.shared.extensions.mapStateIn @@ -25,38 +23,38 @@ import org.readium.r2.shared.publication.Publication import org.readium.r2.shared.publication.services.content.ContentService import org.readium.r2.shared.publication.services.content.ContentTokenizer import org.readium.r2.shared.util.Language +import timber.log.Timber @ExperimentalReadiumApi -class TtsNavigator> private constructor( +class TtsNavigator, + E : TtsEngine.Error, V : TtsEngine.Voice> private constructor( override val publication: Publication, - private val ttsNavigator: TtsNavigatorInternal -) : MediaNavigator, Navigator, Configurable by ttsNavigator { + private val ttsNavigator: TtsNavigatorInternal +) : SynchronizedMediaNavigator, Navigator, Configurable by ttsNavigator { companion object { - suspend operator fun > invoke( + suspend operator fun , + E : TtsEngine.Error, V : TtsEngine.Voice> invoke( application: Application, publication: Publication, - ttsEngineProvider: TtsEngineProvider, + ttsEngineProvider: TtsEngineProvider, tokenizerFactory: (defaultLanguage: Language?) -> ContentTokenizer, - metadataProvider: MetadataProvider, + metadataProvider: MediaMetadataProvider, listener: TtsNavigatorListener, initialPreferences: P? = null, initialLocator: Locator? = null, - ): TtsNavigator? { + ): TtsNavigator? { if (publication.findService(ContentService::class) == null) { return null } - val actualInitialLocator = initialLocator - ?.toTtsLocator(publication) - val actualInitialPreferences = initialPreferences ?: ttsEngineProvider.createEmptyPreferences() val contentIterator = - TtsContentIterator(publication, tokenizerFactory, actualInitialLocator) + TtsContentIterator(publication, tokenizerFactory, initialLocator) val ttsEngine = ttsEngineProvider.createEngine(publication, actualInitialPreferences) @@ -77,42 +75,44 @@ class TtsNavigator> private constructor( } val internalNavigator = - TtsNavigatorInternal(application, ttsEngine, contentIterator, playlistMetadata, mediaItems, listener) - ?: return null + TtsNavigatorInternal( + application, + ttsEngine, + contentIterator, + playlistMetadata, + mediaItems, + ttsEngineProvider::getPlaybackParameters, + ttsEngineProvider::updatePlaybackParameters, + ttsEngineProvider::mapEngineError, + actualInitialPreferences, + listener, + ) ?: return null return TtsNavigator(publication, internalNavigator) } } - data class Playback( - override val state: MediaNavigator.State, - override val locator: Locator, - override val token: Locator? - ) : MediaNavigator.Playback, MediaNavigator.TextSynchronization + sealed class Error : MediaNavigator.Error { + + data class EngineError (val error: E) : Error() + + data class ContentError(val exception: Exception) : Error() + } private val coroutineScope: CoroutineScope = MainScope() - override val playback: StateFlow = + override val playback: StateFlow> = ttsNavigator.playback.mapStateIn(coroutineScope) { it.toPlayback() } - private fun TtsPlayback.toPlayback() = - Playback( - state = when (state) { - MediaNavigatorInternal.State.Playing -> MediaNavigator.State.Playing - MediaNavigatorInternal.State.Paused -> MediaNavigator.State.Paused - MediaNavigatorInternal.State.Ended -> MediaNavigator.State.Ended - }, - locator = locator.toLocator(publication), - token = token?.toLocator(publication) - ) + override val utterance: StateFlow = + ttsNavigator.utterance.mapStateIn(coroutineScope) { Timber.d("utterance $it"); it.toUtterance() } override val currentLocator: StateFlow = - playback.mapStateIn(coroutineScope) { it.locator } + ttsNavigator.utterance.mapStateIn(coroutineScope) { it.toLocator() } override fun go(locator: Locator, animated: Boolean, completion: () -> Unit): Boolean { - val ttsLocator = locator.toTtsLocator(publication) ?: return false - ttsNavigator.go(ttsLocator) + ttsNavigator.go(TtsNavigatorInternal.RelaxedPosition(locator)) return true } @@ -145,4 +145,48 @@ class TtsNavigator> private constructor( override fun asPlayer(): Player = ttsNavigator.asPlayer() + + private fun MediaNavigatorInternal.Playback.toPlayback() = + MediaNavigator.Playback( + state = state.toState(), + playWhenReady = playWhenReady, + error = error?.toError() + ) + + private fun MediaNavigatorInternal.State.toState() = + when (this) { + MediaNavigatorInternal.State.Ready -> MediaNavigator.State.Ready + MediaNavigatorInternal.State.Ended -> MediaNavigator.State.Ended + MediaNavigatorInternal.State.Buffering -> MediaNavigator.State.Buffering + MediaNavigatorInternal.State.Error -> MediaNavigator.State.Error + } + + private fun TtsNavigatorInternal.Error.toError(): Error = + when (this) { + is TtsNavigatorInternal.Error.ContentError -> Error.ContentError(exception) + is TtsNavigatorInternal.Error.EngineError<*> -> Error.EngineError(error) + } + + private fun SynchronizedMediaNavigatorInternal.Utterance.toUtterance() = + SynchronizedMediaNavigator.Utterance( + locator = toLocator(), + range = range + ) + + private fun SynchronizedMediaNavigatorInternal.Utterance.toLocator() = + publication + .locatorFromLink(publication.readingOrder[position.resourceIndex])!! + .copyWithLocations( + progression = null, + otherLocations = buildMap { + put("cssSelector", position.cssSelector) + } + ).copy( + text = + Locator.Text( + highlight = text, + before = position.textBefore, + after = position.textAfter + ) + ) } diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsNavigatorFactory.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsNavigatorFactory.kt index 3fca996067..d6cf0a703b 100644 --- a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsNavigatorFactory.kt +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsNavigatorFactory.kt @@ -7,8 +7,9 @@ package org.readium.r2.navigator.media3.tts2 import android.app.Application -import org.readium.r2.navigator.media3.api.DefaultMetadataProvider -import org.readium.r2.navigator.media3.api.MetadataProvider +import org.readium.r2.navigator.media3.api.DefaultMediaMetadataFactory +import org.readium.r2.navigator.media3.api.MediaMetadataFactory +import org.readium.r2.navigator.media3.api.MediaMetadataProvider import org.readium.r2.navigator.preferences.PreferencesEditor import org.readium.r2.shared.ExperimentalReadiumApi import org.readium.r2.shared.publication.Locator @@ -18,22 +19,24 @@ import org.readium.r2.shared.util.Language import org.readium.r2.shared.util.tokenizer.TextUnit @ExperimentalReadiumApi -class TtsNavigatorFactory, E : PreferencesEditor

>( +class TtsNavigatorFactory, E : PreferencesEditor

, + F : TtsEngine.Error, V : TtsEngine.Voice>( private val application: Application, private val publication: Publication, - private val ttsEngineProvider: TtsEngineProvider, + private val ttsEngineProvider: TtsEngineProvider, private val tokenizerFactory: (defaultLanguage: Language?) -> ContentTokenizer, - private val metadataProvider: MetadataProvider + private val metadataProvider: MediaMetadataProvider ) { companion object { - suspend operator fun , E : PreferencesEditor

> invoke( + suspend operator fun , E : PreferencesEditor

, + F : TtsEngine.Error, V : TtsEngine.Voice> invoke( application: Application, publication: Publication, - ttsEngineProvider: TtsEngineProvider, + ttsEngineProvider: TtsEngineProvider, tokenizerFactory: (defaultLanguage: Language?) -> ContentTokenizer = defaultTokenizerFactory, - metadataProvider: MetadataProvider = defaultMetadataProvider - ): TtsNavigatorFactory? { + metadataProvider: MediaMetadataProvider = defaultMediaMetadataProvider + ): TtsNavigatorFactory? { publication.content() ?.iterator() @@ -54,14 +57,19 @@ class TtsNavigatorFactory, E : Preference ) } - val defaultMetadataProvider: MetadataProvider = DefaultMetadataProvider() + val defaultMediaMetadataProvider: MediaMetadataProvider = + object : MediaMetadataProvider { + override fun createMetadataFactory(publication: Publication): MediaMetadataFactory { + return DefaultMediaMetadataFactory(publication) + } + } } suspend fun createNavigator( listener: TtsNavigatorListener, initialPreferences: P? = null, initialLocator: Locator? = null - ): TtsNavigator { + ): TtsNavigator { return TtsNavigator( application, publication, diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsNavigatorInternal.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsNavigatorInternal.kt index bcd0ee080d..70f4173732 100644 --- a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsNavigatorInternal.kt +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsNavigatorInternal.kt @@ -9,75 +9,103 @@ package org.readium.r2.navigator.media3.tts2 import android.app.Application import androidx.media3.common.MediaItem import androidx.media3.common.MediaMetadata +import androidx.media3.common.PlaybackException +import androidx.media3.common.PlaybackParameters import androidx.media3.common.Player import kotlinx.coroutines.CoroutineScope -import kotlinx.coroutines.ExperimentalCoroutinesApi import kotlinx.coroutines.MainScope import kotlinx.coroutines.flow.StateFlow import org.readium.r2.navigator.media3.api.MediaNavigatorInternal +import org.readium.r2.navigator.media3.api.SynchronizedMediaNavigatorInternal import org.readium.r2.navigator.preferences.Configurable import org.readium.r2.shared.ExperimentalReadiumApi import org.readium.r2.shared.extensions.mapStateIn +import org.readium.r2.shared.publication.Locator -@OptIn(ExperimentalCoroutinesApi::class) @ExperimentalReadiumApi -internal class TtsNavigatorInternal>( - private val ttsPlayer: TtsPlayer, - private val sessionAdapter: TtsSessionAdapter, -) : MediaNavigatorInternal, Configurable by ttsPlayer { +internal class TtsNavigatorInternal, + E : TtsEngine.Error, V : TtsEngine.Voice>( + private val ttsPlayer: TtsPlayer, + private val sessionAdapter: TtsSessionAdapter, + coroutineScope: CoroutineScope, +) : SynchronizedMediaNavigatorInternal, + Configurable by ttsPlayer { companion object { - suspend operator fun > invoke( + suspend operator fun , + E : TtsEngine.Error, V : TtsEngine.Voice> invoke( application: Application, - ttsEngine: TtsEngine, + ttsEngine: TtsEngine, ttsContentIterator: TtsContentIterator, playlistMetadata: MediaMetadata, mediaItems: List, + getPlaybackParameters: (S) -> PlaybackParameters, + updatePlaybackParameters: (P, PlaybackParameters) -> P, + mapEngineError: (E) -> PlaybackException, + initialPreferences: P, listener: TtsNavigatorListener - ): TtsNavigatorInternal? { + ): TtsNavigatorInternal? { + val ttsPlayer = + TtsPlayer(ttsEngine, ttsContentIterator, initialPreferences) + ?: return null - val playerListener = object : TtsPlayer.Listener { + val coroutineScope = + MainScope() - override fun onPlaybackException() { - listener.onPlaybackException() + val playbackParameters = + ttsPlayer.settings.mapStateIn(coroutineScope) { + getPlaybackParameters(it) } - } - val ttsPlayer = - TtsPlayer(ttsEngine, ttsContentIterator, playerListener) - ?: return null + val onSetPlaybackParameters = { parameters: PlaybackParameters -> + val newPreferences = updatePlaybackParameters(ttsPlayer.lastPreferences, parameters) + ttsPlayer.submitPreferences(newPreferences) + } val sessionAdapter = - TtsSessionAdapter(application, ttsPlayer, playlistMetadata, mediaItems, listener::onStopRequested) + TtsSessionAdapter( + application, + ttsPlayer, + playlistMetadata, + mediaItems, + listener::onStopRequested, + playbackParameters, + onSetPlaybackParameters, + mapEngineError + ) - return TtsNavigatorInternal(ttsPlayer, sessionAdapter) + return TtsNavigatorInternal(ttsPlayer, sessionAdapter, coroutineScope) } } - private val coroutineScope: CoroutineScope = - MainScope() - - override val playback: StateFlow = - ttsPlayer.playback - .mapStateIn(coroutineScope) { playback -> - val state = when (playback.state) { - TtsPlayer.Playback.State.READY -> - if (playback.playWhenReady) MediaNavigatorInternal.State.Playing - else MediaNavigatorInternal.State.Paused - TtsPlayer.Playback.State.ENDED -> - MediaNavigatorInternal.State.Ended - } + data class RelaxedPosition( + val locator: Locator + ) : MediaNavigatorInternal.RelaxedPosition - val token = playback.range - ?.let { playback.locator.substring(playback.range) } + data class Position( + val resourceIndex: Int, + val cssSelector: String, + val textBefore: String?, + val textAfter: String?, + ) : MediaNavigatorInternal.Position - TtsPlayback( - state = state, - locator = playback.locator, - token = token - ) - } + sealed class Error : MediaNavigatorInternal.Error { + + data class EngineError (val error: E) : Error() + + data class ContentError(val exception: Exception) : Error() + } + + override val playback: StateFlow> = + ttsPlayer.playback.mapStateIn(coroutineScope) { it.toPlayback() } + + override val utterance: StateFlow> = + ttsPlayer.utterance.mapStateIn(coroutineScope) { it.toUtterance() } + + override val progression: StateFlow = + utterance.mapStateIn(coroutineScope) { it.position } override fun play() { ttsPlayer.play() @@ -87,8 +115,8 @@ internal class TtsNavigatorInternal>( ttsPlayer.pause() } - override fun go(locator: TtsLocator) { - ttsPlayer.go(locator) + override fun go(position: RelaxedPosition) { + ttsPlayer.go(position.locator) } override fun goForward() { @@ -106,4 +134,36 @@ internal class TtsNavigatorInternal>( fun close() { ttsPlayer.close() } + + private fun TtsPlayer.Utterance.toUtterance() = + SynchronizedMediaNavigatorInternal.Utterance( + text = text, + range = range, + position = Position( + resourceIndex = position.resourceIndex, + cssSelector = position.cssSelector, + textBefore = position.textBefore, + textAfter = position.textAfter + ) + ) + + private fun TtsPlayer.Playback.toPlayback() = + MediaNavigatorInternal.Playback( + state = state.toState(), + playWhenReady = playWhenReady, + error = error?.toError() + ) + + private fun TtsPlayer.Playback.State.toState() = + when (this) { + TtsPlayer.Playback.State.Ready -> MediaNavigatorInternal.State.Ready + TtsPlayer.Playback.State.Ended -> MediaNavigatorInternal.State.Ended + TtsPlayer.Playback.State.Error -> MediaNavigatorInternal.State.Error + } + + private fun TtsPlayer.Error.toError(): Error = + when (this) { + is TtsPlayer.Error.ContentError -> Error.ContentError(exception) + is TtsPlayer.Error.EngineError<*> -> Error.EngineError(error) + } } diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsNavigatorListener.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsNavigatorListener.kt index af67a340f0..0917d65452 100644 --- a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsNavigatorListener.kt +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsNavigatorListener.kt @@ -9,6 +9,4 @@ package org.readium.r2.navigator.media3.tts2 interface TtsNavigatorListener { fun onStopRequested() - - fun onPlaybackException() } diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsPlayback.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsPlayback.kt deleted file mode 100644 index e853533b45..0000000000 --- a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsPlayback.kt +++ /dev/null @@ -1,18 +0,0 @@ -/* - * Copyright 2022 Readium Foundation. All rights reserved. - * Use of this source code is governed by the BSD-style license - * available in the top-level LICENSE file of the project. - */ - -package org.readium.r2.navigator.media3.tts2 - -import org.readium.r2.navigator.media3.api.MediaNavigatorInternal -import org.readium.r2.navigator.media3.api.SynchronizedPlayback -import org.readium.r2.shared.ExperimentalReadiumApi - -@ExperimentalReadiumApi -internal data class TtsPlayback( - override val state: MediaNavigatorInternal.State, - override val locator: TtsLocator, - override val token: TtsLocator? -) : SynchronizedPlayback diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsPlayer.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsPlayer.kt index dcc0d0e9f6..c6d5081c89 100644 --- a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsPlayer.kt +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsPlayer.kt @@ -6,168 +6,402 @@ package org.readium.r2.navigator.media3.tts2 -import androidx.media3.common.Player import kotlinx.coroutines.* import kotlinx.coroutines.flow.MutableStateFlow import kotlinx.coroutines.flow.StateFlow import kotlinx.coroutines.flow.asStateFlow +import kotlinx.coroutines.sync.Mutex +import kotlinx.coroutines.sync.withLock import org.readium.r2.navigator.preferences.Configurable import org.readium.r2.shared.ExperimentalReadiumApi +import org.readium.r2.shared.publication.Locator import timber.log.Timber @ExperimentalReadiumApi -internal class TtsPlayer>( - private val engineFacade: TtsEngineFacade, +internal class TtsPlayer, + E : TtsEngine.Error, V : TtsEngine.Voice> private constructor( + private val engineFacade: TtsEngineFacade, private val contentIterator: TtsContentIterator, - private val listener: Listener, - firstUtterance: TtsUtterance, -) : Configurable by engineFacade { + initialContext: Context, + initialPreferences: P +) : Configurable { companion object { - suspend operator fun > invoke( - engine: TtsEngine, + suspend operator fun , + E : TtsEngine.Error, V : TtsEngine.Voice> invoke( + engine: TtsEngine, contentIterator: TtsContentIterator, - listener: Listener - ): TtsPlayer? { + initialPreferences: P, + ): TtsPlayer? { - val firstUtterance = contentIterator.nextUtterance() - ?: run { - contentIterator.seekToBeginning() - contentIterator.nextUtterance() - } ?: return null + val initialContext = contentIterator.startContext() + ?: return null val ttsEngineFacade = TtsEngineFacade(engine) + ttsEngineFacade.submitPreferences(initialPreferences) + contentIterator.language = ttsEngineFacade.settings.value.language - return TtsPlayer(ttsEngineFacade, contentIterator, listener, firstUtterance) + return TtsPlayer(ttsEngineFacade, contentIterator, initialContext, initialPreferences) + } + + private suspend fun TtsContentIterator.startContext(): Context? { + val previousUtterance = previousUtterance() + val currentUtterance = nextUtterance() + + return if (currentUtterance != null) { + Context( + previousUtterance = previousUtterance, + currentUtterance = currentUtterance, + nextUtterance = nextUtterance(), + ended = false + ) + } else { + Context( + previousUtterance = previousUtterance(), + currentUtterance = previousUtterance ?: return null, + nextUtterance = null, + ended = true + ) + } } } - interface Listener { + sealed class Error { + + data class EngineError (val error: E) : Error() - fun onPlaybackException() + data class ContentError(val exception: Exception) : Error() } - @ExperimentalReadiumApi data class Playback( val state: State, - val isPlaying: Boolean, val playWhenReady: Boolean, - val index: Int, - val locator: TtsLocator, - val range: IntRange? + val error: Error? ) { - enum class State(val value: Int) { - READY(Player.STATE_READY), - ENDED(Player.STATE_ENDED); + enum class State { + Ready, + Ended, + Error; } } + data class Utterance( + val text: String, + val position: Position, + val range: IntRange? + ) { + + data class Position( + val resourceIndex: Int, + val cssSelector: String, + val textBefore: String?, + val textAfter: String?, + ) + } + + private data class Context( + val previousUtterance: TtsContentIterator.Utterance?, + val currentUtterance: TtsContentIterator.Utterance, + val nextUtterance: TtsContentIterator.Utterance?, + val ended: Boolean = false + ) + private val coroutineScope: CoroutineScope = MainScope() private val playbackMutable: MutableStateFlow = MutableStateFlow( Playback( - index = firstUtterance.locator.resourceIndex, - state = Playback.State.READY, - isPlaying = false, + state = if (initialContext.ended) Playback.State.Ended else Playback.State.Ready, playWhenReady = false, - locator = firstUtterance.locator, - range = null + error = null ) ) - private var pendingUtterance: TtsUtterance? = - firstUtterance + private val utteranceMutable: MutableStateFlow = + MutableStateFlow(initialContext.currentUtterance.ttsPlayerUtterance()) + + private var context: Context = + initialContext private var playbackJob: Job? = null - init { - contentIterator.language = engineFacade.settings.value.language - } + private val mutex = Mutex() + + val voices: Set get() = + engineFacade.voices val playback: StateFlow = playbackMutable.asStateFlow() + val utterance: StateFlow = + utteranceMutable.asStateFlow() + fun play() { - replacePlaybackJob { - playbackMutable.value = - playbackMutable.value.copy( - state = Playback.State.READY, - playWhenReady = true, - isPlaying = true - ) - playContinuous() + coroutineScope.launch { + playAsync() + } + } + + private suspend fun playAsync() = mutex.withLock { + if (isPlaying()) { + return } + + playbackMutable.value = playbackMutable.value.copy(playWhenReady = true) + playIfReadyAndNotPaused() } fun pause() { - replacePlaybackJob { - playbackMutable.value = - playbackMutable.value.copy( - playWhenReady = false, - isPlaying = false - ) + coroutineScope.launch { + pauseAsync() + } + } + + private suspend fun pauseAsync() = mutex.withLock { + if (!playbackMutable.value.playWhenReady) { + return + } + + playbackJob?.cancelAndJoin() + playbackMutable.value = playbackMutable.value.copy(playWhenReady = false) + utteranceMutable.value = utteranceMutable.value.copy(range = null) + } + + fun go(locator: Locator) { + coroutineScope.launch { + goAsync(locator) + } + } + + private suspend fun goAsync(locator: Locator) = mutex.withLock { + playbackJob?.cancelAndJoin() + contentIterator.seek(locator) + resetContext() + playIfReadyAndNotPaused() + } + + fun go(resourceIndex: Int) { + coroutineScope.launch { + goAsync(resourceIndex) } } - fun go(locator: TtsLocator) { - replacePlaybackJob { - pendingUtterance = null - contentIterator.seek(locator) - playContinuous() + private suspend fun goAsync(resourceIndex: Int) = mutex.withLock { + playbackJob?.cancelAndJoin() + contentIterator.seekToResource(resourceIndex) + resetContext() + playIfReadyAndNotPaused() + } + + fun restartUtterance() { + coroutineScope.launch { + restartUtteranceAsync() } } + private suspend fun restartUtteranceAsync() = mutex.withLock { + playbackJob?.cancelAndJoin() + if (playbackMutable.value.state == Playback.State.Ended) { + playbackMutable.value = playbackMutable.value.copy(state = Playback.State.Ready) + } + playIfReadyAndNotPaused() + } + + fun hasNextUtterance() = + context.nextUtterance != null + fun nextUtterance() { - replacePlaybackJob { - pendingUtterance = null - playContinuous() + coroutineScope.launch { + nextUtteranceAsync() + } + } + + private suspend fun nextUtteranceAsync() = mutex.withLock { + if (context.nextUtterance == null) { + return } + + playbackJob?.cancelAndJoin() + tryLoadNextContext() + playIfReadyAndNotPaused() } + fun hasPreviousUtterance() = + context.previousUtterance != null + fun previousUtterance() { - replacePlaybackJob { - pendingUtterance = null - pendingUtterance = contentIterator.previousUtterance() - playContinuous() + coroutineScope.launch { + previousUtteranceAsync() + } + } + + private suspend fun previousUtteranceAsync() = mutex.withLock { + if (context.previousUtterance == null) { + return + } + playbackJob?.cancelAndJoin() + tryLoadPreviousContext() + playIfReadyAndNotPaused() + } + + fun hasNextResource(): Boolean = + utteranceMutable.value.position.resourceIndex + 1 < contentIterator.resourceCount + + fun nextResource() { + coroutineScope.launch { + nextResourceAsync() + } + } + + private suspend fun nextResourceAsync() = mutex.withLock { + if (!hasNextUtterance()) { + return } + + playbackJob?.cancelAndJoin() + val currentIndex = utteranceMutable.value.position.resourceIndex + contentIterator.seekToResource(currentIndex + 1) + resetContext() + playIfReadyAndNotPaused() } - private fun replacePlaybackJob(block: suspend CoroutineScope.() -> Unit) { + fun hasPreviousResource(): Boolean = + utteranceMutable.value.position.resourceIndex > 0 + + fun previousResource() { coroutineScope.launch { - playbackJob?.cancelAndJoin() - playbackJob = launch { - block() + previousResourceAsync() + } + } + + private suspend fun previousResourceAsync() = mutex.withLock { + if (!hasPreviousResource()) { + return + } + playbackJob?.cancelAndJoin() + val currentIndex = utteranceMutable.value.position.resourceIndex + contentIterator.seekToResource(currentIndex - 1) + resetContext() + playIfReadyAndNotPaused() + } + + private fun playIfReadyAndNotPaused() { + check(playbackJob?.isCompleted ?: true) + if (playback.value.playWhenReady && playback.value.state == Playback.State.Ready) { + playbackJob = coroutineScope.launch { + playContinuous() } } } - private suspend fun playContinuous() { - if (pendingUtterance == null) { - pendingUtterance = contentIterator.nextUtterance() - } - pendingUtterance?.let { - Timber.d("Setting playback to locator ${it.locator}") - playbackMutable.value = playbackMutable.value.copy(range = null, locator = it.locator, isPlaying = true) - engineFacade.speak(it.text, it.language, ::onRangeChanged) - pendingUtterance = null - playContinuous() - } ?: run { - playbackMutable.value = playbackMutable.value.copy( - isPlaying = false, playWhenReady = false, state = Playback.State.ENDED + private suspend fun tryLoadPreviousContext() { + val contextNow = context + // Get previously nextUtterance once more + contentIterator.previousUtterance() + + // Get previously currentUtterance once more + val currentUtterance = checkNotNull(contentIterator.previousUtterance()) + + // Get previous utterance + val previousUtterance = contentIterator.previousUtterance() + + // Get to nextUtterance position + contentIterator.nextUtterance() + contentIterator.nextUtterance() + + context = Context( + previousUtterance = previousUtterance, + currentUtterance = currentUtterance, + nextUtterance = contextNow.currentUtterance + ) + utteranceMutable.value = context.currentUtterance.ttsPlayerUtterance() + } + + private suspend fun tryLoadNextContext() { + Timber.d("tryLoadNextContext") + val contextNow = context + Timber.d("contextNow $contextNow") + + if (contextNow.nextUtterance == null) { + onEndReached() + } else { + context = Context( + previousUtterance = contextNow.currentUtterance, + currentUtterance = contextNow.nextUtterance, + nextUtterance = contentIterator.nextUtterance() ) + Timber.d("newContext $context") + utteranceMutable.value = context.currentUtterance.ttsPlayerUtterance() + Timber.d("utterance ${utteranceMutable.value.text}") + if (playbackMutable.value.state == Playback.State.Ended) { + playbackMutable.value = playbackMutable.value.copy(state = Playback.State.Ready) + } } } + private suspend fun resetContext() { + context = checkNotNull(contentIterator.startContext()) + if (context.nextUtterance == null && context.ended) { + onEndReached() + } + } + + private fun onEndReached() { + playbackMutable.value = playbackMutable.value.copy( + state = Playback.State.Ended, + ) + } + + private suspend fun playContinuous() { + engineFacade.speak(context.currentUtterance.text, context.currentUtterance.language, ::onRangeChanged) + ?.let { exception -> onEngineError(exception) } + mutex.withLock { tryLoadNextContext() } + playContinuous() + } + + private fun onEngineError(error: E) { + Timber.d("onEngineError $error") + playbackMutable.value = playbackMutable.value.copy( + state = Playback.State.Error, + error = Error.EngineError(error) + ) + } + private fun onRangeChanged(range: IntRange) { - val newPlayback = playbackMutable.value.copy(range = range) - playbackMutable.value = newPlayback + val newUtterance = utteranceMutable.value.copy(range = range) + utteranceMutable.value = newUtterance } fun close() { engineFacade.close() } + + var lastPreferences: P = + initialPreferences + + override val settings: StateFlow + get() = engineFacade.settings + + override fun submitPreferences(preferences: P) { + lastPreferences = preferences + engineFacade.submitPreferences(preferences) + } + + private fun isPlaying() = + playbackMutable.value.playWhenReady && playback.value.state == Playback.State.Ready + + private fun TtsContentIterator.Utterance.ttsPlayerUtterance(): Utterance = + Utterance( + text = text, + range = null, + position = Utterance.Position( + resourceIndex = resourceIndex, + cssSelector = cssSelector, + textAfter = textAfter, + textBefore = textBefore + ) + ) } diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsPreferences.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsPreferences.kt deleted file mode 100644 index 7e91b3b018..0000000000 --- a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsPreferences.kt +++ /dev/null @@ -1,17 +0,0 @@ -/* - * Copyright 2022 Readium Foundation. All rights reserved. - * Use of this source code is governed by the BSD-style license - * available in the top-level LICENSE file of the project. - */ - -package org.readium.r2.navigator.media3.tts2 - -import org.readium.r2.navigator.preferences.Configurable -import org.readium.r2.shared.ExperimentalReadiumApi -import org.readium.r2.shared.util.Language - -@ExperimentalReadiumApi -interface TtsPreferences

> : Configurable.Preferences

{ - - val language: Language? -} diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsSessionAdapter.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsSessionAdapter.kt index 64851f707d..7d8e7c8f5d 100644 --- a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsSessionAdapter.kt +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsSessionAdapter.kt @@ -7,8 +7,7 @@ package org.readium.r2.navigator.media3.tts2 import android.app.Application -import android.content.Context -import android.media.AudioManager +import android.os.Handler import android.os.Looper import android.view.Surface import android.view.SurfaceHolder @@ -16,85 +15,115 @@ import android.view.SurfaceView import android.view.TextureView import androidx.media3.common.* import androidx.media3.common.C.* +import androidx.media3.common.PlaybackException.* +import androidx.media3.common.Player.* import androidx.media3.common.text.CueGroup import androidx.media3.common.util.Clock import androidx.media3.common.util.ListenerSet import androidx.media3.common.util.Util import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.MainScope -import kotlinx.coroutines.flow.launchIn -import kotlinx.coroutines.flow.onEach +import kotlinx.coroutines.flow.* import org.readium.r2.shared.ExperimentalReadiumApi +import org.readium.r2.shared.fetcher.Resource import timber.log.Timber @ExperimentalReadiumApi @androidx.annotation.OptIn(androidx.media3.common.util.UnstableApi::class) -internal class TtsSessionAdapter( +internal class TtsSessionAdapter( private val application: Application, - private val ttsPlayer: TtsPlayer<*, *>, + private val ttsPlayer: TtsPlayer<*, *, E, *>, private val playlistMetadata: MediaMetadata, private val mediaItems: List, - private val onStop: () -> Unit -) : BasePlayer() { + private val onStop: () -> Unit, + private val playbackParametersState: StateFlow, + private val updatePlaybackParameters: (PlaybackParameters) -> Unit, + private val mapEngineError: (E) -> PlaybackException +) : Player { private val coroutineScope: CoroutineScope = MainScope() + private val window: Timeline.Window = + Timeline.Window() + private var lastPlayback: TtsPlayer.Playback = ttsPlayer.playback.value + private var lastPlaybackParameters: PlaybackParameters = + playbackParametersState.value + + private val volumeManager = TtsStreamVolumeManager( + application, + Handler(Looper.getMainLooper()), + StreamVolumeManagerListener() + ) + + private var deviceInfo: DeviceInfo = + createDeviceInfo(volumeManager) + init { ttsPlayer.playback .onEach { playback -> - notifyListeners(lastPlayback, playback) + notifyListenersPlaybackChanged(lastPlayback, playback) lastPlayback = playback }.launchIn(coroutineScope) + + playbackParametersState + .onEach { playbackParameters -> + notifyListenersPlaybackParametersChanged(lastPlaybackParameters, playbackParameters) + lastPlaybackParameters = playbackParameters + } } - private var listeners: ListenerSet = + private var listeners: ListenerSet = ListenerSet( applicationLooper, Clock.DEFAULT, - ) { listener: Player.Listener, flags: FlagSet? -> - listener.onEvents(this, Player.Events(flags!!)) + ) { listener: Listener, flags: FlagSet? -> + listener.onEvents(this, Events(flags!!)) } private val permanentAvailableCommands = - Player.Commands.Builder() + Commands.Builder() .addAll( COMMAND_PLAY_PAUSE, COMMAND_STOP, + // COMMAND_SEEK_BACK, + // COMMAND_SEEK_FORWARD, + // COMMAND_SEEK_TO_NEXT_MEDIA_ITEM, // COMMAND_SEEK_TO_PREVIOUS_MEDIA_ITEM, - // COMMAND_SEEK_TO_NEXT, - // COMMAND_SEEK_TO_PREVIOUS - // COMMAND_GET_AUDIO_ATTRIBUTES, - // COMMAND_GET_CURRENT_MEDIA_ITEM, - // COMMAND_GET_MEDIA_ITEMS_METADATA, - // COMMAND_GET_TEXT - ).build() + COMMAND_GET_AUDIO_ATTRIBUTES, + COMMAND_GET_DEVICE_VOLUME, + COMMAND_SET_DEVICE_VOLUME, - private val audioManager: AudioManager = - application.getSystemService(Context.AUDIO_SERVICE) as AudioManager + COMMAND_SET_SPEED_AND_PITCH, + COMMAND_GET_CURRENT_MEDIA_ITEM, + COMMAND_GET_MEDIA_ITEMS_METADATA, + COMMAND_GET_TEXT + ).build() override fun getApplicationLooper(): Looper { return Looper.getMainLooper() } - override fun addListener(listener: Player.Listener) { + override fun addListener(listener: Listener) { Timber.d("addListener") listeners.add(listener) } - override fun removeListener(listener: Player.Listener) { + override fun removeListener(listener: Listener) { Timber.d("removeListener") listeners.remove(listener) } + override fun setMediaItems(mediaItems: MutableList) { + } + override fun setMediaItems(mediaItems: MutableList, resetPosition: Boolean) { - throw NotImplementedError() } override fun setMediaItems( @@ -102,43 +131,88 @@ internal class TtsSessionAdapter( startIndex: Int, startPositionMs: Long ) { - throw NotImplementedError() + } + + override fun setMediaItem(mediaItem: MediaItem) { + } + + override fun setMediaItem(mediaItem: MediaItem, startPositionMs: Long) { + } + + override fun setMediaItem(mediaItem: MediaItem, resetPosition: Boolean) { + } + + override fun addMediaItem(mediaItem: MediaItem) { + } + + override fun addMediaItem(index: Int, mediaItem: MediaItem) { + } + + override fun addMediaItems(mediaItems: MutableList) { } override fun addMediaItems(index: Int, mediaItems: MutableList) { - throw NotImplementedError() + } + + override fun moveMediaItem(currentIndex: Int, newIndex: Int) { } override fun moveMediaItems(fromIndex: Int, toIndex: Int, newIndex: Int) { - throw NotImplementedError() + } + + override fun removeMediaItem(index: Int) { } override fun removeMediaItems(fromIndex: Int, toIndex: Int) { - throw NotImplementedError() } - override fun getAvailableCommands(): Player.Commands { - return Player.Commands.Builder() + override fun clearMediaItems() { + } + + override fun isCommandAvailable(command: Int): Boolean { + return command in availableCommands + } + + override fun canAdvertiseSession(): Boolean { + return true + } + + override fun getAvailableCommands(): Commands { + return Commands.Builder() .addAll(permanentAvailableCommands) .build() } override fun prepare() { - throw NotImplementedError() } override fun getPlaybackState(): Int { - return ttsPlayer.playback.value.state.value + return ttsPlayer.playback.value.state.playerCode } override fun getPlaybackSuppressionReason(): Int { return PLAYBACK_SUPPRESSION_REASON_NONE // TODO } + override fun isPlaying(): Boolean { + return ( + playbackState == STATE_READY && playWhenReady && + playbackSuppressionReason == PLAYBACK_SUPPRESSION_REASON_NONE + ) + } + override fun getPlayerError(): PlaybackException? { return null // TODO } + override fun play() { + playWhenReady = true + } + + override fun pause() { + playWhenReady = false + } + override fun setPlayWhenReady(playWhenReady: Boolean) { if (playWhenReady) { ttsPlayer.play() @@ -171,28 +245,148 @@ internal class TtsSessionAdapter( return false } + override fun seekToDefaultPosition() { + seekToDefaultPosition(currentMediaItemIndex) + } + + override fun seekToDefaultPosition(mediaItemIndex: Int) { + seekTo(mediaItemIndex, 0L) + } + + override fun seekTo(positionMs: Long) { + } + override fun seekTo(mediaItemIndex: Int, positionMs: Long) { - throw NotImplementedError() + val timeline: Timeline = currentTimeline + if (mediaItemIndex < 0 || !timeline.isEmpty && mediaItemIndex >= timeline.windowCount + ) { + throw IllegalSeekPositionException(timeline, mediaItemIndex, positionMs) + } + + ttsPlayer.go(mediaItemIndex) } override fun getSeekBackIncrement(): Long { return 0 } + override fun seekBack() { + ttsPlayer.previousUtterance() + } + override fun getSeekForwardIncrement(): Long { return 0 } + override fun seekForward() { + ttsPlayer.nextUtterance() + } + + @Deprecated("Deprecated in Java", ReplaceWith("hasPreviousMediaItem()")) + override fun hasPrevious(): Boolean { + return hasPreviousMediaItem() + } + + @Deprecated("Deprecated in Java", ReplaceWith("hasPreviousMediaItem()")) + override fun hasPreviousWindow(): Boolean { + return hasPreviousMediaItem() + } + + override fun hasPreviousMediaItem(): Boolean { + return previousMediaItemIndex != INDEX_UNSET + } + + @Deprecated("Deprecated in Java", ReplaceWith("TODO(\"Not yet implemented\")")) + override fun previous() { + seekToPreviousMediaItem() + } + + @Deprecated("Deprecated in Java", ReplaceWith("TODO(\"Not yet implemented\")")) + override fun seekToPreviousWindow() { + seekToPreviousMediaItem() + } + + override fun seekToPreviousMediaItem() { + val previousMediaItemIndex = previousMediaItemIndex + if (previousMediaItemIndex != INDEX_UNSET) { + seekToDefaultPosition(previousMediaItemIndex) + } + } + override fun getMaxSeekToPreviousPosition(): Long { return 0 } + override fun seekToPrevious() { + val timeline = currentTimeline + if (timeline.isEmpty || isPlayingAd) { + return + } + val hasPreviousMediaItem = hasPreviousMediaItem() + if (isCurrentMediaItemLive && !isCurrentMediaItemSeekable) { + if (hasPreviousMediaItem) { + seekToPreviousMediaItem() + } + } else if (hasPreviousMediaItem && currentPosition <= maxSeekToPreviousPosition) { + seekToPreviousMediaItem() + } else { + seekTo( /* positionMs= */0) + } + } + + @Deprecated("Deprecated in Java", ReplaceWith("hasNextMediaItem()")) + override fun hasNext(): Boolean { + return hasNextMediaItem() + } + + @Deprecated("Deprecated in Java", ReplaceWith("hasNextMediaItem()")) + override fun hasNextWindow(): Boolean { + return hasNextMediaItem() + } + + override fun hasNextMediaItem(): Boolean { + return nextMediaItemIndex != INDEX_UNSET + } + + @Deprecated("Deprecated in Java", ReplaceWith("seekToNextMediaItem()")) + override fun next() { + seekToNextMediaItem() + } + + @Deprecated("Deprecated in Java", ReplaceWith("seekToNextMediaItem()")) + override fun seekToNextWindow() { + seekToNextMediaItem() + } + + override fun seekToNextMediaItem() { + val nextMediaItemIndex = nextMediaItemIndex + if (nextMediaItemIndex != INDEX_UNSET) { + seekToDefaultPosition(nextMediaItemIndex) + } + } + + override fun seekToNext() { + val timeline = currentTimeline + if (timeline.isEmpty || isPlayingAd) { + return + } + if (hasNextMediaItem()) { + seekToNextMediaItem() + } else if (isCurrentMediaItemLive && isCurrentMediaItemDynamic) { + seekToDefaultPosition() + } + } + override fun setPlaybackParameters(playbackParameters: PlaybackParameters) { - throw NotImplementedError() // TODO + updatePlaybackParameters(playbackParameters) + } + + override fun setPlaybackSpeed(speed: Float) { + updatePlaybackParameters(playbackParametersState.value.withSpeed(speed)) } override fun getPlaybackParameters(): PlaybackParameters { - return PlaybackParameters.DEFAULT + return playbackParametersState.value } override fun stop() { @@ -231,19 +425,70 @@ internal class TtsSessionAdapter( throw NotImplementedError() } + override fun getCurrentManifest(): Any? { + val timeline = currentTimeline + return if (timeline.isEmpty) null else timeline.getWindow( + currentMediaItemIndex, + window + ).manifest + } + override fun getCurrentTimeline(): Timeline { // MediaNotificationManager requires a non-empty timeline to start foreground playing. return TtsSessionTimeline(mediaItems) - /*return SinglePeriodTimeline( - TIME_UNSET, false, false, false, null, mediaItem)*/ } override fun getCurrentPeriodIndex(): Int { - return lastPlayback.index + return ttsPlayer.utterance.value.position.resourceIndex + } + + @Deprecated("Deprecated in Java", ReplaceWith("currentMediaItemIndex")) + override fun getCurrentWindowIndex(): Int { + return currentMediaItemIndex } override fun getCurrentMediaItemIndex(): Int { - return lastPlayback.index + return ttsPlayer.utterance.value.position.resourceIndex + } + + @Deprecated("Deprecated in Java", ReplaceWith("nextMediaItemIndex")) + override fun getNextWindowIndex(): Int { + return nextMediaItemIndex + } + + override fun getNextMediaItemIndex(): Int { + val timeline = currentTimeline + return if (timeline.isEmpty) INDEX_UNSET else timeline.getNextWindowIndex( + currentMediaItemIndex, getRepeatModeForNavigation(), shuffleModeEnabled + ) + } + + @Deprecated("Deprecated in Java", ReplaceWith("previousMediaItemIndex")) + override fun getPreviousWindowIndex(): Int { + return previousMediaItemIndex + } + + override fun getPreviousMediaItemIndex(): Int { + val timeline = currentTimeline + return if (timeline.isEmpty) INDEX_UNSET else timeline.getPreviousWindowIndex( + currentMediaItemIndex, getRepeatModeForNavigation(), shuffleModeEnabled + ) + } + + override fun getCurrentMediaItem(): MediaItem? { + val timeline = currentTimeline + return if (timeline.isEmpty) + null + else + timeline.getWindow(currentMediaItemIndex, window).mediaItem + } + + override fun getMediaItemCount(): Int { + return currentTimeline.windowCount + } + + override fun getMediaItemAt(index: Int): MediaItem { + return currentTimeline.getWindow(index, window).mediaItem } override fun getDuration(): Long { @@ -258,10 +503,65 @@ internal class TtsSessionAdapter( return 0 } + override fun getBufferedPercentage(): Int { + val position = bufferedPosition + val duration = duration + return if (position == TIME_UNSET || duration == TIME_UNSET) + 0 + else if (duration == 0L) + 100 + else Util.constrainValue( + (position * 100 / duration).toInt(), + 0, + 100 + ) + } + override fun getTotalBufferedDuration(): Long { return 0 } + @Deprecated("Deprecated in Java", ReplaceWith("isCurrentMediaItemDynamic")) + override fun isCurrentWindowDynamic(): Boolean { + return isCurrentMediaItemDynamic + } + + override fun isCurrentMediaItemDynamic(): Boolean { + val timeline = currentTimeline + return !timeline.isEmpty && timeline.getWindow(currentMediaItemIndex, window).isDynamic + } + + @Deprecated("Deprecated in Java", ReplaceWith("isCurrentMediaItemLive")) + override fun isCurrentWindowLive(): Boolean { + return isCurrentMediaItemLive + } + + override fun isCurrentMediaItemLive(): Boolean { + val timeline = currentTimeline + return !timeline.isEmpty && timeline.getWindow(currentMediaItemIndex, window).isLive() + } + + override fun getCurrentLiveOffset(): Long { + val timeline = currentTimeline + if (timeline.isEmpty) { + return TIME_UNSET + } + val windowStartTimeMs = timeline.getWindow(currentMediaItemIndex, window).windowStartTimeMs + return if (windowStartTimeMs == TIME_UNSET) { + TIME_UNSET + } else window.currentUnixTimeMs - window.windowStartTimeMs - contentPosition + } + + @Deprecated("Deprecated in Java", ReplaceWith("isCurrentMediaItemSeekable")) + override fun isCurrentWindowSeekable(): Boolean { + return isCurrentMediaItemSeekable + } + + override fun isCurrentMediaItemSeekable(): Boolean { + val timeline = currentTimeline + return !timeline.isEmpty && timeline.getWindow(currentMediaItemIndex, window).isSeekable + } + override fun isPlayingAd(): Boolean { return false } @@ -274,6 +574,13 @@ internal class TtsSessionAdapter( return INDEX_UNSET } + override fun getContentDuration(): Long { + val timeline = currentTimeline + return if (timeline.isEmpty) TIME_UNSET else timeline.getWindow( + currentMediaItemIndex, window + ).durationMs + } + override fun getContentPosition(): Long { return 0 } @@ -286,11 +593,11 @@ internal class TtsSessionAdapter( return AudioAttributes.Builder() .setUsage(USAGE_MEDIA) .setContentType(AUDIO_CONTENT_TYPE_SPEECH) + .setAllowedCapturePolicy(ALLOW_CAPTURE_BY_SYSTEM) .build() } override fun setVolume(volume: Float) { - throw NotImplementedError() } override fun getVolume(): Float { @@ -298,39 +605,30 @@ internal class TtsSessionAdapter( } override fun clearVideoSurface() { - throw NotImplementedError() } override fun clearVideoSurface(surface: Surface?) { - throw NotImplementedError() } override fun setVideoSurface(surface: Surface?) { - throw NotImplementedError() } override fun setVideoSurfaceHolder(surfaceHolder: SurfaceHolder?) { - throw NotImplementedError() } override fun clearVideoSurfaceHolder(surfaceHolder: SurfaceHolder?) { - throw NotImplementedError() } override fun setVideoSurfaceView(surfaceView: SurfaceView?) { - throw NotImplementedError() } override fun clearVideoSurfaceView(surfaceView: SurfaceView?) { - throw NotImplementedError() } override fun setVideoTextureView(textureView: TextureView?) { - throw NotImplementedError() } override fun clearVideoTextureView(textureView: TextureView?) { - throw NotImplementedError() } override fun getVideoSize(): VideoSize { @@ -342,69 +640,63 @@ internal class TtsSessionAdapter( } override fun getDeviceInfo(): DeviceInfo { - val minVolume = if (Util.SDK_INT >= 28) audioManager.getStreamMinVolume(STREAM_TYPE_MUSIC) else 0 - val maxVolume = audioManager.getStreamMaxVolume(STREAM_TYPE_MUSIC) - return DeviceInfo(DeviceInfo.PLAYBACK_TYPE_LOCAL, minVolume, maxVolume) + return deviceInfo } override fun getDeviceVolume(): Int { - return audioManager.getStreamVolume(STREAM_TYPE_MUSIC) + return volumeManager.getVolume() } override fun isDeviceMuted(): Boolean { - return if (Util.SDK_INT >= 23) { - audioManager.isStreamMute(STREAM_TYPE_MUSIC) - } else { - deviceVolume == 0 - } + return volumeManager.isMuted() } override fun setDeviceVolume(volume: Int) { - throw NotImplementedError() + volumeManager.setVolume(volume) } override fun increaseDeviceVolume() { - throw NotImplementedError() + volumeManager.increaseVolume() } override fun decreaseDeviceVolume() { - throw NotImplementedError() + volumeManager.decreaseVolume() } override fun setDeviceMuted(muted: Boolean) { - throw NotImplementedError() + volumeManager.setMuted(muted) } - private fun notifyListeners( + private fun notifyListenersPlaybackChanged( previousPlaybackInfo: TtsPlayer.Playback, playbackInfo: TtsPlayer.Playback, // playWhenReadyChangeReason: @Player.PlayWhenReadyChangeReason Int, ) { - /*if (previousPlaybackInfo.playbackError != playbackInfo.playbackError) { + if (previousPlaybackInfo.error != playbackInfo.error) { listeners.queueEvent( EVENT_PLAYER_ERROR - ) { listener: Player.Listener -> + ) { listener: Listener -> listener.onPlayerErrorChanged( - playbackInfo.playbackError + playbackInfo.error?.toPlaybackException() ) } - if (playbackInfo.playbackError != null) { + if (playbackInfo.error != null) { listeners.queueEvent( EVENT_PLAYER_ERROR - ) { listener: Player.Listener -> + ) { listener: Listener -> listener.onPlayerError( - playbackInfo.playbackError!! + playbackInfo.error.toPlaybackException() ) } } - }*/ + } - if (previousPlaybackInfo.isPlaying != playbackInfo.isPlaying) { + if (previousPlaybackInfo.state != playbackInfo.state) { listeners.queueEvent( EVENT_PLAYBACK_STATE_CHANGED - ) { listener: Player.Listener -> + ) { listener: Listener -> listener.onPlaybackStateChanged( - playbackInfo.state.value + playbackInfo.state.playerCode ) } } @@ -412,10 +704,10 @@ internal class TtsSessionAdapter( if (previousPlaybackInfo.playWhenReady != playbackInfo.playWhenReady) { listeners.queueEvent( EVENT_PLAY_WHEN_READY_CHANGED - ) { listener: Player.Listener -> + ) { listener: Listener -> listener.onPlayWhenReadyChanged( playbackInfo.playWhenReady, - if (playbackInfo.state == TtsPlayer.Playback.State.ENDED) + if (playbackInfo.state == TtsPlayer.Playback.State.Ended) PLAY_WHEN_READY_CHANGE_REASON_END_OF_MEDIA_ITEM else PLAY_WHEN_READY_CHANGE_REASON_USER_REQUEST @@ -428,24 +720,102 @@ internal class TtsSessionAdapter( if (isPlaying(previousPlaybackInfo) != isPlaying(playbackInfo)) { listeners.queueEvent( EVENT_IS_PLAYING_CHANGED - ) { listener: Player.Listener -> + ) { listener: Listener -> listener.onIsPlayingChanged(isPlaying(playbackInfo)) } } - /*if (previousPlaybackInfo.playbackParameters != playbackInfo.playbackParameters) { - listeners.queueEvent( + + listeners.flushEvents() + } + + private fun notifyListenersPlaybackParametersChanged( + previousPlaybackParameters: PlaybackParameters, + playbackParameters: PlaybackParameters + ) { + if (previousPlaybackParameters != playbackParameters) { + listeners.sendEvent( EVENT_PLAYBACK_PARAMETERS_CHANGED - ) { listener: Player.Listener -> + ) { listener: Listener -> listener.onPlaybackParametersChanged( - playbackInfo.playbackParameters + playbackParameters ) } - }*/ + } + } - listeners.flushEvents() + private fun createDeviceInfo(streamVolumeManager: TtsStreamVolumeManager): DeviceInfo { + val newDeviceInfo = DeviceInfo( + DeviceInfo.PLAYBACK_TYPE_LOCAL, + streamVolumeManager.minVolume, + streamVolumeManager.maxVolume + ) + deviceInfo = newDeviceInfo + return newDeviceInfo } private fun isPlaying(playbackInfo: TtsPlayer.Playback): Boolean { - return (playbackInfo.state == TtsPlayer.Playback.State.READY && playbackInfo.playWhenReady) + return (playbackInfo.state == TtsPlayer.Playback.State.Ready && playbackInfo.playWhenReady) + } + + private fun getRepeatModeForNavigation(): @RepeatMode Int { + val repeatMode = repeatMode + return if (repeatMode == REPEAT_MODE_ONE) REPEAT_MODE_OFF else repeatMode + } + + private inner class StreamVolumeManagerListener : TtsStreamVolumeManager.Listener { + override fun onStreamTypeChanged(streamType: @StreamType Int) { + val newDeviceInfo = createDeviceInfo(volumeManager) + if (newDeviceInfo != deviceInfo) { + listeners.sendEvent( + EVENT_DEVICE_INFO_CHANGED + ) { listener: Listener -> + listener.onDeviceInfoChanged( + newDeviceInfo + ) + } + } + } + + override fun onStreamVolumeChanged(streamVolume: Int, streamMuted: Boolean) { + listeners.sendEvent( + EVENT_DEVICE_VOLUME_CHANGED + ) { listener: Listener -> + listener.onDeviceVolumeChanged( + streamVolume, + streamMuted + ) + } + } + } + + private val TtsPlayer.Playback.State.playerCode get() = when (this) { + TtsPlayer.Playback.State.Ready -> STATE_READY + TtsPlayer.Playback.State.Ended -> STATE_ENDED + TtsPlayer.Playback.State.Error -> STATE_IDLE + } + + @Suppress("Unchecked_cast") + private fun TtsPlayer.Error.toPlaybackException(): PlaybackException = when (this) { + is TtsPlayer.Error.EngineError<*> -> mapEngineError(error as E) + is TtsPlayer.Error.ContentError -> when (exception) { + is Resource.Exception.BadRequest -> + PlaybackException(exception.message, exception.cause, ERROR_CODE_IO_BAD_HTTP_STATUS) + is Resource.Exception.NotFound -> + PlaybackException(exception.message, exception.cause, ERROR_CODE_IO_BAD_HTTP_STATUS) + is Resource.Exception.Forbidden -> + PlaybackException(exception.message, exception.cause, ERROR_CODE_DRM_DISALLOWED_OPERATION) + is Resource.Exception.Unavailable -> + PlaybackException(exception.message, exception.cause, ERROR_CODE_IO_NETWORK_CONNECTION_FAILED) + is Resource.Exception.Offline -> + PlaybackException(exception.message, exception.cause, ERROR_CODE_IO_NETWORK_CONNECTION_FAILED) + is Resource.Exception.OutOfMemory -> + PlaybackException(exception.message, exception.cause, ERROR_CODE_UNSPECIFIED) + is Resource.Exception.Cancelled -> + PlaybackException(exception.message, exception.cause, ERROR_CODE_IO_UNSPECIFIED) + is Resource.Exception.Other -> + PlaybackException(exception.message, exception.cause, ERROR_CODE_UNSPECIFIED) + else -> + PlaybackException(exception.message, exception.cause, ERROR_CODE_UNSPECIFIED) + } } } diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsSessionTimeline.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsSessionTimeline.kt index 85f232d9ee..7e129b2203 100644 --- a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsSessionTimeline.kt +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsSessionTimeline.kt @@ -31,6 +31,7 @@ internal class TtsSessionTimeline( window.firstPeriodIndex = windowIndex window.lastPeriodIndex = windowIndex window.mediaItem = mediaItems[windowIndex] + window.isSeekable = false return window } diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsSettings.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsSettings.kt deleted file mode 100644 index ebe4731983..0000000000 --- a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsSettings.kt +++ /dev/null @@ -1,17 +0,0 @@ -/* - * Copyright 2022 Readium Foundation. All rights reserved. - * Use of this source code is governed by the BSD-style license - * available in the top-level LICENSE file of the project. - */ - -package org.readium.r2.navigator.media3.tts2 - -import org.readium.r2.navigator.preferences.Configurable -import org.readium.r2.shared.ExperimentalReadiumApi -import org.readium.r2.shared.util.Language - -@ExperimentalReadiumApi -interface TtsSettings : Configurable.Settings { - - val language: Language? -} diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsStreamVolumeManager.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsStreamVolumeManager.kt new file mode 100644 index 0000000000..f78456c267 --- /dev/null +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsStreamVolumeManager.kt @@ -0,0 +1,219 @@ +/* + * Copyright 2020 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.readium.r2.navigator.media3.tts2 + +import android.content.BroadcastReceiver +import android.content.Context +import android.content.Intent +import android.content.IntentFilter +import android.media.AudioManager +import android.os.Handler +import androidx.media3.common.C +import androidx.media3.common.util.Assertions +import androidx.media3.common.util.Log +import androidx.media3.common.util.Util + +@androidx.annotation.OptIn(androidx.media3.common.util.UnstableApi::class) +/** A manager that wraps [AudioManager] to control/listen audio stream volume. */ /* package */ +internal class TtsStreamVolumeManager(context: Context, eventHandler: Handler, listener: Listener) { + /** A listener for changes in the manager. */ + interface Listener { + /** Called when the audio stream type is changed. */ + fun onStreamTypeChanged(streamType: @C.StreamType Int) + + /** Called when the audio stream volume or mute state is changed. */ + fun onStreamVolumeChanged(streamVolume: Int, streamMuted: Boolean) + } + + private val applicationContext: Context + private val eventHandler: Handler + private val listener: Listener + private val audioManager: AudioManager + private var receiver: VolumeChangeReceiver? = null + private var streamType: @C.StreamType Int + private var volume: Int + private var muted: Boolean + + /** Creates a manager. */ + init { + applicationContext = context.applicationContext + this.eventHandler = eventHandler + this.listener = listener + audioManager = Assertions.checkStateNotNull( + applicationContext.getSystemService(Context.AUDIO_SERVICE) as AudioManager + ) + streamType = C.STREAM_TYPE_DEFAULT + volume = getVolumeFromManager(audioManager, streamType) + muted = getMutedFromManager(audioManager, streamType) + val receiver = VolumeChangeReceiver() + val filter = IntentFilter(VOLUME_CHANGED_ACTION) + try { + applicationContext.registerReceiver(receiver, filter) + this.receiver = receiver + } catch (e: RuntimeException) { + Log.w(TAG, "Error registering stream volume receiver", e) + } + } + + /** Sets the audio stream type. */ + fun setStreamType(streamType: @C.StreamType Int) { + if (this.streamType == streamType) { + return + } + this.streamType = streamType + updateVolumeAndNotifyIfChanged() + listener.onStreamTypeChanged(streamType) + } + + /** + * Gets the minimum volume for the current audio stream. It can be changed if [ ][.setStreamType] is called. + */ + val minVolume: Int + get() = if (Util.SDK_INT >= 28) audioManager.getStreamMinVolume(streamType) else 0 + + /** + * Gets the maximum volume for the current audio stream. It can be changed if [ ][.setStreamType] is called. + */ + val maxVolume: Int + get() = audioManager.getStreamMaxVolume(streamType) + + /** Gets the current volume for the current audio stream. */ + fun getVolume(): Int { + return volume + } + + /** Gets whether the current audio stream is muted or not. */ + fun isMuted(): Boolean { + return muted + } + + /** + * Sets the volume with the given value for the current audio stream. The value should be between + * [.getMinVolume] and [.getMaxVolume], otherwise it will be ignored. + */ + fun setVolume(volume: Int) { + if (volume < minVolume || volume > maxVolume) { + return + } + audioManager.setStreamVolume(streamType, volume, VOLUME_FLAGS) + updateVolumeAndNotifyIfChanged() + } + + /** + * Increases the volume by one for the current audio stream. It will be ignored if the current + * volume is equal to [.getMaxVolume]. + */ + fun increaseVolume() { + if (volume >= maxVolume) { + return + } + audioManager.adjustStreamVolume(streamType, AudioManager.ADJUST_RAISE, VOLUME_FLAGS) + updateVolumeAndNotifyIfChanged() + } + + /** + * Decreases the volume by one for the current audio stream. It will be ignored if the current + * volume is equal to [.getMinVolume]. + */ + fun decreaseVolume() { + if (volume <= minVolume) { + return + } + audioManager.adjustStreamVolume(streamType, AudioManager.ADJUST_LOWER, VOLUME_FLAGS) + updateVolumeAndNotifyIfChanged() + } + + /** Sets the mute state of the current audio stream. */ + fun setMuted(muted: Boolean) { + if (Util.SDK_INT >= 23) { + audioManager.adjustStreamVolume( + streamType, + if (muted) AudioManager.ADJUST_MUTE else AudioManager.ADJUST_UNMUTE, + VOLUME_FLAGS + ) + } else { + @Suppress("Deprecation") + audioManager.setStreamMute(streamType, muted) + } + updateVolumeAndNotifyIfChanged() + } + + /** Releases the manager. It must be called when the manager is no longer required. */ + fun release() { + if (receiver != null) { + try { + applicationContext.unregisterReceiver(receiver) + } catch (e: RuntimeException) { + Log.w(TAG, "Error unregistering stream volume receiver", e) + } + receiver = null + } + } + + private fun updateVolumeAndNotifyIfChanged() { + val newVolume = getVolumeFromManager(audioManager, streamType) + val newMuted = getMutedFromManager(audioManager, streamType) + if (volume != newVolume || muted != newMuted) { + volume = newVolume + muted = newMuted + listener.onStreamVolumeChanged(newVolume, newMuted) + } + } + + private inner class VolumeChangeReceiver : BroadcastReceiver() { + override fun onReceive(context: Context, intent: Intent) { + eventHandler.post { updateVolumeAndNotifyIfChanged() } + } + } + + companion object { + private const val TAG = "StreamVolumeManager" + + // TODO(b/151280453): Replace the hidden intent action with an official one. + // Copied from AudioManager#VOLUME_CHANGED_ACTION + private const val VOLUME_CHANGED_ACTION = "android.media.VOLUME_CHANGED_ACTION" + + // TODO(b/153317944): Allow users to override these flags. + private const val VOLUME_FLAGS = AudioManager.FLAG_SHOW_UI + private fun getVolumeFromManager( + audioManager: AudioManager, + streamType: @C.StreamType Int + ): Int { + // AudioManager#getStreamVolume(int) throws an exception on some devices. See + // https://github.com/google/ExoPlayer/issues/8191. + return try { + audioManager.getStreamVolume(streamType) + } catch (e: RuntimeException) { + Log.w( + TAG, + "Could not retrieve stream volume for stream type $streamType", e + ) + audioManager.getStreamMaxVolume(streamType) + } + } + + private fun getMutedFromManager( + audioManager: AudioManager, + streamType: @C.StreamType Int + ): Boolean { + return if (Util.SDK_INT >= 23) { + audioManager.isStreamMute(streamType) + } else { + getVolumeFromManager(audioManager, streamType) == 0 + } + } + } +} diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsUtterance.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsUtterance.kt deleted file mode 100644 index 8da9a53b36..0000000000 --- a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsUtterance.kt +++ /dev/null @@ -1,17 +0,0 @@ -/* - * Copyright 2022 Readium Foundation. All rights reserved. - * Use of this source code is governed by the BSD-style license - * available in the top-level LICENSE file of the project. - */ - -package org.readium.r2.navigator.media3.tts2 - -import org.readium.r2.shared.ExperimentalReadiumApi -import org.readium.r2.shared.util.Language - -@ExperimentalReadiumApi -data class TtsUtterance( - val text: String, - val locator: TtsLocator, - val language: Language? -) diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/tts/AndroidTtsEngine.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/tts/AndroidTtsEngine.kt index cfe696fb59..501ede587d 100644 --- a/readium/navigator/src/main/java/org/readium/r2/navigator/tts/AndroidTtsEngine.kt +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/tts/AndroidTtsEngine.kt @@ -190,7 +190,7 @@ class AndroidTtsEngine( engine.setup() engine.setOnUtteranceProgressListener(Listener(id)) engine.speak(utterance.text, TextToSpeech.QUEUE_FLUSH, null, id) - } catch (e: Exception) { + } catch (e: java.lang.Exception) { finish(TtsEngine.Exception.wrap(e)) } } diff --git a/test-app/src/main/java/org/readium/r2/testapp/Application.kt b/test-app/src/main/java/org/readium/r2/testapp/Application.kt index 6a4b65a0c5..57aaa248c0 100755 --- a/test-app/src/main/java/org/readium/r2/testapp/Application.kt +++ b/test-app/src/main/java/org/readium/r2/testapp/Application.kt @@ -69,7 +69,7 @@ class Application : android.app.Application() { this@Application, readium, bookRepository, - navigatorPreferences + navigatorPreferences, ) } } diff --git a/test-app/src/main/java/org/readium/r2/testapp/reader/ReaderInitData.kt b/test-app/src/main/java/org/readium/r2/testapp/reader/ReaderInitData.kt index 296bbfcfcf..5a964c44ac 100644 --- a/test-app/src/main/java/org/readium/r2/testapp/reader/ReaderInitData.kt +++ b/test-app/src/main/java/org/readium/r2/testapp/reader/ReaderInitData.kt @@ -16,14 +16,12 @@ import org.readium.navigator.media2.MediaNavigator import org.readium.r2.navigator.epub.EpubNavigatorFactory import org.readium.r2.navigator.epub.EpubPreferences import org.readium.r2.navigator.media3.androidtts.AndroidTtsPreferences -import org.readium.r2.navigator.media3.androidtts.AndroidTtsPreferencesEditor -import org.readium.r2.navigator.media3.androidtts.AndroidTtsSettings -import org.readium.r2.navigator.media3.tts2.TtsNavigatorFactory import org.readium.r2.navigator.pdf.PdfNavigatorFactory import org.readium.r2.shared.ExperimentalReadiumApi import org.readium.r2.shared.publication.* import org.readium.r2.testapp.reader.preferences.PreferencesManager -import org.readium.r2.testapp.reader.tts.TtsService +import org.readium.r2.testapp.reader.tts.AndroidTtsNavigatorFactory +import org.readium.r2.testapp.reader.tts.TtsServiceFacade sealed class ReaderInitData { abstract val bookId: Long @@ -83,7 +81,7 @@ class DummyReaderInitData( } class TtsInitData( - val sessionBinder: TtsService.Binder, - val ttsNavigatorFactory: TtsNavigatorFactory, + val ttsServiceFacade: TtsServiceFacade, + val ttsNavigatorFactory: AndroidTtsNavigatorFactory, val preferencesManager: PreferencesManager, ) diff --git a/test-app/src/main/java/org/readium/r2/testapp/reader/ReaderRepository.kt b/test-app/src/main/java/org/readium/r2/testapp/reader/ReaderRepository.kt index 43f16de7b5..427d29f3dc 100644 --- a/test-app/src/main/java/org/readium/r2/testapp/reader/ReaderRepository.kt +++ b/test-app/src/main/java/org/readium/r2/testapp/reader/ReaderRepository.kt @@ -32,7 +32,7 @@ import org.readium.r2.testapp.bookshelf.BookRepository import org.readium.r2.testapp.reader.preferences.AndroidTtsPreferencesManagerFactory import org.readium.r2.testapp.reader.preferences.EpubPreferencesManagerFactory import org.readium.r2.testapp.reader.preferences.PdfiumPreferencesManagerFactory -import org.readium.r2.testapp.reader.tts.TtsService +import org.readium.r2.testapp.reader.tts.TtsServiceFacade import timber.log.Timber /** @@ -54,6 +54,9 @@ class ReaderRepository( private val repository: MutableMap = mutableMapOf() + private val ttsServiceFacade: TtsServiceFacade = + TtsServiceFacade(application) + operator fun get(bookId: Long): ReaderInitData? = repository[bookId] @@ -211,27 +214,24 @@ class ReaderRepository( bookId: Long, publication: Publication, ): TtsInitData? { - TtsService.start(application) - val sessionBinder = TtsService.bind(application) val preferencesManager = AndroidTtsPreferencesManagerFactory(preferencesDataStore) .createPreferenceManager(bookId) val ttsEngine = AndroidTtsEngineProvider(application) val navigatorFactory = TtsNavigatorFactory(application, publication, ttsEngine) ?: return null - return TtsInitData(sessionBinder, navigatorFactory, preferencesManager) + return TtsInitData(ttsServiceFacade, navigatorFactory, preferencesManager) } - fun close(bookId: Long) { + suspend fun close(bookId: Long) { Timber.d("Closing Publication") when (val initData = repository.remove(bookId)) { is MediaReaderInitData -> { - initData.publication.close() initData.sessionBinder.closeNavigator() MediaService.stop(application) + initData.publication.close() } is VisualReaderInitData -> { + initData.ttsInitData?.ttsServiceFacade?.closeSession() initData.publication.close() - initData.ttsInitData?.sessionBinder?.closeNavigator() - TtsService.stop(application) } null, is DummyReaderInitData -> { // Do nothing diff --git a/test-app/src/main/java/org/readium/r2/testapp/reader/ReaderViewModel.kt b/test-app/src/main/java/org/readium/r2/testapp/reader/ReaderViewModel.kt index 0be6cb2e81..2ddf3868ef 100644 --- a/test-app/src/main/java/org/readium/r2/testapp/reader/ReaderViewModel.kt +++ b/test-app/src/main/java/org/readium/r2/testapp/reader/ReaderViewModel.kt @@ -74,8 +74,10 @@ class ReaderViewModel( ) fun close() { - tts?.stop() - readerRepository.close(bookId) + viewModelScope.launch { + tts?.stop() + readerRepository.close(bookId) + } } fun saveProgression(locator: Locator) = viewModelScope.launch { diff --git a/test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsEngine.kt b/test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsEngine.kt new file mode 100644 index 0000000000..b015c1c681 --- /dev/null +++ b/test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsEngine.kt @@ -0,0 +1,21 @@ +/* + * Copyright 2022 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + +package org.readium.r2.testapp.reader.tts + +import org.readium.r2.navigator.media3.androidtts.AndroidTtsEngine +import org.readium.r2.navigator.media3.androidtts.AndroidTtsPreferences +import org.readium.r2.navigator.media3.androidtts.AndroidTtsPreferencesEditor +import org.readium.r2.navigator.media3.androidtts.AndroidTtsSettings +import org.readium.r2.navigator.media3.tts2.TtsNavigator +import org.readium.r2.navigator.media3.tts2.TtsNavigatorFactory +import org.readium.r2.shared.ExperimentalReadiumApi + +@OptIn(ExperimentalReadiumApi::class) +typealias AndroidTtsNavigatorFactory = TtsNavigatorFactory + +@OptIn(ExperimentalReadiumApi::class) +typealias AndroidTtsNavigator = TtsNavigator diff --git a/test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsService.kt b/test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsService.kt index 3e2e8c379b..7c47c81b47 100644 --- a/test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsService.kt +++ b/test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsService.kt @@ -22,9 +22,6 @@ import kotlinx.coroutines.* import kotlinx.coroutines.flow.launchIn import kotlinx.coroutines.flow.onEach import kotlinx.coroutines.flow.sample -import org.readium.r2.navigator.media3.androidtts.AndroidTtsPreferences -import org.readium.r2.navigator.media3.androidtts.AndroidTtsSettings -import org.readium.r2.navigator.media3.tts2.TtsNavigator import org.readium.r2.shared.ExperimentalReadiumApi import org.readium.r2.testapp.reader.ReaderActivityContract import org.readium.r2.testapp.utils.LifecycleMedia3SessionService @@ -36,7 +33,7 @@ class TtsService : LifecycleMedia3SessionService() { class Session( val bookId: Long, - val navigator: TtsNavigator, + val navigator: AndroidTtsNavigator, val mediaSession: MediaSession, ) { val coroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.Main) @@ -52,7 +49,7 @@ class TtsService : LifecycleMedia3SessionService() { var session: Session? = null - fun closeNavigator() { + fun closeSession() { stopForeground(true) session?.mediaSession?.release() session?.navigator?.close() @@ -60,8 +57,8 @@ class TtsService : LifecycleMedia3SessionService() { session = null } - fun bindNavigator( - navigator: TtsNavigator, + fun openSession( + navigator: AndroidTtsNavigator, bookId: Long ): Session { val activityIntent = createSessionActivityIntent(bookId) @@ -188,7 +185,7 @@ class TtsService : LifecycleMedia3SessionService() { super.onTaskRemoved(rootIntent) Timber.d("Task removed. Stopping session and service.") // Close the navigator to allow the service to be stopped. - binder.closeNavigator() + binder.closeSession() stopSelf() } diff --git a/test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsServiceFacade.kt b/test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsServiceFacade.kt new file mode 100644 index 0000000000..1477888f2f --- /dev/null +++ b/test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsServiceFacade.kt @@ -0,0 +1,53 @@ +package org.readium.r2.testapp.reader.tts + +import android.app.Application +import kotlinx.coroutines.sync.Mutex +import kotlinx.coroutines.sync.withLock +import org.readium.r2.shared.ExperimentalReadiumApi + +@OptIn(ExperimentalReadiumApi::class) +class TtsServiceFacade( + private val application: Application +) { + + private val mutex = Mutex() + + private var binder: TtsService.Binder? = null + + fun sessionNow(): TtsService.Session? = + binder?.session + + suspend fun getSession(): TtsService.Session? = mutex.withLock { + binder?.session + } + + suspend fun openSession( + bookId: Long, + navigator: AndroidTtsNavigator + ): TtsService.Session = mutex.withLock { + startService() + val binderNow = checkNotNull(binder) + binderNow.openSession(navigator, bookId) + } + + private suspend fun startService() { + if (binder != null) + return + + TtsService.start(application) + binder = TtsService.bind(application) + } + + suspend fun closeSession() = mutex.withLock { + binder?.closeSession() + stopService() + } + + private fun stopService() { + if (binder == null) + return + + binder = null + TtsService.stop(application) + } +} diff --git a/test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsViewModel.kt b/test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsViewModel.kt index b2c3177b32..ef67158608 100644 --- a/test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsViewModel.kt +++ b/test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsViewModel.kt @@ -7,19 +7,16 @@ package org.readium.r2.testapp.reader.tts import android.content.Context -import kotlinx.coroutines.CoroutineScope -import kotlinx.coroutines.Job +import kotlinx.coroutines.* import kotlinx.coroutines.channels.Channel import kotlinx.coroutines.flow.* -import kotlinx.coroutines.launch import org.readium.r2.navigator.Navigator import org.readium.r2.navigator.VisualNavigator import org.readium.r2.navigator.media3.androidtts.AndroidTtsPreferences import org.readium.r2.navigator.media3.androidtts.AndroidTtsPreferencesEditor -import org.readium.r2.navigator.media3.androidtts.AndroidTtsSettings import org.readium.r2.navigator.media3.api.MediaNavigator +import org.readium.r2.navigator.media3.api.SynchronizedMediaNavigator import org.readium.r2.navigator.media3.tts2.TtsNavigator -import org.readium.r2.navigator.media3.tts2.TtsNavigatorFactory import org.readium.r2.navigator.media3.tts2.TtsNavigatorListener import org.readium.r2.navigator.tts.PublicationSpeechSynthesizer import org.readium.r2.shared.DelicateReadiumApi @@ -44,8 +41,8 @@ class TtsViewModel private constructor( private val viewModelScope: CoroutineScope, private val bookId: Long, private val publication: Publication, - private val ttsNavigatorFactory: TtsNavigatorFactory, - private val ttsSessionBinder: TtsService.Binder, + private val ttsNavigatorFactory: AndroidTtsNavigatorFactory, + private val ttsServiceFacade: TtsServiceFacade, private val preferencesManager: PreferencesManager, private val createPreferencesEditor: (AndroidTtsPreferences) -> AndroidTtsPreferencesEditor ) { @@ -68,7 +65,7 @@ class TtsViewModel private constructor( bookId = readerInitData.bookId, publication = readerInitData.publication, ttsNavigatorFactory = readerInitData.ttsInitData.ttsNavigatorFactory, - ttsSessionBinder = readerInitData.ttsInitData.sessionBinder, + ttsServiceFacade = readerInitData.ttsInitData.ttsServiceFacade, preferencesManager = readerInitData.ttsInitData.preferencesManager, createPreferencesEditor = readerInitData.ttsInitData.ttsNavigatorFactory::createTtsPreferencesEditor ) @@ -112,7 +109,7 @@ class TtsViewModel private constructor( * Current state of the view model. */ private val _state: MutableStateFlow = MutableStateFlow( - stateFromPlayback(ttsNavigator?.playback?.value) + stateFromPlayback(navigatorNow?.playback?.value, navigatorNow?.utterance?.value) ) val state: StateFlow = _state.asStateFlow() @@ -120,24 +117,25 @@ class TtsViewModel private constructor( private val _events: Channel = Channel(Channel.BUFFERED) val events: Flow = _events.receiveAsFlow() - private val ttsSession: TtsService.Session? get() = - ttsSessionBinder.session + private val navigatorNow: AndroidTtsNavigator? get() = + ttsServiceFacade.sessionNow()?.navigator - private val ttsNavigator: TtsNavigator? get() = - ttsSession?.navigator - - private var binding: Binding? = - ttsSession?.let { bindSession(it) } + private var binding: Deferred = + viewModelScope.async { + ttsServiceFacade.getSession()?.let { bindSession(it) } + } /** * Starts the TTS using the first visible locator in the given [navigator]. */ fun start(navigator: Navigator) { - if (ttsNavigator != null) return - viewModelScope.launch { + if (ttsServiceFacade.getSession() != null) + return@launch + val session = openSession(navigator) - binding = bindSession(session) + binding.cancelAndJoin() + binding = async { bindSession(session) } } } @@ -149,10 +147,6 @@ class TtsViewModel private constructor( override fun onStopRequested() { stop() } - - override fun onPlaybackException() { - TODO("Not yet implemented") - } } val ttsNavigator = ttsNavigatorFactory.createNavigator( @@ -164,16 +158,17 @@ class TtsViewModel private constructor( // playWhenReady must be true for the MediaSessionService to call Service.startForeground // and prevent crashing ttsNavigator.play() - return ttsSessionBinder.bindNavigator(ttsNavigator, bookId) + return ttsServiceFacade.openSession(bookId, ttsNavigator) } private fun bindSession( ttsSession: TtsService.Session ): Binding { val playbackJob = ttsSession.navigator.playback - .onEach { playback -> - Timber.d("new TTS playback $playback") - _state.value = stateFromPlayback(playback) + .combine(ttsSession.navigator.utterance) { playback, utterance -> + stateFromPlayback(playback, utterance) + }.onEach { state -> + _state.value = state Timber.d("new TTS state ${_state.value}") }.launchIn(viewModelScope) @@ -184,51 +179,53 @@ class TtsViewModel private constructor( return Binding(playbackJob, preferencesJob) } - private fun stateFromPlayback(playback: TtsNavigator.Playback?): State { - if (playback == null) + private fun stateFromPlayback( + playback: MediaNavigator.Playback?, + utterance: SynchronizedMediaNavigator.Utterance? + ): State { + if (playback == null || utterance == null) return State() return State( showControls = playback.state != MediaNavigator.State.Ended, - isPlaying = playback.state == MediaNavigator.State.Playing, - playingWordRange = playback.token, - playingUtterance = playback.locator + isPlaying = playback.playWhenReady, + playingWordRange = utterance.rangeLocator, + playingUtterance = utterance.locator ) } fun stop() { - ttsSession ?: return + viewModelScope.launch { + binding.await()?.apply { + playbackJob.cancel() + submitSettingsJob.cancel() + } - _state.value = State( - showControls = false, - isPlaying = false, - playingWordRange = null, - playingUtterance = null, - ) + ttsServiceFacade.closeSession() - binding?.apply { - playbackJob.cancel() - submitSettingsJob.cancel() + _state.value = State( + showControls = false, + isPlaying = false, + playingWordRange = null, + playingUtterance = null, + ) } - binding = null - - ttsSessionBinder.closeNavigator() } fun play() { - ttsNavigator?.play() + navigatorNow?.play() } fun pause() { - ttsNavigator?.pause() + navigatorNow?.pause() } fun previous() { - ttsNavigator?.goBackward() + navigatorNow?.goBackward() } fun next() { - ttsNavigator?.goForward() + navigatorNow?.goForward() } fun commit() { From 92522e4b9e24c985fb167f221030624a0410268e Mon Sep 17 00:00:00 2001 From: Quentin Gliosca Date: Thu, 26 Jan 2023 14:25:09 +0100 Subject: [PATCH 05/27] Specify correctly iterators --- .../media3/tts2/TtsContentIterator.kt | 12 +++-- .../r2/navigator/media3/tts2/TtsNavigator.kt | 2 + .../r2/navigator/media3/tts2/TtsPlayer.kt | 8 +++- .../iterators/HtmlResourceContentIterator.kt | 38 +++++++-------- .../iterators/PublicationContentIterator.kt | 3 ++ .../org/readium/r2/shared/util/CursorList.kt | 46 +++++++++---------- 6 files changed, 62 insertions(+), 47 deletions(-) diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsContentIterator.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsContentIterator.kt index 70f9b800fa..d6a9961897 100644 --- a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsContentIterator.kt +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsContentIterator.kt @@ -18,6 +18,12 @@ import org.readium.r2.shared.util.CursorList import org.readium.r2.shared.util.Language @ExperimentalReadiumApi + +/** + * A Content Iterator able to provide short utterances. + * + * It is not safe for several coroutines to use this at the same time. + */ internal class TtsContentIterator( private val publication: Publication, private val tokenizerFactory: (language: Language?) -> ContentTokenizer, @@ -137,7 +143,7 @@ internal class TtsContentIterator( list = nextUtterances, startIndex = when (direction) { Direction.Forward -> 0 - Direction.Backward -> nextUtterances.size - 1 + Direction.Backward -> nextUtterances.size + 1 } ) @@ -203,8 +209,8 @@ internal class TtsContentIterator( private fun CursorList.nextIn(direction: Direction): E? = when (direction) { - Direction.Forward -> next() - Direction.Backward -> previous() + Direction.Forward -> if (hasNext()) next() else null + Direction.Backward -> if (hasPrevious()) previous() else null } private suspend fun Content.Iterator.nextIn(direction: Direction): Content.Element? = diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsNavigator.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsNavigator.kt index ab4a81d942..c0ba2d12b8 100644 --- a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsNavigator.kt +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsNavigator.kt @@ -50,6 +50,8 @@ class TtsNavigator, return null } + Timber.d("initialLocator $initialLocator") + val actualInitialPreferences = initialPreferences ?: ttsEngineProvider.createEmptyPreferences() diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsPlayer.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsPlayer.kt index c6d5081c89..d1ace9e5dc 100644 --- a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsPlayer.kt +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsPlayer.kt @@ -49,7 +49,7 @@ internal class TtsPlayer, val previousUtterance = previousUtterance() val currentUtterance = nextUtterance() - return if (currentUtterance != null) { + val context = if (currentUtterance != null) { Context( previousUtterance = previousUtterance, currentUtterance = currentUtterance, @@ -59,11 +59,15 @@ internal class TtsPlayer, } else { Context( previousUtterance = previousUtterance(), - currentUtterance = previousUtterance ?: return null, + currentUtterance = nextUtterance() ?: return null, nextUtterance = null, ended = true ) } + + Timber.d("startContext $context") + + return context } } diff --git a/readium/shared/src/main/java/org/readium/r2/shared/publication/services/content/iterators/HtmlResourceContentIterator.kt b/readium/shared/src/main/java/org/readium/r2/shared/publication/services/content/iterators/HtmlResourceContentIterator.kt index e581b772f1..5340e13398 100644 --- a/readium/shared/src/main/java/org/readium/r2/shared/publication/services/content/iterators/HtmlResourceContentIterator.kt +++ b/readium/shared/src/main/java/org/readium/r2/shared/publication/services/content/iterators/HtmlResourceContentIterator.kt @@ -65,8 +65,15 @@ class HtmlResourceContentIterator( private var currentElement: ContentWithDelta? = null override suspend fun hasPrevious(): Boolean { - currentElement = nextBy(-1) - return currentElement != null + val elements = elements() + val index = (currentIndex ?: elements.startIndex) - 1 + + val content = elements.elements.getOrNull(index) + ?: return false + + currentIndex = index + currentElement = ContentWithDelta(content, -1) + return true } override fun previous(): Content.Element = @@ -75,27 +82,22 @@ class HtmlResourceContentIterator( ?: throw IllegalStateException("Called previous() without a successful call to hasPrevious() first") override suspend fun hasNext(): Boolean { - currentElement = nextBy(+1) - return currentElement != null - } - - override fun next(): Content.Element = - currentElement - ?.takeIf { it.delta == +1 }?.element - ?: throw IllegalStateException("Called next() without a successful call to hasNext() first") - - private suspend fun nextBy(delta: Int): ContentWithDelta? { val elements = elements() - val index = currentIndex?.let { it + delta } - ?: elements.startIndex + val index = (currentIndex ?: (elements.startIndex - 1)) + 1 val content = elements.elements.getOrNull(index) - ?: return null + ?: return false currentIndex = index - return ContentWithDelta(content, delta) + currentElement = ContentWithDelta(content, +1) + return true } + override fun next(): Content.Element = + currentElement + ?.takeIf { it.delta == +1 }?.element + ?: throw IllegalStateException("Called next() without a successful call to hasNext() first") + private var currentIndex: Int? = null private suspend fun elements(): ParsedElements = @@ -123,7 +125,7 @@ class HtmlResourceContentIterator( } /** - * Holds the result of parsing the HTML resource into a list of [ContentElement]. + * Holds the result of parsing the HTML resource into a list of [Content.Element]. * * The [startIndex] will be calculated from the element matched by the base [locator], if * possible. Defaults to 0. @@ -140,7 +142,7 @@ class HtmlResourceContentIterator( fun result() = ParsedElements( elements = elements, - startIndex = if (baseLocator.locations.progression == 1.0) elements.size - 1 + startIndex = if (baseLocator.locations.progression == 1.0) elements.size else startIndex ) diff --git a/readium/shared/src/main/java/org/readium/r2/shared/publication/services/content/iterators/PublicationContentIterator.kt b/readium/shared/src/main/java/org/readium/r2/shared/publication/services/content/iterators/PublicationContentIterator.kt index 6127040281..aadddfef24 100644 --- a/readium/shared/src/main/java/org/readium/r2/shared/publication/services/content/iterators/PublicationContentIterator.kt +++ b/readium/shared/src/main/java/org/readium/r2/shared/publication/services/content/iterators/PublicationContentIterator.kt @@ -14,6 +14,7 @@ import org.readium.r2.shared.publication.Publication import org.readium.r2.shared.publication.indexOfFirstWithHref import org.readium.r2.shared.publication.services.content.Content import org.readium.r2.shared.util.Either +import timber.log.Timber /** * Creates a [Content.Iterator] instance for the [Resource], starting from the given [Locator]. @@ -62,6 +63,7 @@ class PublicationContentIterator( override suspend fun hasPrevious(): Boolean { currentElement = nextIn(Direction.Backward) + Timber.d("hasPrevious ${currentElement?.element?.locator}") return currentElement != null } @@ -72,6 +74,7 @@ class PublicationContentIterator( override suspend fun hasNext(): Boolean { currentElement = nextIn(Direction.Forward) + Timber.d("hasNext ${currentElement?.element?.locator}") return currentElement != null } diff --git a/readium/shared/src/main/java/org/readium/r2/shared/util/CursorList.kt b/readium/shared/src/main/java/org/readium/r2/shared/util/CursorList.kt index cd23198885..c5f28ba4f0 100644 --- a/readium/shared/src/main/java/org/readium/r2/shared/util/CursorList.kt +++ b/readium/shared/src/main/java/org/readium/r2/shared/util/CursorList.kt @@ -8,40 +8,38 @@ import org.readium.r2.shared.InternalReadiumApi @InternalReadiumApi class CursorList( private val list: List = emptyList(), - private val startIndex: Int = 0 + startIndex: Int = 0 ) : List by list { - private var index: Int? = null - /** - * Returns the current element. - */ - fun current(): E? = - moveAndGet(index ?: startIndex) + private var index: Int = startIndex - 1 + + fun hasPrevious(): Boolean { + return index > 0 + } /** * Moves the cursor backward and returns the element, or null when reaching the beginning. */ - fun previous(): E? = - moveAndGet( - index - ?.let { it - 1 } - ?: startIndex - ) + fun previous(): E? { + if (!hasPrevious()) + return null + + index-- + return list[index] + } + + fun hasNext(): Boolean { + return index + 1 < list.size + } /** * Moves the cursor forward and returns the element, or null when reaching the end. */ - fun next(): E? = - moveAndGet( - index?.let { it + 1 } - ?: startIndex - ) - - private fun moveAndGet(index: Int): E? { - if (!list.indices.contains(index)) { + fun next(): E? { + if (!hasNext()) return null - } - this.index = index - return get(index) + + index++ + return list[index] } } From b8379afdafc75240ebc1da7dc1ad8f74e0d7ee23 Mon Sep 17 00:00:00 2001 From: Quentin Gliosca Date: Thu, 26 Jan 2023 15:50:47 +0100 Subject: [PATCH 06/27] Upgrade to media3-alpha03 --- gradle/libs.versions.toml | 2 +- .../navigator/epub/EpubNavigatorFragment.kt | 46 +- .../media3/androidtts/AndroidTtsEngine.kt | 2 +- .../androidtts/AndroidTtsEngineProvider.kt | 2 +- .../androidtts/AndroidTtsPreferences.kt | 2 +- .../media3/androidtts/AndroidTtsSettings.kt | 2 +- .../{tts2 => tts}/TtsContentIterator.kt | 8 +- .../media3/{tts2 => tts}/TtsEngine.kt | 2 +- .../media3/{tts2 => tts}/TtsEngineFacade.kt | 2 +- .../media3/{tts2 => tts}/TtsEngineProvider.kt | 2 +- .../media3/{tts2 => tts}/TtsNavigator.kt | 2 +- .../{tts2 => tts}/TtsNavigatorFactory.kt | 2 +- .../{tts2 => tts}/TtsNavigatorInternal.kt | 2 +- .../{tts2 => tts}/TtsNavigatorListener.kt | 2 +- .../media3/{tts2 => tts}/TtsPlayer.kt | 2 +- .../media3/{tts2 => tts}/TtsSessionAdapter.kt | 15 +- .../{tts2 => tts}/TtsSessionTimeline.kt | 2 +- .../{tts2 => tts}/TtsStreamVolumeManager.kt | 2 +- .../r2/navigator/tts/AndroidTtsEngine.kt | 288 --------- .../tts/PublicationSpeechSynthesizer.kt | 557 ------------------ .../org/readium/r2/navigator/tts/TtsEngine.kt | 149 ----- .../publication/services/content/Content.kt | 3 + .../iterators/PublicationContentIterator.kt | 3 - .../org/readium/r2/shared/util/CursorList.kt | 13 +- test-app/build.gradle.kts | 1 + .../r2/testapp/reader/ReaderRepository.kt | 2 +- .../r2/testapp/reader/tts/TtsEngine.kt | 4 +- .../r2/testapp/reader/tts/TtsViewModel.kt | 5 +- 28 files changed, 82 insertions(+), 1042 deletions(-) rename readium/navigator/src/main/java/org/readium/r2/navigator/media3/{tts2 => tts}/TtsContentIterator.kt (97%) rename readium/navigator/src/main/java/org/readium/r2/navigator/media3/{tts2 => tts}/TtsEngine.kt (97%) rename readium/navigator/src/main/java/org/readium/r2/navigator/media3/{tts2 => tts}/TtsEngineFacade.kt (98%) rename readium/navigator/src/main/java/org/readium/r2/navigator/media3/{tts2 => tts}/TtsEngineProvider.kt (96%) rename readium/navigator/src/main/java/org/readium/r2/navigator/media3/{tts2 => tts}/TtsNavigator.kt (99%) rename readium/navigator/src/main/java/org/readium/r2/navigator/media3/{tts2 => tts}/TtsNavigatorFactory.kt (98%) rename readium/navigator/src/main/java/org/readium/r2/navigator/media3/{tts2 => tts}/TtsNavigatorInternal.kt (99%) rename readium/navigator/src/main/java/org/readium/r2/navigator/media3/{tts2 => tts}/TtsNavigatorListener.kt (84%) rename readium/navigator/src/main/java/org/readium/r2/navigator/media3/{tts2 => tts}/TtsPlayer.kt (99%) rename readium/navigator/src/main/java/org/readium/r2/navigator/media3/{tts2 => tts}/TtsSessionAdapter.kt (98%) rename readium/navigator/src/main/java/org/readium/r2/navigator/media3/{tts2 => tts}/TtsSessionTimeline.kt (97%) rename readium/navigator/src/main/java/org/readium/r2/navigator/media3/{tts2 => tts}/TtsStreamVolumeManager.kt (99%) delete mode 100644 readium/navigator/src/main/java/org/readium/r2/navigator/tts/AndroidTtsEngine.kt delete mode 100644 readium/navigator/src/main/java/org/readium/r2/navigator/tts/PublicationSpeechSynthesizer.kt delete mode 100644 readium/navigator/src/main/java/org/readium/r2/navigator/tts/TtsEngine.kt diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index f601553cf2..886bd8d1c3 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -23,7 +23,7 @@ androidx-lifecycle = "2.5.1" androidx-lifecycle-extensions = "2.2.0" androidx-media = "1.6.0" androidx-media2 = "1.2.1" -androidx-media3 = "1.0.0-beta02" +androidx-media3 = "1.0.0-beta03" androidx-navigation = "2.5.2" androidx-paging = "3.1.1" androidx-recyclerview = "1.2.1" diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/epub/EpubNavigatorFragment.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/epub/EpubNavigatorFragment.kt index 43f298df56..8a3fb9804b 100644 --- a/readium/navigator/src/main/java/org/readium/r2/navigator/epub/EpubNavigatorFragment.kt +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/epub/EpubNavigatorFragment.kt @@ -86,7 +86,7 @@ typealias JavascriptInterfaceFactory = (resource: Link) -> Any? * * To use this [Fragment], create a factory with `EpubNavigatorFragment.createFactory()`. */ -@OptIn(ExperimentalDecorator::class, ExperimentalReadiumApi::class) +@OptIn(ExperimentalDecorator::class, ExperimentalReadiumApi::class, DelicateReadiumApi::class) class EpubNavigatorFragment internal constructor( override val publication: Publication, private val baseUrl: String?, @@ -132,6 +132,22 @@ class EpubNavigatorFragment internal constructor( @ExperimentalReadiumApi var readiumCssRsProperties: RsProperties, + /** + * When disabled, the Android web view's `WebSettings.textZoom` will be used to adjust the + * font size, instead of using the Readium CSS's `--USER__fontSize` variable. + * + * `WebSettings.textZoom` will work with more publications than `--USER__fontSize`, even the + * ones poorly authored. However, the page width is not adjusted when changing the font + * size to keep the optimal line length. + * + * See: + * - https://github.com/readium/mobile/issues/5 + * - https://github.com/readium/mobile/issues/1#issuecomment-652431984 + */ + @ExperimentalReadiumApi + @DelicateReadiumApi + var useReadiumCssFontSize: Boolean = true, + /** * Supported HTML decoration templates. */ @@ -492,29 +508,32 @@ class EpubNavigatorFragment internal constructor( } private fun onSettingsChange(previous: EpubSettings, new: EpubSettings) { - if (viewModel.layout == EpubLayout.FIXED) return - - if (previous.fontSize != new.fontSize) { - r2PagerAdapter?.setFontSize(new.fontSize) - } if (previous.effectiveBackgroundColor != new.effectiveBackgroundColor) { resourcePager.setBackgroundColor(new.effectiveBackgroundColor) } + + if (viewModel.layout == EpubLayout.REFLOWABLE) { + if (previous.fontSize != new.fontSize) { + r2PagerAdapter?.setFontSize(new.fontSize) + } + } } private fun R2PagerAdapter.setFontSize(fontSize: Double) { - r2PagerAdapter?.mFragments?.forEach { _, fragment -> + if (config.useReadiumCssFontSize) return + + mFragments.forEach { _, fragment -> (fragment as? R2EpubPageFragment)?.setFontSize(fontSize) } } private inner class PagerAdapterListener : R2PagerAdapter.Listener { override fun onCreatePageFragment(fragment: Fragment) { - if (viewModel.layout == EpubLayout.FIXED) { - return + if (viewModel.layout == EpubLayout.REFLOWABLE) { + if (!config.useReadiumCssFontSize) { + (fragment as? R2EpubPageFragment)?.setFontSize(settings.value.fontSize) + } } - - (fragment as? R2EpubPageFragment)?.setFontSize(settings.value.fontSize) } } @@ -930,9 +949,8 @@ class EpubNavigatorFragment internal constructor( } private fun notifyCurrentLocation() { - if (view == null) { - return - } + // Make sure viewLifecycleOwner is accessible. + view ?: return val navigator = this debounceLocationNotificationJob?.cancel() diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/androidtts/AndroidTtsEngine.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/androidtts/AndroidTtsEngine.kt index cbe60aeaf9..374bb1c2dc 100644 --- a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/androidtts/AndroidTtsEngine.kt +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/androidtts/AndroidTtsEngine.kt @@ -17,7 +17,7 @@ import kotlinx.coroutines.* import kotlinx.coroutines.flow.MutableStateFlow import kotlinx.coroutines.flow.StateFlow import kotlinx.coroutines.flow.asStateFlow -import org.readium.r2.navigator.media3.tts2.TtsEngine +import org.readium.r2.navigator.media3.tts.TtsEngine import org.readium.r2.shared.ExperimentalReadiumApi import org.readium.r2.shared.publication.Metadata import org.readium.r2.shared.util.Language diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/androidtts/AndroidTtsEngineProvider.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/androidtts/AndroidTtsEngineProvider.kt index 48e37b3bc8..e31916b01c 100644 --- a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/androidtts/AndroidTtsEngineProvider.kt +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/androidtts/AndroidTtsEngineProvider.kt @@ -10,7 +10,7 @@ import android.content.Context import androidx.media3.common.PlaybackException import androidx.media3.common.PlaybackException.* import androidx.media3.common.PlaybackParameters -import org.readium.r2.navigator.media3.tts2.TtsEngineProvider +import org.readium.r2.navigator.media3.tts.TtsEngineProvider import org.readium.r2.shared.ExperimentalReadiumApi import org.readium.r2.shared.publication.Metadata import org.readium.r2.shared.publication.Publication diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/androidtts/AndroidTtsPreferences.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/androidtts/AndroidTtsPreferences.kt index cb6d81fc68..8ac8a410c5 100644 --- a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/androidtts/AndroidTtsPreferences.kt +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/androidtts/AndroidTtsPreferences.kt @@ -7,7 +7,7 @@ package org.readium.r2.navigator.media3.androidtts import kotlinx.serialization.Serializable -import org.readium.r2.navigator.media3.tts2.TtsEngine +import org.readium.r2.navigator.media3.tts.TtsEngine import org.readium.r2.shared.ExperimentalReadiumApi import org.readium.r2.shared.util.Language diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/androidtts/AndroidTtsSettings.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/androidtts/AndroidTtsSettings.kt index 93c53861bd..1dd0e9a18e 100644 --- a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/androidtts/AndroidTtsSettings.kt +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/androidtts/AndroidTtsSettings.kt @@ -6,7 +6,7 @@ package org.readium.r2.navigator.media3.androidtts -import org.readium.r2.navigator.media3.tts2.TtsEngine +import org.readium.r2.navigator.media3.tts.TtsEngine import org.readium.r2.shared.ExperimentalReadiumApi import org.readium.r2.shared.util.Language diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsContentIterator.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsContentIterator.kt similarity index 97% rename from readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsContentIterator.kt rename to readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsContentIterator.kt index d6a9961897..b46a8bbea7 100644 --- a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsContentIterator.kt +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsContentIterator.kt @@ -4,7 +4,7 @@ * available in the top-level LICENSE file of the project. */ -package org.readium.r2.navigator.media3.tts2 +package org.readium.r2.navigator.media3.tts import org.readium.r2.shared.ExperimentalReadiumApi import org.readium.r2.shared.publication.Locator @@ -141,9 +141,9 @@ internal class TtsContentIterator( utterances = CursorList( list = nextUtterances, - startIndex = when (direction) { - Direction.Forward -> 0 - Direction.Backward -> nextUtterances.size + 1 + index = when (direction) { + Direction.Forward -> -1 + Direction.Backward -> nextUtterances.size } ) diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsEngine.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsEngine.kt similarity index 97% rename from readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsEngine.kt rename to readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsEngine.kt index f491d3268c..0a0b77b246 100644 --- a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsEngine.kt +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsEngine.kt @@ -4,7 +4,7 @@ * available in the top-level LICENSE file of the project. */ -package org.readium.r2.navigator.media3.tts2 +package org.readium.r2.navigator.media3.tts import org.readium.r2.navigator.preferences.Configurable import org.readium.r2.shared.ExperimentalReadiumApi diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsEngineFacade.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsEngineFacade.kt similarity index 98% rename from readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsEngineFacade.kt rename to readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsEngineFacade.kt index 402672d62c..cc3a51621f 100644 --- a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsEngineFacade.kt +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsEngineFacade.kt @@ -4,7 +4,7 @@ * available in the top-level LICENSE file of the project. */ -package org.readium.r2.navigator.media3.tts2 +package org.readium.r2.navigator.media3.tts import java.util.* import kotlinx.coroutines.CancellableContinuation diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsEngineProvider.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsEngineProvider.kt similarity index 96% rename from readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsEngineProvider.kt rename to readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsEngineProvider.kt index b0716c0e97..19639bd947 100644 --- a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsEngineProvider.kt +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsEngineProvider.kt @@ -4,7 +4,7 @@ * available in the top-level LICENSE file of the project. */ -package org.readium.r2.navigator.media3.tts2 +package org.readium.r2.navigator.media3.tts import androidx.media3.common.PlaybackException import androidx.media3.common.PlaybackParameters diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsNavigator.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsNavigator.kt similarity index 99% rename from readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsNavigator.kt rename to readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsNavigator.kt index c0ba2d12b8..8266e6fa80 100644 --- a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsNavigator.kt +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsNavigator.kt @@ -4,7 +4,7 @@ * available in the top-level LICENSE file of the project. */ -package org.readium.r2.navigator.media3.tts2 +package org.readium.r2.navigator.media3.tts import android.app.Application import androidx.media3.common.MediaItem diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsNavigatorFactory.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsNavigatorFactory.kt similarity index 98% rename from readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsNavigatorFactory.kt rename to readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsNavigatorFactory.kt index d6cf0a703b..52b936b953 100644 --- a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsNavigatorFactory.kt +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsNavigatorFactory.kt @@ -4,7 +4,7 @@ * available in the top-level LICENSE file of the project. */ -package org.readium.r2.navigator.media3.tts2 +package org.readium.r2.navigator.media3.tts import android.app.Application import org.readium.r2.navigator.media3.api.DefaultMediaMetadataFactory diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsNavigatorInternal.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsNavigatorInternal.kt similarity index 99% rename from readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsNavigatorInternal.kt rename to readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsNavigatorInternal.kt index 70f4173732..ccb332c593 100644 --- a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsNavigatorInternal.kt +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsNavigatorInternal.kt @@ -4,7 +4,7 @@ * available in the top-level LICENSE file of the project. */ -package org.readium.r2.navigator.media3.tts2 +package org.readium.r2.navigator.media3.tts import android.app.Application import androidx.media3.common.MediaItem diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsNavigatorListener.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsNavigatorListener.kt similarity index 84% rename from readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsNavigatorListener.kt rename to readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsNavigatorListener.kt index 0917d65452..e7a6f26e4b 100644 --- a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsNavigatorListener.kt +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsNavigatorListener.kt @@ -4,7 +4,7 @@ * available in the top-level LICENSE file of the project. */ -package org.readium.r2.navigator.media3.tts2 +package org.readium.r2.navigator.media3.tts interface TtsNavigatorListener { diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsPlayer.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsPlayer.kt similarity index 99% rename from readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsPlayer.kt rename to readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsPlayer.kt index d1ace9e5dc..7a6c9a6fea 100644 --- a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsPlayer.kt +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsPlayer.kt @@ -4,7 +4,7 @@ * available in the top-level LICENSE file of the project. */ -package org.readium.r2.navigator.media3.tts2 +package org.readium.r2.navigator.media3.tts import kotlinx.coroutines.* import kotlinx.coroutines.flow.MutableStateFlow diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsSessionAdapter.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsSessionAdapter.kt similarity index 98% rename from readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsSessionAdapter.kt rename to readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsSessionAdapter.kt index 7d8e7c8f5d..de0f014d2a 100644 --- a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsSessionAdapter.kt +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsSessionAdapter.kt @@ -4,7 +4,7 @@ * available in the top-level LICENSE file of the project. */ -package org.readium.r2.navigator.media3.tts2 +package org.readium.r2.navigator.media3.tts import android.app.Application import android.os.Handler @@ -20,10 +20,13 @@ import androidx.media3.common.Player.* import androidx.media3.common.text.CueGroup import androidx.media3.common.util.Clock import androidx.media3.common.util.ListenerSet +import androidx.media3.common.util.Size import androidx.media3.common.util.Util import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.MainScope -import kotlinx.coroutines.flow.* +import kotlinx.coroutines.flow.StateFlow +import kotlinx.coroutines.flow.launchIn +import kotlinx.coroutines.flow.onEach import org.readium.r2.shared.ExperimentalReadiumApi import org.readium.r2.shared.fetcher.Resource import timber.log.Timber @@ -401,7 +404,7 @@ internal class TtsSessionAdapter( } override fun getCurrentTracks(): Tracks { - throw NotImplementedError() + return Tracks.EMPTY } override fun getTrackSelectionParameters(): TrackSelectionParameters { @@ -635,8 +638,12 @@ internal class TtsSessionAdapter( return VideoSize.UNKNOWN } + override fun getSurfaceSize(): Size { + return Size.UNKNOWN + } + override fun getCurrentCues(): CueGroup { - return CueGroup(emptyList()) + return CueGroup.EMPTY_TIME_ZERO } override fun getDeviceInfo(): DeviceInfo { diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsSessionTimeline.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsSessionTimeline.kt similarity index 97% rename from readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsSessionTimeline.kt rename to readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsSessionTimeline.kt index 7e129b2203..ec4a2055c5 100644 --- a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsSessionTimeline.kt +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsSessionTimeline.kt @@ -4,7 +4,7 @@ * available in the top-level LICENSE file of the project. */ -package org.readium.r2.navigator.media3.tts2 +package org.readium.r2.navigator.media3.tts import androidx.media3.common.MediaItem import androidx.media3.common.Timeline diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsStreamVolumeManager.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsStreamVolumeManager.kt similarity index 99% rename from readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsStreamVolumeManager.kt rename to readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsStreamVolumeManager.kt index f78456c267..f484b1bffa 100644 --- a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts2/TtsStreamVolumeManager.kt +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsStreamVolumeManager.kt @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.readium.r2.navigator.media3.tts2 +package org.readium.r2.navigator.media3.tts import android.content.BroadcastReceiver import android.content.Context diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/tts/AndroidTtsEngine.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/tts/AndroidTtsEngine.kt deleted file mode 100644 index 501ede587d..0000000000 --- a/readium/navigator/src/main/java/org/readium/r2/navigator/tts/AndroidTtsEngine.kt +++ /dev/null @@ -1,288 +0,0 @@ -/* - * Copyright 2022 Readium Foundation. All rights reserved. - * Use of this source code is governed by the BSD-style license - * available in the top-level LICENSE file of the project. - */ - -package org.readium.r2.navigator.tts - -import android.content.Context -import android.content.Intent -import android.speech.tts.TextToSpeech -import android.speech.tts.UtteranceProgressListener -import android.speech.tts.Voice as AndroidVoice -import java.util.* -import kotlin.Exception -import kotlin.coroutines.resume -import kotlinx.coroutines.* -import kotlinx.coroutines.channels.Channel -import kotlinx.coroutines.channels.onFailure -import org.readium.r2.navigator.tts.TtsEngine.Voice -import org.readium.r2.shared.ExperimentalReadiumApi -import org.readium.r2.shared.extensions.tryOrLog -import org.readium.r2.shared.util.Language -import org.readium.r2.shared.util.Try - -/** - * Default [TtsEngine] implementation using Android's native text to speech engine. - */ -@ExperimentalReadiumApi -class AndroidTtsEngine( - context: Context, - private val listener: TtsEngine.Listener -) : TtsEngine { - - /** - * Android's TTS error code. - * See https://developer.android.com/reference/android/speech/tts/TextToSpeech#ERROR - */ - enum class EngineError(val code: Int) { - /** Denotes a generic operation failure. */ - Unknown(-1), - /** Denotes a failure caused by an invalid request. */ - InvalidRequest(-8), - /** Denotes a failure caused by a network connectivity problems. */ - Network(-6), - /** Denotes a failure caused by network timeout. */ - NetworkTimeout(-7), - /** Denotes a failure caused by an unfinished download of the voice data. */ - NotInstalledYet(-9), - /** Denotes a failure related to the output (audio device or a file). */ - Output(-5), - /** Denotes a failure of a TTS service. */ - Service(-4), - /** Denotes a failure of a TTS engine to synthesize the given input. */ - Synthesis(-3); - - companion object { - fun getOrDefault(key: Int): EngineError = - values() - .firstOrNull { it.code == key } - ?: Unknown - } - } - - class EngineException(code: Int) : Exception("Android TTS engine error: $code") { - val error: EngineError = - EngineError.getOrDefault(code) - } - - private val scope = CoroutineScope(SupervisorJob() + Dispatchers.Default) - - /** - * Utterances to be synthesized, in order of [speak] calls. - */ - private val tasks = Channel(Channel.BUFFERED) - - /** Future completed when the [engine] is fully initialized. */ - private val init = CompletableDeferred() - - init { - scope.launch { - init.await() - - for (task in tasks) { - ensureActive() - task.run() - } - } - } - - override val rateMultiplierRange: ClosedRange = 0.1..4.0 - - override var availableVoices: List = emptyList() - private set(value) { - field = value - listener.onAvailableVoicesChange(value) - } - - override suspend fun close() { - scope.cancel() - tasks.cancel() - engine.shutdown() - } - - override suspend fun speak( - utterance: TtsEngine.Utterance, - onSpeakRange: (IntRange) -> Unit - ): TtsTry = - suspendCancellableCoroutine { cont -> - val result = tasks.trySend( - UtteranceTask( - utterance = utterance, - continuation = cont, - onSpeakRange = onSpeakRange - ) - ) - - result.onFailure { - listener.onEngineError( - TtsEngine.Exception.Other(IllegalStateException("Failed to schedule a new utterance task")) - ) - } - } - - /** - * Start the activity to install additional language data. - * To be called if you receive a [TtsEngine.Exception.LanguageSupportIncomplete] error. - * - * Returns whether the request was successful. - * - * See https://android-developers.googleblog.com/2009/09/introduction-to-text-to-speech-in.html - */ - fun requestInstallMissingVoice( - context: Context, - intentFlags: Int = Intent.FLAG_ACTIVITY_NEW_TASK - ): Boolean { - val intent = Intent() - .setAction(TextToSpeech.Engine.ACTION_INSTALL_TTS_DATA) - .setFlags(intentFlags) - - if (context.packageManager.queryIntentActivities(intent, 0).isEmpty()) { - return false - } - - context.startActivity(intent) - return true - } - - // Engine - - /** Underlying Android [TextToSpeech] engine. */ - private val engine = TextToSpeech(context, EngineInitListener()) - - private inner class EngineInitListener : TextToSpeech.OnInitListener { - override fun onInit(status: Int) { - if (status == TextToSpeech.SUCCESS) { - scope.launch { - tryOrLog { - availableVoices = engine.voices.map { it.toVoice() } - } - init.complete(Unit) - } - } else { - listener.onEngineError(TtsEngine.Exception.InitializationFailed()) - } - } - } - - /** - * Holds a single utterance to be synthesized and the continuation for the [speak] call. - */ - private inner class UtteranceTask( - val utterance: TtsEngine.Utterance, - val continuation: CancellableContinuation>, - val onSpeakRange: (IntRange) -> Unit, - ) { - fun run() { - if (!continuation.isActive) return - - // Interrupt the engine when the task is cancelled. - continuation.invokeOnCancellation { - tryOrLog { - engine.stop() - engine.setOnUtteranceProgressListener(null) - } - } - - try { - val id = UUID.randomUUID().toString() - engine.setup() - engine.setOnUtteranceProgressListener(Listener(id)) - engine.speak(utterance.text, TextToSpeech.QUEUE_FLUSH, null, id) - } catch (e: java.lang.Exception) { - finish(TtsEngine.Exception.wrap(e)) - } - } - - /** - * Terminates this task. - */ - private fun finish(error: TtsEngine.Exception? = null) { - continuation.resume( - error?.let { Try.failure(error) } - ?: Try.success(Unit) - ) - } - - /** - * Setups the [engine] using the [utterance]'s configuration. - */ - private fun TextToSpeech.setup() { - setSpeechRate(utterance.rateMultiplier.toFloat()) - - utterance.voiceOrLanguage - .onLeft { voice -> - // Setup the user selected voice. - engine.voice = engine.voices - .firstOrNull { it.name == voice.id } - ?: throw IllegalStateException("Unknown Android voice: ${voice.id}") - } - .onRight { language -> - // Or fallback on the language. - val localeResult = engine.setLanguage(language.locale) - if (localeResult < TextToSpeech.LANG_AVAILABLE) { - if (localeResult == TextToSpeech.LANG_MISSING_DATA) - throw TtsEngine.Exception.LanguageSupportIncomplete(language) - else - throw TtsEngine.Exception.LanguageNotSupported(language) - } - } - } - - inner class Listener(val id: String) : UtteranceProgressListener() { - override fun onStart(utteranceId: String?) {} - - override fun onStop(utteranceId: String?, interrupted: Boolean) { - require(utteranceId == id) - finish() - } - - override fun onDone(utteranceId: String?) { - require(utteranceId == id) - finish() - } - - @Deprecated("Deprecated in the interface", ReplaceWith("onError(utteranceId, -1)")) - override fun onError(utteranceId: String?) { - onError(utteranceId, -1) - } - - override fun onError(utteranceId: String?, errorCode: Int) { - require(utteranceId == id) - - val error = EngineException(errorCode) - finish( - when (error.error) { - EngineError.Network, EngineError.NetworkTimeout -> - TtsEngine.Exception.Network(error) - EngineError.NotInstalledYet -> - TtsEngine.Exception.LanguageSupportIncomplete(utterance.language, cause = error) - else -> TtsEngine.Exception.Other(error) - } - ) - } - - override fun onRangeStart(utteranceId: String?, start: Int, end: Int, frame: Int) { - require(utteranceId == id) - onSpeakRange(start until end) - } - } - } -} - -@OptIn(ExperimentalReadiumApi::class) -private fun AndroidVoice.toVoice(): Voice = - Voice( - id = name, - name = null, - language = Language(locale), - quality = when (quality) { - AndroidVoice.QUALITY_VERY_HIGH -> Voice.Quality.Highest - AndroidVoice.QUALITY_HIGH -> Voice.Quality.High - AndroidVoice.QUALITY_LOW -> Voice.Quality.Low - AndroidVoice.QUALITY_VERY_LOW -> Voice.Quality.Lowest - else -> Voice.Quality.Normal - }, - requiresNetwork = isNetworkConnectionRequired - ) diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/tts/PublicationSpeechSynthesizer.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/tts/PublicationSpeechSynthesizer.kt deleted file mode 100644 index 0d32c68956..0000000000 --- a/readium/navigator/src/main/java/org/readium/r2/navigator/tts/PublicationSpeechSynthesizer.kt +++ /dev/null @@ -1,557 +0,0 @@ -/* - * Copyright 2022 Readium Foundation. All rights reserved. - * Use of this source code is governed by the BSD-style license - * available in the top-level LICENSE file of the project. - */ - -package org.readium.r2.navigator.tts - -import android.content.Context -import java.util.* -import kotlinx.coroutines.* -import kotlinx.coroutines.flow.MutableStateFlow -import kotlinx.coroutines.flow.StateFlow -import kotlinx.coroutines.flow.asStateFlow -import kotlinx.coroutines.flow.update -import org.readium.r2.shared.DelicateReadiumApi -import org.readium.r2.shared.ExperimentalReadiumApi -import org.readium.r2.shared.extensions.tryOrLog -import org.readium.r2.shared.publication.Locator -import org.readium.r2.shared.publication.Publication -import org.readium.r2.shared.publication.services.content.Content -import org.readium.r2.shared.publication.services.content.ContentTokenizer -import org.readium.r2.shared.publication.services.content.TextContentTokenizer -import org.readium.r2.shared.publication.services.content.content -import org.readium.r2.shared.util.* -import org.readium.r2.shared.util.tokenizer.TextUnit - -/** - * [PublicationSpeechSynthesizer] orchestrates the rendition of a [publication] by iterating through - * its content, splitting it into individual utterances using a [ContentTokenizer], then using a - * [TtsEngine] to read them aloud. - * - * Don't forget to call [close] when you are done using the [PublicationSpeechSynthesizer]. - */ -@OptIn(DelicateReadiumApi::class) -@ExperimentalReadiumApi -@Deprecated("The API described in this guide will be changed in the next version of the Kotlin toolkit to support background TTS playback and media notifications. It is recommended that you wait before integrating it in your app.") -class PublicationSpeechSynthesizer private constructor( - private val publication: Publication, - config: Configuration, - engineFactory: (listener: TtsEngine.Listener) -> E, - private val tokenizerFactory: (defaultLanguage: Language?) -> ContentTokenizer, - var listener: Listener? = null, -) : SuspendingCloseable { - - companion object { - - /** - * Creates a [PublicationSpeechSynthesizer] using the default native [AndroidTtsEngine]. - * - * @param publication Publication which will be iterated through and synthesized. - * @param config Initial TTS configuration. - * @param tokenizerFactory Factory to create a [ContentTokenizer] which will be used to - * split each [Content.Element] item into smaller chunks. Splits by sentences by default. - * @param listener Optional callbacks listener. - */ - operator fun invoke( - context: Context, - publication: Publication, - config: Configuration = Configuration(), - tokenizerFactory: (defaultLanguage: Language?) -> ContentTokenizer = defaultTokenizerFactory, - listener: Listener? = null, - ): PublicationSpeechSynthesizer? = invoke( - publication, - config = config, - engineFactory = { AndroidTtsEngine(context, listener = it) }, - tokenizerFactory = tokenizerFactory, - listener = listener - ) - - /** - * Creates a [PublicationSpeechSynthesizer] using a custom [TtsEngine]. - * - * @param publication Publication which will be iterated through and synthesized. - * @param config Initial TTS configuration. - * @param engineFactory Factory to create an instance of [TtsEngine]. - * @param tokenizerFactory Factory to create a [ContentTokenizer] which will be used to - * split each [Content.Element] item into smaller chunks. Splits by sentences by default. - * @param listener Optional callbacks listener. - */ - operator fun invoke( - publication: Publication, - config: Configuration = Configuration(), - engineFactory: (TtsEngine.Listener) -> E, - tokenizerFactory: (defaultLanguage: Language?) -> ContentTokenizer = defaultTokenizerFactory, - listener: Listener? = null, - ): PublicationSpeechSynthesizer? { - if (!canSpeak(publication)) return null - - return PublicationSpeechSynthesizer(publication, config, engineFactory, tokenizerFactory, listener) - } - - /** - * The default content tokenizer will split the [Content.Element] items into individual sentences. - */ - val defaultTokenizerFactory: (Language?) -> ContentTokenizer = { language -> - TextContentTokenizer( - unit = TextUnit.Sentence, - language = language, - overrideContentLanguage = false - ) - } - - /** - * Returns whether the [publication] can be played with a [PublicationSpeechSynthesizer]. - */ - fun canSpeak(publication: Publication): Boolean = - publication.content() != null - } - - @ExperimentalReadiumApi - interface Listener { - /** Called when an [error] occurs while speaking [utterance]. */ - fun onUtteranceError(utterance: Utterance, error: Exception) - - /** Called when a global [error] occurs. */ - fun onError(error: Exception) - } - - @ExperimentalReadiumApi - sealed class Exception private constructor( - override val message: String, - cause: Throwable? = null - ) : kotlin.Exception(message, cause) { - - /** Underlying [TtsEngine] error. */ - class Engine(val error: TtsEngine.Exception) : - Exception(error.message, error) - } - - /** - * An utterance is an arbitrary text (e.g. sentence) extracted from the [publication], that can - * be synthesized by the TTS [engine]. - * - * @param text Text to be spoken. - * @param locator Locator to the utterance in the [publication]. - * @param language Language of this utterance, if it differs from the default publication - * language. - */ - @ExperimentalReadiumApi - data class Utterance( - val text: String, - val locator: Locator, - val language: Language?, - ) - - /** - * Represents a state of the [PublicationSpeechSynthesizer]. - */ - sealed class State { - /** The [PublicationSpeechSynthesizer] is completely stopped and must be (re)started from a given locator. */ - object Stopped : State() - - /** The [PublicationSpeechSynthesizer] is paused at the given utterance. */ - data class Paused(val utterance: Utterance) : State() - - /** - * The TTS engine is synthesizing [utterance]. - * - * [range] will be regularly updated while the [utterance] is being played. - */ - data class Playing(val utterance: Utterance, val range: Locator? = null) : State() - } - - private val _state = MutableStateFlow(State.Stopped) - - /** - * Current state of the [PublicationSpeechSynthesizer]. - */ - val state: StateFlow = _state.asStateFlow() - - private val scope = MainScope() - - init { - require(canSpeak(publication)) { - "The content of the publication cannot be synthesized, as it is not iterable" - } - } - - /** - * Underlying [TtsEngine] instance. - * - * WARNING: Don't control the playback or set the config directly with the engine. Use the - * [PublicationSpeechSynthesizer] APIs instead. This property is used to access engine-specific APIs such as - * [AndroidTtsEngine.requestInstallMissingVoice]. - */ - @DelicateReadiumApi - val engine: E by lazy { - engineFactory(object : TtsEngine.Listener { - override fun onEngineError(error: TtsEngine.Exception) { - listener?.onError(Exception.Engine(error)) - stop() - } - - override fun onAvailableVoicesChange(voices: List) { - _availableVoices.value = voices - } - }) - } - - /** - * Interrupts the [TtsEngine] and closes this [PublicationSpeechSynthesizer]. - */ - override suspend fun close() { - tryOrLog { - scope.cancel() - if (::engine.isLazyInitialized) { - engine.close() - } - } - } - - /** - * User configuration for the text-to-speech engine. - * - * @param defaultLanguage Language overriding the publication one. - * @param voiceId Identifier for the voice used to speak the utterances. - * @param rateMultiplier Multiplier for the voice speech rate. Normal is 1.0. See [rateMultiplierRange] - * for the range of values supported by the [TtsEngine]. - * @param extras Extensibility for custom TTS engines. - */ - @ExperimentalReadiumApi - data class Configuration( - val defaultLanguage: Language? = null, - val voiceId: String? = null, - val rateMultiplier: Double = 1.0, - val extras: Any? = null - ) - - private val _config = MutableStateFlow(config) - - /** - * Current user configuration. - */ - val config: StateFlow = _config.asStateFlow() - - /** - * Updates the user configuration. - * - * The change is not immediate, it will be applied for the next utterance. - */ - fun setConfig(config: Configuration) { - _config.value = config.copy( - rateMultiplier = config.rateMultiplier.coerceIn(engine.rateMultiplierRange), - ) - } - - /** - * Range for the speech rate multiplier. Normal is 1.0. - */ - val rateMultiplierRange: ClosedRange - get() = engine.rateMultiplierRange - - private val _availableVoices = MutableStateFlow>(emptyList()) - - /** - * List of synthesizer voices supported by the TTS [engine]. - */ - val availableVoices: StateFlow> = _availableVoices.asStateFlow() - - /** - * Returns the first voice with the given [id] supported by the TTS [engine]. - * - * This can be used to restore the user selected voice after storing it in the shared - * preferences. - */ - fun voiceWithId(id: String): TtsEngine.Voice? { - val voice = lastUsedVoice?.takeIf { it.id == id } - ?: engine.voiceWithId(id) - ?: return null - - lastUsedVoice = voice - return voice - } - - /** - * Cache for the last requested voice, for performance. - */ - private var lastUsedVoice: TtsEngine.Voice? = null - - /** - * (Re)starts the TTS from the given locator or the beginning of the publication. - */ - fun start(fromLocator: Locator? = null) { - replacePlaybackJob { - publicationIterator = publication.content(fromLocator)?.iterator() - playNextUtterance(Direction.Forward) - } - } - - /** - * Stops the synthesizer. - * - * Use [start] to restart it. - */ - fun stop() { - replacePlaybackJob { - _state.value = State.Stopped - publicationIterator = null - } - } - - /** - * Interrupts a played utterance. - * - * Use [resume] to restart the playback from the same utterance. - */ - fun pause() { - replacePlaybackJob { - _state.update { state -> - when (state) { - is State.Playing -> State.Paused(state.utterance) - else -> state - } - } - } - } - - /** - * Resumes an utterance interrupted with [pause]. - */ - fun resume() { - replacePlaybackJob { - (state.value as? State.Paused)?.let { paused -> - play(paused.utterance) - } - } - } - - /** - * Pauses or resumes the playback of the current utterance. - */ - fun pauseOrResume() { - when (state.value) { - is State.Stopped -> return - is State.Playing -> pause() - is State.Paused -> resume() - } - } - - /** - * Skips to the previous utterance. - */ - fun previous() { - replacePlaybackJob { - playNextUtterance(Direction.Backward) - } - } - - /** - * Skips to the next utterance. - */ - fun next() { - replacePlaybackJob { - playNextUtterance(Direction.Forward) - } - } - - /** - * [Content.Iterator] used to iterate through the [publication]. - */ - private var publicationIterator: Content.Iterator? = null - set(value) { - field = value - utterances = CursorList() - } - - /** - * Utterances for the current publication [Content.Element] item. - */ - private var utterances: CursorList = CursorList() - - /** - * Plays the next utterance in the given [direction]. - */ - private suspend fun playNextUtterance(direction: Direction) { - val utterance = nextUtterance(direction) - if (utterance == null) { - _state.value = State.Stopped - return - } - play(utterance) - } - - /** - * Plays the given [utterance] with the TTS [engine]. - */ - private suspend fun play(utterance: Utterance) { - _state.value = State.Playing(utterance) - - engine - .speak( - utterance = TtsEngine.Utterance( - text = utterance.text, - rateMultiplier = config.value.rateMultiplier, - voiceOrLanguage = utterance.voiceOrLanguage() - ), - onSpeakRange = { range -> - _state.value = State.Playing( - utterance = utterance, - range = utterance.locator.copy( - text = utterance.locator.text.substring(range) - ) - ) - } - ) - .onSuccess { - playNextUtterance(Direction.Forward) - } - .onFailure { - _state.value = State.Paused(utterance) - listener?.onUtteranceError(utterance, Exception.Engine(it)) - } - } - - /** - * Returns the user selected voice if it's compatible with the utterance language. Otherwise, - * falls back on the languages. - */ - private fun Utterance.voiceOrLanguage(): Either { - // User selected voice, if it's compatible with the utterance language. - // Or fallback on the languages. - val voice = config.value.voiceId - ?.let { voiceWithId(it) } - ?.takeIf { language == null || it.language.removeRegion() == language.removeRegion() } - - return Either( - voice - ?: language - ?: config.value.defaultLanguage - ?: publication.metadata.language - ?: Language(Locale.getDefault()) - ) - } - - /** - * Gets the next utterance in the given [direction], or null when reaching the beginning or the - * end. - */ - private suspend fun nextUtterance(direction: Direction): Utterance? { - val utterance = utterances.nextIn(direction) - if (utterance == null && loadNextUtterances(direction)) { - return nextUtterance(direction) - } - return utterance - } - - /** - * Loads the utterances for the next publication [Content.Element] item in the given [direction]. - */ - private suspend fun loadNextUtterances(direction: Direction): Boolean { - val content = publicationIterator?.nextIn(direction) - ?: return false - - val nextUtterances = content - .tokenize() - .flatMap { it.utterances() } - - if (nextUtterances.isEmpty()) { - return loadNextUtterances(direction) - } - - utterances = CursorList( - list = nextUtterances, - startIndex = when (direction) { - Direction.Forward -> 0 - Direction.Backward -> nextUtterances.size - 1 - } - ) - - return true - } - - /** - * Splits a publication [Content.Element] item into smaller chunks using the provided tokenizer. - * - * This is used to split a paragraph into sentences, for example. - */ - private fun Content.Element.tokenize(): List = - tokenizerFactory(config.value.defaultLanguage ?: publication.metadata.language) - .tokenize(this) - - /** - * Splits a publication [Content.Element] item into the utterances to be spoken. - */ - private fun Content.Element.utterances(): List { - fun utterance(text: String, locator: Locator, language: Language? = null): Utterance? { - if (!text.any { it.isLetterOrDigit() }) - return null - - return Utterance( - text = text, - locator = locator, - language = language - // If the language is the same as the one declared globally in the publication, - // we omit it. This way, the app can customize the default language used in the - // configuration. - ?.takeIf { it != publication.metadata.language } - ) - } - - return when (this) { - is Content.TextElement -> { - segments.mapNotNull { segment -> - utterance( - text = segment.text, - locator = segment.locator, - language = segment.language - ) - } - } - - is Content.TextualElement -> { - listOfNotNull( - text - ?.takeIf { it.isNotBlank() } - ?.let { utterance(text = it, locator = locator) } - ) - } - - else -> emptyList() - } - } - - /** - * Cancels the previous playback-related job and starts a new one with the given suspending - * [block]. - * - * This is used to interrupt on-going commands. - */ - private fun replacePlaybackJob(block: suspend CoroutineScope.() -> Unit) { - scope.launch { - playbackJob?.cancelAndJoin() - playbackJob = launch { - block() - } - } - } - - private var playbackJob: Job? = null - - private enum class Direction { - Forward, Backward; - } - - private fun CursorList.nextIn(direction: Direction): E? = - when (direction) { - Direction.Forward -> next() - Direction.Backward -> previous() - } - - private suspend fun Content.Iterator.nextIn(direction: Direction): Content.Element? = - when (direction) { - Direction.Forward -> nextOrNull() - Direction.Backward -> previousOrNull() - } -} diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/tts/TtsEngine.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/tts/TtsEngine.kt deleted file mode 100644 index ae3a454f97..0000000000 --- a/readium/navigator/src/main/java/org/readium/r2/navigator/tts/TtsEngine.kt +++ /dev/null @@ -1,149 +0,0 @@ -/* - * Copyright 2022 Readium Foundation. All rights reserved. - * Use of this source code is governed by the BSD-style license - * available in the top-level LICENSE file of the project. - */ - -package org.readium.r2.navigator.tts - -import org.readium.r2.shared.ExperimentalReadiumApi -import org.readium.r2.shared.util.Either -import org.readium.r2.shared.util.Language -import org.readium.r2.shared.util.SuspendingCloseable -import org.readium.r2.shared.util.Try - -@ExperimentalReadiumApi -typealias TtsTry = Try - -/** - * A text-to-speech engine synthesizes text utterances (e.g. sentence). - * - * Implement this interface to support third-party engines with [PublicationSpeechSynthesizer]. - */ -@ExperimentalReadiumApi -interface TtsEngine : SuspendingCloseable { - - @ExperimentalReadiumApi - sealed class Exception private constructor( - override val message: String, - cause: Throwable? = null - ) : kotlin.Exception(message, cause) { - /** Failed to initialize the TTS engine. */ - class InitializationFailed(cause: Throwable? = null) : - Exception("The TTS engine failed to initialize", cause) - - /** Tried to synthesize an utterance with an unsupported language. */ - class LanguageNotSupported(val language: Language, cause: Throwable? = null) : - Exception("The language ${language.code} is not supported by the TTS engine", cause) - - /** The selected language is missing downloadable data. */ - class LanguageSupportIncomplete(val language: Language, cause: Throwable? = null) : - Exception("The language ${language.code} requires additional files by the TTS engine", cause) - - /** Error during network calls. */ - class Network(cause: Throwable? = null) : - Exception("A network error occurred", cause) - - /** Other engine-specific errors. */ - class Other(override val cause: Throwable) : - Exception(cause.message ?: "An unknown error occurred", cause) - - companion object { - fun wrap(e: Throwable): Exception = when (e) { - is Exception -> e - else -> Other(e) - } - } - } - - /** - * TTS engine callbacks. - */ - @ExperimentalReadiumApi - interface Listener { - /** - * Called when a general engine error occurred. - */ - fun onEngineError(error: Exception) - - /** - * Called when the list of available voices is updated. - */ - fun onAvailableVoicesChange(voices: List) - } - - /** - * An utterance is an arbitrary text (e.g. sentence) that can be synthesized by the TTS engine. - * - * @param text Text to be spoken. - * @param rateMultiplier Multiplier for the speech rate. - * @param voiceOrLanguage Either an explicit voice or the language of the text. If a language - * is provided, the default voice for this language will be used. - */ - @ExperimentalReadiumApi - data class Utterance( - val text: String, - val rateMultiplier: Double, - val voiceOrLanguage: Either - ) { - val language: Language = - when (val vl = voiceOrLanguage) { - is Either.Left -> vl.value.language - is Either.Right -> vl.value - } - } - - /** - * Represents a voice provided by the TTS engine which can speak an utterance. - * - * @param id Unique and stable identifier for this voice. Can be used to store and retrieve the - * voice from the user preferences. - * @param name Human-friendly name for this voice, when available. - * @param language Language (and region) this voice belongs to. - * @param quality Voice quality. - * @param requiresNetwork Indicates whether using this voice requires an Internet connection. - */ - @ExperimentalReadiumApi - data class Voice( - val id: String, - val name: String? = null, - val language: Language, - val quality: Quality = Quality.Normal, - val requiresNetwork: Boolean = false, - ) { - enum class Quality { - Lowest, Low, Normal, High, Highest - } - } - - /** - * Synthesizes the given [utterance] and returns its status. - * - * [onSpeakRange] is called repeatedly while the engine plays portions (e.g. words) of the - * utterance. - * - * To interrupt the utterance, cancel the parent coroutine job. - */ - suspend fun speak( - utterance: Utterance, - onSpeakRange: (IntRange) -> Unit = { _ -> } - ): TtsTry - - /** - * Supported range for the speech rate multiplier. - */ - val rateMultiplierRange: ClosedRange - - /** - * List of available synthesizer voices. - * - * Implement [Listener.onAvailableVoicesChange] to be aware of changes in the available voices. - */ - val availableVoices: List - - /** - * Returns the voice with given identifier, if it exists. - */ - fun voiceWithId(id: String): Voice? = - availableVoices.firstOrNull { it.id == id } -} diff --git a/readium/shared/src/main/java/org/readium/r2/shared/publication/services/content/Content.kt b/readium/shared/src/main/java/org/readium/r2/shared/publication/services/content/Content.kt index 7f2ac53895..94804c77ee 100644 --- a/readium/shared/src/main/java/org/readium/r2/shared/publication/services/content/Content.kt +++ b/readium/shared/src/main/java/org/readium/r2/shared/publication/services/content/Content.kt @@ -197,6 +197,9 @@ interface Content { /** * Iterates through a list of [Element] items asynchronously. + * + * [hasNext] and [hasPrevious] refer to the last element computed by a previous call + * to any of both methods. */ @ExperimentalReadiumApi interface Iterator { diff --git a/readium/shared/src/main/java/org/readium/r2/shared/publication/services/content/iterators/PublicationContentIterator.kt b/readium/shared/src/main/java/org/readium/r2/shared/publication/services/content/iterators/PublicationContentIterator.kt index aadddfef24..6127040281 100644 --- a/readium/shared/src/main/java/org/readium/r2/shared/publication/services/content/iterators/PublicationContentIterator.kt +++ b/readium/shared/src/main/java/org/readium/r2/shared/publication/services/content/iterators/PublicationContentIterator.kt @@ -14,7 +14,6 @@ import org.readium.r2.shared.publication.Publication import org.readium.r2.shared.publication.indexOfFirstWithHref import org.readium.r2.shared.publication.services.content.Content import org.readium.r2.shared.util.Either -import timber.log.Timber /** * Creates a [Content.Iterator] instance for the [Resource], starting from the given [Locator]. @@ -63,7 +62,6 @@ class PublicationContentIterator( override suspend fun hasPrevious(): Boolean { currentElement = nextIn(Direction.Backward) - Timber.d("hasPrevious ${currentElement?.element?.locator}") return currentElement != null } @@ -74,7 +72,6 @@ class PublicationContentIterator( override suspend fun hasNext(): Boolean { currentElement = nextIn(Direction.Forward) - Timber.d("hasNext ${currentElement?.element?.locator}") return currentElement != null } diff --git a/readium/shared/src/main/java/org/readium/r2/shared/util/CursorList.kt b/readium/shared/src/main/java/org/readium/r2/shared/util/CursorList.kt index c5f28ba4f0..73e615aa6a 100644 --- a/readium/shared/src/main/java/org/readium/r2/shared/util/CursorList.kt +++ b/readium/shared/src/main/java/org/readium/r2/shared/util/CursorList.kt @@ -4,14 +4,23 @@ import org.readium.r2.shared.InternalReadiumApi /** * A [List] with a mutable cursor index. + * + * [next] and [previous] refer to the last element returned by a previous call + * to any of both methods. + * + * @param list the content of the [CursorList] + * @param index the index of the element that will initially be considered + * as the last returned element. May be -1 or the size of the list as well. */ @InternalReadiumApi class CursorList( private val list: List = emptyList(), - startIndex: Int = 0 + private var index: Int = -1 ) : List by list { - private var index: Int = startIndex - 1 + init { + check(index in -1..list.size) + } fun hasPrevious(): Boolean { return index > 0 diff --git a/test-app/build.gradle.kts b/test-app/build.gradle.kts index 018fa7d8cd..6fb8969723 100644 --- a/test-app/build.gradle.kts +++ b/test-app/build.gradle.kts @@ -102,6 +102,7 @@ dependencies { implementation(libs.kotlinx.coroutines.core) implementation(libs.jsoup) + implementation(libs.bundles.media2) implementation(libs.bundles.media3) // Room database diff --git a/test-app/src/main/java/org/readium/r2/testapp/reader/ReaderRepository.kt b/test-app/src/main/java/org/readium/r2/testapp/reader/ReaderRepository.kt index 427d29f3dc..c6e76cf1d0 100644 --- a/test-app/src/main/java/org/readium/r2/testapp/reader/ReaderRepository.kt +++ b/test-app/src/main/java/org/readium/r2/testapp/reader/ReaderRepository.kt @@ -17,7 +17,7 @@ import org.readium.navigator.media2.ExperimentalMedia2 import org.readium.navigator.media2.MediaNavigator import org.readium.r2.navigator.epub.EpubNavigatorFactory import org.readium.r2.navigator.media3.androidtts.AndroidTtsEngineProvider -import org.readium.r2.navigator.media3.tts2.TtsNavigatorFactory +import org.readium.r2.navigator.media3.tts.TtsNavigatorFactory import org.readium.r2.navigator.pdf.PdfNavigatorFactory import org.readium.r2.shared.ExperimentalReadiumApi import org.readium.r2.shared.publication.Locator diff --git a/test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsEngine.kt b/test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsEngine.kt index b015c1c681..cffe2a3ffa 100644 --- a/test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsEngine.kt +++ b/test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsEngine.kt @@ -10,8 +10,8 @@ import org.readium.r2.navigator.media3.androidtts.AndroidTtsEngine import org.readium.r2.navigator.media3.androidtts.AndroidTtsPreferences import org.readium.r2.navigator.media3.androidtts.AndroidTtsPreferencesEditor import org.readium.r2.navigator.media3.androidtts.AndroidTtsSettings -import org.readium.r2.navigator.media3.tts2.TtsNavigator -import org.readium.r2.navigator.media3.tts2.TtsNavigatorFactory +import org.readium.r2.navigator.media3.tts.TtsNavigator +import org.readium.r2.navigator.media3.tts.TtsNavigatorFactory import org.readium.r2.shared.ExperimentalReadiumApi @OptIn(ExperimentalReadiumApi::class) diff --git a/test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsViewModel.kt b/test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsViewModel.kt index ef67158608..1641aa7f33 100644 --- a/test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsViewModel.kt +++ b/test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsViewModel.kt @@ -16,9 +16,8 @@ import org.readium.r2.navigator.media3.androidtts.AndroidTtsPreferences import org.readium.r2.navigator.media3.androidtts.AndroidTtsPreferencesEditor import org.readium.r2.navigator.media3.api.MediaNavigator import org.readium.r2.navigator.media3.api.SynchronizedMediaNavigator -import org.readium.r2.navigator.media3.tts2.TtsNavigator -import org.readium.r2.navigator.media3.tts2.TtsNavigatorListener -import org.readium.r2.navigator.tts.PublicationSpeechSynthesizer +import org.readium.r2.navigator.media3.tts.TtsNavigator +import org.readium.r2.navigator.media3.tts.TtsNavigatorListener import org.readium.r2.shared.DelicateReadiumApi import org.readium.r2.shared.ExperimentalReadiumApi import org.readium.r2.shared.UserException From 4150a47f25175668dd08cb0c8300ad7e7cb7d597 Mon Sep 17 00:00:00 2001 From: Quentin Gliosca Date: Thu, 26 Jan 2023 18:28:34 +0100 Subject: [PATCH 07/27] Various changes --- .../media3/api/DefaultMediaMetadataFactory.kt | 25 +++++++------------ .../media3/api/MediaMetadataProvider.kt | 5 +++- .../r2/navigator/media3/api/MediaNavigator.kt | 14 ++++++++++- .../r2/navigator/media3/tts/TtsNavigator.kt | 3 +++ .../media3/tts/TtsNavigatorFactory.kt | 6 +---- .../media3/tts/TtsNavigatorInternal.kt | 3 +++ .../r2/navigator/media3/tts/TtsPlayer.kt | 4 +-- .../navigator/media3/tts/TtsSessionAdapter.kt | 2 +- .../org/readium/r2/shared/util/CursorList.kt | 6 ++--- .../reader/preferences/UserPreferences.kt | 2 +- .../r2/testapp/reader/tts/TtsControls.kt | 7 +++++- .../r2/testapp/reader/tts/TtsViewModel.kt | 4 +++ 12 files changed, 49 insertions(+), 32 deletions(-) diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/api/DefaultMediaMetadataFactory.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/api/DefaultMediaMetadataFactory.kt index bfc39f6f78..7d5cb62ab6 100644 --- a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/api/DefaultMediaMetadataFactory.kt +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/api/DefaultMediaMetadataFactory.kt @@ -15,14 +15,17 @@ import kotlinx.coroutines.async import org.readium.r2.shared.publication.Publication @androidx.annotation.OptIn(androidx.media3.common.util.UnstableApi::class) -internal class DefaultMediaMetadataFactory(private val publication: Publication) : MediaMetadataFactory { +internal class DefaultMediaMetadataFactory( + private val publication: Publication +) : MediaMetadataFactory { private val coroutineScope = CoroutineScope(Dispatchers.Default) - private val authors: String? - get() = publication.metadata.authors - .joinToString(", ") { it.name }.takeIf { it.isNotBlank() } + private val authors: String? = + publication.metadata.authors + .joinToString(", ") { it.name } + .takeIf { it.isNotBlank() } private val cover: Deferred = coroutineScope.async { publication.linkWithRel("cover") @@ -34,14 +37,10 @@ internal class DefaultMediaMetadataFactory(private val publication: Publication) override suspend fun publicationMetadata(): MediaMetadata { val builder = MediaMetadata.Builder() .setTitle(publication.metadata.title) - .setAlbumTitle(publication.metadata.title) .setTotalTrackCount(publication.readingOrder.size) authors - ?.let { - builder.setArtist(it) - builder.setAlbumArtist(it) - } + ?.let { builder.setArtist(it) } cover.await() ?.let { builder.maybeSetArtworkData(it, PICTURE_TYPE_FRONT_COVER) } @@ -55,17 +54,11 @@ internal class DefaultMediaMetadataFactory(private val publication: Publication) val link = publication.readingOrder[index] builder.setTrackNumber(index) - // builder.setMediaUri(link.href) builder.setTitle(link.title) builder.setTitle(publication.metadata.title) - builder.setAlbumTitle(publication.metadata.title) - // builder.setDuration(MediaMetadata.METADATA_KEY_DURATION, (link.duration?.toLong() ?: 0) * 1000) authors - ?.let { - builder.setArtist(it) - builder.setAlbumArtist(it) - } + ?.let { builder.setArtist(it) } cover.await() ?.let { builder.maybeSetArtworkData(it, PICTURE_TYPE_FRONT_COVER) } diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/api/MediaMetadataProvider.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/api/MediaMetadataProvider.kt index 0f2acd9170..fb1a45d8d1 100644 --- a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/api/MediaMetadataProvider.kt +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/api/MediaMetadataProvider.kt @@ -8,7 +8,10 @@ package org.readium.r2.navigator.media3.api import org.readium.r2.shared.publication.Publication -interface MediaMetadataProvider { +/** + * To be implemented to use a custom [MediaMetadataFactory]. + */ +fun interface MediaMetadataProvider { fun createMetadataFactory(publication: Publication): MediaMetadataFactory } diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/api/MediaNavigator.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/api/MediaNavigator.kt index 979befbd29..26b3773287 100644 --- a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/api/MediaNavigator.kt +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/api/MediaNavigator.kt @@ -15,8 +15,14 @@ import org.readium.r2.shared.util.Closeable @ExperimentalReadiumApi interface MediaNavigator : Navigator, Closeable { + /** + * Marker interface for the [Playback.error] property. + */ interface Error + /** + * State of the player. + */ enum class State { Ready, Buffering, @@ -24,16 +30,22 @@ interface MediaNavigator : Navigator, Closeable { Error; } + /** + * State of the playback. + */ data class Playback( val state: State, val playWhenReady: Boolean, val error: E? ) + /** + * Indicates the current state of the playback. + */ val playback: StateFlow> /** - * Resumes the playback at the current location or start it again from the beginning if it has finished. + * Resumes the playback at the current location. */ fun play() diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsNavigator.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsNavigator.kt index 8266e6fa80..edf7cef6ed 100644 --- a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsNavigator.kt +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsNavigator.kt @@ -104,6 +104,9 @@ class TtsNavigator, private val coroutineScope: CoroutineScope = MainScope() + val voices: Set get() = + ttsNavigator.voices + override val playback: StateFlow> = ttsNavigator.playback.mapStateIn(coroutineScope) { it.toPlayback() } diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsNavigatorFactory.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsNavigatorFactory.kt index 52b936b953..999004bf25 100644 --- a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsNavigatorFactory.kt +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsNavigatorFactory.kt @@ -58,11 +58,7 @@ class TtsNavigatorFactory, } val defaultMediaMetadataProvider: MediaMetadataProvider = - object : MediaMetadataProvider { - override fun createMetadataFactory(publication: Publication): MediaMetadataFactory { - return DefaultMediaMetadataFactory(publication) - } - } + MediaMetadataProvider { publication -> DefaultMediaMetadataFactory(publication) } } suspend fun createNavigator( diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsNavigatorInternal.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsNavigatorInternal.kt index ccb332c593..61e28b6a38 100644 --- a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsNavigatorInternal.kt +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsNavigatorInternal.kt @@ -98,6 +98,9 @@ internal class TtsNavigatorInternal get() = + ttsPlayer.voices + override val playback: StateFlow> = ttsPlayer.playback.mapStateIn(coroutineScope) { it.toPlayback() } diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsPlayer.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsPlayer.kt index 7a6c9a6fea..e0000326df 100644 --- a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsPlayer.kt +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsPlayer.kt @@ -65,8 +65,6 @@ internal class TtsPlayer, ) } - Timber.d("startContext $context") - return context } } @@ -367,7 +365,7 @@ internal class TtsPlayer, } private fun onEngineError(error: E) { - Timber.d("onEngineError $error") + Timber.e("onEngineError $error") playbackMutable.value = playbackMutable.value.copy( state = Playback.State.Error, error = Error.EngineError(error) diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsSessionAdapter.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsSessionAdapter.kt index de0f014d2a..fbc09288f8 100644 --- a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsSessionAdapter.kt +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsSessionAdapter.kt @@ -400,7 +400,7 @@ internal class TtsSessionAdapter( override fun stop(reset: Boolean) {} override fun release() { - ttsPlayer.close() + // Do nothing. This object does not own the TtsPlayer instance. } override fun getCurrentTracks(): Tracks { diff --git a/readium/shared/src/main/java/org/readium/r2/shared/util/CursorList.kt b/readium/shared/src/main/java/org/readium/r2/shared/util/CursorList.kt index 73e615aa6a..41ebeaa398 100644 --- a/readium/shared/src/main/java/org/readium/r2/shared/util/CursorList.kt +++ b/readium/shared/src/main/java/org/readium/r2/shared/util/CursorList.kt @@ -18,9 +18,9 @@ class CursorList( private var index: Int = -1 ) : List by list { - init { - check(index in -1..list.size) - } + init { + check(index in -1..list.size) + } fun hasPrevious(): Boolean { return index > 0 diff --git a/test-app/src/main/java/org/readium/r2/testapp/reader/preferences/UserPreferences.kt b/test-app/src/main/java/org/readium/r2/testapp/reader/preferences/UserPreferences.kt index 059136fd41..234ae48d52 100644 --- a/test-app/src/main/java/org/readium/r2/testapp/reader/preferences/UserPreferences.kt +++ b/test-app/src/main/java/org/readium/r2/testapp/reader/preferences/UserPreferences.kt @@ -136,7 +136,7 @@ private fun ColumnScope.FixedLayoutUserPreferences( commit: () -> Unit, language: Preference? = null, readingProgression: EnumPreference? = null, - backgroundColor: Preference? = null, + backgroundColor: Preference? = null, scroll: Preference? = null, scrollAxis: EnumPreference? = null, fit: EnumPreference? = null, diff --git a/test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsControls.kt b/test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsControls.kt index 450a3fa9f4..45f32cf99e 100644 --- a/test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsControls.kt +++ b/test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsControls.kt @@ -18,6 +18,7 @@ import androidx.compose.ui.Modifier import androidx.compose.ui.res.stringResource import androidx.compose.ui.text.style.TextAlign import androidx.compose.ui.unit.dp +import org.readium.r2.navigator.media3.androidtts.AndroidTtsEngine import org.readium.r2.navigator.media3.androidtts.AndroidTtsPreferencesEditor import org.readium.r2.navigator.preferences.* import org.readium.r2.shared.ExperimentalReadiumApi @@ -40,6 +41,7 @@ fun TtsControls(model: TtsViewModel, modifier: Modifier = Modifier) { TtsControls( playing = isPlaying, editor = editor, + availableVoices = model.voices, commit = model::commit, onPlayPause = { if (isPlaying) model.pause() else model.play() }, onStop = model::stop, @@ -55,6 +57,7 @@ fun TtsControls(model: TtsViewModel, modifier: Modifier = Modifier) { fun TtsControls( playing: Boolean, editor: AndroidTtsPreferencesEditor, + availableVoices: Set, commit: () -> Unit, onPlayPause: () -> Unit, onStop: () -> Unit, @@ -70,6 +73,7 @@ fun TtsControls( pitch = editor.pitch, language = editor.language, voices = editor.voices, + availableVoices = availableVoices, commit = commit, onDismiss = { showSettings = false } ) @@ -138,6 +142,7 @@ private fun TtsPreferencesDialog( pitch: RangePreference, language: Preference, voices: Preference>, + availableVoices: Set, commit: () -> Unit, onDismiss: () -> Unit ) { @@ -193,7 +198,7 @@ private fun TtsPreferencesDialog( } } } - ).withSupportedValues("a", "b", "c"), + ).withSupportedValues(availableVoices.map { it.name }), formatValue = { it ?: "Default" }, // it?.name ?: it?.id ?: stringResource(R.string.auto) }, commit = commit ) diff --git a/test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsViewModel.kt b/test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsViewModel.kt index 1641aa7f33..62455deeac 100644 --- a/test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsViewModel.kt +++ b/test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsViewModel.kt @@ -12,6 +12,7 @@ import kotlinx.coroutines.channels.Channel import kotlinx.coroutines.flow.* import org.readium.r2.navigator.Navigator import org.readium.r2.navigator.VisualNavigator +import org.readium.r2.navigator.media3.androidtts.AndroidTtsEngine import org.readium.r2.navigator.media3.androidtts.AndroidTtsPreferences import org.readium.r2.navigator.media3.androidtts.AndroidTtsPreferencesEditor import org.readium.r2.navigator.media3.api.MediaNavigator @@ -119,6 +120,9 @@ class TtsViewModel private constructor( private val navigatorNow: AndroidTtsNavigator? get() = ttsServiceFacade.sessionNow()?.navigator + val voices: Set get() = + navigatorNow!!.voices + private var binding: Deferred = viewModelScope.async { ttsServiceFacade.getSession()?.let { bindSession(it) } From aff1ffcd80dfa4fc318beac3175614249d6bacef Mon Sep 17 00:00:00 2001 From: Quentin Gliosca Date: Fri, 27 Jan 2023 09:54:54 +0100 Subject: [PATCH 08/27] Filter voices by language --- .../media3/tts/TtsNavigatorFactory.kt | 1 - .../r2/testapp/reader/tts/TtsControls.kt | 19 ++++++++++++++----- test-app/src/main/res/values/strings.xml | 2 +- 3 files changed, 15 insertions(+), 7 deletions(-) diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsNavigatorFactory.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsNavigatorFactory.kt index 999004bf25..703d4686e3 100644 --- a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsNavigatorFactory.kt +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsNavigatorFactory.kt @@ -8,7 +8,6 @@ package org.readium.r2.navigator.media3.tts import android.app.Application import org.readium.r2.navigator.media3.api.DefaultMediaMetadataFactory -import org.readium.r2.navigator.media3.api.MediaMetadataFactory import org.readium.r2.navigator.media3.api.MediaMetadataProvider import org.readium.r2.navigator.preferences.PreferencesEditor import org.readium.r2.shared.ExperimentalReadiumApi diff --git a/test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsControls.kt b/test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsControls.kt index 45f32cf99e..b9fe1b8414 100644 --- a/test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsControls.kt +++ b/test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsControls.kt @@ -15,6 +15,7 @@ import androidx.compose.material.icons.filled.* import androidx.compose.runtime.* import androidx.compose.ui.Alignment import androidx.compose.ui.Modifier +import androidx.compose.ui.platform.LocalContext import androidx.compose.ui.res.stringResource import androidx.compose.ui.text.style.TextAlign import androidx.compose.ui.unit.dp @@ -180,26 +181,34 @@ private fun TtsPreferencesDialog( preference = language, commit = commit ) + + val context = LocalContext.current + MenuItem( title = stringResource(R.string.tts_voice), preference = voices.map( from = { voices -> - language.effectiveValue?.let { voices[it] } + language.effectiveValue?.let { voices[it.removeRegion()] } }, to = { voice -> buildMap { voices.value?.let { putAll(it) } language.effectiveValue?.let { + val withoutRegion = it.removeRegion() if (voice == null) { - remove(it) + remove(withoutRegion) } else { - put(it, voice) + put(withoutRegion, voice) } } } } - ).withSupportedValues(availableVoices.map { it.name }), - formatValue = { it ?: "Default" }, // it?.name ?: it?.id ?: stringResource(R.string.auto) }, + ).withSupportedValues( + availableVoices + .filter { it.language.removeRegion() == language.effectiveValue } + .map { it.name } + ), + formatValue = { it ?: context.getString(R.string.defaultValue) }, commit = commit ) } diff --git a/test-app/src/main/res/values/strings.xml b/test-app/src/main/res/values/strings.xml index 592869d255..6a4bbec932 100644 --- a/test-app/src/main/res/values/strings.xml +++ b/test-app/src/main/res/values/strings.xml @@ -197,7 +197,7 @@ Select a font Close Language - Auto + Default Failed to initialize the TTS engine The language %s is not supported From 60901b10ac44d46be6c17d42f00d1c06d327d25a Mon Sep 17 00:00:00 2001 From: Quentin Gliosca Date: Fri, 27 Jan 2023 11:28:26 +0100 Subject: [PATCH 09/27] Remove MediaNavigatorInternal --- .../r2/navigator/media3/api/MediaNavigator.kt | 9 +- .../media3/api/SynchronizedMediaNavigator.kt | 18 +- .../api/SynchronizedMediaNavigatorInternal.kt | 22 --- .../navigator/media3/player/PlayerLocator.kt | 26 --- .../SynchronizedNarrationNavigator.kt | 61 ------ .../r2/navigator/media3/tts/TtsNavigator.kt | 186 ++++++++++++------ .../media3/tts/TtsNavigatorFactory.kt | 2 +- .../media3/tts/TtsNavigatorInternal.kt | 172 ---------------- .../media3/tts/TtsNavigatorListener.kt | 12 -- .../r2/testapp/reader/tts/TtsControls.kt | 2 +- .../r2/testapp/reader/tts/TtsViewModel.kt | 5 +- 11 files changed, 148 insertions(+), 367 deletions(-) delete mode 100644 readium/navigator/src/main/java/org/readium/r2/navigator/media3/api/SynchronizedMediaNavigatorInternal.kt delete mode 100644 readium/navigator/src/main/java/org/readium/r2/navigator/media3/player/PlayerLocator.kt delete mode 100644 readium/navigator/src/main/java/org/readium/r2/navigator/media3/syncnarr/SynchronizedNarrationNavigator.kt delete mode 100644 readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsNavigatorInternal.kt delete mode 100644 readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsNavigatorListener.kt diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/api/MediaNavigator.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/api/MediaNavigator.kt index 26b3773287..6e26a66a12 100644 --- a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/api/MediaNavigator.kt +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/api/MediaNavigator.kt @@ -13,7 +13,12 @@ import org.readium.r2.shared.ExperimentalReadiumApi import org.readium.r2.shared.util.Closeable @ExperimentalReadiumApi -interface MediaNavigator : Navigator, Closeable { +interface MediaNavigator : Navigator, Closeable { + + /** + * Marker interface for the [position] flow. + */ + interface Position /** * Marker interface for the [Playback.error] property. @@ -44,6 +49,8 @@ interface MediaNavigator : Navigator, Closeable { */ val playback: StateFlow> + val position: StateFlow

+ /** * Resumes the playback at the current location. */ diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/api/SynchronizedMediaNavigator.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/api/SynchronizedMediaNavigator.kt index 3c8b56cf26..0247c1927e 100644 --- a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/api/SynchronizedMediaNavigator.kt +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/api/SynchronizedMediaNavigator.kt @@ -11,16 +11,22 @@ import org.readium.r2.shared.ExperimentalReadiumApi import org.readium.r2.shared.publication.Locator @ExperimentalReadiumApi -interface SynchronizedMediaNavigator : MediaNavigator { +interface SynchronizedMediaNavigator + : MediaNavigator { + + interface Utterance

{ + val text: String + + val position: P - data class Utterance( - val locator: Locator, val range: IntRange? - ) { - val rangeLocator: Locator? = range + val locator: Locator + + val rangeLocator: Locator? get() = range ?.let { locator.copy(text = locator.text.substring(it)) } } - val utterance: StateFlow + + val utterance: StateFlow> } diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/api/SynchronizedMediaNavigatorInternal.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/api/SynchronizedMediaNavigatorInternal.kt deleted file mode 100644 index c45d0f8a6b..0000000000 --- a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/api/SynchronizedMediaNavigatorInternal.kt +++ /dev/null @@ -1,22 +0,0 @@ -/* - * Copyright 2022 Readium Foundation. All rights reserved. - * Use of this source code is governed by the BSD-style license - * available in the top-level LICENSE file of the project. - */ - -package org.readium.r2.navigator.media3.api - -import kotlinx.coroutines.flow.StateFlow - -interface SynchronizedMediaNavigatorInternal

: - MediaNavigatorInternal { - - data class Utterance

( - val text: String, - val position: P, - val range: IntRange? - ) - - val utterance: StateFlow> -} diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/player/PlayerLocator.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/player/PlayerLocator.kt deleted file mode 100644 index b78d5fb3e4..0000000000 --- a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/player/PlayerLocator.kt +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Copyright 2022 Readium Foundation. All rights reserved. - * Use of this source code is governed by the BSD-style license - * available in the top-level LICENSE file of the project. - */ - -package org.readium.r2.navigator.media3.player - -import kotlin.time.Duration -import kotlin.time.Duration.Companion.seconds -import kotlinx.serialization.Serializable -import org.readium.r2.navigator.extensions.fragmentParameters -import org.readium.r2.navigator.media3.api.MediaNavigatorInternal -import org.readium.r2.shared.ExperimentalReadiumApi -import org.readium.r2.shared.publication.Locator - -@ExperimentalReadiumApi -@Serializable -data class PlayerLocator( - val index: Int, - @Serializable(with = DurationSerializer::class) - val position: Duration -) : MediaNavigatorInternal.Position - -internal val Locator.Locations.time: Duration? get() = - fragmentParameters["t"]?.toIntOrNull()?.seconds diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/syncnarr/SynchronizedNarrationNavigator.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/syncnarr/SynchronizedNarrationNavigator.kt deleted file mode 100644 index 64158df9dc..0000000000 --- a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/syncnarr/SynchronizedNarrationNavigator.kt +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Copyright 2022 Readium Foundation. All rights reserved. - * Use of this source code is governed by the BSD-style license - * available in the top-level LICENSE file of the project. - */ - -package org.readium.r2.navigator.media3.syncnarr - -/*@ExperimentalReadiumApi -class SynchronizedNarrationNavigator>( - private val internalNavigator: SynchronizedNarrationNavigatorInternal -) : MediaNavigator { - - data class Playback( - override val state: MediaNavigator.State, - override val locator: Locator, - override val token: Locator?, - override val buffer: MediaNavigator.Buffer - ) : MediaNavigator.Playback, MediaNavigator.BufferProvider, MediaNavigator.TextSynchronization - - override val publication: Publication - get() = TODO("Not yet implemented") - - override val currentLocator: StateFlow - get() = TODO("Not yet implemented") - - override fun go(locator: Locator, animated: Boolean, completion: () -> Unit): Boolean { - TODO("Not yet implemented") - } - - override fun go(link: Link, animated: Boolean, completion: () -> Unit): Boolean { - TODO("Not yet implemented") - } - - override fun goForward(animated: Boolean, completion: () -> Unit): Boolean { - TODO("Not yet implemented") - } - - override fun goBackward(animated: Boolean, completion: () -> Unit): Boolean { - TODO("Not yet implemented") - } - - override fun close() { - TODO("Not yet implemented") - } - - override val playback: StateFlow - get() = TODO("Not yet implemented") - - override fun play() { - TODO("Not yet implemented") - } - - override fun pause() { - TODO("Not yet implemented") - } - - override fun asPlayer(): Player { - TODO("Not yet implemented") - } -}*/ diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsNavigator.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsNavigator.kt index edf7cef6ed..d9465ebb27 100644 --- a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsNavigator.kt +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsNavigator.kt @@ -8,12 +8,14 @@ package org.readium.r2.navigator.media3.tts import android.app.Application import androidx.media3.common.MediaItem +import androidx.media3.common.PlaybackParameters import androidx.media3.common.Player import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.MainScope import kotlinx.coroutines.flow.StateFlow -import org.readium.r2.navigator.Navigator -import org.readium.r2.navigator.media3.api.* +import org.readium.r2.navigator.media3.api.MediaMetadataProvider +import org.readium.r2.navigator.media3.api.MediaNavigator +import org.readium.r2.navigator.media3.api.SynchronizedMediaNavigator import org.readium.r2.navigator.preferences.Configurable import org.readium.r2.shared.ExperimentalReadiumApi import org.readium.r2.shared.extensions.mapStateIn @@ -23,14 +25,15 @@ import org.readium.r2.shared.publication.Publication import org.readium.r2.shared.publication.services.content.ContentService import org.readium.r2.shared.publication.services.content.ContentTokenizer import org.readium.r2.shared.util.Language -import timber.log.Timber @ExperimentalReadiumApi class TtsNavigator, E : TtsEngine.Error, V : TtsEngine.Voice> private constructor( + coroutineScope: CoroutineScope, override val publication: Publication, - private val ttsNavigator: TtsNavigatorInternal -) : SynchronizedMediaNavigator, Navigator, Configurable by ttsNavigator { + private val player: TtsPlayer, + private val sessionAdapter: TtsSessionAdapter, +) : SynchronizedMediaNavigator, Configurable by player { companion object { @@ -41,7 +44,7 @@ class TtsNavigator, ttsEngineProvider: TtsEngineProvider, tokenizerFactory: (defaultLanguage: Language?) -> ContentTokenizer, metadataProvider: MediaMetadataProvider, - listener: TtsNavigatorListener, + listener: Listener, initialPreferences: P? = null, initialLocator: Locator? = null, ): TtsNavigator? { @@ -50,8 +53,6 @@ class TtsNavigator, return null } - Timber.d("initialLocator $initialLocator") - val actualInitialPreferences = initialPreferences ?: ttsEngineProvider.createEmptyPreferences() @@ -76,24 +77,65 @@ class TtsNavigator, .build() } - val internalNavigator = - TtsNavigatorInternal( + val ttsPlayer = + TtsPlayer(ttsEngine, contentIterator, actualInitialPreferences) + ?: return null + + val coroutineScope = + MainScope() + + val playbackParameters = + ttsPlayer.settings.mapStateIn(coroutineScope) { + ttsEngineProvider.getPlaybackParameters(it) + } + + val onSetPlaybackParameters = { parameters: PlaybackParameters -> + val newPreferences = ttsEngineProvider.updatePlaybackParameters(ttsPlayer.lastPreferences, parameters) + ttsPlayer.submitPreferences(newPreferences) + } + + val sessionAdapter = + TtsSessionAdapter( application, - ttsEngine, - contentIterator, + ttsPlayer, playlistMetadata, mediaItems, - ttsEngineProvider::getPlaybackParameters, - ttsEngineProvider::updatePlaybackParameters, - ttsEngineProvider::mapEngineError, - actualInitialPreferences, - listener, - ) ?: return null - - return TtsNavigator(publication, internalNavigator) + listener::onStopRequested, + playbackParameters, + onSetPlaybackParameters, + ttsEngineProvider::mapEngineError + ) + + return TtsNavigator(coroutineScope, publication, ttsPlayer, sessionAdapter) } } + interface Listener { + + fun onStopRequested() + + fun onMissingLanguageData() + } + + data class Position( + val resourceIndex: Int, + val cssSelector: String, + val textBefore: String?, + val textAfter: String?, + ) : MediaNavigator.Position + + + data class Utterance( + override val text: String, + override val position: Position, + override val range: IntRange?, + override val locator: Locator + ) : SynchronizedMediaNavigator.Utterance { + + override val rangeLocator: Locator? = range + ?.let { locator.copy(text = locator.text.substring(it)) } + } + sealed class Error : MediaNavigator.Error { data class EngineError (val error: E) : Error() @@ -101,85 +143,97 @@ class TtsNavigator, data class ContentError(val exception: Exception) : Error() } - private val coroutineScope: CoroutineScope = - MainScope() - val voices: Set get() = - ttsNavigator.voices + player.voices override val playback: StateFlow> = - ttsNavigator.playback.mapStateIn(coroutineScope) { it.toPlayback() } + player.playback.mapStateIn(coroutineScope) { it.toPlayback() } + + override val utterance: StateFlow = + player.utterance.mapStateIn(coroutineScope) { it.toUtterance() } + + override val position: StateFlow = + utterance.mapStateIn(coroutineScope) { it.position } + + override fun play() { + player.play() + } + + override fun pause() { + player.pause() + } + + fun go(locator: Locator) { + player.go(locator) + } + + fun previousUtterance() { + player.previousUtterance() + } + + fun nextUtterance() { + player.nextUtterance() + } - override val utterance: StateFlow = - ttsNavigator.utterance.mapStateIn(coroutineScope) { Timber.d("utterance $it"); it.toUtterance() } + override fun asPlayer(): Player = + sessionAdapter + + override fun close() { + player.close() + } override val currentLocator: StateFlow = - ttsNavigator.utterance.mapStateIn(coroutineScope) { it.toLocator() } + utterance.mapStateIn(coroutineScope) { it.locator } override fun go(locator: Locator, animated: Boolean, completion: () -> Unit): Boolean { - ttsNavigator.go(TtsNavigatorInternal.RelaxedPosition(locator)) + player.go(locator) return true } override fun go(link: Link, animated: Boolean, completion: () -> Unit): Boolean { val locator = publication.locatorFromLink(link) ?: return false - return go(locator) + return go(locator, animated, completion) } override fun goForward(animated: Boolean, completion: () -> Unit): Boolean { - ttsNavigator.goForward() + player.nextUtterance() return true } override fun goBackward(animated: Boolean, completion: () -> Unit): Boolean { - ttsNavigator.goBackward() + player.previousUtterance() return true } - override fun close() { - ttsNavigator.close() - } - - override fun play() { - ttsNavigator.play() - } - - override fun pause() { - ttsNavigator.pause() - } - - override fun asPlayer(): Player = - ttsNavigator.asPlayer() - - private fun MediaNavigatorInternal.Playback.toPlayback() = + private fun TtsPlayer.Playback.toPlayback() = MediaNavigator.Playback( state = state.toState(), playWhenReady = playWhenReady, error = error?.toError() ) - private fun MediaNavigatorInternal.State.toState() = + private fun TtsPlayer.Playback.State.toState() = when (this) { - MediaNavigatorInternal.State.Ready -> MediaNavigator.State.Ready - MediaNavigatorInternal.State.Ended -> MediaNavigator.State.Ended - MediaNavigatorInternal.State.Buffering -> MediaNavigator.State.Buffering - MediaNavigatorInternal.State.Error -> MediaNavigator.State.Error + TtsPlayer.Playback.State.Ready -> MediaNavigator.State.Ready + TtsPlayer.Playback.State.Ended -> MediaNavigator.State.Ended + TtsPlayer.Playback.State.Error -> MediaNavigator.State.Error } - private fun TtsNavigatorInternal.Error.toError(): Error = + private fun TtsPlayer.Error.toError(): Error = when (this) { - is TtsNavigatorInternal.Error.ContentError -> Error.ContentError(exception) - is TtsNavigatorInternal.Error.EngineError<*> -> Error.EngineError(error) + is TtsPlayer.Error.ContentError -> Error.ContentError(exception) + is TtsPlayer.Error.EngineError<*> -> Error.EngineError(error) } - private fun SynchronizedMediaNavigatorInternal.Utterance.toUtterance() = - SynchronizedMediaNavigator.Utterance( - locator = toLocator(), - range = range + private fun TtsPlayer.Utterance.toUtterance(): Utterance { + val position = Position( + resourceIndex = position.resourceIndex, + cssSelector = position.cssSelector, + textBefore = position.textBefore, + textAfter = position.textAfter ) - private fun SynchronizedMediaNavigatorInternal.Utterance.toLocator() = - publication + val locator = publication .locatorFromLink(publication.readingOrder[position.resourceIndex])!! .copyWithLocations( progression = null, @@ -194,4 +248,12 @@ class TtsNavigator, after = position.textAfter ) ) + + return Utterance( + text = text, + position = position, + locator = locator, + range = range + ) + } } diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsNavigatorFactory.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsNavigatorFactory.kt index 703d4686e3..c62ff222d7 100644 --- a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsNavigatorFactory.kt +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsNavigatorFactory.kt @@ -61,7 +61,7 @@ class TtsNavigatorFactory, } suspend fun createNavigator( - listener: TtsNavigatorListener, + listener: TtsNavigator.Listener, initialPreferences: P? = null, initialLocator: Locator? = null ): TtsNavigator { diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsNavigatorInternal.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsNavigatorInternal.kt deleted file mode 100644 index 61e28b6a38..0000000000 --- a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsNavigatorInternal.kt +++ /dev/null @@ -1,172 +0,0 @@ -/* - * Copyright 2022 Readium Foundation. All rights reserved. - * Use of this source code is governed by the BSD-style license - * available in the top-level LICENSE file of the project. - */ - -package org.readium.r2.navigator.media3.tts - -import android.app.Application -import androidx.media3.common.MediaItem -import androidx.media3.common.MediaMetadata -import androidx.media3.common.PlaybackException -import androidx.media3.common.PlaybackParameters -import androidx.media3.common.Player -import kotlinx.coroutines.CoroutineScope -import kotlinx.coroutines.MainScope -import kotlinx.coroutines.flow.StateFlow -import org.readium.r2.navigator.media3.api.MediaNavigatorInternal -import org.readium.r2.navigator.media3.api.SynchronizedMediaNavigatorInternal -import org.readium.r2.navigator.preferences.Configurable -import org.readium.r2.shared.ExperimentalReadiumApi -import org.readium.r2.shared.extensions.mapStateIn -import org.readium.r2.shared.publication.Locator - -@ExperimentalReadiumApi -internal class TtsNavigatorInternal, - E : TtsEngine.Error, V : TtsEngine.Voice>( - private val ttsPlayer: TtsPlayer, - private val sessionAdapter: TtsSessionAdapter, - coroutineScope: CoroutineScope, -) : SynchronizedMediaNavigatorInternal, - Configurable by ttsPlayer { - - companion object { - - suspend operator fun , - E : TtsEngine.Error, V : TtsEngine.Voice> invoke( - application: Application, - ttsEngine: TtsEngine, - ttsContentIterator: TtsContentIterator, - playlistMetadata: MediaMetadata, - mediaItems: List, - getPlaybackParameters: (S) -> PlaybackParameters, - updatePlaybackParameters: (P, PlaybackParameters) -> P, - mapEngineError: (E) -> PlaybackException, - initialPreferences: P, - listener: TtsNavigatorListener - ): TtsNavigatorInternal? { - val ttsPlayer = - TtsPlayer(ttsEngine, ttsContentIterator, initialPreferences) - ?: return null - - val coroutineScope = - MainScope() - - val playbackParameters = - ttsPlayer.settings.mapStateIn(coroutineScope) { - getPlaybackParameters(it) - } - - val onSetPlaybackParameters = { parameters: PlaybackParameters -> - val newPreferences = updatePlaybackParameters(ttsPlayer.lastPreferences, parameters) - ttsPlayer.submitPreferences(newPreferences) - } - - val sessionAdapter = - TtsSessionAdapter( - application, - ttsPlayer, - playlistMetadata, - mediaItems, - listener::onStopRequested, - playbackParameters, - onSetPlaybackParameters, - mapEngineError - ) - - return TtsNavigatorInternal(ttsPlayer, sessionAdapter, coroutineScope) - } - } - - data class RelaxedPosition( - val locator: Locator - ) : MediaNavigatorInternal.RelaxedPosition - - data class Position( - val resourceIndex: Int, - val cssSelector: String, - val textBefore: String?, - val textAfter: String?, - ) : MediaNavigatorInternal.Position - - sealed class Error : MediaNavigatorInternal.Error { - - data class EngineError (val error: E) : Error() - - data class ContentError(val exception: Exception) : Error() - } - - val voices: Set get() = - ttsPlayer.voices - - override val playback: StateFlow> = - ttsPlayer.playback.mapStateIn(coroutineScope) { it.toPlayback() } - - override val utterance: StateFlow> = - ttsPlayer.utterance.mapStateIn(coroutineScope) { it.toUtterance() } - - override val progression: StateFlow = - utterance.mapStateIn(coroutineScope) { it.position } - - override fun play() { - ttsPlayer.play() - } - - override fun pause() { - ttsPlayer.pause() - } - - override fun go(position: RelaxedPosition) { - ttsPlayer.go(position.locator) - } - - override fun goForward() { - ttsPlayer.nextUtterance() - } - - override fun goBackward() { - ttsPlayer.previousUtterance() - } - - override fun asPlayer(): Player { - return sessionAdapter - } - - fun close() { - ttsPlayer.close() - } - - private fun TtsPlayer.Utterance.toUtterance() = - SynchronizedMediaNavigatorInternal.Utterance( - text = text, - range = range, - position = Position( - resourceIndex = position.resourceIndex, - cssSelector = position.cssSelector, - textBefore = position.textBefore, - textAfter = position.textAfter - ) - ) - - private fun TtsPlayer.Playback.toPlayback() = - MediaNavigatorInternal.Playback( - state = state.toState(), - playWhenReady = playWhenReady, - error = error?.toError() - ) - - private fun TtsPlayer.Playback.State.toState() = - when (this) { - TtsPlayer.Playback.State.Ready -> MediaNavigatorInternal.State.Ready - TtsPlayer.Playback.State.Ended -> MediaNavigatorInternal.State.Ended - TtsPlayer.Playback.State.Error -> MediaNavigatorInternal.State.Error - } - - private fun TtsPlayer.Error.toError(): Error = - when (this) { - is TtsPlayer.Error.ContentError -> Error.ContentError(exception) - is TtsPlayer.Error.EngineError<*> -> Error.EngineError(error) - } -} diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsNavigatorListener.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsNavigatorListener.kt deleted file mode 100644 index e7a6f26e4b..0000000000 --- a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsNavigatorListener.kt +++ /dev/null @@ -1,12 +0,0 @@ -/* - * Copyright 2022 Readium Foundation. All rights reserved. - * Use of this source code is governed by the BSD-style license - * available in the top-level LICENSE file of the project. - */ - -package org.readium.r2.navigator.media3.tts - -interface TtsNavigatorListener { - - fun onStopRequested() -} diff --git a/test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsControls.kt b/test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsControls.kt index b9fe1b8414..28ee923d3b 100644 --- a/test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsControls.kt +++ b/test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsControls.kt @@ -208,7 +208,7 @@ private fun TtsPreferencesDialog( .filter { it.language.removeRegion() == language.effectiveValue } .map { it.name } ), - formatValue = { it ?: context.getString(R.string.defaultValue) }, + formatValue = { it ?: context.getString(R.string.defaultValue) }, commit = commit ) } diff --git a/test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsViewModel.kt b/test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsViewModel.kt index 62455deeac..f51ac41774 100644 --- a/test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsViewModel.kt +++ b/test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsViewModel.kt @@ -18,7 +18,6 @@ import org.readium.r2.navigator.media3.androidtts.AndroidTtsPreferencesEditor import org.readium.r2.navigator.media3.api.MediaNavigator import org.readium.r2.navigator.media3.api.SynchronizedMediaNavigator import org.readium.r2.navigator.media3.tts.TtsNavigator -import org.readium.r2.navigator.media3.tts.TtsNavigatorListener import org.readium.r2.shared.DelicateReadiumApi import org.readium.r2.shared.ExperimentalReadiumApi import org.readium.r2.shared.UserException @@ -145,7 +144,7 @@ class TtsViewModel private constructor( private suspend fun openSession(navigator: Navigator): TtsService.Session { val start = (navigator as? VisualNavigator)?.firstVisibleElementLocator() - val listener = object : TtsNavigatorListener { + val listener = object : TtsNavigator.Listener { override fun onStopRequested() { stop() @@ -184,7 +183,7 @@ class TtsViewModel private constructor( private fun stateFromPlayback( playback: MediaNavigator.Playback?, - utterance: SynchronizedMediaNavigator.Utterance? + utterance: SynchronizedMediaNavigator.Utterance? ): State { if (playback == null || utterance == null) return State() From e1e0363c2aa7f5ec77bf2b243fa2822f9cdc7f5f Mon Sep 17 00:00:00 2001 From: Quentin Gliosca Date: Fri, 27 Jan 2023 14:52:59 +0100 Subject: [PATCH 10/27] Fix exoplayer --- .../media3/api/MediaNavigatorInternal.kt | 69 ---------- .../media3/api/SynchronizedMediaNavigator.kt | 5 +- .../r2/navigator/media3/audio/AudioEngine.kt | 46 +++++++ .../AudioEngineProvider.kt} | 8 +- .../navigator/media3/audio/AudioNavigator.kt | 70 ++++++++++ .../{player => audio}/DurationSerializer.kt | 2 +- .../media3/exoplayer/ExoPlayerEngine.kt | 85 ++++++++++++ .../exoplayer/ExoPlayerEngineProvider.kt | 27 +--- .../media3/player/PlayerNavigatorFactory.kt | 59 --------- .../r2/navigator/media3/tts/TtsNavigator.kt | 15 ++- .../media3/tts/TtsNavigatorFactory.kt | 49 ++++++- .../android}/AndroidTtsEngine.kt | 4 +- .../android}/AndroidTtsEngineProvider.kt | 4 +- .../android}/AndroidTtsPreferences.kt | 2 +- .../android}/AndroidTtsPreferencesEditor.kt | 2 +- .../android}/AndroidTtsPreferencesFilters.kt | 2 +- .../AndroidTtsPreferencesSerializer.kt | 2 +- .../android}/AndroidTtsSettings.kt | 2 +- .../android}/AndroidTtsSettingsResolver.kt | 2 +- .../r2/testapp/reader/ReaderInitData.kt | 2 +- .../r2/testapp/reader/ReaderRepository.kt | 6 +- .../reader/preferences/PreferencesManagers.kt | 8 +- .../r2/testapp/reader/tts/TtsControls.kt | 4 +- .../r2/testapp/reader/tts/TtsEngine.kt | 8 +- .../r2/testapp/reader/tts/TtsViewModel.kt | 123 ++++++++---------- 25 files changed, 344 insertions(+), 262 deletions(-) delete mode 100644 readium/navigator/src/main/java/org/readium/r2/navigator/media3/api/MediaNavigatorInternal.kt create mode 100644 readium/navigator/src/main/java/org/readium/r2/navigator/media3/audio/AudioEngine.kt rename readium/navigator/src/main/java/org/readium/r2/navigator/media3/{player/MediaEngineProvider.kt => audio/AudioEngineProvider.kt} (77%) create mode 100644 readium/navigator/src/main/java/org/readium/r2/navigator/media3/audio/AudioNavigator.kt rename readium/navigator/src/main/java/org/readium/r2/navigator/media3/{player => audio}/DurationSerializer.kt (94%) create mode 100644 readium/navigator/src/main/java/org/readium/r2/navigator/media3/exoplayer/ExoPlayerEngine.kt delete mode 100644 readium/navigator/src/main/java/org/readium/r2/navigator/media3/player/PlayerNavigatorFactory.kt rename readium/navigator/src/main/java/org/readium/r2/navigator/media3/{androidtts => tts/android}/AndroidTtsEngine.kt (98%) rename readium/navigator/src/main/java/org/readium/r2/navigator/media3/{androidtts => tts/android}/AndroidTtsEngineProvider.kt (96%) rename readium/navigator/src/main/java/org/readium/r2/navigator/media3/{androidtts => tts/android}/AndroidTtsPreferences.kt (94%) rename readium/navigator/src/main/java/org/readium/r2/navigator/media3/{androidtts => tts/android}/AndroidTtsPreferencesEditor.kt (98%) rename readium/navigator/src/main/java/org/readium/r2/navigator/media3/{androidtts => tts/android}/AndroidTtsPreferencesFilters.kt (95%) rename readium/navigator/src/main/java/org/readium/r2/navigator/media3/{androidtts => tts/android}/AndroidTtsPreferencesSerializer.kt (94%) rename readium/navigator/src/main/java/org/readium/r2/navigator/media3/{androidtts => tts/android}/AndroidTtsSettings.kt (91%) rename readium/navigator/src/main/java/org/readium/r2/navigator/media3/{androidtts => tts/android}/AndroidTtsSettingsResolver.kt (93%) diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/api/MediaNavigatorInternal.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/api/MediaNavigatorInternal.kt deleted file mode 100644 index 19a22d2da3..0000000000 --- a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/api/MediaNavigatorInternal.kt +++ /dev/null @@ -1,69 +0,0 @@ -/* - * Copyright 2022 Readium Foundation. All rights reserved. - * Use of this source code is governed by the BSD-style license - * available in the top-level LICENSE file of the project. - */ - -package org.readium.r2.navigator.media3.api - -import androidx.media3.common.Player -import kotlinx.coroutines.flow.StateFlow -import org.readium.r2.shared.InternalReadiumApi - -@InternalReadiumApi -interface MediaNavigatorInternal

{ - - interface Position - - interface RelaxedPosition - - interface Error - - enum class State { - Ready, - Buffering, - Ended, - Error; - } - - data class Playback( - val state: State, - val playWhenReady: Boolean, - val error: E? - ) - - val playback: StateFlow> - - val progression: StateFlow

- - /** - * Resumes the playback at the current location. - */ - fun play() - - /** - * Pauses the playback. - */ - fun pause() - - /** - * Seeks to the given locator. - */ - fun go(position: R) - - /** - * Skips forward - */ - fun goForward() - - /** - * Skips backward. - */ - fun goBackward() - - /** - * Adapts this navigator to the media3 [Player] interface. - */ - fun asPlayer(): Player -} diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/api/SynchronizedMediaNavigator.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/api/SynchronizedMediaNavigator.kt index 0247c1927e..8d3da873bc 100644 --- a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/api/SynchronizedMediaNavigator.kt +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/api/SynchronizedMediaNavigator.kt @@ -21,12 +21,11 @@ interface SynchronizedMediaNavigator> } diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/audio/AudioEngine.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/audio/AudioEngine.kt new file mode 100644 index 0000000000..a167bd13ea --- /dev/null +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/audio/AudioEngine.kt @@ -0,0 +1,46 @@ +/* + * Copyright 2022 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + +package org.readium.r2.navigator.media3.audio + +import androidx.media3.common.Player +import kotlin.time.Duration +import kotlinx.coroutines.flow.StateFlow +import org.readium.r2.navigator.media3.api.MediaNavigator +import org.readium.r2.navigator.preferences.Configurable +import org.readium.r2.shared.ExperimentalReadiumApi + +@ExperimentalReadiumApi +interface AudioEngine, E : AudioEngine.Error> + : Configurable { + + interface Error + + data class Playback( + val state: MediaNavigator.State, + val playWhenReady: Boolean, + val error: E? + ) + + data class Position( + val index: Int, + val duration: Duration + ) + + val playback: StateFlow> + + val position: StateFlow + + fun play() + + fun pause() + + fun seek(index: Long, position: Duration) + + fun close() + + fun asPlayer(): Player +} diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/player/MediaEngineProvider.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/audio/AudioEngineProvider.kt similarity index 77% rename from readium/navigator/src/main/java/org/readium/r2/navigator/media3/player/MediaEngineProvider.kt rename to readium/navigator/src/main/java/org/readium/r2/navigator/media3/audio/AudioEngineProvider.kt index e6e5b4e3f3..4413448bea 100644 --- a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/player/MediaEngineProvider.kt +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/audio/AudioEngineProvider.kt @@ -4,9 +4,8 @@ * available in the top-level LICENSE file of the project. */ -package org.readium.r2.navigator.media3.player +package org.readium.r2.navigator.media3.audio -import androidx.media3.common.Player import org.readium.r2.navigator.preferences.Configurable import org.readium.r2.navigator.preferences.PreferencesEditor import org.readium.r2.shared.ExperimentalReadiumApi @@ -14,9 +13,10 @@ import org.readium.r2.shared.publication.Metadata import org.readium.r2.shared.publication.Publication @ExperimentalReadiumApi -interface MediaEngineProvider, E : PreferencesEditor

> { +interface AudioEngineProvider, + E : PreferencesEditor

, F : AudioEngine.Error> { - suspend fun createPlayer(publication: Publication): Player + suspend fun createEngine(publication: Publication): AudioEngine /** * Creates settings for [metadata] and [preferences]. diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/audio/AudioNavigator.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/audio/AudioNavigator.kt new file mode 100644 index 0000000000..7faddfd7d1 --- /dev/null +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/audio/AudioNavigator.kt @@ -0,0 +1,70 @@ +/* + * Copyright 2022 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + +package org.readium.r2.navigator.media3.audio + +import androidx.media3.common.Player +import kotlinx.coroutines.flow.StateFlow +import org.readium.r2.navigator.media3.api.MediaNavigator +import org.readium.r2.navigator.preferences.Configurable +import org.readium.r2.shared.ExperimentalReadiumApi +import org.readium.r2.shared.publication.Link +import org.readium.r2.shared.publication.Locator +import org.readium.r2.shared.publication.Publication + +@ExperimentalReadiumApi +class AudioNavigator, E : AudioEngine.Error>( + private val mediaEngine: AudioEngine +) : MediaNavigator, Configurable by mediaEngine { + + class Position: MediaNavigator.Position + + class Error : MediaNavigator.Error + + override val publication: Publication + get() = TODO("Not yet implemented") + + override val currentLocator: StateFlow + get() = TODO("Not yet implemented") + + override fun go(locator: Locator, animated: Boolean, completion: () -> Unit): Boolean { + TODO("Not yet implemented") + } + + override fun go(link: Link, animated: Boolean, completion: () -> Unit): Boolean { + TODO("Not yet implemented") + } + + override fun goForward(animated: Boolean, completion: () -> Unit): Boolean { + TODO("Not yet implemented") + } + + override fun goBackward(animated: Boolean, completion: () -> Unit): Boolean { + TODO("Not yet implemented") + } + + override fun close() { + TODO("Not yet implemented") + } + + override val playback: StateFlow> + get() = TODO("Not yet implemented") + + override val position: StateFlow + get() = TODO("Not yet implemented") + + override fun play() { + TODO("Not yet implemented") + } + + override fun pause() { + TODO("Not yet implemented") + } + + override fun asPlayer(): Player { + TODO("Not yet implemented") + } +} diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/player/DurationSerializer.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/audio/DurationSerializer.kt similarity index 94% rename from readium/navigator/src/main/java/org/readium/r2/navigator/media3/player/DurationSerializer.kt rename to readium/navigator/src/main/java/org/readium/r2/navigator/media3/audio/DurationSerializer.kt index 9a717708a3..ea9836096b 100644 --- a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/player/DurationSerializer.kt +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/audio/DurationSerializer.kt @@ -4,7 +4,7 @@ * available in the top-level LICENSE file of the project. */ -package org.readium.r2.navigator.media3.player +package org.readium.r2.navigator.media3.audio import kotlin.time.Duration import kotlinx.serialization.KSerializer diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/exoplayer/ExoPlayerEngine.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/exoplayer/ExoPlayerEngine.kt new file mode 100644 index 0000000000..3022884ce5 --- /dev/null +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/exoplayer/ExoPlayerEngine.kt @@ -0,0 +1,85 @@ +/* + * Copyright 2022 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + +package org.readium.r2.navigator.media3.exoplayer + +import android.app.Application +import androidx.media3.common.AudioAttributes +import androidx.media3.common.C +import androidx.media3.common.Player +import androidx.media3.datasource.DataSource +import androidx.media3.exoplayer.ExoPlayer +import androidx.media3.exoplayer.source.DefaultMediaSourceFactory +import kotlin.time.Duration +import kotlinx.coroutines.flow.StateFlow +import org.readium.r2.navigator.media3.audio.AudioEngine +import org.readium.r2.shared.ExperimentalReadiumApi +import org.readium.r2.shared.publication.Publication + +@ExperimentalReadiumApi +@androidx.annotation.OptIn(androidx.media3.common.util.UnstableApi::class) +class ExoPlayerEngine( + private val application: Application, + private val publication: Publication, + private val exoPlayer: ExoPlayer, +) : AudioEngine { + + companion object { + + private fun createExoPlayer( + application: Application, + publication: Publication, + ): ExoPlayer { + val dataSourceFactory: DataSource.Factory = ExoPlayerDataSource.Factory(publication) + return ExoPlayer.Builder(application) + .setMediaSourceFactory(DefaultMediaSourceFactory(dataSourceFactory)) + .setAudioAttributes( + AudioAttributes.Builder() + .setContentType(C.AUDIO_CONTENT_TYPE_MUSIC) + .setUsage(C.USAGE_MEDIA) + .build(), + true + ) + .setHandleAudioBecomingNoisy(true) + .build() + } + } + + class Error : AudioEngine.Error + + override val playback: StateFlow> + get() = TODO("Not yet implemented") + + override val position: StateFlow + get() = TODO("Not yet implemented") + + override val settings: StateFlow + get() = TODO("Not yet implemented") + + override fun play() { + TODO("Not yet implemented") + } + + override fun pause() { + TODO("Not yet implemented") + } + + override fun seek(index: Long, position: Duration) { + TODO("Not yet implemented") + } + + override fun close() { + TODO("Not yet implemented") + } + + override fun asPlayer(): Player { + return exoPlayer + } + + override fun submitPreferences(preferences: ExoPlayerPreferences) { + TODO("Not yet implemented") + } +} diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/exoplayer/ExoPlayerEngineProvider.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/exoplayer/ExoPlayerEngineProvider.kt index 3bae8b85c1..34529b8503 100644 --- a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/exoplayer/ExoPlayerEngineProvider.kt +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/exoplayer/ExoPlayerEngineProvider.kt @@ -6,36 +6,17 @@ package org.readium.r2.navigator.media3.exoplayer -import android.content.Context -import androidx.media3.common.AudioAttributes -import androidx.media3.common.C -import androidx.media3.datasource.DataSource -import androidx.media3.exoplayer.ExoPlayer -import androidx.media3.exoplayer.source.DefaultMediaSourceFactory -import org.readium.r2.navigator.media3.player.MediaEngineProvider +import org.readium.r2.navigator.media3.audio.AudioEngineProvider import org.readium.r2.shared.ExperimentalReadiumApi import org.readium.r2.shared.publication.Metadata import org.readium.r2.shared.publication.Publication @ExperimentalReadiumApi @androidx.annotation.OptIn(androidx.media3.common.util.UnstableApi::class) -class ExoPlayerEngineProvider( - private val context: Context, -) : MediaEngineProvider { +class ExoPlayerEngineProvider() : AudioEngineProvider { - override suspend fun createPlayer(publication: Publication): ExoPlayer { - val dataSourceFactory: DataSource.Factory = ExoPlayerDataSource.Factory(publication) - return ExoPlayer.Builder(context) - .setMediaSourceFactory(DefaultMediaSourceFactory(dataSourceFactory)) - .setAudioAttributes( - AudioAttributes.Builder() - .setContentType(C.AUDIO_CONTENT_TYPE_MUSIC) - .setUsage(C.USAGE_MEDIA) - .build(), - true - ) - .setHandleAudioBecomingNoisy(true) - .build() + override suspend fun createEngine(publication: Publication):ExoPlayerEngine { + TODO("Not yet implemented") } override fun computeSettings( diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/player/PlayerNavigatorFactory.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/player/PlayerNavigatorFactory.kt deleted file mode 100644 index b1002b22b8..0000000000 --- a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/player/PlayerNavigatorFactory.kt +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Copyright 2022 Readium Foundation. All rights reserved. - * Use of this source code is governed by the BSD-style license - * available in the top-level LICENSE file of the project. - */ - -package org.readium.r2.navigator.media3.player - -/*import org.readium.r2.navigator.media3.api.MediaMetadataProvider -import org.readium.r2.navigator.preferences.Configurable -import org.readium.r2.navigator.preferences.PreferencesEditor -import org.readium.r2.shared.ExperimentalReadiumApi -import org.readium.r2.shared.publication.Locator -import org.readium.r2.shared.publication.Publication - -@ExperimentalReadiumApi -class PlayerNavigatorFactory, E : PreferencesEditor

>( - private val publication: Publication, - private val mediaEngineProvider: MediaEngineProvider, - private val playerNavigator: PlayerNavigator, -) { - - companion object { - - suspend operator fun , E : PreferencesEditor

> invoke( - publication: Publication, - mediaEngineProvider: MediaEngineProvider, - metadataProvider: MediaMetadataProvider, - initialPreferences: P, - initialLocator: Locator - ): PlayerNavigatorFactory { - - val navigator = PlayerNavigator( - publication, - mediaEngineProvider, - metadataProvider.createMetadataFactory(publication), - initialPreferences, - initialLocator, - ) - - return PlayerNavigatorFactory( - publication, - mediaEngineProvider, - navigator - ) - } - } - - fun getMediaNavigator(): PlayerNavigator = - playerNavigator - - fun createPreferencesEditor( - initialPreferences: P - ): E = - mediaEngineProvider.createPreferenceEditor( - publication, - initialPreferences - ) -}*/ diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsNavigator.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsNavigator.kt index d9465ebb27..23beb240ee 100644 --- a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsNavigator.kt +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsNavigator.kt @@ -90,7 +90,10 @@ class TtsNavigator, } val onSetPlaybackParameters = { parameters: PlaybackParameters -> - val newPreferences = ttsEngineProvider.updatePlaybackParameters(ttsPlayer.lastPreferences, parameters) + val newPreferences = ttsEngineProvider.updatePlaybackParameters( + ttsPlayer.lastPreferences, + parameters + ) ttsPlayer.submitPreferences(newPreferences) } @@ -114,7 +117,7 @@ class TtsNavigator, fun onStopRequested() - fun onMissingLanguageData() + fun onMissingLanguageData(language: Language) } data class Position( @@ -129,11 +132,11 @@ class TtsNavigator, override val text: String, override val position: Position, override val range: IntRange?, - override val locator: Locator + override val utteranceLocator: Locator ) : SynchronizedMediaNavigator.Utterance { override val rangeLocator: Locator? = range - ?.let { locator.copy(text = locator.text.substring(it)) } + ?.let { utteranceLocator.copy(text = utteranceLocator.text.substring(it)) } } sealed class Error : MediaNavigator.Error { @@ -183,7 +186,7 @@ class TtsNavigator, } override val currentLocator: StateFlow = - utterance.mapStateIn(coroutineScope) { it.locator } + utterance.mapStateIn(coroutineScope) { it.utteranceLocator } override fun go(locator: Locator, animated: Boolean, completion: () -> Unit): Boolean { player.go(locator) @@ -252,7 +255,7 @@ class TtsNavigator, return Utterance( text = text, position = position, - locator = locator, + utteranceLocator = locator, range = range ) } diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsNavigatorFactory.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsNavigatorFactory.kt index c62ff222d7..40a6b911fc 100644 --- a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsNavigatorFactory.kt +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsNavigatorFactory.kt @@ -7,6 +7,7 @@ package org.readium.r2.navigator.media3.tts import android.app.Application +import org.readium.r2.navigator.media3.tts.android.* import org.readium.r2.navigator.media3.api.DefaultMediaMetadataFactory import org.readium.r2.navigator.media3.api.MediaMetadataProvider import org.readium.r2.navigator.preferences.PreferencesEditor @@ -19,7 +20,7 @@ import org.readium.r2.shared.util.tokenizer.TextUnit @ExperimentalReadiumApi class TtsNavigatorFactory, E : PreferencesEditor

, - F : TtsEngine.Error, V : TtsEngine.Voice>( + F : TtsEngine.Error, V : TtsEngine.Voice> private constructor( private val application: Application, private val publication: Publication, private val ttsEngineProvider: TtsEngineProvider, @@ -28,6 +29,26 @@ class TtsNavigatorFactory, ) { companion object { + suspend operator fun invoke( + application: Application, + publication: Publication, + tokenizerFactory: (defaultLanguage: Language?) -> ContentTokenizer = defaultTokenizerFactory, + metadataProvider: MediaMetadataProvider = defaultMediaMetadataProvider, + defaultVoiceProvider: AndroidTtsEngine.DefaultVoiceProvider? = null + ): TtsNavigatorFactory? { + + val engineProvider = AndroidTtsEngineProvider(application, defaultVoiceProvider) + + return createNavigatorFactory( + application, + publication, + engineProvider, + tokenizerFactory, + metadataProvider + ) + } + suspend operator fun , E : PreferencesEditor

, F : TtsEngine.Error, V : TtsEngine.Voice> invoke( application: Application, @@ -37,12 +58,36 @@ class TtsNavigatorFactory, metadataProvider: MediaMetadataProvider = defaultMediaMetadataProvider ): TtsNavigatorFactory? { + return createNavigatorFactory( + application, + publication, + ttsEngineProvider, + tokenizerFactory, + metadataProvider + ) + } + + suspend fun , E : PreferencesEditor

, + F : TtsEngine.Error, V : TtsEngine.Voice> createNavigatorFactory( + application: Application, + publication: Publication, + ttsEngineProvider: TtsEngineProvider, + tokenizerFactory: (defaultLanguage: Language?) -> ContentTokenizer = defaultTokenizerFactory, + metadataProvider: MediaMetadataProvider = defaultMediaMetadataProvider + ): TtsNavigatorFactory? { + publication.content() ?.iterator() ?.takeIf { it.hasNext() } ?: return null - return TtsNavigatorFactory(application, publication, ttsEngineProvider, tokenizerFactory, metadataProvider) + return TtsNavigatorFactory( + application, + publication, + ttsEngineProvider, + tokenizerFactory, + metadataProvider + ) } /** diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/androidtts/AndroidTtsEngine.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/android/AndroidTtsEngine.kt similarity index 98% rename from readium/navigator/src/main/java/org/readium/r2/navigator/media3/androidtts/AndroidTtsEngine.kt rename to readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/android/AndroidTtsEngine.kt index 374bb1c2dc..98933a1e07 100644 --- a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/androidtts/AndroidTtsEngine.kt +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/android/AndroidTtsEngine.kt @@ -4,7 +4,7 @@ * available in the top-level LICENSE file of the project. */ -package org.readium.r2.navigator.media3.androidtts +package org.readium.r2.navigator.media3.tts.android import android.content.Context import android.speech.tts.TextToSpeech @@ -32,7 +32,7 @@ class AndroidTtsEngine( private val defaultVoiceProvider: DefaultVoiceProvider?, initialPreferences: AndroidTtsPreferences ) : TtsEngine { + AndroidTtsEngine.Exception, AndroidTtsEngine.Voice> { companion object { diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/androidtts/AndroidTtsEngineProvider.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/android/AndroidTtsEngineProvider.kt similarity index 96% rename from readium/navigator/src/main/java/org/readium/r2/navigator/media3/androidtts/AndroidTtsEngineProvider.kt rename to readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/android/AndroidTtsEngineProvider.kt index e31916b01c..0943173eb0 100644 --- a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/androidtts/AndroidTtsEngineProvider.kt +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/android/AndroidTtsEngineProvider.kt @@ -4,7 +4,7 @@ * available in the top-level LICENSE file of the project. */ -package org.readium.r2.navigator.media3.androidtts +package org.readium.r2.navigator.media3.tts.android import android.content.Context import androidx.media3.common.PlaybackException @@ -21,7 +21,7 @@ class AndroidTtsEngineProvider( private val context: Context, private val defaultVoiceProvider: AndroidTtsEngine.DefaultVoiceProvider? = null ) : TtsEngineProvider { + AndroidTtsEngine.Exception, AndroidTtsEngine.Voice> { override suspend fun createEngine( publication: Publication, diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/androidtts/AndroidTtsPreferences.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/android/AndroidTtsPreferences.kt similarity index 94% rename from readium/navigator/src/main/java/org/readium/r2/navigator/media3/androidtts/AndroidTtsPreferences.kt rename to readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/android/AndroidTtsPreferences.kt index 8ac8a410c5..f9e782bc18 100644 --- a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/androidtts/AndroidTtsPreferences.kt +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/android/AndroidTtsPreferences.kt @@ -4,7 +4,7 @@ * available in the top-level LICENSE file of the project. */ -package org.readium.r2.navigator.media3.androidtts +package org.readium.r2.navigator.media3.tts.android import kotlinx.serialization.Serializable import org.readium.r2.navigator.media3.tts.TtsEngine diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/androidtts/AndroidTtsPreferencesEditor.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/android/AndroidTtsPreferencesEditor.kt similarity index 98% rename from readium/navigator/src/main/java/org/readium/r2/navigator/media3/androidtts/AndroidTtsPreferencesEditor.kt rename to readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/android/AndroidTtsPreferencesEditor.kt index 944d570bdf..6d6e7c9d4b 100644 --- a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/androidtts/AndroidTtsPreferencesEditor.kt +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/android/AndroidTtsPreferencesEditor.kt @@ -4,7 +4,7 @@ * available in the top-level LICENSE file of the project. */ -package org.readium.r2.navigator.media3.androidtts +package org.readium.r2.navigator.media3.tts.android import org.readium.r2.navigator.extensions.format import org.readium.r2.navigator.preferences.* diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/androidtts/AndroidTtsPreferencesFilters.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/android/AndroidTtsPreferencesFilters.kt similarity index 95% rename from readium/navigator/src/main/java/org/readium/r2/navigator/media3/androidtts/AndroidTtsPreferencesFilters.kt rename to readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/android/AndroidTtsPreferencesFilters.kt index cebe76957b..971bcd6f52 100644 --- a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/androidtts/AndroidTtsPreferencesFilters.kt +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/android/AndroidTtsPreferencesFilters.kt @@ -4,7 +4,7 @@ * available in the top-level LICENSE file of the project. */ -package org.readium.r2.navigator.media3.androidtts +package org.readium.r2.navigator.media3.tts.android import org.readium.r2.navigator.preferences.PreferencesFilter import org.readium.r2.shared.ExperimentalReadiumApi diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/androidtts/AndroidTtsPreferencesSerializer.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/android/AndroidTtsPreferencesSerializer.kt similarity index 94% rename from readium/navigator/src/main/java/org/readium/r2/navigator/media3/androidtts/AndroidTtsPreferencesSerializer.kt rename to readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/android/AndroidTtsPreferencesSerializer.kt index 40acbc5b2a..e6e0b948b2 100644 --- a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/androidtts/AndroidTtsPreferencesSerializer.kt +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/android/AndroidTtsPreferencesSerializer.kt @@ -4,7 +4,7 @@ * available in the top-level LICENSE file of the project. */ -package org.readium.r2.navigator.media3.androidtts +package org.readium.r2.navigator.media3.tts.android import kotlinx.serialization.json.Json import org.readium.r2.navigator.preferences.PreferencesSerializer diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/androidtts/AndroidTtsSettings.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/android/AndroidTtsSettings.kt similarity index 91% rename from readium/navigator/src/main/java/org/readium/r2/navigator/media3/androidtts/AndroidTtsSettings.kt rename to readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/android/AndroidTtsSettings.kt index 1dd0e9a18e..7e0fa42de2 100644 --- a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/androidtts/AndroidTtsSettings.kt +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/android/AndroidTtsSettings.kt @@ -4,7 +4,7 @@ * available in the top-level LICENSE file of the project. */ -package org.readium.r2.navigator.media3.androidtts +package org.readium.r2.navigator.media3.tts.android import org.readium.r2.navigator.media3.tts.TtsEngine import org.readium.r2.shared.ExperimentalReadiumApi diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/androidtts/AndroidTtsSettingsResolver.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/android/AndroidTtsSettingsResolver.kt similarity index 93% rename from readium/navigator/src/main/java/org/readium/r2/navigator/media3/androidtts/AndroidTtsSettingsResolver.kt rename to readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/android/AndroidTtsSettingsResolver.kt index 01d790ec70..920e7be7d4 100644 --- a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/androidtts/AndroidTtsSettingsResolver.kt +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/android/AndroidTtsSettingsResolver.kt @@ -4,7 +4,7 @@ * available in the top-level LICENSE file of the project. */ -package org.readium.r2.navigator.media3.androidtts +package org.readium.r2.navigator.media3.tts.android import org.readium.r2.shared.ExperimentalReadiumApi import org.readium.r2.shared.publication.Metadata diff --git a/test-app/src/main/java/org/readium/r2/testapp/reader/ReaderInitData.kt b/test-app/src/main/java/org/readium/r2/testapp/reader/ReaderInitData.kt index 5a964c44ac..93a9fa96da 100644 --- a/test-app/src/main/java/org/readium/r2/testapp/reader/ReaderInitData.kt +++ b/test-app/src/main/java/org/readium/r2/testapp/reader/ReaderInitData.kt @@ -15,7 +15,7 @@ import org.readium.navigator.media2.ExperimentalMedia2 import org.readium.navigator.media2.MediaNavigator import org.readium.r2.navigator.epub.EpubNavigatorFactory import org.readium.r2.navigator.epub.EpubPreferences -import org.readium.r2.navigator.media3.androidtts.AndroidTtsPreferences +import org.readium.r2.navigator.media3.tts.android.AndroidTtsPreferences import org.readium.r2.navigator.pdf.PdfNavigatorFactory import org.readium.r2.shared.ExperimentalReadiumApi import org.readium.r2.shared.publication.* diff --git a/test-app/src/main/java/org/readium/r2/testapp/reader/ReaderRepository.kt b/test-app/src/main/java/org/readium/r2/testapp/reader/ReaderRepository.kt index c6e76cf1d0..715dd47fde 100644 --- a/test-app/src/main/java/org/readium/r2/testapp/reader/ReaderRepository.kt +++ b/test-app/src/main/java/org/readium/r2/testapp/reader/ReaderRepository.kt @@ -6,17 +6,16 @@ package org.readium.r2.testapp.reader +import androidx.datastore.preferences.core.Preferences as JetpackPreferences import android.app.Activity import android.app.Application import androidx.datastore.core.DataStore -import androidx.datastore.preferences.core.Preferences as JetpackPreferences import java.io.File import org.json.JSONObject import org.readium.adapters.pdfium.navigator.PdfiumEngineProvider import org.readium.navigator.media2.ExperimentalMedia2 import org.readium.navigator.media2.MediaNavigator import org.readium.r2.navigator.epub.EpubNavigatorFactory -import org.readium.r2.navigator.media3.androidtts.AndroidTtsEngineProvider import org.readium.r2.navigator.media3.tts.TtsNavigatorFactory import org.readium.r2.navigator.pdf.PdfNavigatorFactory import org.readium.r2.shared.ExperimentalReadiumApi @@ -216,8 +215,7 @@ class ReaderRepository( ): TtsInitData? { val preferencesManager = AndroidTtsPreferencesManagerFactory(preferencesDataStore) .createPreferenceManager(bookId) - val ttsEngine = AndroidTtsEngineProvider(application) - val navigatorFactory = TtsNavigatorFactory(application, publication, ttsEngine) ?: return null + val navigatorFactory = TtsNavigatorFactory(application, publication) ?: return null return TtsInitData(ttsServiceFacade, navigatorFactory, preferencesManager) } diff --git a/test-app/src/main/java/org/readium/r2/testapp/reader/preferences/PreferencesManagers.kt b/test-app/src/main/java/org/readium/r2/testapp/reader/preferences/PreferencesManagers.kt index 2452c6e45e..4e1b8224d4 100644 --- a/test-app/src/main/java/org/readium/r2/testapp/reader/preferences/PreferencesManagers.kt +++ b/test-app/src/main/java/org/readium/r2/testapp/reader/preferences/PreferencesManagers.kt @@ -27,10 +27,10 @@ import org.readium.r2.navigator.epub.EpubPreferences import org.readium.r2.navigator.epub.EpubPreferencesSerializer import org.readium.r2.navigator.epub.EpubPublicationPreferencesFilter import org.readium.r2.navigator.epub.EpubSharedPreferencesFilter -import org.readium.r2.navigator.media3.androidtts.AndroidTtsPreferences -import org.readium.r2.navigator.media3.androidtts.AndroidTtsPreferencesSerializer -import org.readium.r2.navigator.media3.androidtts.AndroidTtsPublicationPreferencesFilter -import org.readium.r2.navigator.media3.androidtts.AndroidTtsSharedPreferencesFilter +import org.readium.r2.navigator.media3.tts.android.AndroidTtsPreferences +import org.readium.r2.navigator.media3.tts.android.AndroidTtsPreferencesSerializer +import org.readium.r2.navigator.media3.tts.android.AndroidTtsPublicationPreferencesFilter +import org.readium.r2.navigator.media3.tts.android.AndroidTtsSharedPreferencesFilter import org.readium.r2.navigator.media3.exoplayer.ExoPlayerPreferences import org.readium.r2.navigator.media3.exoplayer.ExoPlayerPreferencesSerializer import org.readium.r2.navigator.media3.exoplayer.ExoPlayerPublicationPreferencesFilter diff --git a/test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsControls.kt b/test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsControls.kt index 28ee923d3b..552a43d4af 100644 --- a/test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsControls.kt +++ b/test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsControls.kt @@ -19,8 +19,8 @@ import androidx.compose.ui.platform.LocalContext import androidx.compose.ui.res.stringResource import androidx.compose.ui.text.style.TextAlign import androidx.compose.ui.unit.dp -import org.readium.r2.navigator.media3.androidtts.AndroidTtsEngine -import org.readium.r2.navigator.media3.androidtts.AndroidTtsPreferencesEditor +import org.readium.r2.navigator.media3.tts.android.AndroidTtsEngine +import org.readium.r2.navigator.media3.tts.android.AndroidTtsPreferencesEditor import org.readium.r2.navigator.preferences.* import org.readium.r2.shared.ExperimentalReadiumApi import org.readium.r2.shared.util.Language diff --git a/test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsEngine.kt b/test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsEngine.kt index cffe2a3ffa..f89d8685a9 100644 --- a/test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsEngine.kt +++ b/test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsEngine.kt @@ -6,10 +6,10 @@ package org.readium.r2.testapp.reader.tts -import org.readium.r2.navigator.media3.androidtts.AndroidTtsEngine -import org.readium.r2.navigator.media3.androidtts.AndroidTtsPreferences -import org.readium.r2.navigator.media3.androidtts.AndroidTtsPreferencesEditor -import org.readium.r2.navigator.media3.androidtts.AndroidTtsSettings +import org.readium.r2.navigator.media3.tts.android.AndroidTtsEngine +import org.readium.r2.navigator.media3.tts.android.AndroidTtsPreferences +import org.readium.r2.navigator.media3.tts.android.AndroidTtsPreferencesEditor +import org.readium.r2.navigator.media3.tts.android.AndroidTtsSettings import org.readium.r2.navigator.media3.tts.TtsNavigator import org.readium.r2.navigator.media3.tts.TtsNavigatorFactory import org.readium.r2.shared.ExperimentalReadiumApi diff --git a/test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsViewModel.kt b/test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsViewModel.kt index f51ac41774..fd635cba32 100644 --- a/test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsViewModel.kt +++ b/test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsViewModel.kt @@ -7,33 +7,36 @@ package org.readium.r2.testapp.reader.tts import android.content.Context +import android.content.Intent +import android.content.pm.PackageManager +import android.os.Build +import android.speech.tts.TextToSpeech import kotlinx.coroutines.* import kotlinx.coroutines.channels.Channel import kotlinx.coroutines.flow.* import org.readium.r2.navigator.Navigator import org.readium.r2.navigator.VisualNavigator -import org.readium.r2.navigator.media3.androidtts.AndroidTtsEngine -import org.readium.r2.navigator.media3.androidtts.AndroidTtsPreferences -import org.readium.r2.navigator.media3.androidtts.AndroidTtsPreferencesEditor +import org.readium.r2.navigator.media3.tts.TtsNavigator +import org.readium.r2.navigator.media3.tts.android.AndroidTtsEngine +import org.readium.r2.navigator.media3.tts.android.AndroidTtsPreferences +import org.readium.r2.navigator.media3.tts.android.AndroidTtsPreferencesEditor import org.readium.r2.navigator.media3.api.MediaNavigator import org.readium.r2.navigator.media3.api.SynchronizedMediaNavigator -import org.readium.r2.navigator.media3.tts.TtsNavigator -import org.readium.r2.shared.DelicateReadiumApi import org.readium.r2.shared.ExperimentalReadiumApi import org.readium.r2.shared.UserException import org.readium.r2.shared.publication.Locator import org.readium.r2.shared.publication.Publication import org.readium.r2.shared.util.Language +import org.readium.r2.testapp.R import org.readium.r2.testapp.reader.ReaderInitData import org.readium.r2.testapp.reader.VisualReaderInitData import org.readium.r2.testapp.reader.preferences.PreferencesManager import org.readium.r2.testapp.utils.extensions.mapStateIn -import timber.log.Timber /** * View model controlling a [TtsNavigator] to read a publication aloud. * - * Note: This is not an Android [ViewModel], but it is a component of [ReaderViewModel]. + * Note: This is not an Android ViewModel, but it is a component of ReaderViewModel. */ @OptIn(ExperimentalReadiumApi::class) class TtsViewModel private constructor( @@ -91,7 +94,7 @@ class TtsViewModel private constructor( sealed class Event { /** - * Emitted when the [PublicationSpeechSynthesizer] fails with an error. + * Emitted when the [TtsNavigator] fails with an error. */ class OnError(val error: UserException) : Event() @@ -149,6 +152,12 @@ class TtsViewModel private constructor( override fun onStopRequested() { stop() } + + override fun onMissingLanguageData(language: Language) { + viewModelScope.launch { + _events.send(Event.OnMissingVoiceData(language)) + } + } } val ttsNavigator = ttsNavigatorFactory.createNavigator( @@ -167,11 +176,12 @@ class TtsViewModel private constructor( ttsSession: TtsService.Session ): Binding { val playbackJob = ttsSession.navigator.playback - .combine(ttsSession.navigator.utterance) { playback, utterance -> + .onEach { playback -> + playback.error?.let { onPlaybackError(it) } + }.combine(ttsSession.navigator.utterance) { playback, utterance -> stateFromPlayback(playback, utterance) }.onEach { state -> _state.value = state - Timber.d("new TTS state ${_state.value}") }.launchIn(viewModelScope) val preferencesJob = preferencesManager.preferences @@ -192,7 +202,7 @@ class TtsViewModel private constructor( showControls = playback.state != MediaNavigator.State.Ended, isPlaying = playback.playWhenReady, playingWordRange = utterance.rangeLocator, - playingUtterance = utterance.locator + playingUtterance = utterance.utteranceLocator ) } @@ -238,71 +248,44 @@ class TtsViewModel private constructor( /** * Starts the activity to install additional voice data. */ - @OptIn(DelicateReadiumApi::class) fun requestInstallVoice(context: Context) { - // synthesizer.engine.requestInstallMissingVoice(context) - } - - /*private inner class SynthesizerListener : PublicationSpeechSynthesizer.Listener { - override fun onUtteranceError( - utterance: PublicationSpeechSynthesizer.Utterance, - error: PublicationSpeechSynthesizer.Exception - ) { - viewModelScope.launch { - // The synthesizer is paused when encountering an error while playing an - // utterance. Here we will skip to the next utterance unless the exception is - // recoverable. - val shouldContinuePlayback = !handleTtsException(error) - if (shouldContinuePlayback) { - next() - } + val intent = Intent() + .setAction(TextToSpeech.Engine.ACTION_INSTALL_TTS_DATA) + .setFlags(Intent.FLAG_ACTIVITY_NEW_TASK) + + val availableActivities = + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.TIRAMISU) { + context.packageManager.queryIntentActivities( + intent, + PackageManager.ResolveInfoFlags.of(0) + ) + } else { + @Suppress("Deprecation") + context.packageManager.queryIntentActivities(intent, 0) } - } - override fun onError(error: PublicationSpeechSynthesizer.Exception) { - viewModelScope.launch { - handleTtsException(error) - } + if (availableActivities.isNotEmpty()) { + context.startActivity(intent) } + } - /** - * Handles the given error and returns whether it was recovered from. - */ - private suspend fun handleTtsException(error: PublicationSpeechSynthesizer.Exception): Boolean = - when (error) { - is PublicationSpeechSynthesizer.Exception.Engine -> when (val err = error.error) { - // The `LanguageSupportIncomplete` exception is a special case. We can recover from - // it by asking the user to download the missing voice data. - is TtsEngine.Exception.LanguageSupportIncomplete -> { - _events.send(Event.OnMissingVoiceData(err.language)) - true - } - - else -> { - _events.send(Event.OnError(err.toUserException())) - false - } + private fun onPlaybackError(error: TtsNavigator.Error) { + val exception = when (error) { + is TtsNavigator.Error.ContentError -> { + UserException(R.string.tts_error_other) + } + is TtsNavigator.Error.EngineError<*> -> { + when ((error.error as AndroidTtsEngine.Exception).error) { + AndroidTtsEngine.EngineError.Network -> + UserException(R.string.tts_error_network) + else -> + UserException(R.string.tts_error_other) } } + } - private fun TtsEngine.Exception.toUserException(): UserException = - when (this) { - is TtsEngine.Exception.InitializationFailed -> - UserException(R.string.tts_error_initialization) - is TtsEngine.Exception.LanguageNotSupported -> - UserException( - R.string.tts_error_language_not_supported, - language.locale.displayName - ) - is TtsEngine.Exception.LanguageSupportIncomplete -> - UserException( - R.string.tts_error_language_support_incomplete, - language.locale.displayName - ) - is TtsEngine.Exception.Network -> - UserException(R.string.tts_error_network) - is TtsEngine.Exception.Other -> - UserException(R.string.tts_error_other) - } - }*/ + viewModelScope.launch { + _events.send(Event.OnError(exception)) + } + } } From 98786bd4bbe96ac9239e61d45cc531abd4c396dc Mon Sep 17 00:00:00 2001 From: Quentin Gliosca Date: Sun, 29 Jan 2023 20:25:10 +0100 Subject: [PATCH 11/27] Polishing --- .../r2/navigator/media3/api/MediaNavigator.kt | 2 +- .../media3/api/SynchronizedMediaNavigator.kt | 9 +- .../r2/navigator/media3/audio/AudioEngine.kt | 4 +- .../navigator/media3/audio/AudioNavigator.kt | 2 +- .../exoplayer/ExoPlayerEngineProvider.kt | 2 +- .../media3/tts/TtsContentIterator.kt | 12 +- .../r2/navigator/media3/tts/TtsEngine.kt | 47 +++++ .../navigator/media3/tts/TtsEngineFacade.kt | 6 +- .../navigator/media3/tts/TtsEngineProvider.kt | 18 ++ .../r2/navigator/media3/tts/TtsNavigator.kt | 41 +++-- .../media3/tts/TtsNavigatorFactory.kt | 16 +- .../r2/navigator/media3/tts/TtsPlayer.kt | 129 +++++++++----- .../navigator/media3/tts/TtsSessionAdapter.kt | 51 +++--- .../media3/tts/android/AndroidTtsEngine.kt | 118 +++++++------ .../tts/android/AndroidTtsEngineProvider.kt | 43 +++-- .../tts/android/AndroidTtsPreferences.kt | 19 +- .../android/AndroidTtsPreferencesEditor.kt | 11 +- .../media3/tts/android/AndroidTtsSettings.kt | 8 +- .../tts/android/AndroidTtsSettingsResolver.kt | 7 +- .../r2/testapp/reader/ReaderRepository.kt | 2 +- .../r2/testapp/reader/VisualReaderFragment.kt | 13 +- .../reader/preferences/PreferencesManagers.kt | 8 +- .../r2/testapp/reader/tts/TtsControls.kt | 6 +- .../r2/testapp/reader/tts/TtsEngine.kt | 8 +- .../r2/testapp/reader/tts/TtsServiceFacade.kt | 3 + .../r2/testapp/reader/tts/TtsViewModel.kt | 167 ++++++++---------- 26 files changed, 452 insertions(+), 300 deletions(-) diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/api/MediaNavigator.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/api/MediaNavigator.kt index 6e26a66a12..6926626246 100644 --- a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/api/MediaNavigator.kt +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/api/MediaNavigator.kt @@ -13,7 +13,7 @@ import org.readium.r2.shared.ExperimentalReadiumApi import org.readium.r2.shared.util.Closeable @ExperimentalReadiumApi -interface MediaNavigator : Navigator, Closeable { +interface MediaNavigator

: Navigator, Closeable { /** * Marker interface for the [position] flow. diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/api/SynchronizedMediaNavigator.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/api/SynchronizedMediaNavigator.kt index 8d3da873bc..3fa9c1ef8a 100644 --- a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/api/SynchronizedMediaNavigator.kt +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/api/SynchronizedMediaNavigator.kt @@ -11,8 +11,8 @@ import org.readium.r2.shared.ExperimentalReadiumApi import org.readium.r2.shared.publication.Locator @ExperimentalReadiumApi -interface SynchronizedMediaNavigator - : MediaNavigator { +interface SynchronizedMediaNavigator

: + MediaNavigator { interface Utterance

{ val text: String @@ -21,10 +21,9 @@ interface SynchronizedMediaNavigator> diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/audio/AudioEngine.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/audio/AudioEngine.kt index a167bd13ea..9c6465b1bc 100644 --- a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/audio/AudioEngine.kt +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/audio/AudioEngine.kt @@ -14,8 +14,8 @@ import org.readium.r2.navigator.preferences.Configurable import org.readium.r2.shared.ExperimentalReadiumApi @ExperimentalReadiumApi -interface AudioEngine, E : AudioEngine.Error> - : Configurable { +interface AudioEngine, E : AudioEngine.Error> : + Configurable { interface Error diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/audio/AudioNavigator.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/audio/AudioNavigator.kt index 7faddfd7d1..2c8cd01c0d 100644 --- a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/audio/AudioNavigator.kt +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/audio/AudioNavigator.kt @@ -20,7 +20,7 @@ class AudioNavigator, private val mediaEngine: AudioEngine ) : MediaNavigator, Configurable by mediaEngine { - class Position: MediaNavigator.Position + class Position : MediaNavigator.Position class Error : MediaNavigator.Error diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/exoplayer/ExoPlayerEngineProvider.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/exoplayer/ExoPlayerEngineProvider.kt index 34529b8503..bbfa3d9211 100644 --- a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/exoplayer/ExoPlayerEngineProvider.kt +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/exoplayer/ExoPlayerEngineProvider.kt @@ -15,7 +15,7 @@ import org.readium.r2.shared.publication.Publication @androidx.annotation.OptIn(androidx.media3.common.util.UnstableApi::class) class ExoPlayerEngineProvider() : AudioEngineProvider { - override suspend fun createEngine(publication: Publication):ExoPlayerEngine { + override suspend fun createEngine(publication: Publication): ExoPlayerEngine { TODO("Not yet implemented") } diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsContentIterator.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsContentIterator.kt index b46a8bbea7..1180cad1ca 100644 --- a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsContentIterator.kt +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsContentIterator.kt @@ -12,8 +12,8 @@ import org.readium.r2.shared.publication.Publication import org.readium.r2.shared.publication.html.cssSelector import org.readium.r2.shared.publication.indexOfFirstWithHref import org.readium.r2.shared.publication.services.content.Content +import org.readium.r2.shared.publication.services.content.ContentService import org.readium.r2.shared.publication.services.content.ContentTokenizer -import org.readium.r2.shared.publication.services.content.content import org.readium.r2.shared.util.CursorList import org.readium.r2.shared.util.Language @@ -22,7 +22,7 @@ import org.readium.r2.shared.util.Language /** * A Content Iterator able to provide short utterances. * - * It is not safe for several coroutines to use this at the same time. + * Not thread-safe. */ internal class TtsContentIterator( private val publication: Publication, @@ -38,6 +38,10 @@ internal class TtsContentIterator( val language: Language? ) + private val contentService: ContentService = + publication.findService(ContentService::class) + ?: throw IllegalStateException("No ContentService.") + /** * Current subset of utterances with a cursor. */ @@ -92,9 +96,7 @@ internal class TtsContentIterator( */ private fun createIterator(locator: Locator?): Content.Iterator = - publication.content(locator) - ?.iterator() - ?: throw IllegalStateException("No ContentService.") + contentService.content(locator).iterator() /** * Advances to the previous item and returns it, or null if we reached the beginning. diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsEngine.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsEngine.kt index 0a0b77b246..c590aef3ea 100644 --- a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsEngine.kt +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsEngine.kt @@ -20,19 +20,31 @@ interface TtsEngine, interface Preferences

> : Configurable.Preferences

{ + /** + * The default language to use when no language information is passed to [speak]. + */ val language: Language? } interface Settings : Configurable.Settings { + /** + * The default language to use when no language information is passed to [speak]. + */ val language: Language? } interface Voice { + /** + * The voice's language. + * */ val language: Language } + /** + * Marker interface for the errors that the [TtsEngine] returns. + */ interface Error /** @@ -40,24 +52,59 @@ interface TtsEngine, */ interface Listener { + /** + * Called when the utterance with the given id starts as perceived by the caller. + */ fun onStart(requestId: String) + /** + * Called when the [TtsEngine] is about to speak the specified [range] of the utterance with + * the given id. + * + * This callback may not be called if the [TtsEngine] does not provide range information. + */ fun onRange(requestId: String, range: IntRange) + /** + * Called if the utterance with the given id has been stopped while in progress + * by a call to [stop]. + */ fun onInterrupted(requestId: String) + /** + * Called when the utterance with the given id has been flushed from the synthesis queue + * by a call to [stop]. + */ fun onFlushed(requestId: String) + /** + * Called when the utterance with the given id has successfully completed processing. + */ fun onDone(requestId: String) + /** + * Called when an error has occurred during processing of te utterance with the given id. + */ fun onError(requestId: String, error: E) } + /** + * Sets of voices available with this [TtsEngine]. + */ val voices: Set + /** + * Submits a new speak request. + */ fun speak(requestId: String, text: String, language: Language?) + /** + * Stops the [TtsEngine]. + */ fun stop() + /** + * Sets a new listener or removes the current one. + */ fun setListener(listener: Listener?) } diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsEngineFacade.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsEngineFacade.kt index cc3a51621f..6c9b4e6d15 100644 --- a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsEngineFacade.kt +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsEngineFacade.kt @@ -22,7 +22,7 @@ internal class TtsEngineFacade by engine { init { - val listener = TtsEngineListener() + val listener = EngineListener() engine.setListener(listener) } @@ -34,8 +34,8 @@ internal class TtsEngineFacade Unit): E? = suspendCancellableCoroutine { continuation -> continuation.invokeOnCancellation { engine.stop() } - val id = UUID.randomUUID().toString() currentTask?.continuation?.cancel() + val id = UUID.randomUUID().toString() currentTask = UtteranceTask(id, continuation, onRange) engine.speak(id, text, language) } @@ -51,7 +51,7 @@ internal class TtsEngineFacade Unit ) - private inner class TtsEngineListener : TtsEngine.Listener { + private inner class EngineListener : TtsEngine.Listener { override fun onStart(requestId: String) { } diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsEngineProvider.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsEngineProvider.kt index 19639bd947..5e1d26e580 100644 --- a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsEngineProvider.kt +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsEngineProvider.kt @@ -19,15 +19,33 @@ import org.readium.r2.shared.publication.Publication interface TtsEngineProvider, E : PreferencesEditor

, F : TtsEngine.Error, V : TtsEngine.Voice> { + /** + * Creates a [TtsEngine] for [publication] and [initialPreferences]. + */ suspend fun createEngine(publication: Publication, initialPreferences: P): TtsEngine? + /** + * Creates a preferences editor for [publication] and [initialPreferences]. + */ fun createPreferencesEditor(publication: Publication, initialPreferences: P): E + /** + * Creates an empty set of preferences of this TTS engine provider. + */ fun createEmptyPreferences(): P + /** + * Computes Media3 [PlaybackParameters] from the given [settings]. + */ fun getPlaybackParameters(settings: S): PlaybackParameters + /** + * Updates [previousPreferences] to honor the given Media3 [playbackParameters]. + */ fun updatePlaybackParameters(previousPreferences: P, playbackParameters: PlaybackParameters): P + /** + * Maps an engine-specific error to Media3 [PlaybackException]. + */ fun mapEngineError(error: F): PlaybackException } diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsNavigator.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsNavigator.kt index 23beb240ee..a7f71332a5 100644 --- a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsNavigator.kt +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsNavigator.kt @@ -127,17 +127,13 @@ class TtsNavigator, val textAfter: String?, ) : MediaNavigator.Position - data class Utterance( override val text: String, override val position: Position, override val range: IntRange?, - override val utteranceLocator: Locator - ) : SynchronizedMediaNavigator.Utterance { - - override val rangeLocator: Locator? = range - ?.let { utteranceLocator.copy(text = utteranceLocator.text.substring(it)) } - } + override val utteranceHighlight: Locator, + override val tokenHighlight: Locator? + ) : SynchronizedMediaNavigator.Utterance sealed class Error : MediaNavigator.Error { @@ -156,7 +152,9 @@ class TtsNavigator, player.utterance.mapStateIn(coroutineScope) { it.toUtterance() } override val position: StateFlow = - utterance.mapStateIn(coroutineScope) { it.position } + utterance.mapStateIn(coroutineScope) { utterance -> + utterance.position.copy(textAfter = utterance.text + utterance.position.textAfter) + } override fun play() { player.play() @@ -186,7 +184,7 @@ class TtsNavigator, } override val currentLocator: StateFlow = - utterance.mapStateIn(coroutineScope) { it.utteranceLocator } + utterance.mapStateIn(coroutineScope) { it.tokenHighlight ?: it.utteranceHighlight } override fun go(locator: Locator, animated: Boolean, completion: () -> Unit): Boolean { player.go(locator) @@ -228,15 +226,16 @@ class TtsNavigator, is TtsPlayer.Error.EngineError<*> -> Error.EngineError(error) } - private fun TtsPlayer.Utterance.toUtterance(): Utterance { - val position = Position( - resourceIndex = position.resourceIndex, - cssSelector = position.cssSelector, - textBefore = position.textBefore, - textAfter = position.textAfter + private fun TtsPlayer.Utterance.Position.toPosition(): Position = + Position( + resourceIndex = resourceIndex, + cssSelector = cssSelector, + textBefore = textBefore, + textAfter = textAfter ) - val locator = publication + private fun TtsPlayer.Utterance.toUtterance(): Utterance { + val utteranceHighlight = publication .locatorFromLink(publication.readingOrder[position.resourceIndex])!! .copyWithLocations( progression = null, @@ -252,11 +251,15 @@ class TtsNavigator, ) ) + val tokenHighlight = range + ?.let { utteranceHighlight.copy(text = utteranceHighlight.text.substring(it)) } + return Utterance( text = text, - position = position, - utteranceLocator = locator, - range = range + position = position.toPosition(), + range = range, + utteranceHighlight = utteranceHighlight, + tokenHighlight = tokenHighlight, ) } } diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsNavigatorFactory.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsNavigatorFactory.kt index 40a6b911fc..5cda7e2bdd 100644 --- a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsNavigatorFactory.kt +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsNavigatorFactory.kt @@ -7,9 +7,13 @@ package org.readium.r2.navigator.media3.tts import android.app.Application -import org.readium.r2.navigator.media3.tts.android.* import org.readium.r2.navigator.media3.api.DefaultMediaMetadataFactory import org.readium.r2.navigator.media3.api.MediaMetadataProvider +import org.readium.r2.navigator.media3.tts.android.AndroidTtsEngine +import org.readium.r2.navigator.media3.tts.android.AndroidTtsEngineProvider +import org.readium.r2.navigator.media3.tts.android.AndroidTtsPreferences +import org.readium.r2.navigator.media3.tts.android.AndroidTtsPreferencesEditor +import org.readium.r2.navigator.media3.tts.android.AndroidTtsSettings import org.readium.r2.navigator.preferences.PreferencesEditor import org.readium.r2.shared.ExperimentalReadiumApi import org.readium.r2.shared.publication.Locator @@ -35,8 +39,8 @@ class TtsNavigatorFactory, tokenizerFactory: (defaultLanguage: Language?) -> ContentTokenizer = defaultTokenizerFactory, metadataProvider: MediaMetadataProvider = defaultMediaMetadataProvider, defaultVoiceProvider: AndroidTtsEngine.DefaultVoiceProvider? = null - ): TtsNavigatorFactory? { + ): TtsNavigatorFactory? { val engineProvider = AndroidTtsEngineProvider(application, defaultVoiceProvider) @@ -67,13 +71,13 @@ class TtsNavigatorFactory, ) } - suspend fun , E : PreferencesEditor

, + private suspend fun , E : PreferencesEditor

, F : TtsEngine.Error, V : TtsEngine.Voice> createNavigatorFactory( application: Application, publication: Publication, ttsEngineProvider: TtsEngineProvider, - tokenizerFactory: (defaultLanguage: Language?) -> ContentTokenizer = defaultTokenizerFactory, - metadataProvider: MediaMetadataProvider = defaultMediaMetadataProvider + tokenizerFactory: (defaultLanguage: Language?) -> ContentTokenizer, + metadataProvider: MediaMetadataProvider ): TtsNavigatorFactory? { publication.content() diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsPlayer.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsPlayer.kt index e0000326df..5408cbec8f 100644 --- a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsPlayer.kt +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsPlayer.kt @@ -6,16 +6,21 @@ package org.readium.r2.navigator.media3.tts -import kotlinx.coroutines.* +import kotlin.coroutines.coroutineContext +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Job +import kotlinx.coroutines.MainScope +import kotlinx.coroutines.cancelAndJoin import kotlinx.coroutines.flow.MutableStateFlow import kotlinx.coroutines.flow.StateFlow import kotlinx.coroutines.flow.asStateFlow +import kotlinx.coroutines.isActive +import kotlinx.coroutines.launch import kotlinx.coroutines.sync.Mutex import kotlinx.coroutines.sync.withLock import org.readium.r2.navigator.preferences.Configurable import org.readium.r2.shared.ExperimentalReadiumApi import org.readium.r2.shared.publication.Locator -import timber.log.Timber @ExperimentalReadiumApi internal class TtsPlayer, @@ -39,8 +44,6 @@ internal class TtsPlayer, ?: return null val ttsEngineFacade = TtsEngineFacade(engine) - ttsEngineFacade.submitPreferences(initialPreferences) - contentIterator.language = ttsEngineFacade.settings.value.language return TtsPlayer(ttsEngineFacade, contentIterator, initialContext, initialPreferences) } @@ -57,9 +60,15 @@ internal class TtsPlayer, ended = false ) } else { + val actualCurrentUtterance = previousUtterance ?: return null + val actualPreviousUtterance = previousUtterance() + + // Go back to the end of the iterator. + nextUtterance() + Context( - previousUtterance = previousUtterance(), - currentUtterance = nextUtterance() ?: return null, + previousUtterance = actualPreviousUtterance, + currentUtterance = actualCurrentUtterance, nextUtterance = null, ended = true ) @@ -113,6 +122,15 @@ internal class TtsPlayer, private val coroutineScope: CoroutineScope = MainScope() + private var context: Context = + initialContext + + private var playbackJob: Job? = + null + + private val mutex: Mutex = + Mutex() + private val playbackMutable: MutableStateFlow = MutableStateFlow( Playback( @@ -125,14 +143,10 @@ internal class TtsPlayer, private val utteranceMutable: MutableStateFlow = MutableStateFlow(initialContext.currentUtterance.ttsPlayerUtterance()) - private var context: Context = - initialContext - - private var playbackJob: Job? = null + override val settings: StateFlow = + engineFacade.settings - private val mutex = Mutex() - - val voices: Set get() = + val voices: Set = engineFacade.voices val playback: StateFlow = @@ -141,6 +155,13 @@ internal class TtsPlayer, val utterance: StateFlow = utteranceMutable.asStateFlow() + var lastPreferences: P = + initialPreferences + + init { + submitPreferences(initialPreferences) + } + fun play() { coroutineScope.launch { playAsync() @@ -167,9 +188,23 @@ internal class TtsPlayer, return } - playbackJob?.cancelAndJoin() playbackMutable.value = playbackMutable.value.copy(playWhenReady = false) utteranceMutable.value = utteranceMutable.value.copy(range = null) + playbackJob?.cancelAndJoin() + Unit + } + + fun tryRecover() { + coroutineScope.launch { + tryRecoverAsync() + } + } + + private suspend fun tryRecoverAsync() = mutex.withLock { + playbackMutable.value = playbackMutable.value.copy(error = null) + utteranceMutable.value = utteranceMutable.value.copy(range = null) + playbackJob?.join() + playIfReadyAndNotPaused() } fun go(locator: Locator) { @@ -179,9 +214,10 @@ internal class TtsPlayer, } private suspend fun goAsync(locator: Locator) = mutex.withLock { - playbackJob?.cancelAndJoin() + playbackJob?.cancel() contentIterator.seek(locator) resetContext() + playbackJob?.join() playIfReadyAndNotPaused() } @@ -192,9 +228,10 @@ internal class TtsPlayer, } private suspend fun goAsync(resourceIndex: Int) = mutex.withLock { - playbackJob?.cancelAndJoin() + playbackJob?.cancel() contentIterator.seekToResource(resourceIndex) resetContext() + playbackJob?.join() playIfReadyAndNotPaused() } @@ -205,10 +242,12 @@ internal class TtsPlayer, } private suspend fun restartUtteranceAsync() = mutex.withLock { - playbackJob?.cancelAndJoin() + playbackJob?.cancel() if (playbackMutable.value.state == Playback.State.Ended) { playbackMutable.value = playbackMutable.value.copy(state = Playback.State.Ready) } + utteranceMutable.value = utteranceMutable.value.copy(range = null) + playbackJob?.join() playIfReadyAndNotPaused() } @@ -226,8 +265,9 @@ internal class TtsPlayer, return } - playbackJob?.cancelAndJoin() + playbackJob?.cancel() tryLoadNextContext() + playbackJob?.join() playIfReadyAndNotPaused() } @@ -244,8 +284,9 @@ internal class TtsPlayer, if (context.previousUtterance == null) { return } - playbackJob?.cancelAndJoin() + playbackJob?.cancel() tryLoadPreviousContext() + playbackJob?.join() playIfReadyAndNotPaused() } @@ -263,10 +304,11 @@ internal class TtsPlayer, return } - playbackJob?.cancelAndJoin() + playbackJob?.cancel() val currentIndex = utteranceMutable.value.position.resourceIndex contentIterator.seekToResource(currentIndex + 1) resetContext() + playbackJob?.join() playIfReadyAndNotPaused() } @@ -283,10 +325,11 @@ internal class TtsPlayer, if (!hasPreviousResource()) { return } - playbackJob?.cancelAndJoin() + playbackJob?.cancel() val currentIndex = utteranceMutable.value.position.resourceIndex contentIterator.seekToResource(currentIndex - 1) resetContext() + playbackJob?.join() playIfReadyAndNotPaused() } @@ -301,31 +344,32 @@ internal class TtsPlayer, private suspend fun tryLoadPreviousContext() { val contextNow = context - // Get previously nextUtterance once more - contentIterator.previousUtterance() // Get previously currentUtterance once more - val currentUtterance = checkNotNull(contentIterator.previousUtterance()) + contentIterator.previousUtterance() + + // Get previously previousUtterance once more + contentIterator.previousUtterance() - // Get previous utterance + // Get new previous utterance val previousUtterance = contentIterator.previousUtterance() - // Get to nextUtterance position + // Go to currentUtterance position contentIterator.nextUtterance() + + // Go to nextUtterance position contentIterator.nextUtterance() context = Context( previousUtterance = previousUtterance, - currentUtterance = currentUtterance, + currentUtterance = checkNotNull(contextNow.previousUtterance), nextUtterance = contextNow.currentUtterance ) utteranceMutable.value = context.currentUtterance.ttsPlayerUtterance() } private suspend fun tryLoadNextContext() { - Timber.d("tryLoadNextContext") val contextNow = context - Timber.d("contextNow $contextNow") if (contextNow.nextUtterance == null) { onEndReached() @@ -335,9 +379,7 @@ internal class TtsPlayer, currentUtterance = contextNow.nextUtterance, nextUtterance = contentIterator.nextUtterance() ) - Timber.d("newContext $context") utteranceMutable.value = context.currentUtterance.ttsPlayerUtterance() - Timber.d("utterance ${utteranceMutable.value.text}") if (playbackMutable.value.state == Playback.State.Ended) { playbackMutable.value = playbackMutable.value.copy(state = Playback.State.Ready) } @@ -358,18 +400,28 @@ internal class TtsPlayer, } private suspend fun playContinuous() { - engineFacade.speak(context.currentUtterance.text, context.currentUtterance.language, ::onRangeChanged) - ?.let { exception -> onEngineError(exception) } - mutex.withLock { tryLoadNextContext() } + if (!coroutineContext.isActive) { + return + } + + val error = speakUtterance(context.currentUtterance) + + mutex.withLock { + error?.let { exception -> onEngineError(exception) } + tryLoadNextContext() + } playContinuous() } + private suspend fun speakUtterance(utterance: TtsContentIterator.Utterance): E? = + engineFacade.speak(utterance.text, utterance.language, ::onRangeChanged) + private fun onEngineError(error: E) { - Timber.e("onEngineError $error") playbackMutable.value = playbackMutable.value.copy( state = Playback.State.Error, error = Error.EngineError(error) ) + playbackJob?.cancel() } private fun onRangeChanged(range: IntRange) { @@ -381,15 +433,10 @@ internal class TtsPlayer, engineFacade.close() } - var lastPreferences: P = - initialPreferences - - override val settings: StateFlow - get() = engineFacade.settings - override fun submitPreferences(preferences: P) { lastPreferences = preferences engineFacade.submitPreferences(preferences) + contentIterator.language = engineFacade.settings.value.language } private fun isPlaying() = diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsSessionAdapter.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsSessionAdapter.kt index fbc09288f8..72b4421ebc 100644 --- a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsSessionAdapter.kt +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsSessionAdapter.kt @@ -29,7 +29,6 @@ import kotlinx.coroutines.flow.launchIn import kotlinx.coroutines.flow.onEach import org.readium.r2.shared.ExperimentalReadiumApi import org.readium.r2.shared.fetcher.Resource -import timber.log.Timber @ExperimentalReadiumApi @androidx.annotation.OptIn(androidx.media3.common.util.UnstableApi::class) @@ -58,7 +57,7 @@ internal class TtsSessionAdapter( private val volumeManager = TtsStreamVolumeManager( application, - Handler(Looper.getMainLooper()), + Handler(applicationLooper), StreamVolumeManagerListener() ) @@ -114,12 +113,10 @@ internal class TtsSessionAdapter( } override fun addListener(listener: Listener) { - Timber.d("addListener") listeners.add(listener) } override fun removeListener(listener: Listener) { - Timber.d("removeListener") listeners.remove(listener) } @@ -187,6 +184,7 @@ internal class TtsSessionAdapter( } override fun prepare() { + ttsPlayer.tryRecover() } override fun getPlaybackState(): Int { @@ -194,7 +192,7 @@ internal class TtsSessionAdapter( } override fun getPlaybackSuppressionReason(): Int { - return PLAYBACK_SUPPRESSION_REASON_NONE // TODO + return PLAYBACK_SUPPRESSION_REASON_NONE } override fun isPlaying(): Boolean { @@ -205,7 +203,7 @@ internal class TtsSessionAdapter( } override fun getPlayerError(): PlaybackException? { - return null // TODO + return lastPlayback.error?.toPlaybackException() } override fun play() { @@ -229,7 +227,6 @@ internal class TtsSessionAdapter( } override fun setRepeatMode(repeatMode: Int) { - throw NotImplementedError() } override fun getRepeatMode(): Int { @@ -237,7 +234,6 @@ internal class TtsSessionAdapter( } override fun setShuffleModeEnabled(shuffleModeEnabled: Boolean) { - throw NotImplementedError() } override fun getShuffleModeEnabled(): Boolean { @@ -413,11 +409,11 @@ internal class TtsSessionAdapter( } override fun setTrackSelectionParameters(parameters: TrackSelectionParameters) { - throw NotImplementedError() } override fun getMediaMetadata(): MediaMetadata { - return currentTimeline.getWindow(currentMediaItemIndex, window).mediaItem.mediaMetadata + return currentTimeline.getWindow(currentMediaItemIndex, window) + .mediaItem.mediaMetadata } override fun getPlaylistMetadata(): MediaMetadata { @@ -430,10 +426,10 @@ internal class TtsSessionAdapter( override fun getCurrentManifest(): Any? { val timeline = currentTimeline - return if (timeline.isEmpty) null else timeline.getWindow( - currentMediaItemIndex, - window - ).manifest + return if (timeline.isEmpty) + null + else + timeline.getWindow(currentMediaItemIndex, window).manifest } override fun getCurrentTimeline(): Timeline { @@ -461,9 +457,14 @@ internal class TtsSessionAdapter( override fun getNextMediaItemIndex(): Int { val timeline = currentTimeline - return if (timeline.isEmpty) INDEX_UNSET else timeline.getNextWindowIndex( - currentMediaItemIndex, getRepeatModeForNavigation(), shuffleModeEnabled - ) + return if (timeline.isEmpty) + INDEX_UNSET + else + timeline.getNextWindowIndex( + currentMediaItemIndex, + getRepeatModeForNavigation(), + shuffleModeEnabled + ) } @Deprecated("Deprecated in Java", ReplaceWith("previousMediaItemIndex")) @@ -473,9 +474,12 @@ internal class TtsSessionAdapter( override fun getPreviousMediaItemIndex(): Int { val timeline = currentTimeline - return if (timeline.isEmpty) INDEX_UNSET else timeline.getPreviousWindowIndex( - currentMediaItemIndex, getRepeatModeForNavigation(), shuffleModeEnabled - ) + return if (timeline.isEmpty) + INDEX_UNSET + else + timeline.getPreviousWindowIndex( + currentMediaItemIndex, getRepeatModeForNavigation(), shuffleModeEnabled + ) } override fun getCurrentMediaItem(): MediaItem? { @@ -579,9 +583,10 @@ internal class TtsSessionAdapter( override fun getContentDuration(): Long { val timeline = currentTimeline - return if (timeline.isEmpty) TIME_UNSET else timeline.getWindow( - currentMediaItemIndex, window - ).durationMs + return if (timeline.isEmpty) + TIME_UNSET + else + timeline.getWindow(currentMediaItemIndex, window).durationMs } override fun getContentPosition(): Long { diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/android/AndroidTtsEngine.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/android/AndroidTtsEngine.kt index 98933a1e07..b19cd29a8e 100644 --- a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/android/AndroidTtsEngine.kt +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/android/AndroidTtsEngine.kt @@ -12,8 +12,7 @@ import android.speech.tts.TextToSpeech.QUEUE_ADD import android.speech.tts.UtteranceProgressListener import android.speech.tts.Voice as AndroidVoice import android.speech.tts.Voice.* -import java.util.* -import kotlinx.coroutines.* +import kotlinx.coroutines.CompletableDeferred import kotlinx.coroutines.flow.MutableStateFlow import kotlinx.coroutines.flow.StateFlow import kotlinx.coroutines.flow.asStateFlow @@ -26,13 +25,13 @@ import org.readium.r2.shared.util.Language * Default [TtsEngine] implementation using Android's native text to speech engine. */ @ExperimentalReadiumApi -class AndroidTtsEngine( +class AndroidTtsEngine private constructor( private val engine: TextToSpeech, metadata: Metadata, private val defaultVoiceProvider: DefaultVoiceProvider?, initialPreferences: AndroidTtsPreferences ) : TtsEngine { + AndroidTtsEngine.Error, AndroidTtsEngine.Voice> { companion object { @@ -62,43 +61,43 @@ class AndroidTtsEngine( fun chooseVoice(language: Language?, availableVoices: Set): Voice? } - /** - * Android's TTS error code. - * See https://developer.android.com/reference/android/speech/tts/TextToSpeech#ERROR - */ - enum class EngineError(val code: Int) { - /** Denotes a generic operation failure. */ - Unknown(-1), - /** Denotes a failure caused by an invalid request. */ - InvalidRequest(-8), - /** Denotes a failure caused by a network connectivity problems. */ - Network(-6), - /** Denotes a failure caused by network timeout. */ - NetworkTimeout(-7), - /** Denotes a failure caused by an unfinished download of the voice data. */ - NotInstalledYet(-9), - /** Denotes a failure related to the output (audio device or a file). */ - Output(-5), - /** Denotes a failure of a TTS service. */ - Service(-4), - /** Denotes a failure of a TTS engine to synthesize the given input. */ - Synthesis(-3); - - companion object { - fun getOrDefault(key: Int): EngineError = - values() - .firstOrNull { it.code == key } - ?: Unknown + class Error(code: Int) : TtsEngine.Error { + + val kind: Kind = + Kind.getOrDefault(code) + + /** + * Android's TTS error code. + * See https://developer.android.com/reference/android/speech/tts/TextToSpeech#ERROR + */ + enum class Kind(val code: Int) { + /** Denotes a generic operation failure. */ + Unknown(-1), + /** Denotes a failure caused by an invalid request. */ + InvalidRequest(-8), + /** Denotes a failure caused by a network connectivity problems. */ + Network(-6), + /** Denotes a failure caused by network timeout. */ + NetworkTimeout(-7), + /** Denotes a failure caused by an unfinished download of the voice data. */ + NotInstalledYet(-9), + /** Denotes a failure related to the output (audio device or a file). */ + Output(-5), + /** Denotes a failure of a TTS service. */ + Service(-4), + /** Denotes a failure of a TTS engine to synthesize the given input. */ + Synthesis(-3); + + companion object { + + fun getOrDefault(key: Int): Kind = + values() + .firstOrNull { it.code == key } + ?: Unknown + } } } - class Exception(code: Int) : - kotlin.Exception("Android TTS engine error: $code"), TtsEngine.Error { - - val error: EngineError = - EngineError.getOrDefault(code) - } - /** * Represents a voice provided by the TTS engine which can speak an utterance. * @@ -119,13 +118,6 @@ class AndroidTtsEngine( } } - init { - engine.setOnUtteranceProgressListener(Listener()) - } - - private var listener: TtsEngine.Listener? = - null - private val settingsResolver: AndroidTtsSettingsResolver = AndroidTtsSettingsResolver(metadata) @@ -134,11 +126,18 @@ class AndroidTtsEngine( override val voices: Set get() = engine.voices - .map { it.toTtsEngineVoice() } - .toSet() + ?.map { it.toTtsEngineVoice() } + ?.toSet() + .orEmpty() - override fun setListener(listener: TtsEngine.Listener?) { - this.listener = listener + override fun setListener( + listener: TtsEngine.Listener? + ) { + if (listener == null) { + engine.setOnUtteranceProgressListener(null) + } else { + engine.setOnUtteranceProgressListener(UtteranceListener(listener)) + } } override fun speak( @@ -180,16 +179,21 @@ class AndroidTtsEngine( val language = utteranceLanguage ?: settings.language - val preferredVoice = language - ?.let { settings.voices[it] } - ?.let { voiceForName(it) } + val preferredVoiceWithRegion = + settings.voices[language] + ?.let { voiceForName(it) } + + val preferredVoiceWithoutRegion = + settings.voices[language.removeRegion()] + ?.let { voiceForName(it) } - val voice = preferredVoice + val voice = preferredVoiceWithRegion + ?: preferredVoiceWithoutRegion ?: defaultVoice(language, voices) voice ?.let { engine.voice = it } - ?: run { engine.language = language?.locale ?: Locale.getDefault() } + ?: run { engine.language = language.locale } } private fun defaultVoice(language: Language?, voices: Set): AndroidVoice? = @@ -216,7 +220,9 @@ class AndroidTtsEngine( requiresNetwork = isNetworkConnectionRequired ) - inner class Listener : UtteranceProgressListener() { + class UtteranceListener( + private val listener: TtsEngine.Listener? + ) : UtteranceProgressListener() { override fun onStart(utteranceId: String) { listener?.onStart(utteranceId) } @@ -243,7 +249,7 @@ class AndroidTtsEngine( override fun onError(utteranceId: String, errorCode: Int) { listener?.onError( utteranceId, - Exception(errorCode) + Error(errorCode) ) } diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/android/AndroidTtsEngineProvider.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/android/AndroidTtsEngineProvider.kt index 0943173eb0..1f23ae3763 100644 --- a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/android/AndroidTtsEngineProvider.kt +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/android/AndroidTtsEngineProvider.kt @@ -21,7 +21,7 @@ class AndroidTtsEngineProvider( private val context: Context, private val defaultVoiceProvider: AndroidTtsEngine.DefaultVoiceProvider? = null ) : TtsEngineProvider { + AndroidTtsEngine.Error, AndroidTtsEngine.Voice> { override suspend fun createEngine( publication: Publication, @@ -66,23 +66,28 @@ class AndroidTtsEngineProvider( ) } - override fun mapEngineError(error: AndroidTtsEngine.Exception): PlaybackException = - when (error.error) { - AndroidTtsEngine.EngineError.Unknown -> - PlaybackException(error.message, error.cause, ERROR_CODE_UNSPECIFIED) - AndroidTtsEngine.EngineError.InvalidRequest -> - PlaybackException(error.message, error.cause, ERROR_CODE_IO_BAD_HTTP_STATUS) - AndroidTtsEngine.EngineError.Network -> - PlaybackException(error.message, error.cause, ERROR_CODE_IO_NETWORK_CONNECTION_FAILED) - AndroidTtsEngine.EngineError.NetworkTimeout -> - PlaybackException(error.message, error.cause, ERROR_CODE_IO_NETWORK_CONNECTION_TIMEOUT) - AndroidTtsEngine.EngineError.NotInstalledYet -> - PlaybackException(error.message, error.cause, ERROR_CODE_UNSPECIFIED) - AndroidTtsEngine.EngineError.Output -> - PlaybackException(error.message, error.cause, ERROR_CODE_UNSPECIFIED) - AndroidTtsEngine.EngineError.Service -> - PlaybackException(error.message, error.cause, ERROR_CODE_UNSPECIFIED) - AndroidTtsEngine.EngineError.Synthesis -> - PlaybackException(error.message, error.cause, ERROR_CODE_UNSPECIFIED) + override fun mapEngineError(error: AndroidTtsEngine.Error): PlaybackException { + val errorCode = when (error.kind) { + AndroidTtsEngine.Error.Kind.Unknown -> + ERROR_CODE_UNSPECIFIED + AndroidTtsEngine.Error.Kind.InvalidRequest -> + ERROR_CODE_IO_BAD_HTTP_STATUS + AndroidTtsEngine.Error.Kind.Network -> + ERROR_CODE_IO_NETWORK_CONNECTION_FAILED + AndroidTtsEngine.Error.Kind.NetworkTimeout -> + ERROR_CODE_IO_NETWORK_CONNECTION_TIMEOUT + AndroidTtsEngine.Error.Kind.NotInstalledYet -> + ERROR_CODE_UNSPECIFIED + AndroidTtsEngine.Error.Kind.Output -> + ERROR_CODE_UNSPECIFIED + AndroidTtsEngine.Error.Kind.Service -> + ERROR_CODE_UNSPECIFIED + AndroidTtsEngine.Error.Kind.Synthesis -> + ERROR_CODE_UNSPECIFIED } + + val message = "Android TTS engine error: ${error.kind.code}" + + return PlaybackException(message, null, errorCode) + } } diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/android/AndroidTtsPreferences.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/android/AndroidTtsPreferences.kt index f9e782bc18..08f3222fc8 100644 --- a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/android/AndroidTtsPreferences.kt +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/android/AndroidTtsPreferences.kt @@ -11,20 +11,33 @@ import org.readium.r2.navigator.media3.tts.TtsEngine import org.readium.r2.shared.ExperimentalReadiumApi import org.readium.r2.shared.util.Language +/** + * Preferences for the TTS navigator with the Android built-in engine. + * + * @param language Language of the publication content. + * @param pitch Playback pitch rate. + * @param speed Playback speed rate. + * @param voices Map of preferred voices for specific languages. + */ @ExperimentalReadiumApi @Serializable data class AndroidTtsPreferences( override val language: Language? = null, - val voices: Map? = null, val pitch: Double? = null, val speed: Double? = null, + val voices: Map? = null, ) : TtsEngine.Preferences { + init { + require(pitch == null || pitch > 0) + require(speed == null || speed > 0) + } + override fun plus(other: AndroidTtsPreferences): AndroidTtsPreferences = AndroidTtsPreferences( language = other.language ?: language, - voices = other.voices ?: voices, + pitch = other.pitch ?: pitch, speed = other.speed ?: speed, - pitch = other.pitch ?: pitch + voices = other.voices ?: voices, ) } diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/android/AndroidTtsPreferencesEditor.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/android/AndroidTtsPreferencesEditor.kt index 6d6e7c9d4b..11ae51f3c8 100644 --- a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/android/AndroidTtsPreferencesEditor.kt +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/android/AndroidTtsPreferencesEditor.kt @@ -12,6 +12,13 @@ import org.readium.r2.shared.ExperimentalReadiumApi import org.readium.r2.shared.publication.Metadata import org.readium.r2.shared.util.Language +/** + * Editor for a set of [AndroidTtsPreferences]. + * + * Use [AndroidTtsPreferencesEditor] to assist you in building a preferences user interface or modifying + * existing preferences. It includes rules for adjusting preferences, such as the supported values + * or ranges. + */ @ExperimentalReadiumApi class AndroidTtsPreferencesEditor( initialPreferences: AndroidTtsPreferences, @@ -50,7 +57,7 @@ class AndroidTtsPreferencesEditor( getEffectiveValue = { state.settings.pitch }, getIsEffective = { true }, updateValue = { value -> updateValues { it.copy(pitch = value) } }, - supportedRange = 0.0..Double.MAX_VALUE, + supportedRange = 0.1..Double.MAX_VALUE, progressionStrategy = DoubleIncrement(0.1), valueFormatter = { "${it.format(2)}x" }, ) @@ -61,7 +68,7 @@ class AndroidTtsPreferencesEditor( getEffectiveValue = { state.settings.speed }, getIsEffective = { true }, updateValue = { value -> updateValues { it.copy(speed = value) } }, - supportedRange = 0.0..Double.MAX_VALUE, + supportedRange = 0.1..Double.MAX_VALUE, progressionStrategy = DoubleIncrement(0.1), valueFormatter = { "${it.format(2)}x" }, ) diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/android/AndroidTtsSettings.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/android/AndroidTtsSettings.kt index 7e0fa42de2..493dd0649d 100644 --- a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/android/AndroidTtsSettings.kt +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/android/AndroidTtsSettings.kt @@ -10,10 +10,14 @@ import org.readium.r2.navigator.media3.tts.TtsEngine import org.readium.r2.shared.ExperimentalReadiumApi import org.readium.r2.shared.util.Language +/* Settings values of the TTS navigator with the Android built-in engine. +* +* @see AndroidTtsPreferences +*/ @ExperimentalReadiumApi data class AndroidTtsSettings( - override val language: Language?, - val voices: Map, + override val language: Language, val pitch: Double, val speed: Double, + val voices: Map, ) : TtsEngine.Settings diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/android/AndroidTtsSettingsResolver.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/android/AndroidTtsSettingsResolver.kt index 920e7be7d4..c8cd00eeb5 100644 --- a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/android/AndroidTtsSettingsResolver.kt +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/android/AndroidTtsSettingsResolver.kt @@ -6,8 +6,10 @@ package org.readium.r2.navigator.media3.tts.android +import androidx.compose.ui.text.intl.Locale import org.readium.r2.shared.ExperimentalReadiumApi import org.readium.r2.shared.publication.Metadata +import org.readium.r2.shared.util.Language @ExperimentalReadiumApi internal class AndroidTtsSettingsResolver( @@ -15,9 +17,12 @@ internal class AndroidTtsSettingsResolver( ) { fun settings(preferences: AndroidTtsPreferences): AndroidTtsSettings { + val language = preferences.language + ?: metadata.language + ?: Language(Locale.current.toLanguageTag()) return AndroidTtsSettings( - language = preferences.language ?: metadata.language, + language = language, voices = preferences.voices ?: emptyMap(), pitch = preferences.pitch ?: 1.0, speed = preferences.speed ?: 1.0, diff --git a/test-app/src/main/java/org/readium/r2/testapp/reader/ReaderRepository.kt b/test-app/src/main/java/org/readium/r2/testapp/reader/ReaderRepository.kt index 715dd47fde..db187d7d25 100644 --- a/test-app/src/main/java/org/readium/r2/testapp/reader/ReaderRepository.kt +++ b/test-app/src/main/java/org/readium/r2/testapp/reader/ReaderRepository.kt @@ -6,10 +6,10 @@ package org.readium.r2.testapp.reader -import androidx.datastore.preferences.core.Preferences as JetpackPreferences import android.app.Activity import android.app.Application import androidx.datastore.core.DataStore +import androidx.datastore.preferences.core.Preferences as JetpackPreferences import java.io.File import org.json.JSONObject import org.readium.adapters.pdfium.navigator.PdfiumEngineProvider diff --git a/test-app/src/main/java/org/readium/r2/testapp/reader/VisualReaderFragment.kt b/test-app/src/main/java/org/readium/r2/testapp/reader/VisualReaderFragment.kt index 6ef8169725..e09f0382e4 100644 --- a/test-app/src/main/java/org/readium/r2/testapp/reader/VisualReaderFragment.kt +++ b/test-app/src/main/java/org/readium/r2/testapp/reader/VisualReaderFragment.kt @@ -52,7 +52,6 @@ import org.readium.r2.testapp.reader.tts.TtsViewModel import org.readium.r2.testapp.utils.* import org.readium.r2.testapp.utils.extensions.confirmDialog import org.readium.r2.testapp.utils.extensions.throttleLatest -import timber.log.Timber /* * Base reader fragment class @@ -187,7 +186,7 @@ abstract class VisualReaderFragment : BaseReaderFragment(), VisualNavigator.List // Navigate to the currently spoken word. // This will automatically turn pages when needed. - state.map { it.playingWordRange } + position .filterNotNull() // Improve performances by throttling the moves to maximum one per second. .throttleLatest(1.seconds) @@ -198,8 +197,7 @@ abstract class VisualReaderFragment : BaseReaderFragment(), VisualNavigator.List // Prevent interacting with the publication (including page turns) while the TTS is // playing. - state.map { it.isPlaying } - .distinctUntilChanged() + isPlaying .onEach { isPlaying -> disableTouches = isPlaying } @@ -207,10 +205,8 @@ abstract class VisualReaderFragment : BaseReaderFragment(), VisualNavigator.List // Highlight the currently spoken utterance. (navigator as? DecorableNavigator)?.let { navigator -> - state.map { it.playingUtterance } - .distinctUntilChanged() + highlight .onEach { locator -> - Timber.d("Highlighting $locator") val decoration = locator?.let { Decoration( id = "tts", @@ -223,8 +219,7 @@ abstract class VisualReaderFragment : BaseReaderFragment(), VisualNavigator.List .launchIn(scope) } - state.map { it.showControls } - .distinctUntilChanged() + showControls .onEach { showControls -> preventProgressionSaving = showControls } diff --git a/test-app/src/main/java/org/readium/r2/testapp/reader/preferences/PreferencesManagers.kt b/test-app/src/main/java/org/readium/r2/testapp/reader/preferences/PreferencesManagers.kt index 4e1b8224d4..9cd54fb689 100644 --- a/test-app/src/main/java/org/readium/r2/testapp/reader/preferences/PreferencesManagers.kt +++ b/test-app/src/main/java/org/readium/r2/testapp/reader/preferences/PreferencesManagers.kt @@ -27,14 +27,14 @@ import org.readium.r2.navigator.epub.EpubPreferences import org.readium.r2.navigator.epub.EpubPreferencesSerializer import org.readium.r2.navigator.epub.EpubPublicationPreferencesFilter import org.readium.r2.navigator.epub.EpubSharedPreferencesFilter -import org.readium.r2.navigator.media3.tts.android.AndroidTtsPreferences -import org.readium.r2.navigator.media3.tts.android.AndroidTtsPreferencesSerializer -import org.readium.r2.navigator.media3.tts.android.AndroidTtsPublicationPreferencesFilter -import org.readium.r2.navigator.media3.tts.android.AndroidTtsSharedPreferencesFilter import org.readium.r2.navigator.media3.exoplayer.ExoPlayerPreferences import org.readium.r2.navigator.media3.exoplayer.ExoPlayerPreferencesSerializer import org.readium.r2.navigator.media3.exoplayer.ExoPlayerPublicationPreferencesFilter import org.readium.r2.navigator.media3.exoplayer.ExoPlayerSharedPreferencesFilter +import org.readium.r2.navigator.media3.tts.android.AndroidTtsPreferences +import org.readium.r2.navigator.media3.tts.android.AndroidTtsPreferencesSerializer +import org.readium.r2.navigator.media3.tts.android.AndroidTtsPublicationPreferencesFilter +import org.readium.r2.navigator.media3.tts.android.AndroidTtsSharedPreferencesFilter import org.readium.r2.navigator.preferences.Configurable import org.readium.r2.navigator.preferences.PreferencesFilter import org.readium.r2.navigator.preferences.PreferencesSerializer diff --git a/test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsControls.kt b/test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsControls.kt index 552a43d4af..8ba11cc0b5 100644 --- a/test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsControls.kt +++ b/test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsControls.kt @@ -34,9 +34,9 @@ import org.readium.r2.testapp.utils.extensions.asStateWhenStarted */ @Composable fun TtsControls(model: TtsViewModel, modifier: Modifier = Modifier) { - val showControls by model.state.asStateWhenStarted { it.showControls } - val isPlaying by model.state.asStateWhenStarted { it.isPlaying } - val editor by model.editor.collectAsState() + val showControls by model.showControls.asStateWhenStarted() + val isPlaying by model.isPlaying.asStateWhenStarted() + val editor by model.editor.asStateWhenStarted() if (showControls) { TtsControls( diff --git a/test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsEngine.kt b/test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsEngine.kt index f89d8685a9..7012a7d993 100644 --- a/test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsEngine.kt +++ b/test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsEngine.kt @@ -6,16 +6,16 @@ package org.readium.r2.testapp.reader.tts +import org.readium.r2.navigator.media3.tts.TtsNavigator +import org.readium.r2.navigator.media3.tts.TtsNavigatorFactory import org.readium.r2.navigator.media3.tts.android.AndroidTtsEngine import org.readium.r2.navigator.media3.tts.android.AndroidTtsPreferences import org.readium.r2.navigator.media3.tts.android.AndroidTtsPreferencesEditor import org.readium.r2.navigator.media3.tts.android.AndroidTtsSettings -import org.readium.r2.navigator.media3.tts.TtsNavigator -import org.readium.r2.navigator.media3.tts.TtsNavigatorFactory import org.readium.r2.shared.ExperimentalReadiumApi @OptIn(ExperimentalReadiumApi::class) -typealias AndroidTtsNavigatorFactory = TtsNavigatorFactory +typealias AndroidTtsNavigatorFactory = TtsNavigatorFactory @OptIn(ExperimentalReadiumApi::class) -typealias AndroidTtsNavigator = TtsNavigator +typealias AndroidTtsNavigator = TtsNavigator diff --git a/test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsServiceFacade.kt b/test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsServiceFacade.kt index 1477888f2f..41d52d484a 100644 --- a/test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsServiceFacade.kt +++ b/test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsServiceFacade.kt @@ -5,6 +5,9 @@ import kotlinx.coroutines.sync.Mutex import kotlinx.coroutines.sync.withLock import org.readium.r2.shared.ExperimentalReadiumApi +/** + * Enables to try to close a session without starting the [TtsService] if it is not started. + */ @OptIn(ExperimentalReadiumApi::class) class TtsServiceFacade( private val application: Application diff --git a/test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsViewModel.kt b/test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsViewModel.kt index fd635cba32..150d59f8f8 100644 --- a/test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsViewModel.kt +++ b/test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsViewModel.kt @@ -16,12 +16,11 @@ import kotlinx.coroutines.channels.Channel import kotlinx.coroutines.flow.* import org.readium.r2.navigator.Navigator import org.readium.r2.navigator.VisualNavigator +import org.readium.r2.navigator.media3.api.MediaNavigator import org.readium.r2.navigator.media3.tts.TtsNavigator import org.readium.r2.navigator.media3.tts.android.AndroidTtsEngine import org.readium.r2.navigator.media3.tts.android.AndroidTtsPreferences import org.readium.r2.navigator.media3.tts.android.AndroidTtsPreferencesEditor -import org.readium.r2.navigator.media3.api.MediaNavigator -import org.readium.r2.navigator.media3.api.SynchronizedMediaNavigator import org.readium.r2.shared.ExperimentalReadiumApi import org.readium.r2.shared.UserException import org.readium.r2.shared.publication.Locator @@ -47,7 +46,7 @@ class TtsViewModel private constructor( private val ttsServiceFacade: TtsServiceFacade, private val preferencesManager: PreferencesManager, private val createPreferencesEditor: (AndroidTtsPreferences) -> AndroidTtsPreferencesEditor -) { +) : TtsNavigator.Listener { companion object { /** @@ -74,24 +73,6 @@ class TtsViewModel private constructor( } } - /** - * @param showControls Whether the TTS was enabled by the user. - * @param isPlaying Whether the TTS is currently speaking. - * @param playingWordRange Locator to the currently spoken word. - * @param playingUtterance Locator for the currently spoken utterance (e.g. sentence). - */ - data class State( - val showControls: Boolean = false, - val isPlaying: Boolean = false, - val playingWordRange: Locator? = null, - val playingUtterance: Locator? = null, - ) - - data class Binding( - val playbackJob: Job, - val submitSettingsJob: Job - ) - sealed class Event { /** * Emitted when the [TtsNavigator] fails with an error. @@ -104,20 +85,44 @@ class TtsViewModel private constructor( class OnMissingVoiceData(val language: Language) : Event() } + private var binding: Deferred = + viewModelScope.async { + ttsServiceFacade.getSession() + ?.let { bindSession(it) } + } + + private val _showControls: MutableStateFlow = + MutableStateFlow(navigatorNow != null) + + private val _isPlaying: MutableStateFlow = + MutableStateFlow(navigatorNow?.playback?.value?.playWhenReady ?: false) + + private val _position: MutableStateFlow = + MutableStateFlow(navigatorNow?.currentLocator?.value) + + private val _highlight: MutableStateFlow = + MutableStateFlow(navigatorNow?.utterance?.value?.utteranceHighlight) + + private val _events: Channel = + Channel(Channel.BUFFERED) + val editor: StateFlow = preferencesManager.preferences .mapStateIn(viewModelScope, createPreferencesEditor) - /** - * Current state of the view model. - */ - private val _state: MutableStateFlow = MutableStateFlow( - stateFromPlayback(navigatorNow?.playback?.value, navigatorNow?.utterance?.value) - ) + val showControls: StateFlow = + _showControls.asStateFlow() - val state: StateFlow = _state.asStateFlow() + val isPlaying: StateFlow = + _isPlaying.asStateFlow() - private val _events: Channel = Channel(Channel.BUFFERED) - val events: Flow = _events.receiveAsFlow() + val position: StateFlow = + _position.asStateFlow() + + val highlight: StateFlow = + _highlight.asStateFlow() + + val events: Flow = + _events.receiveAsFlow() private val navigatorNow: AndroidTtsNavigator? get() = ttsServiceFacade.sessionNow()?.navigator @@ -125,11 +130,6 @@ class TtsViewModel private constructor( val voices: Set get() = navigatorNow!!.voices - private var binding: Deferred = - viewModelScope.async { - ttsServiceFacade.getSession()?.let { bindSession(it) } - } - /** * Starts the TTS using the first visible locator in the given [navigator]. */ @@ -147,21 +147,8 @@ class TtsViewModel private constructor( private suspend fun openSession(navigator: Navigator): TtsService.Session { val start = (navigator as? VisualNavigator)?.firstVisibleElementLocator() - val listener = object : TtsNavigator.Listener { - - override fun onStopRequested() { - stop() - } - - override fun onMissingLanguageData(language: Language) { - viewModelScope.launch { - _events.send(Event.OnMissingVoiceData(language)) - } - } - } - val ttsNavigator = ttsNavigatorFactory.createNavigator( - listener, + this, preferencesManager.preferences.value, start ) @@ -174,53 +161,45 @@ class TtsViewModel private constructor( private fun bindSession( ttsSession: TtsService.Session - ): Binding { - val playbackJob = ttsSession.navigator.playback - .onEach { playback -> + ): Job { + val job = Job() + val scope = viewModelScope + job + + _showControls.value = true + + ttsSession.navigator.playback + .onEach { playback -> + if (playback.state == MediaNavigator.State.Ended) { + stop() + } + + _isPlaying.value = playback.playWhenReady playback.error?.let { onPlaybackError(it) } - }.combine(ttsSession.navigator.utterance) { playback, utterance -> - stateFromPlayback(playback, utterance) - }.onEach { state -> - _state.value = state - }.launchIn(viewModelScope) + }.launchIn(scope) + + ttsSession.navigator.utterance + .onEach { utterance -> + _highlight.value = utterance.utteranceHighlight + }.launchIn(scope) - val preferencesJob = preferencesManager.preferences + preferencesManager.preferences .onEach { ttsSession.navigator.submitPreferences(it) } - .launchIn(viewModelScope) + .launchIn(scope) - return Binding(playbackJob, preferencesJob) - } + ttsSession.navigator.currentLocator + .onEach { _position.value = it } + .launchIn(scope) - private fun stateFromPlayback( - playback: MediaNavigator.Playback?, - utterance: SynchronizedMediaNavigator.Utterance? - ): State { - if (playback == null || utterance == null) - return State() - - return State( - showControls = playback.state != MediaNavigator.State.Ended, - isPlaying = playback.playWhenReady, - playingWordRange = utterance.rangeLocator, - playingUtterance = utterance.utteranceLocator - ) + return job } fun stop() { viewModelScope.launch { - binding.await()?.apply { - playbackJob.cancel() - submitSettingsJob.cancel() - } - + binding.cancelAndJoin() + _highlight.value = null + _showControls.value = false + _isPlaying.value = false ttsServiceFacade.closeSession() - - _state.value = State( - showControls = false, - isPlaying = false, - playingWordRange = null, - playingUtterance = null, - ) } } @@ -269,14 +248,24 @@ class TtsViewModel private constructor( } } + override fun onStopRequested() { + stop() + } + + override fun onMissingLanguageData(language: Language) { + viewModelScope.launch { + _events.send(Event.OnMissingVoiceData(language)) + } + } + private fun onPlaybackError(error: TtsNavigator.Error) { val exception = when (error) { is TtsNavigator.Error.ContentError -> { - UserException(R.string.tts_error_other) + UserException(R.string.tts_error_other, cause = error.exception) } is TtsNavigator.Error.EngineError<*> -> { - when ((error.error as AndroidTtsEngine.Exception).error) { - AndroidTtsEngine.EngineError.Network -> + when ((error.error as AndroidTtsEngine.Error).kind) { + AndroidTtsEngine.Error.Kind.Network -> UserException(R.string.tts_error_network) else -> UserException(R.string.tts_error_other) From e07b4591fa99b5f67df6cf03b3995ea8524cf086 Mon Sep 17 00:00:00 2001 From: Quentin Gliosca Date: Tue, 31 Jan 2023 11:36:34 +0100 Subject: [PATCH 12/27] Small fix --- .../java/org/readium/r2/testapp/reader/MediaService.kt | 8 +------- .../java/org/readium/r2/testapp/reader/tts/TtsService.kt | 9 +-------- 2 files changed, 2 insertions(+), 15 deletions(-) diff --git a/test-app/src/main/java/org/readium/r2/testapp/reader/MediaService.kt b/test-app/src/main/java/org/readium/r2/testapp/reader/MediaService.kt index e8469aae1e..29e0159a03 100644 --- a/test-app/src/main/java/org/readium/r2/testapp/reader/MediaService.kt +++ b/test-app/src/main/java/org/readium/r2/testapp/reader/MediaService.kt @@ -78,13 +78,7 @@ class MediaService : LifecycleMedia2SessionService() { flags = flags or PendingIntent.FLAG_IMMUTABLE } - val intent = - ReaderActivityContract().createIntent( - applicationContext, - ReaderActivityContract.Arguments(bookId) - ) - /* intent.addFlags(Intent.FLAG_ACTIVITY_CLEAR_TOP) - .addFlags(Intent.FLAG_ACTIVITY_SINGLE_TOP) */ + val intent = application.packageManager.getLaunchIntentForPackage(application.packageName) return PendingIntent.getActivity(applicationContext, 0, intent, flags) } } diff --git a/test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsService.kt b/test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsService.kt index 7c47c81b47..0011de9930 100644 --- a/test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsService.kt +++ b/test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsService.kt @@ -23,7 +23,6 @@ import kotlinx.coroutines.flow.launchIn import kotlinx.coroutines.flow.onEach import kotlinx.coroutines.flow.sample import org.readium.r2.shared.ExperimentalReadiumApi -import org.readium.r2.testapp.reader.ReaderActivityContract import org.readium.r2.testapp.utils.LifecycleMedia3SessionService import timber.log.Timber @@ -98,13 +97,7 @@ class TtsService : LifecycleMedia3SessionService() { flags = flags or PendingIntent.FLAG_IMMUTABLE } - val intent = - ReaderActivityContract().createIntent( - applicationContext, - ReaderActivityContract.Arguments(bookId) - ) - /*intent.addFlags(Intent.FLAG_ACTIVITY_CLEAR_TOP) - intent.addFlags(Intent.FLAG_ACTIVITY_SINGLE_TOP)*/ + val intent = application.packageManager.getLaunchIntentForPackage(application.packageName) return PendingIntent.getActivity(applicationContext, 0, intent, flags) } From fde1cbc4316f64ccf950a5ce6a00c26d05a52bce Mon Sep 17 00:00:00 2001 From: Quentin Gliosca Date: Tue, 31 Jan 2023 17:58:47 +0100 Subject: [PATCH 13/27] Improve error catching --- .../media3/tts/TtsNavigatorFactory.kt | 4 +- .../r2/navigator/media3/tts/TtsPlayer.kt | 75 +++++++++++++------ .../media3/tts/android/AndroidTtsEngine.kt | 11 ++- .../r2/testapp/reader/tts/TtsViewModel.kt | 10 ++- 4 files changed, 73 insertions(+), 27 deletions(-) diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsNavigatorFactory.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsNavigatorFactory.kt index 5cda7e2bdd..f0885b411f 100644 --- a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsNavigatorFactory.kt +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsNavigatorFactory.kt @@ -113,7 +113,7 @@ class TtsNavigatorFactory, listener: TtsNavigator.Listener, initialPreferences: P? = null, initialLocator: Locator? = null - ): TtsNavigator { + ): TtsNavigator? { return TtsNavigator( application, publication, @@ -123,7 +123,7 @@ class TtsNavigatorFactory, listener, initialPreferences, initialLocator - )!! + ) } fun createTtsPreferencesEditor( diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsPlayer.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsPlayer.kt index 5408cbec8f..653bd341c4 100644 --- a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsPlayer.kt +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsPlayer.kt @@ -20,6 +20,7 @@ import kotlinx.coroutines.sync.Mutex import kotlinx.coroutines.sync.withLock import org.readium.r2.navigator.preferences.Configurable import org.readium.r2.shared.ExperimentalReadiumApi +import org.readium.r2.shared.extensions.tryOrNull import org.readium.r2.shared.publication.Locator @ExperimentalReadiumApi @@ -40,7 +41,7 @@ internal class TtsPlayer, initialPreferences: P, ): TtsPlayer? { - val initialContext = contentIterator.startContext() + val initialContext = tryOrNull { contentIterator.startContext() } ?: return null val ttsEngineFacade = TtsEngineFacade(engine) @@ -345,20 +346,28 @@ internal class TtsPlayer, private suspend fun tryLoadPreviousContext() { val contextNow = context - // Get previously currentUtterance once more - contentIterator.previousUtterance() + val previousUtterance = + try { + // Get previously currentUtterance once more + contentIterator.previousUtterance() - // Get previously previousUtterance once more - contentIterator.previousUtterance() + // Get previously previousUtterance once more + contentIterator.previousUtterance() - // Get new previous utterance - val previousUtterance = contentIterator.previousUtterance() + // Get new previous utterance + val previousUtterance = contentIterator.previousUtterance() - // Go to currentUtterance position - contentIterator.nextUtterance() + // Go to currentUtterance position + contentIterator.nextUtterance() - // Go to nextUtterance position - contentIterator.nextUtterance() + // Go to nextUtterance position + contentIterator.nextUtterance() + + previousUtterance + } catch (e: Exception) { + onContentError(e) + return + } context = Context( previousUtterance = previousUtterance, @@ -373,21 +382,35 @@ internal class TtsPlayer, if (contextNow.nextUtterance == null) { onEndReached() - } else { - context = Context( - previousUtterance = contextNow.currentUtterance, - currentUtterance = contextNow.nextUtterance, - nextUtterance = contentIterator.nextUtterance() - ) - utteranceMutable.value = context.currentUtterance.ttsPlayerUtterance() - if (playbackMutable.value.state == Playback.State.Ended) { - playbackMutable.value = playbackMutable.value.copy(state = Playback.State.Ready) - } + return + } + + val nextUtterance = try { + contentIterator.nextUtterance() + } catch (e: Exception) { + onContentError(e) + return + } + + context = Context( + previousUtterance = contextNow.currentUtterance, + currentUtterance = contextNow.nextUtterance, + nextUtterance = nextUtterance + ) + utteranceMutable.value = context.currentUtterance.ttsPlayerUtterance() + if (playbackMutable.value.state == Playback.State.Ended) { + playbackMutable.value = playbackMutable.value.copy(state = Playback.State.Ready) } } private suspend fun resetContext() { - context = checkNotNull(contentIterator.startContext()) + val startContext = try { + contentIterator.startContext() + } catch (e: Exception) { + onContentError(e) + return + } + context = checkNotNull(startContext) if (context.nextUtterance == null && context.ended) { onEndReached() } @@ -424,6 +447,14 @@ internal class TtsPlayer, playbackJob?.cancel() } + private fun onContentError(exception: Exception) { + playbackMutable.value = playbackMutable.value.copy( + state = Playback.State.Error, + error = Error.ContentError(exception) + ) + playbackJob?.cancel() + } + private fun onRangeChanged(range: IntRange) { val newUtterance = utteranceMutable.value.copy(range = range) utteranceMutable.value = newUtterance diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/android/AndroidTtsEngine.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/android/AndroidTtsEngine.kt index b19cd29a8e..2b67097fee 100644 --- a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/android/AndroidTtsEngine.kt +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/android/AndroidTtsEngine.kt @@ -8,6 +8,7 @@ package org.readium.r2.navigator.media3.tts.android import android.content.Context import android.speech.tts.TextToSpeech +import android.speech.tts.TextToSpeech.ERROR import android.speech.tts.TextToSpeech.QUEUE_ADD import android.speech.tts.UtteranceProgressListener import android.speech.tts.Voice as AndroidVoice @@ -124,6 +125,9 @@ class AndroidTtsEngine private constructor( private val _settings: MutableStateFlow = MutableStateFlow(settingsResolver.settings(initialPreferences)) + private var listener: TtsEngine.Listener? = + null + override val voices: Set get() = engine.voices ?.map { it.toTtsEngineVoice() } @@ -135,7 +139,9 @@ class AndroidTtsEngine private constructor( ) { if (listener == null) { engine.setOnUtteranceProgressListener(null) + this@AndroidTtsEngine.listener = null } else { + this@AndroidTtsEngine.listener = listener engine.setOnUtteranceProgressListener(UtteranceListener(listener)) } } @@ -146,7 +152,10 @@ class AndroidTtsEngine private constructor( language: Language? ) { engine.setupVoice(settings.value, language, voices) - engine.speak(text, QUEUE_ADD, null, requestId) + val queued = engine.speak(text, QUEUE_ADD, null, requestId) + if (queued == ERROR) { + listener?.onError(requestId, Error(Error.Kind.Unknown.code)) + } } override fun stop() { diff --git a/test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsViewModel.kt b/test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsViewModel.kt index 150d59f8f8..95be42bcc3 100644 --- a/test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsViewModel.kt +++ b/test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsViewModel.kt @@ -139,19 +139,25 @@ class TtsViewModel private constructor( return@launch val session = openSession(navigator) + ?: run { + val exception = UserException(R.string.tts_error_initialization) + _events.send(Event.OnError(exception)) + return@launch + } + binding.cancelAndJoin() binding = async { bindSession(session) } } } - private suspend fun openSession(navigator: Navigator): TtsService.Session { + private suspend fun openSession(navigator: Navigator): TtsService.Session? { val start = (navigator as? VisualNavigator)?.firstVisibleElementLocator() val ttsNavigator = ttsNavigatorFactory.createNavigator( this, preferencesManager.preferences.value, start - ) + ) ?: return null // playWhenReady must be true for the MediaSessionService to call Service.startForeground // and prevent crashing From b14405a74ddf0f60156acdd4fd59e10278e9dc11 Mon Sep 17 00:00:00 2001 From: Quentin Gliosca Date: Wed, 15 Feb 2023 17:25:16 +0100 Subject: [PATCH 14/27] Various changes --- .../media3/api/SynchronizedMediaNavigator.kt | 7 +- .../r2/navigator/media3/tts/TtsAliases.kt | 4 +- .../r2/navigator/media3/tts/TtsEngine.kt | 6 +- .../r2/navigator/media3/tts/TtsNavigator.kt | 11 +- .../r2/navigator/media3/tts/TtsPlayer.kt | 63 +-- .../media3/tts/android/AndroidTtsEngine.kt | 27 ++ .../tts/android/AndroidTtsPreferences.kt | 2 +- .../media3/tts/android/AndroidTtsSettings.kt | 9 +- .../tts/session/AudioBecomingNoisyManager.kt | 79 ++++ .../media3/tts/session/AudioFocusManager.kt | 421 ++++++++++++++++++ .../StreamVolumeManager.kt} | 4 +- .../tts/{ => session}/TtsSessionAdapter.kt | 85 +++- .../TtsTimeline.kt} | 4 +- .../services/content/ContentTokenizer.kt | 2 + test-app/src/main/AndroidManifest.xml | 1 + .../r2/testapp/bookshelf/BookshelfFragment.kt | 1 - .../testapp/bookshelf/BookshelfViewModel.kt | 46 -- .../readium/r2/testapp/reader/MediaService.kt | 4 +- .../r2/testapp/reader/ReaderInitData.kt | 2 +- .../r2/testapp/reader/VisualReaderFragment.kt | 21 +- .../r2/testapp/reader/tts/TtsControls.kt | 67 +-- .../r2/testapp/reader/tts/TtsService.kt | 64 +-- .../r2/testapp/reader/tts/TtsServiceFacade.kt | 31 +- .../r2/testapp/reader/tts/TtsViewModel.kt | 34 +- .../utils/LifecycleMedia3SessionService.kt | 63 --- 25 files changed, 742 insertions(+), 316 deletions(-) rename test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsEngine.kt => readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsAliases.kt (85%) create mode 100644 readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/session/AudioBecomingNoisyManager.kt create mode 100644 readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/session/AudioFocusManager.kt rename readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/{TtsStreamVolumeManager.kt => session/StreamVolumeManager.kt} (98%) rename readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/{ => session}/TtsSessionAdapter.kt (90%) rename readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/{TtsSessionTimeline.kt => session/TtsTimeline.kt} (94%) delete mode 100644 test-app/src/main/java/org/readium/r2/testapp/utils/LifecycleMedia3SessionService.kt diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/api/SynchronizedMediaNavigator.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/api/SynchronizedMediaNavigator.kt index 3fa9c1ef8a..6574965bf3 100644 --- a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/api/SynchronizedMediaNavigator.kt +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/api/SynchronizedMediaNavigator.kt @@ -10,6 +10,9 @@ import kotlinx.coroutines.flow.StateFlow import org.readium.r2.shared.ExperimentalReadiumApi import org.readium.r2.shared.publication.Locator +/** + * A [MediaNavigator] aware of the utterances that are being read aloud. + */ @ExperimentalReadiumApi interface SynchronizedMediaNavigator

: MediaNavigator { @@ -21,9 +24,9 @@ interface SynchronizedMediaNavigator

> diff --git a/test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsEngine.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsAliases.kt similarity index 85% rename from test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsEngine.kt rename to readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsAliases.kt index 7012a7d993..717a2e74c0 100644 --- a/test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsEngine.kt +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsAliases.kt @@ -4,10 +4,8 @@ * available in the top-level LICENSE file of the project. */ -package org.readium.r2.testapp.reader.tts +package org.readium.r2.navigator.media3.tts -import org.readium.r2.navigator.media3.tts.TtsNavigator -import org.readium.r2.navigator.media3.tts.TtsNavigatorFactory import org.readium.r2.navigator.media3.tts.android.AndroidTtsEngine import org.readium.r2.navigator.media3.tts.android.AndroidTtsPreferences import org.readium.r2.navigator.media3.tts.android.AndroidTtsPreferencesEditor diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsEngine.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsEngine.kt index c590aef3ea..51c87f7a47 100644 --- a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsEngine.kt +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsEngine.kt @@ -38,7 +38,7 @@ interface TtsEngine, /** * The voice's language. - * */ + */ val language: Language } @@ -83,7 +83,7 @@ interface TtsEngine, fun onDone(requestId: String) /** - * Called when an error has occurred during processing of te utterance with the given id. + * Called when an error has occurred during processing of the utterance with the given id. */ fun onError(requestId: String, error: E) } @@ -94,7 +94,7 @@ interface TtsEngine, val voices: Set /** - * Submits a new speak request. + * Enqueues a new speak request. */ fun speak(requestId: String, text: String, language: Language?) diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsNavigator.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsNavigator.kt index a7f71332a5..f10b5a7ca2 100644 --- a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsNavigator.kt +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsNavigator.kt @@ -16,6 +16,7 @@ import kotlinx.coroutines.flow.StateFlow import org.readium.r2.navigator.media3.api.MediaMetadataProvider import org.readium.r2.navigator.media3.api.MediaNavigator import org.readium.r2.navigator.media3.api.SynchronizedMediaNavigator +import org.readium.r2.navigator.media3.tts.session.TtsSessionAdapter import org.readium.r2.navigator.preferences.Configurable import org.readium.r2.shared.ExperimentalReadiumApi import org.readium.r2.shared.extensions.mapStateIn @@ -131,8 +132,8 @@ class TtsNavigator, override val text: String, override val position: Position, override val range: IntRange?, - override val utteranceHighlight: Locator, - override val tokenHighlight: Locator? + override val utteranceLocator: Locator, + override val tokenLocator: Locator? ) : SynchronizedMediaNavigator.Utterance sealed class Error : MediaNavigator.Error { @@ -184,7 +185,7 @@ class TtsNavigator, } override val currentLocator: StateFlow = - utterance.mapStateIn(coroutineScope) { it.tokenHighlight ?: it.utteranceHighlight } + utterance.mapStateIn(coroutineScope) { it.tokenLocator ?: it.utteranceLocator } override fun go(locator: Locator, animated: Boolean, completion: () -> Unit): Boolean { player.go(locator) @@ -258,8 +259,8 @@ class TtsNavigator, text = text, position = position.toPosition(), range = range, - utteranceHighlight = utteranceHighlight, - tokenHighlight = tokenHighlight, + utteranceLocator = utteranceHighlight, + tokenLocator = tokenHighlight, ) } } diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsPlayer.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsPlayer.kt index 653bd341c4..996f1cbef4 100644 --- a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsPlayer.kt +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsPlayer.kt @@ -7,15 +7,10 @@ package org.readium.r2.navigator.media3.tts import kotlin.coroutines.coroutineContext -import kotlinx.coroutines.CoroutineScope -import kotlinx.coroutines.Job -import kotlinx.coroutines.MainScope -import kotlinx.coroutines.cancelAndJoin +import kotlinx.coroutines.* import kotlinx.coroutines.flow.MutableStateFlow import kotlinx.coroutines.flow.StateFlow import kotlinx.coroutines.flow.asStateFlow -import kotlinx.coroutines.isActive -import kotlinx.coroutines.launch import kotlinx.coroutines.sync.Mutex import kotlinx.coroutines.sync.withLock import org.readium.r2.navigator.preferences.Configurable @@ -23,12 +18,15 @@ import org.readium.r2.shared.ExperimentalReadiumApi import org.readium.r2.shared.extensions.tryOrNull import org.readium.r2.shared.publication.Locator +/** + * Plays the content from a [TtsContentIterator] with a [TtsEngine]. + */ @ExperimentalReadiumApi internal class TtsPlayer, E : TtsEngine.Error, V : TtsEngine.Voice> private constructor( private val engineFacade: TtsEngineFacade, private val contentIterator: TtsContentIterator, - initialContext: Context, + initialWindow: UtteranceWindow, initialPreferences: P ) : Configurable { @@ -49,12 +47,12 @@ internal class TtsPlayer, return TtsPlayer(ttsEngineFacade, contentIterator, initialContext, initialPreferences) } - private suspend fun TtsContentIterator.startContext(): Context? { + private suspend fun TtsContentIterator.startContext(): UtteranceWindow? { val previousUtterance = previousUtterance() val currentUtterance = nextUtterance() - val context = if (currentUtterance != null) { - Context( + val startWindow = if (currentUtterance != null) { + UtteranceWindow( previousUtterance = previousUtterance, currentUtterance = currentUtterance, nextUtterance = nextUtterance(), @@ -67,7 +65,7 @@ internal class TtsPlayer, // Go back to the end of the iterator. nextUtterance() - Context( + UtteranceWindow( previousUtterance = actualPreviousUtterance, currentUtterance = actualCurrentUtterance, nextUtterance = null, @@ -75,7 +73,7 @@ internal class TtsPlayer, ) } - return context + return startWindow } } @@ -113,7 +111,7 @@ internal class TtsPlayer, ) } - private data class Context( + private data class UtteranceWindow( val previousUtterance: TtsContentIterator.Utterance?, val currentUtterance: TtsContentIterator.Utterance, val nextUtterance: TtsContentIterator.Utterance?, @@ -123,8 +121,8 @@ internal class TtsPlayer, private val coroutineScope: CoroutineScope = MainScope() - private var context: Context = - initialContext + private var utteranceWindow: UtteranceWindow = + initialWindow private var playbackJob: Job? = null @@ -135,14 +133,14 @@ internal class TtsPlayer, private val playbackMutable: MutableStateFlow = MutableStateFlow( Playback( - state = if (initialContext.ended) Playback.State.Ended else Playback.State.Ready, + state = if (initialWindow.ended) Playback.State.Ended else Playback.State.Ready, playWhenReady = false, error = null ) ) private val utteranceMutable: MutableStateFlow = - MutableStateFlow(initialContext.currentUtterance.ttsPlayerUtterance()) + MutableStateFlow(initialWindow.currentUtterance.ttsPlayerUtterance()) override val settings: StateFlow = engineFacade.settings @@ -156,6 +154,10 @@ internal class TtsPlayer, val utterance: StateFlow = utteranceMutable.asStateFlow() + /** + * We need to keep the last submitted preferences because TtsSessionAdapter deals with + * preferences, not settings. + */ var lastPreferences: P = initialPreferences @@ -253,7 +255,7 @@ internal class TtsPlayer, } fun hasNextUtterance() = - context.nextUtterance != null + utteranceWindow.nextUtterance != null fun nextUtterance() { coroutineScope.launch { @@ -262,7 +264,7 @@ internal class TtsPlayer, } private suspend fun nextUtteranceAsync() = mutex.withLock { - if (context.nextUtterance == null) { + if (utteranceWindow.nextUtterance == null) { return } @@ -273,7 +275,7 @@ internal class TtsPlayer, } fun hasPreviousUtterance() = - context.previousUtterance != null + utteranceWindow.previousUtterance != null fun previousUtterance() { coroutineScope.launch { @@ -282,7 +284,7 @@ internal class TtsPlayer, } private suspend fun previousUtteranceAsync() = mutex.withLock { - if (context.previousUtterance == null) { + if (utteranceWindow.previousUtterance == null) { return } playbackJob?.cancel() @@ -344,7 +346,7 @@ internal class TtsPlayer, } private suspend fun tryLoadPreviousContext() { - val contextNow = context + val contextNow = utteranceWindow val previousUtterance = try { @@ -369,16 +371,16 @@ internal class TtsPlayer, return } - context = Context( + utteranceWindow = UtteranceWindow( previousUtterance = previousUtterance, currentUtterance = checkNotNull(contextNow.previousUtterance), nextUtterance = contextNow.currentUtterance ) - utteranceMutable.value = context.currentUtterance.ttsPlayerUtterance() + utteranceMutable.value = utteranceWindow.currentUtterance.ttsPlayerUtterance() } private suspend fun tryLoadNextContext() { - val contextNow = context + val contextNow = utteranceWindow if (contextNow.nextUtterance == null) { onEndReached() @@ -392,12 +394,12 @@ internal class TtsPlayer, return } - context = Context( + utteranceWindow = UtteranceWindow( previousUtterance = contextNow.currentUtterance, currentUtterance = contextNow.nextUtterance, nextUtterance = nextUtterance ) - utteranceMutable.value = context.currentUtterance.ttsPlayerUtterance() + utteranceMutable.value = utteranceWindow.currentUtterance.ttsPlayerUtterance() if (playbackMutable.value.state == Playback.State.Ended) { playbackMutable.value = playbackMutable.value.copy(state = Playback.State.Ready) } @@ -410,8 +412,8 @@ internal class TtsPlayer, onContentError(e) return } - context = checkNotNull(startContext) - if (context.nextUtterance == null && context.ended) { + utteranceWindow = checkNotNull(startContext) + if (utteranceWindow.nextUtterance == null && utteranceWindow.ended) { onEndReached() } } @@ -427,7 +429,7 @@ internal class TtsPlayer, return } - val error = speakUtterance(context.currentUtterance) + val error = speakUtterance(utteranceWindow.currentUtterance) mutex.withLock { error?.let { exception -> onEngineError(exception) } @@ -461,6 +463,7 @@ internal class TtsPlayer, } fun close() { + coroutineScope.cancel() engineFacade.close() } diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/android/AndroidTtsEngine.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/android/AndroidTtsEngine.kt index 2b67097fee..597ab128a9 100644 --- a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/android/AndroidTtsEngine.kt +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/android/AndroidTtsEngine.kt @@ -7,6 +7,9 @@ package org.readium.r2.navigator.media3.tts.android import android.content.Context +import android.content.Intent +import android.content.pm.PackageManager +import android.os.Build import android.speech.tts.TextToSpeech import android.speech.tts.TextToSpeech.ERROR import android.speech.tts.TextToSpeech.QUEUE_ADD @@ -55,6 +58,30 @@ class AndroidTtsEngine private constructor( else null } + + /** + * Starts the activity to install additional voice data. + */ + fun requestInstallVoice(context: Context) { + val intent = Intent() + .setAction(TextToSpeech.Engine.ACTION_INSTALL_TTS_DATA) + .setFlags(Intent.FLAG_ACTIVITY_NEW_TASK) + + val availableActivities = + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.TIRAMISU) { + context.packageManager.queryIntentActivities( + intent, + PackageManager.ResolveInfoFlags.of(0) + ) + } else { + @Suppress("Deprecation") + context.packageManager.queryIntentActivities(intent, 0) + } + + if (availableActivities.isNotEmpty()) { + context.startActivity(intent) + } + } } fun interface DefaultVoiceProvider { diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/android/AndroidTtsPreferences.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/android/AndroidTtsPreferences.kt index 08f3222fc8..946504d218 100644 --- a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/android/AndroidTtsPreferences.kt +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/android/AndroidTtsPreferences.kt @@ -12,7 +12,7 @@ import org.readium.r2.shared.ExperimentalReadiumApi import org.readium.r2.shared.util.Language /** - * Preferences for the TTS navigator with the Android built-in engine. + * Preferences for the the Android built-in TTS engine. * * @param language Language of the publication content. * @param pitch Playback pitch rate. diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/android/AndroidTtsSettings.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/android/AndroidTtsSettings.kt index 493dd0649d..9617ff1af3 100644 --- a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/android/AndroidTtsSettings.kt +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/android/AndroidTtsSettings.kt @@ -10,10 +10,11 @@ import org.readium.r2.navigator.media3.tts.TtsEngine import org.readium.r2.shared.ExperimentalReadiumApi import org.readium.r2.shared.util.Language -/* Settings values of the TTS navigator with the Android built-in engine. -* -* @see AndroidTtsPreferences -*/ +/** + * Settings values of the Android built-in TTS engine. + * + * @see AndroidTtsPreferences + */ @ExperimentalReadiumApi data class AndroidTtsSettings( override val language: Language, diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/session/AudioBecomingNoisyManager.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/session/AudioBecomingNoisyManager.kt new file mode 100644 index 0000000000..57df1b096d --- /dev/null +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/session/AudioBecomingNoisyManager.kt @@ -0,0 +1,79 @@ +/* + * Copyright (C) 2019 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.readium.r2.navigator.media3.tts.session + +import android.content.BroadcastReceiver +import android.content.Context +import android.content.Intent +import android.content.IntentFilter +import android.media.AudioManager +import android.os.Handler + +internal class AudioBecomingNoisyManager( + context: Context, + eventHandler: Handler, + listener: EventListener +) { + private val context: Context + private val receiver: AudioBecomingNoisyReceiver + private var receiverRegistered = false + + interface EventListener { + fun onAudioBecomingNoisy() + } + + init { + this.context = context.applicationContext + receiver = AudioBecomingNoisyReceiver(eventHandler, listener) + } + + /** + * Enables the [AudioBecomingNoisyManager] which calls [ ][EventListener.onAudioBecomingNoisy] + * upon receiving an intent of [ ][AudioManager.ACTION_AUDIO_BECOMING_NOISY]. + * + * @param enabled True if the listener should be notified when audio is becoming noisy. + */ + fun setEnabled(enabled: Boolean) { + if (enabled && !receiverRegistered) { + context.registerReceiver( + receiver, IntentFilter(AudioManager.ACTION_AUDIO_BECOMING_NOISY) + ) + receiverRegistered = true + } else if (!enabled && receiverRegistered) { + context.unregisterReceiver(receiver) + receiverRegistered = false + } + } + + private inner class AudioBecomingNoisyReceiver( + private val eventHandler: Handler, + private val listener: EventListener + ) : + BroadcastReceiver(), Runnable { + override fun onReceive(context: Context, intent: Intent) { + if (AudioManager.ACTION_AUDIO_BECOMING_NOISY == intent.action) { + eventHandler.post(this) + } + } + + override fun run() { + if (receiverRegistered) { + listener.onAudioBecomingNoisy() + } + } + } +} diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/session/AudioFocusManager.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/session/AudioFocusManager.kt new file mode 100644 index 0000000000..1532391955 --- /dev/null +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/session/AudioFocusManager.kt @@ -0,0 +1,421 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.readium.r2.navigator.media3.tts.session + +import android.content.Context +import android.media.AudioFocusRequest +import android.media.AudioManager +import android.media.AudioManager.OnAudioFocusChangeListener +import android.os.Handler +import androidx.annotation.IntDef +import androidx.annotation.RequiresApi +import androidx.annotation.VisibleForTesting +import androidx.media3.common.AudioAttributes +import androidx.media3.common.C +import androidx.media3.common.Player +import androidx.media3.common.util.Log +import androidx.media3.common.util.Util +import org.readium.r2.navigator.media3.tts.session.AudioFocusManager.PlayerControl + +@androidx.annotation.OptIn(androidx.media3.common.util.UnstableApi::class) +/** Manages requesting and responding to changes in audio focus. + * + * @param context The current context. + * @param eventHandler A [Handler] to for the thread on which the player is used. + * @param playerControl A [PlayerControl] to handle commands from this instance. + */ +internal class AudioFocusManager( + context: Context, + eventHandler: Handler, + playerControl: PlayerControl +) { + /** Interface to allow AudioFocusManager to give commands to a player. */ + interface PlayerControl { + /** + * Called when the volume multiplier on the player should be changed. + * + * @param volumeMultiplier The new volume multiplier. + */ + fun setVolumeMultiplier(volumeMultiplier: Float) + + /** + * Called when a command must be executed on the player. + * + * @param playerCommand The command that must be executed. + */ + fun executePlayerCommand(playerCommand: @PlayerCommand Int) + } + + /** + * Player commands. One of [.PLAYER_COMMAND_DO_NOT_PLAY], [ ][.PLAYER_COMMAND_WAIT_FOR_CALLBACK] + * or [.PLAYER_COMMAND_PLAY_WHEN_READY]. + */ + @Target(AnnotationTarget.TYPE) + @MustBeDocumented + @Retention(AnnotationRetention.SOURCE) + @IntDef( + PLAYER_COMMAND_DO_NOT_PLAY, PLAYER_COMMAND_WAIT_FOR_CALLBACK, PLAYER_COMMAND_PLAY_WHEN_READY + ) + annotation class PlayerCommand + + /** Audio focus state. */ + @Target(AnnotationTarget.TYPE) + @MustBeDocumented + @Retention(AnnotationRetention.SOURCE) + @IntDef( + AUDIO_FOCUS_STATE_NO_FOCUS, + AUDIO_FOCUS_STATE_HAVE_FOCUS, + AUDIO_FOCUS_STATE_LOSS_TRANSIENT, + AUDIO_FOCUS_STATE_LOSS_TRANSIENT_DUCK + ) + private annotation class AudioFocusState + + /** + * Audio focus types. One of [.AUDIOFOCUS_NONE], [.AUDIOFOCUS_GAIN], [ ][.AUDIOFOCUS_GAIN_TRANSIENT], [.AUDIOFOCUS_GAIN_TRANSIENT_MAY_DUCK] or [ ][.AUDIOFOCUS_GAIN_TRANSIENT_EXCLUSIVE]. + */ + @Target(AnnotationTarget.TYPE) + @MustBeDocumented + @Retention(AnnotationRetention.SOURCE) + @IntDef( + AUDIOFOCUS_NONE, + AUDIOFOCUS_GAIN, + AUDIOFOCUS_GAIN_TRANSIENT, + AUDIOFOCUS_GAIN_TRANSIENT_MAY_DUCK, + AUDIOFOCUS_GAIN_TRANSIENT_EXCLUSIVE + ) + private annotation class AudioFocusGain + + private val audioManager: AudioManager + private val focusListener: AudioFocusListener + private var playerControl: PlayerControl? + private var audioAttributes: AudioAttributes? = null + private var audioFocusState: @AudioFocusState Int + private var focusGainToRequest = 0 + + /** Gets the current player volume multiplier. */ + var volumeMultiplier = VOLUME_MULTIPLIER_DEFAULT + private set + private lateinit var audioFocusRequest: AudioFocusRequest + private var rebuildAudioFocusRequest = false + + init { + audioManager = checkNotNull( + context.applicationContext.getSystemService(Context.AUDIO_SERVICE) as AudioManager + ) + this.playerControl = playerControl + focusListener = AudioFocusListener(eventHandler) + audioFocusState = AUDIO_FOCUS_STATE_NO_FOCUS + } + + /** + * Sets audio attributes that should be used to manage audio focus. + * + * + * Call [.updateAudioFocus] to update the audio focus based on these + * attributes. + * + * @param audioAttributes The audio attributes or `null` if audio focus should not be + * managed automatically. + */ + fun setAudioAttributes(audioAttributes: AudioAttributes?) { + if (!Util.areEqual(this.audioAttributes, audioAttributes)) { + this.audioAttributes = audioAttributes + focusGainToRequest = convertAudioAttributesToFocusGain(audioAttributes) + require( + focusGainToRequest == AUDIOFOCUS_GAIN || focusGainToRequest == AUDIOFOCUS_NONE + ) { "Automatic handling of audio focus is only available for USAGE_MEDIA and USAGE_GAME." } + } + } + + /** + * Called by the player to abandon or request audio focus based on the desired player state. + * + * @param playWhenReady The desired value of playWhenReady. + * @param playbackState The desired playback state. + * @return A [PlayerCommand] to execute on the player. + */ + fun updateAudioFocus( + playWhenReady: Boolean, + playbackState: @Player.State Int + ): @PlayerCommand Int { + if (shouldAbandonAudioFocusIfHeld(playbackState)) { + abandonAudioFocusIfHeld() + return if (playWhenReady) PLAYER_COMMAND_PLAY_WHEN_READY else PLAYER_COMMAND_DO_NOT_PLAY + } + return if (playWhenReady) requestAudioFocus() else PLAYER_COMMAND_DO_NOT_PLAY + } + + /** + * Called when the manager is no longer required. Audio focus will be released without making any + * calls to the [PlayerControl]. + */ + fun release() { + playerControl = null + abandonAudioFocusIfHeld() + } + + // Internal methods. + @VisibleForTesting + fun /* package */getFocusListener(): OnAudioFocusChangeListener { + return focusListener + } + + private fun shouldAbandonAudioFocusIfHeld(playbackState: @Player.State Int): Boolean { + return playbackState == Player.STATE_IDLE || focusGainToRequest != AUDIOFOCUS_GAIN + } + + private fun requestAudioFocus(): @PlayerCommand Int { + if (audioFocusState == AUDIO_FOCUS_STATE_HAVE_FOCUS) { + return PLAYER_COMMAND_PLAY_WHEN_READY + } + val requestResult = + if (Util.SDK_INT >= 26) requestAudioFocusV26() else requestAudioFocusDefault() + return if (requestResult == AudioManager.AUDIOFOCUS_REQUEST_GRANTED) { + setAudioFocusState(AUDIO_FOCUS_STATE_HAVE_FOCUS) + PLAYER_COMMAND_PLAY_WHEN_READY + } else { + setAudioFocusState(AUDIO_FOCUS_STATE_NO_FOCUS) + PLAYER_COMMAND_DO_NOT_PLAY + } + } + + private fun abandonAudioFocusIfHeld() { + if (audioFocusState == AUDIO_FOCUS_STATE_NO_FOCUS) { + return + } + if (Util.SDK_INT >= 26) { + abandonAudioFocusV26() + } else { + abandonAudioFocusDefault() + } + setAudioFocusState(AUDIO_FOCUS_STATE_NO_FOCUS) + } + + @Suppress("Deprecation") + private fun requestAudioFocusDefault(): Int { + return audioManager.requestAudioFocus( + focusListener, + Util.getStreamTypeForAudioUsage(checkNotNull(audioAttributes).usage), + focusGainToRequest + ) + } + + @RequiresApi(26) + private fun requestAudioFocusV26(): Int { + if (!::audioFocusRequest.isInitialized || rebuildAudioFocusRequest) { + val builder = + if (!::audioFocusRequest.isInitialized) + AudioFocusRequest.Builder(focusGainToRequest) + else + AudioFocusRequest.Builder( + audioFocusRequest + ) + val willPauseWhenDucked = willPauseWhenDucked() + audioFocusRequest = builder + .setAudioAttributes( + checkNotNull(audioAttributes).audioAttributesV21.audioAttributes + ) + .setWillPauseWhenDucked(willPauseWhenDucked) + .setOnAudioFocusChangeListener(focusListener) + .build() + rebuildAudioFocusRequest = false + } + return audioManager.requestAudioFocus(audioFocusRequest) + } + + @Suppress("Deprecation") + private fun abandonAudioFocusDefault() { + audioManager.abandonAudioFocus(focusListener) + } + + @RequiresApi(26) + private fun abandonAudioFocusV26() { + if (::audioFocusRequest.isInitialized) { + audioManager.abandonAudioFocusRequest(audioFocusRequest) + } + } + + private fun willPauseWhenDucked(): Boolean { + return audioAttributes != null && audioAttributes!!.contentType == C.AUDIO_CONTENT_TYPE_SPEECH + } + + private fun setAudioFocusState(audioFocusState: @AudioFocusState Int) { + if (this.audioFocusState == audioFocusState) { + return + } + this.audioFocusState = audioFocusState + val volumeMultiplier = + if (audioFocusState == AUDIO_FOCUS_STATE_LOSS_TRANSIENT_DUCK) + VOLUME_MULTIPLIER_DUCK + else + VOLUME_MULTIPLIER_DEFAULT + if (this.volumeMultiplier == volumeMultiplier) { + return + } + this.volumeMultiplier = volumeMultiplier + if (playerControl != null) { + playerControl!!.setVolumeMultiplier(volumeMultiplier) + } + } + + private fun handlePlatformAudioFocusChange(focusChange: Int) { + when (focusChange) { + AudioManager.AUDIOFOCUS_GAIN -> { + setAudioFocusState(AUDIO_FOCUS_STATE_HAVE_FOCUS) + executePlayerCommand(PLAYER_COMMAND_PLAY_WHEN_READY) + return + } + AudioManager.AUDIOFOCUS_LOSS -> { + executePlayerCommand(PLAYER_COMMAND_DO_NOT_PLAY) + abandonAudioFocusIfHeld() + return + } + AudioManager.AUDIOFOCUS_LOSS_TRANSIENT, AudioManager.AUDIOFOCUS_LOSS_TRANSIENT_CAN_DUCK -> { + if (focusChange == AudioManager.AUDIOFOCUS_LOSS_TRANSIENT || willPauseWhenDucked()) { + executePlayerCommand(PLAYER_COMMAND_WAIT_FOR_CALLBACK) + setAudioFocusState(AUDIO_FOCUS_STATE_LOSS_TRANSIENT) + } else { + setAudioFocusState(AUDIO_FOCUS_STATE_LOSS_TRANSIENT_DUCK) + } + return + } + else -> Log.w( + TAG, + "Unknown focus change type: $focusChange" + ) + } + } + + private fun executePlayerCommand(playerCommand: @PlayerCommand Int) { + if (playerControl != null) { + playerControl!!.executePlayerCommand(playerCommand) + } + } + + // Internal audio focus listener. + private inner class AudioFocusListener(private val eventHandler: Handler) : + OnAudioFocusChangeListener { + override fun onAudioFocusChange(focusChange: Int) { + eventHandler.post { handlePlatformAudioFocusChange(focusChange) } + } + } + + companion object { + /** Do not play. */ + const val PLAYER_COMMAND_DO_NOT_PLAY = -1 + + /** Do not play now. Wait for callback to play. */ + const val PLAYER_COMMAND_WAIT_FOR_CALLBACK = 0 + + /** Play freely. */ + const val PLAYER_COMMAND_PLAY_WHEN_READY = 1 + + /** No audio focus is currently being held. */ + private const val AUDIO_FOCUS_STATE_NO_FOCUS = 0 + + /** The requested audio focus is currently held. */ + private const val AUDIO_FOCUS_STATE_HAVE_FOCUS = 1 + + /** Audio focus has been temporarily lost. */ + private const val AUDIO_FOCUS_STATE_LOSS_TRANSIENT = 2 + + /** Audio focus has been temporarily lost, but playback may continue with reduced volume. */ + private const val AUDIO_FOCUS_STATE_LOSS_TRANSIENT_DUCK = 3 + + /** + * @see AudioManager.AUDIOFOCUS_NONE + */ + private const val AUDIOFOCUS_NONE = AudioManager.AUDIOFOCUS_NONE + + /** + * @see AudioManager.AUDIOFOCUS_GAIN + */ + private const val AUDIOFOCUS_GAIN = AudioManager.AUDIOFOCUS_GAIN + + /** + * @see AudioManager.AUDIOFOCUS_GAIN_TRANSIENT + */ + private const val AUDIOFOCUS_GAIN_TRANSIENT = AudioManager.AUDIOFOCUS_GAIN_TRANSIENT + + /** + * @see AudioManager.AUDIOFOCUS_GAIN_TRANSIENT_MAY_DUCK + */ + private const val AUDIOFOCUS_GAIN_TRANSIENT_MAY_DUCK = + AudioManager.AUDIOFOCUS_GAIN_TRANSIENT_MAY_DUCK + + /** + * @see AudioManager.AUDIOFOCUS_GAIN_TRANSIENT_EXCLUSIVE + */ + private const val AUDIOFOCUS_GAIN_TRANSIENT_EXCLUSIVE = + AudioManager.AUDIOFOCUS_GAIN_TRANSIENT_EXCLUSIVE + private const val TAG = "AudioFocusManager" + private const val VOLUME_MULTIPLIER_DUCK = 0.2f + private const val VOLUME_MULTIPLIER_DEFAULT = 1.0f + + /** + * Converts [AudioAttributes] to one of the audio focus request. + * + * + * This follows the class Javadoc of [AudioFocusRequest]. + * + * @param audioAttributes The audio attributes associated with this focus request. + * @return The type of audio focus gain that should be requested. + */ + private fun convertAudioAttributesToFocusGain( + audioAttributes: AudioAttributes? + ): @AudioFocusGain Int { + return if (audioAttributes == null) { + // Don't handle audio focus. It may be either video only contents or developers + // want to have more finer grained control. (e.g. adding audio focus listener) + AUDIOFOCUS_NONE + } else when (audioAttributes.usage) { + C.USAGE_VOICE_COMMUNICATION_SIGNALLING -> AUDIOFOCUS_NONE + C.USAGE_GAME, C.USAGE_MEDIA -> AUDIOFOCUS_GAIN + C.USAGE_UNKNOWN -> { + Log.w( + TAG, + "Specify a proper usage in the audio attributes for audio focus" + + " handling. Using AUDIOFOCUS_GAIN by default." + ) + AUDIOFOCUS_GAIN + } + C.USAGE_ALARM, C.USAGE_VOICE_COMMUNICATION -> AUDIOFOCUS_GAIN_TRANSIENT + C.USAGE_ASSISTANCE_NAVIGATION_GUIDANCE, C.USAGE_ASSISTANCE_SONIFICATION, + C.USAGE_NOTIFICATION, C.USAGE_NOTIFICATION_COMMUNICATION_DELAYED, + C.USAGE_NOTIFICATION_COMMUNICATION_INSTANT, C.USAGE_NOTIFICATION_COMMUNICATION_REQUEST, + C.USAGE_NOTIFICATION_EVENT, C.USAGE_NOTIFICATION_RINGTONE -> + AUDIOFOCUS_GAIN_TRANSIENT_MAY_DUCK + C.USAGE_ASSISTANT -> + if (Util.SDK_INT >= 19) { + AUDIOFOCUS_GAIN_TRANSIENT_EXCLUSIVE + } else { + AUDIOFOCUS_GAIN_TRANSIENT + } + C.USAGE_ASSISTANCE_ACCESSIBILITY -> { + if (audioAttributes.contentType == C.AUDIO_CONTENT_TYPE_SPEECH) { + // Voice shouldn't be interrupted by other playback. + AUDIOFOCUS_GAIN_TRANSIENT + } else AUDIOFOCUS_GAIN_TRANSIENT_MAY_DUCK + } + else -> { + Log.w(TAG, "Unidentified audio usage: " + audioAttributes.usage) + AUDIOFOCUS_NONE + } + } + } + } +} diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsStreamVolumeManager.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/session/StreamVolumeManager.kt similarity index 98% rename from readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsStreamVolumeManager.kt rename to readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/session/StreamVolumeManager.kt index f484b1bffa..1190643074 100644 --- a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsStreamVolumeManager.kt +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/session/StreamVolumeManager.kt @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.readium.r2.navigator.media3.tts +package org.readium.r2.navigator.media3.tts.session import android.content.BroadcastReceiver import android.content.Context @@ -28,7 +28,7 @@ import androidx.media3.common.util.Util @androidx.annotation.OptIn(androidx.media3.common.util.UnstableApi::class) /** A manager that wraps [AudioManager] to control/listen audio stream volume. */ /* package */ -internal class TtsStreamVolumeManager(context: Context, eventHandler: Handler, listener: Listener) { +internal class StreamVolumeManager(context: Context, eventHandler: Handler, listener: Listener) { /** A listener for changes in the manager. */ interface Listener { /** Called when the audio stream type is changed. */ diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsSessionAdapter.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/session/TtsSessionAdapter.kt similarity index 90% rename from readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsSessionAdapter.kt rename to readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/session/TtsSessionAdapter.kt index 72b4421ebc..d103c7d498 100644 --- a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsSessionAdapter.kt +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/session/TtsSessionAdapter.kt @@ -4,7 +4,7 @@ * available in the top-level LICENSE file of the project. */ -package org.readium.r2.navigator.media3.tts +package org.readium.r2.navigator.media3.tts.session import android.app.Application import android.os.Handler @@ -27,9 +27,14 @@ import kotlinx.coroutines.MainScope import kotlinx.coroutines.flow.StateFlow import kotlinx.coroutines.flow.launchIn import kotlinx.coroutines.flow.onEach +import org.readium.r2.navigator.media3.tts.TtsEngine +import org.readium.r2.navigator.media3.tts.TtsPlayer import org.readium.r2.shared.ExperimentalReadiumApi import org.readium.r2.shared.fetcher.Resource +/** + * Adapts the [TtsPlayer] to media3 [Player] interface. + */ @ExperimentalReadiumApi @androidx.annotation.OptIn(androidx.media3.common.util.UnstableApi::class) internal class TtsSessionAdapter( @@ -46,6 +51,9 @@ internal class TtsSessionAdapter( private val coroutineScope: CoroutineScope = MainScope() + private val eventHandler: Handler = + Handler(applicationLooper) + private val window: Timeline.Window = Timeline.Window() @@ -55,20 +63,45 @@ internal class TtsSessionAdapter( private var lastPlaybackParameters: PlaybackParameters = playbackParametersState.value - private val volumeManager = TtsStreamVolumeManager( + private val streamVolumeManager = StreamVolumeManager( application, Handler(applicationLooper), StreamVolumeManagerListener() ) + init { + streamVolumeManager.setStreamType(Util.getStreamTypeForAudioUsage(audioAttributes.usage)) + } + + private val audioFocusManager = AudioFocusManager( + application, + eventHandler, + AudioFocusManagerListener() + ) + + init { + audioFocusManager.setAudioAttributes(audioAttributes) + } + + private val audioBecomingNoisyManager = AudioBecomingNoisyManager( + application, + eventHandler, + AudioBecomingNoisyManagerListener() + ) + + init { + audioBecomingNoisyManager.setEnabled(true) + } + private var deviceInfo: DeviceInfo = - createDeviceInfo(volumeManager) + createDeviceInfo(streamVolumeManager) init { ttsPlayer.playback .onEach { playback -> notifyListenersPlaybackChanged(lastPlayback, playback) lastPlayback = playback + audioFocusManager.updateAudioFocus(playback.playWhenReady, playback.state.playerCode) }.launchIn(coroutineScope) playbackParametersState @@ -396,7 +429,11 @@ internal class TtsSessionAdapter( override fun stop(reset: Boolean) {} override fun release() { - // Do nothing. This object does not own the TtsPlayer instance. + streamVolumeManager.release() + audioFocusManager.release() + audioBecomingNoisyManager.setEnabled(false) + eventHandler.removeCallbacksAndMessages(null) + // This object does not own the TtsPlayer instance, do not close it. } override fun getCurrentTracks(): Tracks { @@ -434,7 +471,7 @@ internal class TtsSessionAdapter( override fun getCurrentTimeline(): Timeline { // MediaNotificationManager requires a non-empty timeline to start foreground playing. - return TtsSessionTimeline(mediaItems) + return TtsTimeline(mediaItems) } override fun getCurrentPeriodIndex(): Int { @@ -656,27 +693,27 @@ internal class TtsSessionAdapter( } override fun getDeviceVolume(): Int { - return volumeManager.getVolume() + return streamVolumeManager.getVolume() } override fun isDeviceMuted(): Boolean { - return volumeManager.isMuted() + return streamVolumeManager.isMuted() } override fun setDeviceVolume(volume: Int) { - volumeManager.setVolume(volume) + streamVolumeManager.setVolume(volume) } override fun increaseDeviceVolume() { - volumeManager.increaseVolume() + streamVolumeManager.increaseVolume() } override fun decreaseDeviceVolume() { - volumeManager.decreaseVolume() + streamVolumeManager.decreaseVolume() } override fun setDeviceMuted(muted: Boolean) { - volumeManager.setMuted(muted) + streamVolumeManager.setMuted(muted) } private fun notifyListenersPlaybackChanged( @@ -755,7 +792,7 @@ internal class TtsSessionAdapter( } } - private fun createDeviceInfo(streamVolumeManager: TtsStreamVolumeManager): DeviceInfo { + private fun createDeviceInfo(streamVolumeManager: StreamVolumeManager): DeviceInfo { val newDeviceInfo = DeviceInfo( DeviceInfo.PLAYBACK_TYPE_LOCAL, streamVolumeManager.minVolume, @@ -774,9 +811,10 @@ internal class TtsSessionAdapter( return if (repeatMode == REPEAT_MODE_ONE) REPEAT_MODE_OFF else repeatMode } - private inner class StreamVolumeManagerListener : TtsStreamVolumeManager.Listener { + private inner class StreamVolumeManagerListener : StreamVolumeManager.Listener { + override fun onStreamTypeChanged(streamType: @StreamType Int) { - val newDeviceInfo = createDeviceInfo(volumeManager) + val newDeviceInfo = createDeviceInfo(streamVolumeManager) if (newDeviceInfo != deviceInfo) { listeners.sendEvent( EVENT_DEVICE_INFO_CHANGED @@ -800,6 +838,25 @@ internal class TtsSessionAdapter( } } + private inner class AudioFocusManagerListener : AudioFocusManager.PlayerControl { + + override fun setVolumeMultiplier(volumeMultiplier: Float) { + // Do nothing as we're not supposed to duck volume with + // contentType == C.AUDIO_CONTENT_TYPE_SPEECH + } + + override fun executePlayerCommand(playerCommand: Int) { + playWhenReady = playWhenReady && playerCommand != AudioFocusManager.PLAYER_COMMAND_DO_NOT_PLAY + } + } + + private inner class AudioBecomingNoisyManagerListener : AudioBecomingNoisyManager.EventListener { + + override fun onAudioBecomingNoisy() { + playWhenReady = false + } + } + private val TtsPlayer.Playback.State.playerCode get() = when (this) { TtsPlayer.Playback.State.Ready -> STATE_READY TtsPlayer.Playback.State.Ended -> STATE_ENDED diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsSessionTimeline.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/session/TtsTimeline.kt similarity index 94% rename from readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsSessionTimeline.kt rename to readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/session/TtsTimeline.kt index ec4a2055c5..8cfdacad23 100644 --- a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsSessionTimeline.kt +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/session/TtsTimeline.kt @@ -4,14 +4,14 @@ * available in the top-level LICENSE file of the project. */ -package org.readium.r2.navigator.media3.tts +package org.readium.r2.navigator.media3.tts.session import androidx.media3.common.MediaItem import androidx.media3.common.Timeline import java.util.* @androidx.annotation.OptIn(androidx.media3.common.util.UnstableApi::class) -internal class TtsSessionTimeline( +internal class TtsTimeline( private val mediaItems: List, ) : Timeline() { diff --git a/readium/shared/src/main/java/org/readium/r2/shared/publication/services/content/ContentTokenizer.kt b/readium/shared/src/main/java/org/readium/r2/shared/publication/services/content/ContentTokenizer.kt index 552639aa18..574f772de7 100644 --- a/readium/shared/src/main/java/org/readium/r2/shared/publication/services/content/ContentTokenizer.kt +++ b/readium/shared/src/main/java/org/readium/r2/shared/publication/services/content/ContentTokenizer.kt @@ -23,6 +23,8 @@ fun interface ContentTokenizer : Tokenizer * portions. * * @param contextSnippetLength Length of `before` and `after` snippets in the produced [Locator]s. + * @param overrideContentLanguage If true, let [language] override language information that could be available in + * content. If false, [language] will be used only as a default when there is no data-specific information. */ @ExperimentalReadiumApi class TextContentTokenizer( diff --git a/test-app/src/main/AndroidManifest.xml b/test-app/src/main/AndroidManifest.xml index 1e9738b983..129d540698 100644 --- a/test-app/src/main/AndroidManifest.xml +++ b/test-app/src/main/AndroidManifest.xml @@ -197,6 +197,7 @@ + diff --git a/test-app/src/main/java/org/readium/r2/testapp/bookshelf/BookshelfFragment.kt b/test-app/src/main/java/org/readium/r2/testapp/bookshelf/BookshelfFragment.kt index 8c0374e9f6..d40da78c98 100644 --- a/test-app/src/main/java/org/readium/r2/testapp/bookshelf/BookshelfFragment.kt +++ b/test-app/src/main/java/org/readium/r2/testapp/bookshelf/BookshelfFragment.kt @@ -35,7 +35,6 @@ class BookshelfFragment : Fragment() { private val bookshelfViewModel: BookshelfViewModel by activityViewModels() private lateinit var bookshelfAdapter: BookshelfAdapter private lateinit var documentPickerLauncher: ActivityResultLauncher - private lateinit var readerLauncher: ActivityResultLauncher private var binding: FragmentBookshelfBinding by viewLifecycle() override fun onCreateView( diff --git a/test-app/src/main/java/org/readium/r2/testapp/bookshelf/BookshelfViewModel.kt b/test-app/src/main/java/org/readium/r2/testapp/bookshelf/BookshelfViewModel.kt index a79d414a3e..ec97f4a1ed 100644 --- a/test-app/src/main/java/org/readium/r2/testapp/bookshelf/BookshelfViewModel.kt +++ b/test-app/src/main/java/org/readium/r2/testapp/bookshelf/BookshelfViewModel.kt @@ -9,23 +9,14 @@ package org.readium.r2.testapp.bookshelf import android.app.Activity import android.app.Application import android.content.Context -import android.graphics.Bitmap -import android.graphics.BitmapFactory import android.net.Uri import androidx.lifecycle.AndroidViewModel import androidx.lifecycle.viewModelScope -import java.io.File -import java.io.FileOutputStream -import java.io.IOException -import java.net.HttpURLConnection -import java.net.URL import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.channels.Channel import kotlinx.coroutines.launch import kotlinx.coroutines.withContext import org.readium.r2.shared.UserException -import org.readium.r2.shared.publication.Publication -import org.readium.r2.shared.publication.services.cover import org.readium.r2.testapp.BuildConfig import org.readium.r2.testapp.R import org.readium.r2.testapp.domain.model.Book @@ -119,43 +110,6 @@ class BookshelfViewModel(application: Application) : AndroidViewModel(applicatio } } - fun closePublication(bookId: Long) = viewModelScope.launch { - val readerRepository = app.readerRepository.await() - readerRepository.close(bookId) - } - - private fun storeCoverImage(publication: Publication, imageName: String) = - viewModelScope.launch(Dispatchers.IO) { - // TODO Figure out where to store these cover images - val coverImageDir = File(app.storageDir, "covers/") - if (!coverImageDir.exists()) { - coverImageDir.mkdirs() - } - val coverImageFile = File(app.storageDir, "covers/$imageName.png") - - val bitmap: Bitmap? = publication.cover() - - val resized = bitmap?.let { Bitmap.createScaledBitmap(it, 120, 200, true) } - val fos = FileOutputStream(coverImageFile) - resized?.compress(Bitmap.CompressFormat.PNG, 80, fos) - fos.flush() - fos.close() - } - - private fun getBitmapFromURL(src: String): Bitmap? { - return try { - val url = URL(src) - val connection = url.openConnection() as HttpURLConnection - connection.doInput = true - connection.connect() - val input = connection.inputStream - BitmapFactory.decodeStream(input) - } catch (e: IOException) { - e.printStackTrace() - null - } - } - sealed class Event { object ImportPublicationSuccess : Event() diff --git a/test-app/src/main/java/org/readium/r2/testapp/reader/MediaService.kt b/test-app/src/main/java/org/readium/r2/testapp/reader/MediaService.kt index 29e0159a03..8e420630b4 100644 --- a/test-app/src/main/java/org/readium/r2/testapp/reader/MediaService.kt +++ b/test-app/src/main/java/org/readium/r2/testapp/reader/MediaService.kt @@ -56,7 +56,7 @@ class MediaService : LifecycleMedia2SessionService() { @OptIn(FlowPreview::class) fun bindNavigator(navigator: MediaNavigator, bookId: Long) { - val activityIntent = createSessionActivityIntent(bookId) + val activityIntent = createSessionActivityIntent() mediaNavigator = navigator mediaSession = navigator.session(applicationContext, activityIntent) .also { addSession(it) } @@ -71,7 +71,7 @@ class MediaService : LifecycleMedia2SessionService() { .launchIn(lifecycleScope) } - private fun createSessionActivityIntent(bookId: Long): PendingIntent { + private fun createSessionActivityIntent(): PendingIntent { // This intent will be triggered when the notification is clicked. var flags = PendingIntent.FLAG_UPDATE_CURRENT if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) { diff --git a/test-app/src/main/java/org/readium/r2/testapp/reader/ReaderInitData.kt b/test-app/src/main/java/org/readium/r2/testapp/reader/ReaderInitData.kt index 93a9fa96da..cf042237f6 100644 --- a/test-app/src/main/java/org/readium/r2/testapp/reader/ReaderInitData.kt +++ b/test-app/src/main/java/org/readium/r2/testapp/reader/ReaderInitData.kt @@ -15,12 +15,12 @@ import org.readium.navigator.media2.ExperimentalMedia2 import org.readium.navigator.media2.MediaNavigator import org.readium.r2.navigator.epub.EpubNavigatorFactory import org.readium.r2.navigator.epub.EpubPreferences +import org.readium.r2.navigator.media3.tts.AndroidTtsNavigatorFactory import org.readium.r2.navigator.media3.tts.android.AndroidTtsPreferences import org.readium.r2.navigator.pdf.PdfNavigatorFactory import org.readium.r2.shared.ExperimentalReadiumApi import org.readium.r2.shared.publication.* import org.readium.r2.testapp.reader.preferences.PreferencesManager -import org.readium.r2.testapp.reader.tts.AndroidTtsNavigatorFactory import org.readium.r2.testapp.reader.tts.TtsServiceFacade sealed class ReaderInitData { diff --git a/test-app/src/main/java/org/readium/r2/testapp/reader/VisualReaderFragment.kt b/test-app/src/main/java/org/readium/r2/testapp/reader/VisualReaderFragment.kt index e09f0382e4..5199a11f9d 100644 --- a/test-app/src/main/java/org/readium/r2/testapp/reader/VisualReaderFragment.kt +++ b/test-app/src/main/java/org/readium/r2/testapp/reader/VisualReaderFragment.kt @@ -41,8 +41,11 @@ import kotlinx.coroutines.flow.* import kotlinx.coroutines.launch import kotlinx.parcelize.Parcelize import org.readium.r2.navigator.* +import org.readium.r2.navigator.media3.tts.android.AndroidTtsEngine import org.readium.r2.navigator.util.BaseActionModeCallback import org.readium.r2.navigator.util.EdgeTapNavigation +import org.readium.r2.shared.ExperimentalReadiumApi +import org.readium.r2.shared.publication.Locator import org.readium.r2.shared.util.Language import org.readium.r2.testapp.R import org.readium.r2.testapp.databinding.FragmentReaderBinding @@ -58,7 +61,7 @@ import org.readium.r2.testapp.utils.extensions.throttleLatest * * Provides common menu items and saves last location on stop. */ -@OptIn(ExperimentalDecorator::class) +@OptIn(ExperimentalDecorator::class, ExperimentalReadiumApi::class) abstract class VisualReaderFragment : BaseReaderFragment(), VisualNavigator.Listener { protected var binding: FragmentReaderBinding by viewLifecycle() @@ -79,6 +82,11 @@ abstract class VisualReaderFragment : BaseReaderFragment(), VisualNavigator.List */ private var disableTouches by mutableStateOf(false) + /** + * When true, the fragment won't save progression. + * This is useful in the case where the TTS is on and a service is saving progression + * in background. + */ private var preventProgressionSaving: Boolean = false override fun onViewCreated(view: View, savedInstanceState: Bundle?) { @@ -239,10 +247,15 @@ abstract class VisualReaderFragment : BaseReaderFragment(), VisualNavigator.List getString(R.string.tts_error_language_support_incomplete, language.locale.displayLanguage) ) ) { - tts.requestInstallVoice(activity) + AndroidTtsEngine.requestInstallVoice(activity) } } + override fun go(locator: Locator, animated: Boolean) { + model.tts?.stop() + super.go(locator, animated) + } + override fun onDestroyView() { (navigator as? DecorableNavigator)?.removeDecorationListener(decorationListener) super.onDestroyView() @@ -262,10 +275,6 @@ abstract class VisualReaderFragment : BaseReaderFragment(), VisualNavigator.List override fun onOptionsItemSelected(item: MenuItem): Boolean { when (item.itemId) { R.id.tts -> checkNotNull(model.tts).start(navigator) - R.id.toc -> { - model.tts?.stop() - super.onOptionsItemSelected(item) - } else -> return super.onOptionsItemSelected(item) } return true diff --git a/test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsControls.kt b/test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsControls.kt index 8ba11cc0b5..6decfc1c11 100644 --- a/test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsControls.kt +++ b/test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsControls.kt @@ -73,8 +73,7 @@ fun TtsControls( speed = editor.speed, pitch = editor.pitch, language = editor.language, - voices = editor.voices, - availableVoices = availableVoices, + voice = editor.voice(availableVoices), commit = commit, onDismiss = { showSettings = false } ) @@ -142,8 +141,7 @@ private fun TtsPreferencesDialog( speed: RangePreference, pitch: RangePreference, language: Preference, - voices: Preference>, - availableVoices: Set, + voice: EnumPreference, commit: () -> Unit, onDismiss: () -> Unit ) { @@ -186,28 +184,7 @@ private fun TtsPreferencesDialog( MenuItem( title = stringResource(R.string.tts_voice), - preference = voices.map( - from = { voices -> - language.effectiveValue?.let { voices[it.removeRegion()] } - }, - to = { voice -> - buildMap { - voices.value?.let { putAll(it) } - language.effectiveValue?.let { - val withoutRegion = it.removeRegion() - if (voice == null) { - remove(withoutRegion) - } else { - put(withoutRegion, voice) - } - } - } - } - ).withSupportedValues( - availableVoices - .filter { it.language.removeRegion() == language.effectiveValue } - .map { it.name } - ), + preference = voice, formatValue = { it ?: context.getString(R.string.defaultValue) }, commit = commit ) @@ -215,3 +192,41 @@ private fun TtsPreferencesDialog( } ) } + +/** + * [AndroidTtsPreferencesEditor] supports choosing voices for any language or region. + * For this test app, we've chosen to present to the user only the voice for the + * TTS default language and to ignore regions. + */ +private fun AndroidTtsPreferencesEditor.voice( + availableVoices: Set +): EnumPreference { + + // Recomposition will be triggered higher if the value changes. + val currentLanguage = language.effectiveValue?.removeRegion() + + return voices.map( + from = { voices -> + currentLanguage?.let { voices[it] } + }, + to = { voice -> + currentLanguage + ?.let { voices.value.orEmpty().update(it, voice) } + ?: voices.value.orEmpty() + } + ).withSupportedValues( + availableVoices + .filter { it.language.removeRegion() == currentLanguage } + .map { it.name } + ) +} + +private fun Map.update(key: K, value: V?): Map = + buildMap { + putAll(this@update) + if (value == null) { + remove(key) + } else { + put(key, value) + } + } diff --git a/test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsService.kt b/test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsService.kt index 0011de9930..5e3f0b709d 100644 --- a/test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsService.kt +++ b/test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsService.kt @@ -6,29 +6,27 @@ package org.readium.r2.testapp.reader.tts -import android.app.* +import android.app.Application +import android.app.PendingIntent import android.content.ComponentName -import android.content.Context import android.content.Intent import android.content.ServiceConnection -import android.graphics.Color import android.os.Build import android.os.IBinder -import androidx.annotation.RequiresApi -import androidx.core.app.NotificationCompat import androidx.core.content.ContextCompat import androidx.media3.session.MediaSession +import androidx.media3.session.MediaSessionService import kotlinx.coroutines.* import kotlinx.coroutines.flow.launchIn import kotlinx.coroutines.flow.onEach import kotlinx.coroutines.flow.sample +import org.readium.r2.navigator.media3.tts.AndroidTtsNavigator import org.readium.r2.shared.ExperimentalReadiumApi -import org.readium.r2.testapp.utils.LifecycleMedia3SessionService import timber.log.Timber @OptIn(ExperimentalReadiumApi::class) @androidx.annotation.OptIn(androidx.media3.common.util.UnstableApi::class) -class TtsService : LifecycleMedia3SessionService() { +class TtsService : MediaSessionService() { class Session( val bookId: Long, @@ -60,7 +58,7 @@ class TtsService : LifecycleMedia3SessionService() { navigator: AndroidTtsNavigator, bookId: Long ): Session { - val activityIntent = createSessionActivityIntent(bookId) + val activityIntent = createSessionActivityIntent() val mediaSession = MediaSession.Builder(applicationContext, navigator.asPlayer()) .setSessionActivity(activityIntent) .setId(bookId.toString()) @@ -90,7 +88,7 @@ class TtsService : LifecycleMedia3SessionService() { return session } - private fun createSessionActivityIntent(bookId: Long): PendingIntent { + private fun createSessionActivityIntent(): PendingIntent { // This intent will be triggered when the notification is clicked. var flags = PendingIntent.FLAG_UPDATE_CURRENT if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) { @@ -107,44 +105,6 @@ class TtsService : LifecycleMedia3SessionService() { Binder() } - override fun onCreate() { - super.onCreate() - Timber.d("TtsService created.") - // val initialNotification = createInitialNotification() - // startForeground(1, initialNotification) - } - - private fun createInitialNotification(): Notification { - return if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) { - val notificationChannelId = createNotificationChannel() - Notification.Builder(this, notificationChannelId) - .setContentTitle("R2 testapp") - .setContentText("rgergergergg") - .setAutoCancel(true) - .build() - } else { - NotificationCompat.Builder(this) - .setContentTitle("R2 testapp") - .setContentText("grgrgrgrg") - .setPriority(NotificationCompat.PRIORITY_DEFAULT) - .setAutoCancel(true) - .build() - } - } - - @RequiresApi(Build.VERSION_CODES.O) - private fun createNotificationChannel(): String { - val notificationChannelId = "example.permanence" - val channelName = "Background Service" - val channel = NotificationChannel(notificationChannelId, channelName, NotificationManager.IMPORTANCE_NONE) - channel.lightColor = Color.BLUE - channel.lockscreenVisibility = Notification.VISIBILITY_PRIVATE - - val manager = getSystemService(Context.NOTIFICATION_SERVICE) as NotificationManager - manager.createNotificationChannel(channel) - return notificationChannelId - } - override fun onBind(intent: Intent?): IBinder? { Timber.d("onBind called with $intent") @@ -164,16 +124,6 @@ class TtsService : LifecycleMedia3SessionService() { return binder.session?.mediaSession } - override fun onUpdateNotification(session: MediaSession) { - Timber.d("onUpdateNotification") - super.onUpdateNotification(session) - } - - override fun onDestroy() { - super.onDestroy() - Timber.d("MediaService destroyed.") - } - override fun onTaskRemoved(rootIntent: Intent) { super.onTaskRemoved(rootIntent) Timber.d("Task removed. Stopping session and service.") diff --git a/test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsServiceFacade.kt b/test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsServiceFacade.kt index 41d52d484a..272db24863 100644 --- a/test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsServiceFacade.kt +++ b/test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsServiceFacade.kt @@ -1,8 +1,10 @@ package org.readium.r2.testapp.reader.tts import android.app.Application +import kotlinx.coroutines.CancellationException import kotlinx.coroutines.sync.Mutex import kotlinx.coroutines.sync.withLock +import org.readium.r2.navigator.media3.tts.AndroidTtsNavigator import org.readium.r2.shared.ExperimentalReadiumApi /** @@ -28,28 +30,21 @@ class TtsServiceFacade( bookId: Long, navigator: AndroidTtsNavigator ): TtsService.Session = mutex.withLock { - startService() - val binderNow = checkNotNull(binder) - binderNow.openSession(navigator, bookId) - } - - private suspend fun startService() { - if (binder != null) - return - - TtsService.start(application) - binder = TtsService.bind(application) + try { + if (binder == null) { + TtsService.start(application) + binder = TtsService.bind(application) + } + + binder!!.openSession(navigator, bookId) + } catch (e: CancellationException) { + TtsService.stop(application) + throw e + } } suspend fun closeSession() = mutex.withLock { binder?.closeSession() - stopService() - } - - private fun stopService() { - if (binder == null) - return - binder = null TtsService.stop(application) } diff --git a/test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsViewModel.kt b/test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsViewModel.kt index 95be42bcc3..377f1cbb6b 100644 --- a/test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsViewModel.kt +++ b/test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsViewModel.kt @@ -6,17 +6,14 @@ package org.readium.r2.testapp.reader.tts -import android.content.Context -import android.content.Intent -import android.content.pm.PackageManager -import android.os.Build -import android.speech.tts.TextToSpeech import kotlinx.coroutines.* import kotlinx.coroutines.channels.Channel import kotlinx.coroutines.flow.* import org.readium.r2.navigator.Navigator import org.readium.r2.navigator.VisualNavigator import org.readium.r2.navigator.media3.api.MediaNavigator +import org.readium.r2.navigator.media3.tts.AndroidTtsNavigator +import org.readium.r2.navigator.media3.tts.AndroidTtsNavigatorFactory import org.readium.r2.navigator.media3.tts.TtsNavigator import org.readium.r2.navigator.media3.tts.android.AndroidTtsEngine import org.readium.r2.navigator.media3.tts.android.AndroidTtsPreferences @@ -101,7 +98,7 @@ class TtsViewModel private constructor( MutableStateFlow(navigatorNow?.currentLocator?.value) private val _highlight: MutableStateFlow = - MutableStateFlow(navigatorNow?.utterance?.value?.utteranceHighlight) + MutableStateFlow(navigatorNow?.utterance?.value?.utteranceLocator) private val _events: Channel = Channel(Channel.BUFFERED) @@ -185,7 +182,7 @@ class TtsViewModel private constructor( ttsSession.navigator.utterance .onEach { utterance -> - _highlight.value = utterance.utteranceHighlight + _highlight.value = utterance.utteranceLocator }.launchIn(scope) preferencesManager.preferences @@ -230,29 +227,6 @@ class TtsViewModel private constructor( preferencesManager.setPreferences(editor.value.preferences) } } - /** - * Starts the activity to install additional voice data. - */ - fun requestInstallVoice(context: Context) { - val intent = Intent() - .setAction(TextToSpeech.Engine.ACTION_INSTALL_TTS_DATA) - .setFlags(Intent.FLAG_ACTIVITY_NEW_TASK) - - val availableActivities = - if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.TIRAMISU) { - context.packageManager.queryIntentActivities( - intent, - PackageManager.ResolveInfoFlags.of(0) - ) - } else { - @Suppress("Deprecation") - context.packageManager.queryIntentActivities(intent, 0) - } - - if (availableActivities.isNotEmpty()) { - context.startActivity(intent) - } - } override fun onStopRequested() { stop() diff --git a/test-app/src/main/java/org/readium/r2/testapp/utils/LifecycleMedia3SessionService.kt b/test-app/src/main/java/org/readium/r2/testapp/utils/LifecycleMedia3SessionService.kt deleted file mode 100644 index dda4958b3e..0000000000 --- a/test-app/src/main/java/org/readium/r2/testapp/utils/LifecycleMedia3SessionService.kt +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Copyright 2022 Readium Foundation. All rights reserved. - * Use of this source code is governed by the BSD-style license - * available in the top-level LICENSE file of the project. - */ - -package org.readium.r2.testapp.utils - -import android.content.Intent -import android.os.IBinder -import androidx.annotation.CallSuper -import androidx.lifecycle.Lifecycle -import androidx.lifecycle.LifecycleOwner -import androidx.lifecycle.ServiceLifecycleDispatcher -import androidx.media3.session.MediaSessionService - -/* - * Borrowed from - * https://android.googlesource.com/platform/frameworks/support/+/refs/heads/androidx-main/lifecycle/lifecycle-service/src/main/java/androidx/lifecycle/LifecycleService.java - */ - -abstract class LifecycleMedia3SessionService : MediaSessionService(), LifecycleOwner { - - @Suppress("LeakingThis") - private val lifecycleDispatcher = ServiceLifecycleDispatcher(this) - - @CallSuper - override fun onCreate() { - lifecycleDispatcher.onServicePreSuperOnCreate() - super.onCreate() - } - - @CallSuper - override fun onBind(intent: Intent?): IBinder? { - lifecycleDispatcher.onServicePreSuperOnBind() - return super.onBind(intent) - } - - @CallSuper - override fun onStart(intent: Intent?, startId: Int) { - lifecycleDispatcher.onServicePreSuperOnStart() - super.onStart(intent, startId) - } - - // this method is added only to annotate it with @CallSuper. - // In usual service super.onStartCommand is no-op, but in LifecycleService - // it results in mDispatcher.onServicePreSuperOnStart() call, because - // super.onStartCommand calls onStart(). - @CallSuper - override fun onStartCommand(intent: Intent?, flags: Int, startId: Int): Int { - return super.onStartCommand(intent, flags, startId) - } - - @CallSuper - override fun onDestroy() { - lifecycleDispatcher.onServicePreSuperOnDestroy() - super.onDestroy() - } - - override fun getLifecycle(): Lifecycle { - return lifecycleDispatcher.lifecycle - } -} From c76def0847f60685e88672f9755dae6e5ec5e4d9 Mon Sep 17 00:00:00 2001 From: Quentin Gliosca Date: Thu, 16 Feb 2023 15:40:33 +0100 Subject: [PATCH 15/27] Change MediaNavigator interface --- .../r2/navigator/media3/api/MediaNavigator.kt | 43 ++++++++----- .../media3/api/SynchronizedMediaNavigator.kt | 4 +- .../navigator/media3/audio/AudioNavigator.kt | 6 +- .../r2/navigator/media3/tts/TtsNavigator.kt | 35 +++++++---- .../r2/navigator/media3/tts/TtsPlayer.kt | 62 +++++++++++-------- .../media3/tts/session/TtsSessionAdapter.kt | 28 ++++----- .../r2/testapp/reader/tts/TtsViewModel.kt | 21 ++++--- 7 files changed, 117 insertions(+), 82 deletions(-) diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/api/MediaNavigator.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/api/MediaNavigator.kt index 6926626246..0947ede120 100644 --- a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/api/MediaNavigator.kt +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/api/MediaNavigator.kt @@ -13,41 +13,54 @@ import org.readium.r2.shared.ExperimentalReadiumApi import org.readium.r2.shared.util.Closeable @ExperimentalReadiumApi -interface MediaNavigator

: Navigator, Closeable { +interface MediaNavigator

: Navigator, Closeable { /** * Marker interface for the [position] flow. */ interface Position - /** - * Marker interface for the [Playback.error] property. - */ - interface Error - /** * State of the player. */ - enum class State { - Ready, - Buffering, - Ended, - Error; + sealed interface State { + + /** + * The navigator is ready to play. + */ + interface Ready : State + + /** + * The end of the media has been reached. + */ + interface Ended : State + + /** + * The navigator cannot play because the buffer is starved. + */ + interface Buffering : State + + /** + * The navigator cannot play because an error occurred. + */ + interface Error : State } /** * State of the playback. + * + * @param state The current state. + * @param playWhenReady If the navigator should play as soon as the state is Ready. */ - data class Playback( + data class Playback( val state: State, - val playWhenReady: Boolean, - val error: E? + val playWhenReady: Boolean ) /** * Indicates the current state of the playback. */ - val playback: StateFlow> + val playback: StateFlow val position: StateFlow

diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/api/SynchronizedMediaNavigator.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/api/SynchronizedMediaNavigator.kt index 6574965bf3..6b5bbcb8c8 100644 --- a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/api/SynchronizedMediaNavigator.kt +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/api/SynchronizedMediaNavigator.kt @@ -14,8 +14,8 @@ import org.readium.r2.shared.publication.Locator * A [MediaNavigator] aware of the utterances that are being read aloud. */ @ExperimentalReadiumApi -interface SynchronizedMediaNavigator

: - MediaNavigator { +interface SynchronizedMediaNavigator

: + MediaNavigator

{ interface Utterance

{ val text: String diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/audio/AudioNavigator.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/audio/AudioNavigator.kt index 2c8cd01c0d..69fda4d58d 100644 --- a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/audio/AudioNavigator.kt +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/audio/AudioNavigator.kt @@ -18,11 +18,11 @@ import org.readium.r2.shared.publication.Publication @ExperimentalReadiumApi class AudioNavigator, E : AudioEngine.Error>( private val mediaEngine: AudioEngine -) : MediaNavigator, Configurable by mediaEngine { +) : MediaNavigator, Configurable by mediaEngine { class Position : MediaNavigator.Position - class Error : MediaNavigator.Error + class Error : MediaNavigator.State.Error override val publication: Publication get() = TODO("Not yet implemented") @@ -50,7 +50,7 @@ class AudioNavigator, TODO("Not yet implemented") } - override val playback: StateFlow> + override val playback: StateFlow get() = TODO("Not yet implemented") override val position: StateFlow diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsNavigator.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsNavigator.kt index f10b5a7ca2..a3631dfecb 100644 --- a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsNavigator.kt +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsNavigator.kt @@ -27,6 +27,9 @@ import org.readium.r2.shared.publication.services.content.ContentService import org.readium.r2.shared.publication.services.content.ContentTokenizer import org.readium.r2.shared.util.Language +/** + * A navigator to read aloud a [Publication] with a TTS engine. + */ @ExperimentalReadiumApi class TtsNavigator, E : TtsEngine.Error, V : TtsEngine.Voice> private constructor( @@ -34,7 +37,7 @@ class TtsNavigator, override val publication: Publication, private val player: TtsPlayer, private val sessionAdapter: TtsSessionAdapter, -) : SynchronizedMediaNavigator, Configurable by player { +) : SynchronizedMediaNavigator, Configurable by player { companion object { @@ -136,17 +139,24 @@ class TtsNavigator, override val tokenLocator: Locator? ) : SynchronizedMediaNavigator.Utterance - sealed class Error : MediaNavigator.Error { + sealed class State { + + object Ready : MediaNavigator.State.Ready + + object Ended : MediaNavigator.State.Ended - data class EngineError (val error: E) : Error() + sealed class Error : MediaNavigator.State.Error { - data class ContentError(val exception: Exception) : Error() + data class EngineError (val error: E) : Error() + + data class ContentError(val exception: Exception) : Error() + } } val voices: Set get() = player.voices - override val playback: StateFlow> = + override val playback: StateFlow = player.playback.mapStateIn(coroutineScope) { it.toPlayback() } override val utterance: StateFlow = @@ -211,20 +221,19 @@ class TtsNavigator, MediaNavigator.Playback( state = state.toState(), playWhenReady = playWhenReady, - error = error?.toError() ) - private fun TtsPlayer.Playback.State.toState() = + private fun TtsPlayer.State.toState() = when (this) { - TtsPlayer.Playback.State.Ready -> MediaNavigator.State.Ready - TtsPlayer.Playback.State.Ended -> MediaNavigator.State.Ended - TtsPlayer.Playback.State.Error -> MediaNavigator.State.Error + TtsPlayer.State.Ready -> State.Ready + TtsPlayer.State.Ended -> State.Ended + is TtsPlayer.State.Error -> this.toError() } - private fun TtsPlayer.Error.toError(): Error = + private fun TtsPlayer.State.Error.toError(): State.Error = when (this) { - is TtsPlayer.Error.ContentError -> Error.ContentError(exception) - is TtsPlayer.Error.EngineError<*> -> Error.EngineError(error) + is TtsPlayer.State.Error.ContentError -> State.Error.ContentError(exception) + is TtsPlayer.State.Error.EngineError<*> -> State.Error.EngineError(error) } private fun TtsPlayer.Utterance.Position.toPosition(): Position = diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsPlayer.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsPlayer.kt index 996f1cbef4..25e1bc1267 100644 --- a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsPlayer.kt +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsPlayer.kt @@ -77,25 +77,36 @@ internal class TtsPlayer, } } - sealed class Error { + /** + * State of the player. + */ + sealed interface State { + + /** + * The player is ready to play. + */ + object Ready : State - data class EngineError (val error: E) : Error() + /** + * The end of the media has been reached. + */ + object Ended : State - data class ContentError(val exception: Exception) : Error() + /** + * The player cannot play because an error occurred. + */ + sealed class Error : State { + + data class EngineError (val error: E) : Error() + + data class ContentError(val exception: Exception) : Error() + } } data class Playback( val state: State, val playWhenReady: Boolean, - val error: Error? - ) { - - enum class State { - Ready, - Ended, - Error; - } - } + ) data class Utterance( val text: String, @@ -133,9 +144,8 @@ internal class TtsPlayer, private val playbackMutable: MutableStateFlow = MutableStateFlow( Playback( - state = if (initialWindow.ended) Playback.State.Ended else Playback.State.Ready, - playWhenReady = false, - error = null + state = if (initialWindow.ended) State.Ended else State.Ready, + playWhenReady = false ) ) @@ -204,7 +214,7 @@ internal class TtsPlayer, } private suspend fun tryRecoverAsync() = mutex.withLock { - playbackMutable.value = playbackMutable.value.copy(error = null) + playbackMutable.value = playbackMutable.value.copy(state = State.Ready) utteranceMutable.value = utteranceMutable.value.copy(range = null) playbackJob?.join() playIfReadyAndNotPaused() @@ -246,8 +256,8 @@ internal class TtsPlayer, private suspend fun restartUtteranceAsync() = mutex.withLock { playbackJob?.cancel() - if (playbackMutable.value.state == Playback.State.Ended) { - playbackMutable.value = playbackMutable.value.copy(state = Playback.State.Ready) + if (playbackMutable.value.state == State.Ended) { + playbackMutable.value = playbackMutable.value.copy(state = State.Ready) } utteranceMutable.value = utteranceMutable.value.copy(range = null) playbackJob?.join() @@ -338,7 +348,7 @@ internal class TtsPlayer, private fun playIfReadyAndNotPaused() { check(playbackJob?.isCompleted ?: true) - if (playback.value.playWhenReady && playback.value.state == Playback.State.Ready) { + if (playback.value.playWhenReady && playback.value.state == State.Ready) { playbackJob = coroutineScope.launch { playContinuous() } @@ -400,8 +410,8 @@ internal class TtsPlayer, nextUtterance = nextUtterance ) utteranceMutable.value = utteranceWindow.currentUtterance.ttsPlayerUtterance() - if (playbackMutable.value.state == Playback.State.Ended) { - playbackMutable.value = playbackMutable.value.copy(state = Playback.State.Ready) + if (playbackMutable.value.state == State.Ended) { + playbackMutable.value = playbackMutable.value.copy(state = State.Ready) } } @@ -420,7 +430,7 @@ internal class TtsPlayer, private fun onEndReached() { playbackMutable.value = playbackMutable.value.copy( - state = Playback.State.Ended, + state = State.Ended, ) } @@ -443,16 +453,14 @@ internal class TtsPlayer, private fun onEngineError(error: E) { playbackMutable.value = playbackMutable.value.copy( - state = Playback.State.Error, - error = Error.EngineError(error) + state = State.Error.EngineError(error) ) playbackJob?.cancel() } private fun onContentError(exception: Exception) { playbackMutable.value = playbackMutable.value.copy( - state = Playback.State.Error, - error = Error.ContentError(exception) + state = State.Error.ContentError(exception) ) playbackJob?.cancel() } @@ -474,7 +482,7 @@ internal class TtsPlayer, } private fun isPlaying() = - playbackMutable.value.playWhenReady && playback.value.state == Playback.State.Ready + playbackMutable.value.playWhenReady && playback.value.state == State.Ready private fun TtsContentIterator.Utterance.ttsPlayerUtterance(): Utterance = Utterance( diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/session/TtsSessionAdapter.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/session/TtsSessionAdapter.kt index d103c7d498..a50c7ac9f2 100644 --- a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/session/TtsSessionAdapter.kt +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/session/TtsSessionAdapter.kt @@ -236,7 +236,7 @@ internal class TtsSessionAdapter( } override fun getPlayerError(): PlaybackException? { - return lastPlayback.error?.toPlaybackException() + return (lastPlayback.state as? TtsPlayer.State.Error)?.toPlaybackException() } override fun play() { @@ -721,20 +721,20 @@ internal class TtsSessionAdapter( playbackInfo: TtsPlayer.Playback, // playWhenReadyChangeReason: @Player.PlayWhenReadyChangeReason Int, ) { - if (previousPlaybackInfo.error != playbackInfo.error) { + if (previousPlaybackInfo.state as? TtsPlayer.State.Error != playbackInfo.state as? Error) { listeners.queueEvent( EVENT_PLAYER_ERROR ) { listener: Listener -> listener.onPlayerErrorChanged( - playbackInfo.error?.toPlaybackException() + (playbackInfo.state as? TtsPlayer.State.Error)?.toPlaybackException() ) } - if (playbackInfo.error != null) { + if (playbackInfo.state is TtsPlayer.State.Error) { listeners.queueEvent( EVENT_PLAYER_ERROR ) { listener: Listener -> listener.onPlayerError( - playbackInfo.error.toPlaybackException() + playbackInfo.state.toPlaybackException() ) } } @@ -756,7 +756,7 @@ internal class TtsSessionAdapter( ) { listener: Listener -> listener.onPlayWhenReadyChanged( playbackInfo.playWhenReady, - if (playbackInfo.state == TtsPlayer.Playback.State.Ended) + if (playbackInfo.state == TtsPlayer.State.Ended) PLAY_WHEN_READY_CHANGE_REASON_END_OF_MEDIA_ITEM else PLAY_WHEN_READY_CHANGE_REASON_USER_REQUEST @@ -803,7 +803,7 @@ internal class TtsSessionAdapter( } private fun isPlaying(playbackInfo: TtsPlayer.Playback): Boolean { - return (playbackInfo.state == TtsPlayer.Playback.State.Ready && playbackInfo.playWhenReady) + return (playbackInfo.state == TtsPlayer.State.Ready && playbackInfo.playWhenReady) } private fun getRepeatModeForNavigation(): @RepeatMode Int { @@ -857,16 +857,16 @@ internal class TtsSessionAdapter( } } - private val TtsPlayer.Playback.State.playerCode get() = when (this) { - TtsPlayer.Playback.State.Ready -> STATE_READY - TtsPlayer.Playback.State.Ended -> STATE_ENDED - TtsPlayer.Playback.State.Error -> STATE_IDLE + private val TtsPlayer.State.playerCode get() = when (this) { + TtsPlayer.State.Ready -> STATE_READY + TtsPlayer.State.Ended -> STATE_ENDED + is TtsPlayer.State.Error -> STATE_IDLE } @Suppress("Unchecked_cast") - private fun TtsPlayer.Error.toPlaybackException(): PlaybackException = when (this) { - is TtsPlayer.Error.EngineError<*> -> mapEngineError(error as E) - is TtsPlayer.Error.ContentError -> when (exception) { + private fun TtsPlayer.State.Error.toPlaybackException(): PlaybackException = when (this) { + is TtsPlayer.State.Error.EngineError<*> -> mapEngineError(error as E) + is TtsPlayer.State.Error.ContentError -> when (exception) { is Resource.Exception.BadRequest -> PlaybackException(exception.message, exception.cause, ERROR_CODE_IO_BAD_HTTP_STATUS) is Resource.Exception.NotFound -> diff --git a/test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsViewModel.kt b/test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsViewModel.kt index 377f1cbb6b..7c27eda85a 100644 --- a/test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsViewModel.kt +++ b/test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsViewModel.kt @@ -172,12 +172,17 @@ class TtsViewModel private constructor( ttsSession.navigator.playback .onEach { playback -> - if (playback.state == MediaNavigator.State.Ended) { - stop() - } - _isPlaying.value = playback.playWhenReady - playback.error?.let { onPlaybackError(it) } + when (playback.state) { + is MediaNavigator.State.Ended -> { + stop() + } + is MediaNavigator.State.Error -> { + onPlaybackError(playback.state as TtsNavigator.State.Error) + } + is MediaNavigator.State.Ready -> {} + is MediaNavigator.State.Buffering -> {} + } }.launchIn(scope) ttsSession.navigator.utterance @@ -238,12 +243,12 @@ class TtsViewModel private constructor( } } - private fun onPlaybackError(error: TtsNavigator.Error) { + private fun onPlaybackError(error: TtsNavigator.State.Error) { val exception = when (error) { - is TtsNavigator.Error.ContentError -> { + is TtsNavigator.State.Error.ContentError -> { UserException(R.string.tts_error_other, cause = error.exception) } - is TtsNavigator.Error.EngineError<*> -> { + is TtsNavigator.State.Error.EngineError<*> -> { when ((error.error as AndroidTtsEngine.Error).kind) { AndroidTtsEngine.Error.Kind.Network -> UserException(R.string.tts_error_network) From 141efe63a08d24dad48bfb2e57238fe16eb69b50 Mon Sep 17 00:00:00 2001 From: Quentin Gliosca Date: Thu, 16 Feb 2023 18:52:01 +0100 Subject: [PATCH 16/27] Add AndroidTtsDefaults --- .../media3/tts/TtsNavigatorFactory.kt | 10 ++++-- .../media3/tts/android/AndroidTtsDefaults.kt | 30 +++++++++++++++++ .../media3/tts/android/AndroidTtsEngine.kt | 33 +++++++++++-------- .../tts/android/AndroidTtsEngineProvider.kt | 14 +++++--- .../android/AndroidTtsPreferencesEditor.kt | 3 +- .../tts/android/AndroidTtsSettingsResolver.kt | 10 +++--- 6 files changed, 75 insertions(+), 25 deletions(-) create mode 100644 readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/android/AndroidTtsDefaults.kt diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsNavigatorFactory.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsNavigatorFactory.kt index f0885b411f..f0f45c8c03 100644 --- a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsNavigatorFactory.kt +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsNavigatorFactory.kt @@ -38,11 +38,14 @@ class TtsNavigatorFactory, publication: Publication, tokenizerFactory: (defaultLanguage: Language?) -> ContentTokenizer = defaultTokenizerFactory, metadataProvider: MediaMetadataProvider = defaultMediaMetadataProvider, - defaultVoiceProvider: AndroidTtsEngine.DefaultVoiceProvider? = null + voiceSelector: (Language?, Set) -> AndroidTtsEngine.Voice? = defaultVoiceSelector, ): TtsNavigatorFactory? { - val engineProvider = AndroidTtsEngineProvider(application, defaultVoiceProvider) + val engineProvider = AndroidTtsEngineProvider( + context = application, + voiceSelector = voiceSelector + ) return createNavigatorFactory( application, @@ -107,6 +110,9 @@ class TtsNavigatorFactory, val defaultMediaMetadataProvider: MediaMetadataProvider = MediaMetadataProvider { publication -> DefaultMediaMetadataFactory(publication) } + + val defaultVoiceSelector: (Language?, Set) -> AndroidTtsEngine.Voice? = + { _, _ -> null } } suspend fun createNavigator( diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/android/AndroidTtsDefaults.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/android/AndroidTtsDefaults.kt new file mode 100644 index 0000000000..a46e75eb7b --- /dev/null +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/android/AndroidTtsDefaults.kt @@ -0,0 +1,30 @@ +/* + * Copyright 2022 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + +package org.readium.r2.navigator.media3.tts.android + +import org.readium.r2.shared.ExperimentalReadiumApi +import org.readium.r2.shared.util.Language + +/** + * Default values for the Android TTS engine. + * + * These values will be used as a last resort by [AndroidTtsSettingsResolver] + * when no user preference takes precedence. + * + * @see AndroidTtsPreferences + */ +@ExperimentalReadiumApi +data class AndroidTtsDefaults( + val language: Language? = null, + val pitch: Double? = null, + val speed: Double? = null +) { + init { + require(pitch == null || pitch > 0) + require(speed == null || speed > 0) + } +} diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/android/AndroidTtsEngine.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/android/AndroidTtsEngine.kt index 597ab128a9..cebdb5f0e7 100644 --- a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/android/AndroidTtsEngine.kt +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/android/AndroidTtsEngine.kt @@ -22,7 +22,6 @@ import kotlinx.coroutines.flow.StateFlow import kotlinx.coroutines.flow.asStateFlow import org.readium.r2.navigator.media3.tts.TtsEngine import org.readium.r2.shared.ExperimentalReadiumApi -import org.readium.r2.shared.publication.Metadata import org.readium.r2.shared.util.Language /** @@ -31,8 +30,8 @@ import org.readium.r2.shared.util.Language @ExperimentalReadiumApi class AndroidTtsEngine private constructor( private val engine: TextToSpeech, - metadata: Metadata, - private val defaultVoiceProvider: DefaultVoiceProvider?, + private val settingsResolver: SettingsResolver, + private val voiceSelector: VoiceSelector, initialPreferences: AndroidTtsPreferences ) : TtsEngine { @@ -41,8 +40,8 @@ class AndroidTtsEngine private constructor( suspend operator fun invoke( context: Context, - metadata: Metadata, - defaultVoiceProvider: DefaultVoiceProvider?, + settingsResolver: SettingsResolver, + voiceSelector: VoiceSelector, initialPreferences: AndroidTtsPreferences ): AndroidTtsEngine? { @@ -54,7 +53,7 @@ class AndroidTtsEngine private constructor( val engine = TextToSpeech(context, listener) return if (init.await()) - AndroidTtsEngine(engine, metadata, defaultVoiceProvider, initialPreferences) + AndroidTtsEngine(engine, settingsResolver, voiceSelector, initialPreferences) else null } @@ -84,9 +83,20 @@ class AndroidTtsEngine private constructor( } } - fun interface DefaultVoiceProvider { + fun interface SettingsResolver { - fun chooseVoice(language: Language?, availableVoices: Set): Voice? + /** + * Computes a set of engine settings from the engine preferences. + */ + fun settings(preferences: AndroidTtsPreferences): AndroidTtsSettings + } + + fun interface VoiceSelector { + + /** + * Selects a voice for the given [language]. + */ + fun voice(language: Language?, availableVoices: Set): Voice? } class Error(code: Int) : TtsEngine.Error { @@ -146,9 +156,6 @@ class AndroidTtsEngine private constructor( } } - private val settingsResolver: AndroidTtsSettingsResolver = - AndroidTtsSettingsResolver(metadata) - private val _settings: MutableStateFlow = MutableStateFlow(settingsResolver.settings(initialPreferences)) @@ -233,8 +240,8 @@ class AndroidTtsEngine private constructor( } private fun defaultVoice(language: Language?, voices: Set): AndroidVoice? = - defaultVoiceProvider - ?.chooseVoice(language, voices) + voiceSelector + .voice(language, voices) ?.let { voiceForName(it.name) } private fun voiceForName(name: String) = diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/android/AndroidTtsEngineProvider.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/android/AndroidTtsEngineProvider.kt index 1f23ae3763..bdfbb7cde9 100644 --- a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/android/AndroidTtsEngineProvider.kt +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/android/AndroidTtsEngineProvider.kt @@ -19,7 +19,8 @@ import org.readium.r2.shared.publication.Publication @androidx.annotation.OptIn(androidx.media3.common.util.UnstableApi::class) class AndroidTtsEngineProvider( private val context: Context, - private val defaultVoiceProvider: AndroidTtsEngine.DefaultVoiceProvider? = null + private val defaults: AndroidTtsDefaults = AndroidTtsDefaults(), + private val voiceSelector: AndroidTtsEngine.VoiceSelector = AndroidTtsEngine.VoiceSelector { _, _ -> null } ) : TtsEngineProvider { @@ -27,10 +28,13 @@ class AndroidTtsEngineProvider( publication: Publication, initialPreferences: AndroidTtsPreferences ): AndroidTtsEngine? { + val settingsResolver = + AndroidTtsSettingsResolver(publication.metadata, defaults) + return AndroidTtsEngine( context, - publication.metadata, - defaultVoiceProvider, + settingsResolver, + voiceSelector, initialPreferences ) } @@ -39,13 +43,13 @@ class AndroidTtsEngineProvider( metadata: Metadata, preferences: AndroidTtsPreferences ): AndroidTtsSettings = - AndroidTtsSettingsResolver(metadata).settings(preferences) + AndroidTtsSettingsResolver(metadata, defaults).settings(preferences) override fun createPreferencesEditor( publication: Publication, initialPreferences: AndroidTtsPreferences ): AndroidTtsPreferencesEditor = - AndroidTtsPreferencesEditor(initialPreferences, publication.metadata) + AndroidTtsPreferencesEditor(initialPreferences, publication.metadata, defaults) override fun createEmptyPreferences(): AndroidTtsPreferences = AndroidTtsPreferences() diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/android/AndroidTtsPreferencesEditor.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/android/AndroidTtsPreferencesEditor.kt index 11ae51f3c8..3226c99b57 100644 --- a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/android/AndroidTtsPreferencesEditor.kt +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/android/AndroidTtsPreferencesEditor.kt @@ -23,6 +23,7 @@ import org.readium.r2.shared.util.Language class AndroidTtsPreferencesEditor( initialPreferences: AndroidTtsPreferences, publicationMetadata: Metadata, + defaults: AndroidTtsDefaults, ) : PreferencesEditor { private data class State( @@ -31,7 +32,7 @@ class AndroidTtsPreferencesEditor( ) private val settingsResolver: AndroidTtsSettingsResolver = - AndroidTtsSettingsResolver(publicationMetadata) + AndroidTtsSettingsResolver(publicationMetadata, defaults) private var state: State = initialPreferences.toState() diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/android/AndroidTtsSettingsResolver.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/android/AndroidTtsSettingsResolver.kt index c8cd00eeb5..dd7ae33e84 100644 --- a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/android/AndroidTtsSettingsResolver.kt +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/android/AndroidTtsSettingsResolver.kt @@ -14,18 +14,20 @@ import org.readium.r2.shared.util.Language @ExperimentalReadiumApi internal class AndroidTtsSettingsResolver( private val metadata: Metadata, -) { + private val defaults: AndroidTtsDefaults +) : AndroidTtsEngine.SettingsResolver { - fun settings(preferences: AndroidTtsPreferences): AndroidTtsSettings { + override fun settings(preferences: AndroidTtsPreferences): AndroidTtsSettings { val language = preferences.language ?: metadata.language + ?: defaults.language ?: Language(Locale.current.toLanguageTag()) return AndroidTtsSettings( language = language, voices = preferences.voices ?: emptyMap(), - pitch = preferences.pitch ?: 1.0, - speed = preferences.speed ?: 1.0, + pitch = preferences.pitch ?: defaults.pitch ?: 1.0, + speed = preferences.speed ?: defaults.speed ?: 1.0, ) } } From 23000c601785df9da58a621a0bd6fe79a46e61ac Mon Sep 17 00:00:00 2001 From: Quentin Gliosca Date: Mon, 20 Feb 2023 13:52:26 +0100 Subject: [PATCH 17/27] Enable to easily customize media metadata --- .../media3/api/DefaultMediaMetadataFactory.kt | 29 +++++++++++-------- .../api/DefaultMediaMetadataProvider.kt | 24 +++++++++++++++ .../media3/api/MediaMetadataProvider.kt | 2 +- .../media3/tts/TtsNavigatorFactory.kt | 15 +++++----- 4 files changed, 49 insertions(+), 21 deletions(-) create mode 100644 readium/navigator/src/main/java/org/readium/r2/navigator/media3/api/DefaultMediaMetadataProvider.kt diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/api/DefaultMediaMetadataFactory.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/api/DefaultMediaMetadataFactory.kt index 7d5cb62ab6..292e585273 100644 --- a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/api/DefaultMediaMetadataFactory.kt +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/api/DefaultMediaMetadataFactory.kt @@ -14,21 +14,30 @@ import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.async import org.readium.r2.shared.publication.Publication +/** + * Builds media metadata using the given title, author and cover, + * and fall back on what is in the publication. + */ @androidx.annotation.OptIn(androidx.media3.common.util.UnstableApi::class) internal class DefaultMediaMetadataFactory( - private val publication: Publication + private val publication: Publication, + title: String? = null, + author: String? = null, + cover: ByteArray? = null ) : MediaMetadataFactory { private val coroutineScope = CoroutineScope(Dispatchers.Default) + private val title: String = + title ?: publication.metadata.title + private val authors: String? = - publication.metadata.authors - .joinToString(", ") { it.name } - .takeIf { it.isNotBlank() } + author ?: publication.metadata.authors + .firstOrNull { it.name.isNotBlank() }?.name private val cover: Deferred = coroutineScope.async { - publication.linkWithRel("cover") + cover ?: publication.linkWithRel("cover") ?.let { publication.get(it) } ?.read() ?.getOrNull() @@ -36,7 +45,7 @@ internal class DefaultMediaMetadataFactory( override suspend fun publicationMetadata(): MediaMetadata { val builder = MediaMetadata.Builder() - .setTitle(publication.metadata.title) + .setTitle(title) .setTotalTrackCount(publication.readingOrder.size) authors @@ -49,13 +58,9 @@ internal class DefaultMediaMetadataFactory( } override suspend fun resourceMetadata(index: Int): MediaMetadata { - val builder = MediaMetadata.Builder() - val link = publication.readingOrder[index] - - builder.setTrackNumber(index) - builder.setTitle(link.title) - builder.setTitle(publication.metadata.title) + .setTrackNumber(index) + .setTitle(title) authors ?.let { builder.setArtist(it) } diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/api/DefaultMediaMetadataProvider.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/api/DefaultMediaMetadataProvider.kt new file mode 100644 index 0000000000..573bbc2725 --- /dev/null +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/api/DefaultMediaMetadataProvider.kt @@ -0,0 +1,24 @@ +/* + * Copyright 2022 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + +package org.readium.r2.navigator.media3.api + +import org.readium.r2.shared.publication.Publication + +/** + * Builds a [MediaMetadataFactory] which will use the given title, author and cover, + * and fall back on what is in the publication. + */ +class DefaultMediaMetadataProvider( + private val title: String? = null, + private val author: String? = null, + private val cover: ByteArray? = null +) : MediaMetadataProvider { + + override fun createMetadataFactory(publication: Publication): MediaMetadataFactory { + return DefaultMediaMetadataFactory(publication, title, author, cover) + } +} diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/api/MediaMetadataProvider.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/api/MediaMetadataProvider.kt index fb1a45d8d1..7211cc8b2e 100644 --- a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/api/MediaMetadataProvider.kt +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/api/MediaMetadataProvider.kt @@ -11,7 +11,7 @@ import org.readium.r2.shared.publication.Publication /** * To be implemented to use a custom [MediaMetadataFactory]. */ -fun interface MediaMetadataProvider { +interface MediaMetadataProvider { fun createMetadataFactory(publication: Publication): MediaMetadataFactory } diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsNavigatorFactory.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsNavigatorFactory.kt index f0f45c8c03..7f5760d0d7 100644 --- a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsNavigatorFactory.kt +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsNavigatorFactory.kt @@ -7,18 +7,17 @@ package org.readium.r2.navigator.media3.tts import android.app.Application -import org.readium.r2.navigator.media3.api.DefaultMediaMetadataFactory +import org.readium.r2.navigator.media3.api.DefaultMediaMetadataProvider import org.readium.r2.navigator.media3.api.MediaMetadataProvider -import org.readium.r2.navigator.media3.tts.android.AndroidTtsEngine -import org.readium.r2.navigator.media3.tts.android.AndroidTtsEngineProvider -import org.readium.r2.navigator.media3.tts.android.AndroidTtsPreferences -import org.readium.r2.navigator.media3.tts.android.AndroidTtsPreferencesEditor -import org.readium.r2.navigator.media3.tts.android.AndroidTtsSettings +import org.readium.r2.navigator.media3.tts.android.* import org.readium.r2.navigator.preferences.PreferencesEditor import org.readium.r2.shared.ExperimentalReadiumApi import org.readium.r2.shared.publication.Locator import org.readium.r2.shared.publication.Publication -import org.readium.r2.shared.publication.services.content.* +import org.readium.r2.shared.publication.services.content.Content +import org.readium.r2.shared.publication.services.content.ContentTokenizer +import org.readium.r2.shared.publication.services.content.TextContentTokenizer +import org.readium.r2.shared.publication.services.content.content import org.readium.r2.shared.util.Language import org.readium.r2.shared.util.tokenizer.TextUnit @@ -109,7 +108,7 @@ class TtsNavigatorFactory, } val defaultMediaMetadataProvider: MediaMetadataProvider = - MediaMetadataProvider { publication -> DefaultMediaMetadataFactory(publication) } + DefaultMediaMetadataProvider() val defaultVoiceSelector: (Language?, Set) -> AndroidTtsEngine.Voice? = { _, _ -> null } From 64b52a2f1e97b88665547474532d3c3e5a406cb5 Mon Sep 17 00:00:00 2001 From: Quentin Gliosca Date: Tue, 21 Feb 2023 09:37:22 +0100 Subject: [PATCH 18/27] Add a specific listener to AndroidTtsEngine and refactor TtsViewModel --- .../r2/navigator/media3/tts/TtsNavigator.kt | 7 +- .../media3/tts/TtsNavigatorFactory.kt | 13 +- .../r2/navigator/media3/tts/TtsPlayer.kt | 12 +- .../media3/tts/android/AndroidTtsEngine.kt | 35 +++-- .../tts/android/AndroidTtsEngineProvider.kt | 2 + .../r2/testapp/reader/tts/TtsService.kt | 26 ++-- .../r2/testapp/reader/tts/TtsServiceFacade.kt | 49 ++++-- .../r2/testapp/reader/tts/TtsViewModel.kt | 142 +++++++----------- 8 files changed, 149 insertions(+), 137 deletions(-) diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsNavigator.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsNavigator.kt index a3631dfecb..020e9ebba3 100644 --- a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsNavigator.kt +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsNavigator.kt @@ -57,8 +57,9 @@ class TtsNavigator, return null } - val actualInitialPreferences = initialPreferences - ?: ttsEngineProvider.createEmptyPreferences() + val actualInitialPreferences = + initialPreferences + ?: ttsEngineProvider.createEmptyPreferences() val contentIterator = TtsContentIterator(publication, tokenizerFactory, initialLocator) @@ -120,8 +121,6 @@ class TtsNavigator, interface Listener { fun onStopRequested() - - fun onMissingLanguageData(language: Language) } data class Position( diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsNavigatorFactory.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsNavigatorFactory.kt index 7f5760d0d7..e2d50887b6 100644 --- a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsNavigatorFactory.kt +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsNavigatorFactory.kt @@ -9,7 +9,9 @@ package org.readium.r2.navigator.media3.tts import android.app.Application import org.readium.r2.navigator.media3.api.DefaultMediaMetadataProvider import org.readium.r2.navigator.media3.api.MediaMetadataProvider -import org.readium.r2.navigator.media3.tts.android.* +import org.readium.r2.navigator.media3.tts.android.AndroidTtsDefaults +import org.readium.r2.navigator.media3.tts.android.AndroidTtsEngine +import org.readium.r2.navigator.media3.tts.android.AndroidTtsEngineProvider import org.readium.r2.navigator.preferences.PreferencesEditor import org.readium.r2.shared.ExperimentalReadiumApi import org.readium.r2.shared.publication.Locator @@ -37,13 +39,16 @@ class TtsNavigatorFactory, publication: Publication, tokenizerFactory: (defaultLanguage: Language?) -> ContentTokenizer = defaultTokenizerFactory, metadataProvider: MediaMetadataProvider = defaultMediaMetadataProvider, + defaults: AndroidTtsDefaults = AndroidTtsDefaults(), voiceSelector: (Language?, Set) -> AndroidTtsEngine.Voice? = defaultVoiceSelector, - ): TtsNavigatorFactory? { + listener: AndroidTtsEngine.Listener? = null + ): AndroidTtsNavigatorFactory? { val engineProvider = AndroidTtsEngineProvider( context = application, - voiceSelector = voiceSelector + defaults = defaults, + voiceSelector = voiceSelector, + listener = listener ) return createNavigatorFactory( diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsPlayer.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsPlayer.kt index 25e1bc1267..99391ead07 100644 --- a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsPlayer.kt +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsPlayer.kt @@ -42,9 +42,17 @@ internal class TtsPlayer, val initialContext = tryOrNull { contentIterator.startContext() } ?: return null - val ttsEngineFacade = TtsEngineFacade(engine) + val ttsEngineFacade = + TtsEngineFacade( + engine, + ) - return TtsPlayer(ttsEngineFacade, contentIterator, initialContext, initialPreferences) + return TtsPlayer( + ttsEngineFacade, + contentIterator, + initialContext, + initialPreferences + ) } private suspend fun TtsContentIterator.startContext(): UtteranceWindow? { diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/android/AndroidTtsEngine.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/android/AndroidTtsEngine.kt index cebdb5f0e7..ad64c6dc93 100644 --- a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/android/AndroidTtsEngine.kt +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/android/AndroidTtsEngine.kt @@ -11,10 +11,9 @@ import android.content.Intent import android.content.pm.PackageManager import android.os.Build import android.speech.tts.TextToSpeech -import android.speech.tts.TextToSpeech.ERROR -import android.speech.tts.TextToSpeech.QUEUE_ADD import android.speech.tts.UtteranceProgressListener import android.speech.tts.Voice as AndroidVoice +import android.speech.tts.TextToSpeech.* import android.speech.tts.Voice.* import kotlinx.coroutines.CompletableDeferred import kotlinx.coroutines.flow.MutableStateFlow @@ -32,6 +31,7 @@ class AndroidTtsEngine private constructor( private val engine: TextToSpeech, private val settingsResolver: SettingsResolver, private val voiceSelector: VoiceSelector, + private val listener: Listener?, initialPreferences: AndroidTtsPreferences ) : TtsEngine { @@ -42,18 +42,19 @@ class AndroidTtsEngine private constructor( context: Context, settingsResolver: SettingsResolver, voiceSelector: VoiceSelector, + listener: Listener?, initialPreferences: AndroidTtsPreferences ): AndroidTtsEngine? { val init = CompletableDeferred() - val listener = TextToSpeech.OnInitListener { status -> - init.complete(status == TextToSpeech.SUCCESS) + val initListener = OnInitListener { status -> + init.complete(status == SUCCESS) } - val engine = TextToSpeech(context, listener) + val engine = TextToSpeech(context, initListener) return if (init.await()) - AndroidTtsEngine(engine, settingsResolver, voiceSelector, initialPreferences) + AndroidTtsEngine(engine, settingsResolver, voiceSelector, listener, initialPreferences) else null } @@ -63,7 +64,7 @@ class AndroidTtsEngine private constructor( */ fun requestInstallVoice(context: Context) { val intent = Intent() - .setAction(TextToSpeech.Engine.ACTION_INSTALL_TTS_DATA) + .setAction(Engine.ACTION_INSTALL_TTS_DATA) .setFlags(Intent.FLAG_ACTIVITY_NEW_TASK) val availableActivities = @@ -156,10 +157,17 @@ class AndroidTtsEngine private constructor( } } + interface Listener { + + fun onMissingData(language: Language) + + fun onLanguageNotSupported(language: Language) + } + private val _settings: MutableStateFlow = MutableStateFlow(settingsResolver.settings(initialPreferences)) - private var listener: TtsEngine.Listener? = + private var utteranceListener: TtsEngine.Listener? = null override val voices: Set get() = @@ -173,9 +181,9 @@ class AndroidTtsEngine private constructor( ) { if (listener == null) { engine.setOnUtteranceProgressListener(null) - this@AndroidTtsEngine.listener = null + this@AndroidTtsEngine.utteranceListener = null } else { - this@AndroidTtsEngine.listener = listener + this@AndroidTtsEngine.utteranceListener = listener engine.setOnUtteranceProgressListener(UtteranceListener(listener)) } } @@ -188,7 +196,7 @@ class AndroidTtsEngine private constructor( engine.setupVoice(settings.value, language, voices) val queued = engine.speak(text, QUEUE_ADD, null, requestId) if (queued == ERROR) { - listener?.onError(requestId, Error(Error.Kind.Unknown.code)) + utteranceListener?.onError(requestId, Error(Error.Kind.Unknown.code)) } } @@ -222,6 +230,11 @@ class AndroidTtsEngine private constructor( val language = utteranceLanguage ?: settings.language + when (engine.isLanguageAvailable(language.locale)) { + LANG_MISSING_DATA -> listener?.onMissingData(language) + LANG_NOT_SUPPORTED -> listener?.onLanguageNotSupported(language) + } + val preferredVoiceWithRegion = settings.voices[language] ?.let { voiceForName(it) } diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/android/AndroidTtsEngineProvider.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/android/AndroidTtsEngineProvider.kt index bdfbb7cde9..6b27277e1a 100644 --- a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/android/AndroidTtsEngineProvider.kt +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/android/AndroidTtsEngineProvider.kt @@ -20,6 +20,7 @@ import org.readium.r2.shared.publication.Publication class AndroidTtsEngineProvider( private val context: Context, private val defaults: AndroidTtsDefaults = AndroidTtsDefaults(), + private val listener: AndroidTtsEngine.Listener? = null, private val voiceSelector: AndroidTtsEngine.VoiceSelector = AndroidTtsEngine.VoiceSelector { _, _ -> null } ) : TtsEngineProvider { @@ -35,6 +36,7 @@ class AndroidTtsEngineProvider( context, settingsResolver, voiceSelector, + listener, initialPreferences ) } diff --git a/test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsService.kt b/test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsService.kt index 5e3f0b709d..764317e6be 100644 --- a/test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsService.kt +++ b/test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsService.kt @@ -17,9 +17,7 @@ import androidx.core.content.ContextCompat import androidx.media3.session.MediaSession import androidx.media3.session.MediaSessionService import kotlinx.coroutines.* -import kotlinx.coroutines.flow.launchIn -import kotlinx.coroutines.flow.onEach -import kotlinx.coroutines.flow.sample +import kotlinx.coroutines.flow.* import org.readium.r2.navigator.media3.tts.AndroidTtsNavigator import org.readium.r2.shared.ExperimentalReadiumApi import timber.log.Timber @@ -44,20 +42,24 @@ class TtsService : MediaSessionService() { private val app: org.readium.r2.testapp.Application get() = application as org.readium.r2.testapp.Application - var session: Session? = null + private val sessionMutable: MutableStateFlow = + MutableStateFlow(null) + + val session: StateFlow = + sessionMutable.asStateFlow() fun closeSession() { stopForeground(true) - session?.mediaSession?.release() - session?.navigator?.close() - session?.coroutineScope?.cancel() - session = null + session.value?.mediaSession?.release() + session.value?.navigator?.close() + session.value?.coroutineScope?.cancel() + sessionMutable.value = null } fun openSession( navigator: AndroidTtsNavigator, bookId: Long - ): Session { + ) { val activityIntent = createSessionActivityIntent() val mediaSession = MediaSession.Builder(applicationContext, navigator.asPlayer()) .setSessionActivity(activityIntent) @@ -72,7 +74,7 @@ class TtsService : MediaSessionService() { mediaSession ) - this@Binder.session = session + sessionMutable.value = session /* * Launch a job for saving progression even when playback is going on in the background @@ -84,8 +86,6 @@ class TtsService : MediaSessionService() { Timber.d("Saving TTS progression $locator") app.bookRepository.saveProgression(locator, bookId) }.launchIn(session.coroutineScope) - - return session } private fun createSessionActivityIntent(): PendingIntent { @@ -121,7 +121,7 @@ class TtsService : MediaSessionService() { } override fun onGetSession(controllerInfo: MediaSession.ControllerInfo): MediaSession? { - return binder.session?.mediaSession + return binder.session.value?.mediaSession } override fun onTaskRemoved(rootIntent: Intent) { diff --git a/test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsServiceFacade.kt b/test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsServiceFacade.kt index 272db24863..7bece6912e 100644 --- a/test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsServiceFacade.kt +++ b/test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsServiceFacade.kt @@ -1,7 +1,8 @@ package org.readium.r2.testapp.reader.tts import android.app.Application -import kotlinx.coroutines.CancellationException +import kotlinx.coroutines.* +import kotlinx.coroutines.flow.* import kotlinx.coroutines.sync.Mutex import kotlinx.coroutines.sync.withLock import org.readium.r2.navigator.media3.tts.AndroidTtsNavigator @@ -14,26 +15,40 @@ import org.readium.r2.shared.ExperimentalReadiumApi class TtsServiceFacade( private val application: Application ) { + private val coroutineScope: CoroutineScope = + MainScope() - private val mutex = Mutex() + private val mutex: Mutex = + Mutex() - private var binder: TtsService.Binder? = null + private var binder: TtsService.Binder? = + null - fun sessionNow(): TtsService.Session? = - binder?.session + private var bindingJob: Job? = + null - suspend fun getSession(): TtsService.Session? = mutex.withLock { - binder?.session - } + private val sessionMutable: MutableStateFlow = + MutableStateFlow(null) + + val session: StateFlow = + sessionMutable.asStateFlow() suspend fun openSession( bookId: Long, navigator: AndroidTtsNavigator - ): TtsService.Session = mutex.withLock { + ) = mutex.withLock { + if (session.value != null) { + throw CancellationException("A session is already running.") + } + try { if (binder == null) { TtsService.start(application) - binder = TtsService.bind(application) + val binder = TtsService.bind(application) + this.binder = binder + bindingJob = binder.session + .onEach { sessionMutable.value = it } + .launchIn(coroutineScope) } binder!!.openSession(navigator, bookId) @@ -44,8 +59,16 @@ class TtsServiceFacade( } suspend fun closeSession() = mutex.withLock { - binder?.closeSession() - binder = null - TtsService.stop(application) + if (session.value == null) { + throw CancellationException("No session to close.") + } + + withContext(NonCancellable) { + bindingJob!!.cancelAndJoin() + binder!!.closeSession() + sessionMutable.value = null + binder = null + TtsService.stop(application) + } } } diff --git a/test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsViewModel.kt b/test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsViewModel.kt index 7c27eda85a..f20fbce003 100644 --- a/test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsViewModel.kt +++ b/test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsViewModel.kt @@ -34,7 +34,7 @@ import org.readium.r2.testapp.utils.extensions.mapStateIn * * Note: This is not an Android ViewModel, but it is a component of ReaderViewModel. */ -@OptIn(ExperimentalReadiumApi::class) +@OptIn(ExperimentalReadiumApi::class, ExperimentalCoroutinesApi::class) class TtsViewModel private constructor( private val viewModelScope: CoroutineScope, private val bookId: Long, @@ -82,131 +82,99 @@ class TtsViewModel private constructor( class OnMissingVoiceData(val language: Language) : Event() } - private var binding: Deferred = - viewModelScope.async { - ttsServiceFacade.getSession() - ?.let { bindSession(it) } - } - - private val _showControls: MutableStateFlow = - MutableStateFlow(navigatorNow != null) - - private val _isPlaying: MutableStateFlow = - MutableStateFlow(navigatorNow?.playback?.value?.playWhenReady ?: false) - - private val _position: MutableStateFlow = - MutableStateFlow(navigatorNow?.currentLocator?.value) - - private val _highlight: MutableStateFlow = - MutableStateFlow(navigatorNow?.utterance?.value?.utteranceLocator) + private val navigatorNow: AndroidTtsNavigator? get() = + ttsServiceFacade.session.value?.navigator private val _events: Channel = Channel(Channel.BUFFERED) + val events: Flow = + _events.receiveAsFlow() + val editor: StateFlow = preferencesManager.preferences .mapStateIn(viewModelScope, createPreferencesEditor) val showControls: StateFlow = - _showControls.asStateFlow() + ttsServiceFacade.session.mapStateIn(viewModelScope) { + it != null + } val isPlaying: StateFlow = - _isPlaying.asStateFlow() + ttsServiceFacade.session.flatMapLatest { session -> + session?.navigator?.playback?.map { playback -> playback.playWhenReady } + ?: MutableStateFlow(false) + }.stateIn(viewModelScope, SharingStarted.Eagerly, false) val position: StateFlow = - _position.asStateFlow() + ttsServiceFacade.session.flatMapLatest { session -> + session?.navigator?.currentLocator ?: MutableStateFlow(null) + }.stateIn(viewModelScope, SharingStarted.Eagerly, null) val highlight: StateFlow = - _highlight.asStateFlow() + ttsServiceFacade.session.flatMapLatest { session -> + session?.navigator?.utterance?.map { it.utteranceLocator } + ?: MutableStateFlow(null) + }.stateIn(viewModelScope, SharingStarted.Eagerly, null) - val events: Flow = - _events.receiveAsFlow() + val voices: Set get() = + ttsServiceFacade.session.value?.navigator?.voices.orEmpty() - private val navigatorNow: AndroidTtsNavigator? get() = - ttsServiceFacade.sessionNow()?.navigator + init { + ttsServiceFacade.session + .flatMapLatest { it?.navigator?.playback ?: MutableStateFlow(null) } + .onEach { playback -> + when (playback?.state) { + null -> { + } + is MediaNavigator.State.Ended -> { + stop() + } + is MediaNavigator.State.Error -> { + onPlaybackError(playback.state as TtsNavigator.State.Error) + } + is MediaNavigator.State.Ready -> {} + is MediaNavigator.State.Buffering -> {} + } + }.launchIn(viewModelScope) - val voices: Set get() = - navigatorNow!!.voices + preferencesManager.preferences + .onEach { ttsServiceFacade.session.value?.navigator?.submitPreferences(it) } + .launchIn(viewModelScope) + } /** * Starts the TTS using the first visible locator in the given [navigator]. */ fun start(navigator: Navigator) { viewModelScope.launch { - if (ttsServiceFacade.getSession() != null) + if (ttsServiceFacade.session.value != null) return@launch - val session = openSession(navigator) - ?: run { - val exception = UserException(R.string.tts_error_initialization) - _events.send(Event.OnError(exception)) - return@launch - } - - binding.cancelAndJoin() - binding = async { bindSession(session) } + openSession(navigator) } } - private suspend fun openSession(navigator: Navigator): TtsService.Session? { + private suspend fun openSession(navigator: Navigator) { val start = (navigator as? VisualNavigator)?.firstVisibleElementLocator() val ttsNavigator = ttsNavigatorFactory.createNavigator( this, preferencesManager.preferences.value, start - ) ?: return null + ) ?: run { + val exception = UserException(R.string.tts_error_initialization) + _events.send(Event.OnError(exception)) + return + } // playWhenReady must be true for the MediaSessionService to call Service.startForeground // and prevent crashing ttsNavigator.play() - return ttsServiceFacade.openSession(bookId, ttsNavigator) - } - - private fun bindSession( - ttsSession: TtsService.Session - ): Job { - val job = Job() - val scope = viewModelScope + job - - _showControls.value = true - - ttsSession.navigator.playback - .onEach { playback -> - _isPlaying.value = playback.playWhenReady - when (playback.state) { - is MediaNavigator.State.Ended -> { - stop() - } - is MediaNavigator.State.Error -> { - onPlaybackError(playback.state as TtsNavigator.State.Error) - } - is MediaNavigator.State.Ready -> {} - is MediaNavigator.State.Buffering -> {} - } - }.launchIn(scope) - - ttsSession.navigator.utterance - .onEach { utterance -> - _highlight.value = utterance.utteranceLocator - }.launchIn(scope) - - preferencesManager.preferences - .onEach { ttsSession.navigator.submitPreferences(it) } - .launchIn(scope) - - ttsSession.navigator.currentLocator - .onEach { _position.value = it } - .launchIn(scope) - - return job + ttsServiceFacade.openSession(bookId, ttsNavigator) } fun stop() { viewModelScope.launch { - binding.cancelAndJoin() - _highlight.value = null - _showControls.value = false - _isPlaying.value = false ttsServiceFacade.closeSession() } } @@ -237,12 +205,6 @@ class TtsViewModel private constructor( stop() } - override fun onMissingLanguageData(language: Language) { - viewModelScope.launch { - _events.send(Event.OnMissingVoiceData(language)) - } - } - private fun onPlaybackError(error: TtsNavigator.State.Error) { val exception = when (error) { is TtsNavigator.State.Error.ContentError -> { From 25a69d97003df6b662b8efca7525743855d77664 Mon Sep 17 00:00:00 2001 From: Quentin Gliosca Date: Tue, 21 Feb 2023 09:45:54 +0100 Subject: [PATCH 19/27] Update media3 --- gradle/libs.versions.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 886bd8d1c3..a32be7e185 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -23,7 +23,7 @@ androidx-lifecycle = "2.5.1" androidx-lifecycle-extensions = "2.2.0" androidx-media = "1.6.0" androidx-media2 = "1.2.1" -androidx-media3 = "1.0.0-beta03" +androidx-media3 = "1.0.0-rc01" androidx-navigation = "2.5.2" androidx-paging = "3.1.1" androidx-recyclerview = "1.2.1" From 7699637f6e8c93c08f579592f5cccbc5a88793dd Mon Sep 17 00:00:00 2001 From: Quentin Gliosca Date: Tue, 21 Feb 2023 09:59:15 +0100 Subject: [PATCH 20/27] Introduce TtsEngine.RequestId --- .../r2/navigator/media3/tts/TtsEngine.kt | 20 ++++++++++++------- .../navigator/media3/tts/TtsEngineFacade.kt | 16 +++++++-------- .../media3/tts/android/AndroidTtsEngine.kt | 17 ++++++++-------- 3 files changed, 30 insertions(+), 23 deletions(-) diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsEngine.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsEngine.kt index 51c87f7a47..6851323492 100644 --- a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsEngine.kt +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsEngine.kt @@ -47,6 +47,12 @@ interface TtsEngine, */ interface Error + /** + * An id to identify a request to speak. + */ + @JvmInline + value class RequestId(val id: String) + /** * TTS engine callbacks. */ @@ -55,7 +61,7 @@ interface TtsEngine, /** * Called when the utterance with the given id starts as perceived by the caller. */ - fun onStart(requestId: String) + fun onStart(requestId: RequestId) /** * Called when the [TtsEngine] is about to speak the specified [range] of the utterance with @@ -63,29 +69,29 @@ interface TtsEngine, * * This callback may not be called if the [TtsEngine] does not provide range information. */ - fun onRange(requestId: String, range: IntRange) + fun onRange(requestId: RequestId, range: IntRange) /** * Called if the utterance with the given id has been stopped while in progress * by a call to [stop]. */ - fun onInterrupted(requestId: String) + fun onInterrupted(requestId: RequestId) /** * Called when the utterance with the given id has been flushed from the synthesis queue * by a call to [stop]. */ - fun onFlushed(requestId: String) + fun onFlushed(requestId: RequestId) /** * Called when the utterance with the given id has successfully completed processing. */ - fun onDone(requestId: String) + fun onDone(requestId: RequestId) /** * Called when an error has occurred during processing of the utterance with the given id. */ - fun onError(requestId: String, error: E) + fun onError(requestId: RequestId, error: E) } /** @@ -96,7 +102,7 @@ interface TtsEngine, /** * Enqueues a new speak request. */ - fun speak(requestId: String, text: String, language: Language?) + fun speak(requestId: RequestId, text: String, language: Language?) /** * Stops the [TtsEngine]. diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsEngineFacade.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsEngineFacade.kt index 6c9b4e6d15..5f5151ca28 100644 --- a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsEngineFacade.kt +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsEngineFacade.kt @@ -35,7 +35,7 @@ internal class TtsEngineFacade continuation.invokeOnCancellation { engine.stop() } currentTask?.continuation?.cancel() - val id = UUID.randomUUID().toString() + val id = TtsEngine.RequestId(UUID.randomUUID().toString()) currentTask = UtteranceTask(id, continuation, onRange) engine.speak(id, text, language) } @@ -46,24 +46,24 @@ internal class TtsEngineFacade( - val requestId: String, + val requestId: TtsEngine.RequestId, val continuation: CancellableContinuation, val onRange: (IntRange) -> Unit ) private inner class EngineListener : TtsEngine.Listener { - override fun onStart(requestId: String) { + override fun onStart(requestId: TtsEngine.RequestId) { } - override fun onRange(requestId: String, range: IntRange) { + override fun onRange(requestId: TtsEngine.RequestId, range: IntRange) { currentTask ?.takeIf { it.requestId == requestId } ?.onRange ?.invoke(range) } - override fun onInterrupted(requestId: String) { + override fun onInterrupted(requestId: TtsEngine.RequestId) { currentTask ?.takeIf { it.requestId == requestId } ?.continuation @@ -71,7 +71,7 @@ internal class TtsEngineFacade? ) : UtteranceProgressListener() { override fun onStart(utteranceId: String) { - listener?.onStart(utteranceId) + listener?.onStart(TtsEngine.RequestId(utteranceId)) } override fun onStop(utteranceId: String, interrupted: Boolean) { listener?.let { + val requestId = TtsEngine.RequestId(utteranceId) if (interrupted) { - it.onInterrupted(utteranceId) + it.onInterrupted(requestId) } else { - it.onFlushed(utteranceId) + it.onFlushed(requestId) } } } override fun onDone(utteranceId: String) { - listener?.onDone(utteranceId) + listener?.onDone(TtsEngine.RequestId(utteranceId)) } @Deprecated("Deprecated in the interface", ReplaceWith("onError(utteranceId, -1)")) @@ -304,13 +305,13 @@ class AndroidTtsEngine private constructor( override fun onError(utteranceId: String, errorCode: Int) { listener?.onError( - utteranceId, + TtsEngine.RequestId(utteranceId), Error(errorCode) ) } override fun onRangeStart(utteranceId: String, start: Int, end: Int, frame: Int) { - listener?.onRange(utteranceId, start until end) + listener?.onRange(TtsEngine.RequestId(utteranceId), start until end) } } } From 5c1ff6cb4561c9d478dd0f6a59b5d4dc972fb37f Mon Sep 17 00:00:00 2001 From: Quentin Gliosca Date: Tue, 21 Feb 2023 13:28:49 +0100 Subject: [PATCH 21/27] Add Voice.Id and override content language when language is forced --- .../media3/api/MediaMetadataProvider.kt | 2 +- .../media3/tts/TtsContentIterator.kt | 21 +++++++++++++++---- .../r2/navigator/media3/tts/TtsEngine.kt | 5 +++++ .../r2/navigator/media3/tts/TtsNavigator.kt | 4 ++-- .../media3/tts/TtsNavigatorFactory.kt | 20 +++++++----------- .../r2/navigator/media3/tts/TtsPlayer.kt | 3 ++- .../media3/tts/android/AndroidTtsEngine.kt | 19 ++++++++++------- .../tts/android/AndroidTtsPreferences.kt | 2 +- .../android/AndroidTtsPreferencesEditor.kt | 2 +- .../media3/tts/android/AndroidTtsSettings.kt | 3 ++- .../tts/android/AndroidTtsSettingsResolver.kt | 1 + .../services/content/ContentTokenizer.kt | 4 ++-- .../r2/testapp/reader/tts/TtsControls.kt | 8 +++---- .../r2/testapp/reader/tts/TtsService.kt | 1 + 14 files changed, 59 insertions(+), 36 deletions(-) diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/api/MediaMetadataProvider.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/api/MediaMetadataProvider.kt index 7211cc8b2e..fb1a45d8d1 100644 --- a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/api/MediaMetadataProvider.kt +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/api/MediaMetadataProvider.kt @@ -11,7 +11,7 @@ import org.readium.r2.shared.publication.Publication /** * To be implemented to use a custom [MediaMetadataFactory]. */ -interface MediaMetadataProvider { +fun interface MediaMetadataProvider { fun createMetadataFactory(publication: Publication): MediaMetadataFactory } diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsContentIterator.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsContentIterator.kt index 1180cad1ca..04e738242b 100644 --- a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsContentIterator.kt +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsContentIterator.kt @@ -13,9 +13,10 @@ import org.readium.r2.shared.publication.html.cssSelector import org.readium.r2.shared.publication.indexOfFirstWithHref import org.readium.r2.shared.publication.services.content.Content import org.readium.r2.shared.publication.services.content.ContentService -import org.readium.r2.shared.publication.services.content.ContentTokenizer +import org.readium.r2.shared.publication.services.content.TextContentTokenizer import org.readium.r2.shared.util.CursorList import org.readium.r2.shared.util.Language +import org.readium.r2.shared.util.tokenizer.TextTokenizer @ExperimentalReadiumApi @@ -26,7 +27,7 @@ import org.readium.r2.shared.util.Language */ internal class TtsContentIterator( private val publication: Publication, - private val tokenizerFactory: (language: Language?) -> ContentTokenizer, + private val tokenizerFactory: (language: Language?) -> TextTokenizer, initialLocator: Locator? ) { data class Utterance( @@ -65,6 +66,14 @@ internal class TtsContentIterator( var language: Language? = null + /** + * Whether language information in content should be superseded by [language] while tokenizing. + * + * Modifying this property is not immediate, the new value will be applied as soon as possible. + */ + var overrideContentLanguage: Boolean = + false + val resourceCount: Int = publication.readingOrder.size @@ -158,8 +167,12 @@ internal class TtsContentIterator( * This is used to split a paragraph into sentences, for example. */ private fun Content.Element.tokenize(): List { - val language = this@tokenize.language ?: this@TtsContentIterator.language - return tokenizerFactory(language).tokenize(this) + val contentTokenizer = TextContentTokenizer( + language = this@TtsContentIterator.language, + textTokenizerFactory = tokenizerFactory, + overrideContentLanguage = overrideContentLanguage + ) + return contentTokenizer.tokenize(this) } /** diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsEngine.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsEngine.kt index 6851323492..36ec182284 100644 --- a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsEngine.kt +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsEngine.kt @@ -32,6 +32,11 @@ interface TtsEngine, * The default language to use when no language information is passed to [speak]. */ val language: Language? + + /** + * Whether language information in content should be superseded by [language]. + */ + val overrideContentLanguage: Boolean } interface Voice { diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsNavigator.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsNavigator.kt index 020e9ebba3..8240d0573e 100644 --- a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsNavigator.kt +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsNavigator.kt @@ -24,8 +24,8 @@ import org.readium.r2.shared.publication.Link import org.readium.r2.shared.publication.Locator import org.readium.r2.shared.publication.Publication import org.readium.r2.shared.publication.services.content.ContentService -import org.readium.r2.shared.publication.services.content.ContentTokenizer import org.readium.r2.shared.util.Language +import org.readium.r2.shared.util.tokenizer.TextTokenizer /** * A navigator to read aloud a [Publication] with a TTS engine. @@ -46,7 +46,7 @@ class TtsNavigator, application: Application, publication: Publication, ttsEngineProvider: TtsEngineProvider, - tokenizerFactory: (defaultLanguage: Language?) -> ContentTokenizer, + tokenizerFactory: (language: Language?) -> TextTokenizer, metadataProvider: MediaMetadataProvider, listener: Listener, initialPreferences: P? = null, diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsNavigatorFactory.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsNavigatorFactory.kt index e2d50887b6..a0150a6485 100644 --- a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsNavigatorFactory.kt +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsNavigatorFactory.kt @@ -17,10 +17,10 @@ import org.readium.r2.shared.ExperimentalReadiumApi import org.readium.r2.shared.publication.Locator import org.readium.r2.shared.publication.Publication import org.readium.r2.shared.publication.services.content.Content -import org.readium.r2.shared.publication.services.content.ContentTokenizer -import org.readium.r2.shared.publication.services.content.TextContentTokenizer import org.readium.r2.shared.publication.services.content.content import org.readium.r2.shared.util.Language +import org.readium.r2.shared.util.tokenizer.DefaultTextContentTokenizer +import org.readium.r2.shared.util.tokenizer.TextTokenizer import org.readium.r2.shared.util.tokenizer.TextUnit @ExperimentalReadiumApi @@ -29,7 +29,7 @@ class TtsNavigatorFactory, private val application: Application, private val publication: Publication, private val ttsEngineProvider: TtsEngineProvider, - private val tokenizerFactory: (defaultLanguage: Language?) -> ContentTokenizer, + private val tokenizerFactory: (language: Language?) -> TextTokenizer, private val metadataProvider: MediaMetadataProvider ) { companion object { @@ -37,7 +37,7 @@ class TtsNavigatorFactory, suspend operator fun invoke( application: Application, publication: Publication, - tokenizerFactory: (defaultLanguage: Language?) -> ContentTokenizer = defaultTokenizerFactory, + tokenizerFactory: (language: Language?) -> TextTokenizer = defaultTokenizerFactory, metadataProvider: MediaMetadataProvider = defaultMediaMetadataProvider, defaults: AndroidTtsDefaults = AndroidTtsDefaults(), voiceSelector: (Language?, Set) -> AndroidTtsEngine.Voice? = defaultVoiceSelector, @@ -65,7 +65,7 @@ class TtsNavigatorFactory, application: Application, publication: Publication, ttsEngineProvider: TtsEngineProvider, - tokenizerFactory: (defaultLanguage: Language?) -> ContentTokenizer = defaultTokenizerFactory, + tokenizerFactory: (language: Language?) -> TextTokenizer = defaultTokenizerFactory, metadataProvider: MediaMetadataProvider = defaultMediaMetadataProvider ): TtsNavigatorFactory? { @@ -83,7 +83,7 @@ class TtsNavigatorFactory, application: Application, publication: Publication, ttsEngineProvider: TtsEngineProvider, - tokenizerFactory: (defaultLanguage: Language?) -> ContentTokenizer, + tokenizerFactory: (language: Language?) -> TextTokenizer, metadataProvider: MediaMetadataProvider ): TtsNavigatorFactory? { @@ -104,12 +104,8 @@ class TtsNavigatorFactory, /** * The default content tokenizer will split the [Content.Element] items into individual sentences. */ - val defaultTokenizerFactory: (Language?) -> ContentTokenizer = { language -> - TextContentTokenizer( - unit = TextUnit.Sentence, - language = language, - overrideContentLanguage = false - ) + val defaultTokenizerFactory: (Language?) -> TextTokenizer = { language -> + DefaultTextContentTokenizer(TextUnit.Sentence, language) } val defaultMediaMetadataProvider: MediaMetadataProvider = diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsPlayer.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsPlayer.kt index 99391ead07..d98a450d62 100644 --- a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsPlayer.kt +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsPlayer.kt @@ -44,7 +44,7 @@ internal class TtsPlayer, val ttsEngineFacade = TtsEngineFacade( - engine, + engine ) return TtsPlayer( @@ -487,6 +487,7 @@ internal class TtsPlayer, lastPreferences = preferences engineFacade.submitPreferences(preferences) contentIterator.language = engineFacade.settings.value.language + contentIterator.overrideContentLanguage = engineFacade.settings.value.overrideContentLanguage } private fun isPlaying() = diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/android/AndroidTtsEngine.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/android/AndroidTtsEngine.kt index 0aa35bcb03..289f46c9ae 100644 --- a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/android/AndroidTtsEngine.kt +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/android/AndroidTtsEngine.kt @@ -11,9 +11,9 @@ import android.content.Intent import android.content.pm.PackageManager import android.os.Build import android.speech.tts.TextToSpeech +import android.speech.tts.TextToSpeech.* import android.speech.tts.UtteranceProgressListener import android.speech.tts.Voice as AndroidVoice -import android.speech.tts.TextToSpeech.* import android.speech.tts.Voice.* import kotlinx.coroutines.CompletableDeferred import kotlinx.coroutines.flow.MutableStateFlow @@ -140,18 +140,22 @@ class AndroidTtsEngine private constructor( /** * Represents a voice provided by the TTS engine which can speak an utterance. * - * @param name Unique and stable identifier for this voice + * @param id Unique and stable identifier for this voice * @param language Language (and region) this voice belongs to. * @param quality Voice quality. * @param requiresNetwork Indicates whether using this voice requires an Internet connection. */ data class Voice( - val name: String, + val id: Id, override val language: Language, val quality: Quality = Quality.Normal, val requiresNetwork: Boolean = false, ) : TtsEngine.Voice { + @kotlinx.serialization.Serializable + @JvmInline + value class Id(val value: String) + enum class Quality { Lowest, Low, Normal, High, Highest } @@ -228,6 +232,7 @@ class AndroidTtsEngine private constructor( voices: Set ) { val language = utteranceLanguage + .takeUnless { settings.overrideContentLanguage } ?: settings.language when (engine.isLanguageAvailable(language.locale)) { @@ -237,11 +242,11 @@ class AndroidTtsEngine private constructor( val preferredVoiceWithRegion = settings.voices[language] - ?.let { voiceForName(it) } + ?.let { voiceForName(it.value) } val preferredVoiceWithoutRegion = settings.voices[language.removeRegion()] - ?.let { voiceForName(it) } + ?.let { voiceForName(it.value) } val voice = preferredVoiceWithRegion ?: preferredVoiceWithoutRegion @@ -255,7 +260,7 @@ class AndroidTtsEngine private constructor( private fun defaultVoice(language: Language?, voices: Set): AndroidVoice? = voiceSelector .voice(language, voices) - ?.let { voiceForName(it.name) } + ?.let { voiceForName(it.id.value) } private fun voiceForName(name: String) = engine.voices @@ -263,7 +268,7 @@ class AndroidTtsEngine private constructor( private fun AndroidVoice.toTtsEngineVoice() = Voice( - name = name, + id = Voice.Id(name), language = Language(locale), quality = when (quality) { QUALITY_VERY_HIGH -> Voice.Quality.Highest diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/android/AndroidTtsPreferences.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/android/AndroidTtsPreferences.kt index 946504d218..b397d3712c 100644 --- a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/android/AndroidTtsPreferences.kt +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/android/AndroidTtsPreferences.kt @@ -25,7 +25,7 @@ data class AndroidTtsPreferences( override val language: Language? = null, val pitch: Double? = null, val speed: Double? = null, - val voices: Map? = null, + val voices: Map? = null, ) : TtsEngine.Preferences { init { diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/android/AndroidTtsPreferencesEditor.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/android/AndroidTtsPreferencesEditor.kt index 3226c99b57..b8a47e159e 100644 --- a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/android/AndroidTtsPreferencesEditor.kt +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/android/AndroidTtsPreferencesEditor.kt @@ -74,7 +74,7 @@ class AndroidTtsPreferencesEditor( valueFormatter = { "${it.format(2)}x" }, ) - val voices: Preference> = + val voices: Preference> = PreferenceDelegate( getValue = { preferences.voices }, getEffectiveValue = { state.settings.voices }, diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/android/AndroidTtsSettings.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/android/AndroidTtsSettings.kt index 9617ff1af3..4f0542fd04 100644 --- a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/android/AndroidTtsSettings.kt +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/android/AndroidTtsSettings.kt @@ -18,7 +18,8 @@ import org.readium.r2.shared.util.Language @ExperimentalReadiumApi data class AndroidTtsSettings( override val language: Language, + override val overrideContentLanguage: Boolean, val pitch: Double, val speed: Double, - val voices: Map, + val voices: Map, ) : TtsEngine.Settings diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/android/AndroidTtsSettingsResolver.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/android/AndroidTtsSettingsResolver.kt index dd7ae33e84..5a17cefd8f 100644 --- a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/android/AndroidTtsSettingsResolver.kt +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/android/AndroidTtsSettingsResolver.kt @@ -28,6 +28,7 @@ internal class AndroidTtsSettingsResolver( voices = preferences.voices ?: emptyMap(), pitch = preferences.pitch ?: defaults.pitch ?: 1.0, speed = preferences.speed ?: defaults.speed ?: 1.0, + overrideContentLanguage = preferences.language != null ) } } diff --git a/readium/shared/src/main/java/org/readium/r2/shared/publication/services/content/ContentTokenizer.kt b/readium/shared/src/main/java/org/readium/r2/shared/publication/services/content/ContentTokenizer.kt index 574f772de7..0603894ba8 100644 --- a/readium/shared/src/main/java/org/readium/r2/shared/publication/services/content/ContentTokenizer.kt +++ b/readium/shared/src/main/java/org/readium/r2/shared/publication/services/content/ContentTokenizer.kt @@ -37,10 +37,10 @@ class TextContentTokenizer( /** * A [ContentTokenizer] using the default [TextTokenizer] to split the text of the [Content.Element]. */ - constructor(language: Language?, overrideContentLanguage: Boolean, unit: TextUnit) : this( + constructor(language: Language?, unit: TextUnit, overrideContentLanguage: Boolean = false) : this( language = language, - overrideContentLanguage = overrideContentLanguage, textTokenizerFactory = { contentLanguage -> DefaultTextContentTokenizer(unit, contentLanguage) }, + overrideContentLanguage = overrideContentLanguage ) override fun tokenize(data: Content.Element): List = listOf( diff --git a/test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsControls.kt b/test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsControls.kt index 6decfc1c11..a1fc051a5a 100644 --- a/test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsControls.kt +++ b/test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsControls.kt @@ -141,7 +141,7 @@ private fun TtsPreferencesDialog( speed: RangePreference, pitch: RangePreference, language: Preference, - voice: EnumPreference, + voice: EnumPreference, commit: () -> Unit, onDismiss: () -> Unit ) { @@ -185,7 +185,7 @@ private fun TtsPreferencesDialog( MenuItem( title = stringResource(R.string.tts_voice), preference = voice, - formatValue = { it ?: context.getString(R.string.defaultValue) }, + formatValue = { it?.value ?: context.getString(R.string.defaultValue) }, commit = commit ) } @@ -200,7 +200,7 @@ private fun TtsPreferencesDialog( */ private fun AndroidTtsPreferencesEditor.voice( availableVoices: Set -): EnumPreference { +): EnumPreference { // Recomposition will be triggered higher if the value changes. val currentLanguage = language.effectiveValue?.removeRegion() @@ -217,7 +217,7 @@ private fun AndroidTtsPreferencesEditor.voice( ).withSupportedValues( availableVoices .filter { it.language.removeRegion() == currentLanguage } - .map { it.name } + .map { it.id } ) } diff --git a/test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsService.kt b/test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsService.kt index 764317e6be..b759bfc0e0 100644 --- a/test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsService.kt +++ b/test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsService.kt @@ -56,6 +56,7 @@ class TtsService : MediaSessionService() { sessionMutable.value = null } + @OptIn(FlowPreview::class) fun openSession( navigator: AndroidTtsNavigator, bookId: Long From da0a3ecb573c26b2e497905b369459cc404a7e49 Mon Sep 17 00:00:00 2001 From: Quentin Gliosca Date: Wed, 22 Feb 2023 14:35:56 +0100 Subject: [PATCH 22/27] Make default values internal in TtsNavigatorFactory --- .../readium/r2/navigator/media3/tts/TtsNavigatorFactory.kt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsNavigatorFactory.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsNavigatorFactory.kt index a0150a6485..48201ee251 100644 --- a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsNavigatorFactory.kt +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsNavigatorFactory.kt @@ -104,14 +104,14 @@ class TtsNavigatorFactory, /** * The default content tokenizer will split the [Content.Element] items into individual sentences. */ - val defaultTokenizerFactory: (Language?) -> TextTokenizer = { language -> + private val defaultTokenizerFactory: (Language?) -> TextTokenizer = { language -> DefaultTextContentTokenizer(TextUnit.Sentence, language) } - val defaultMediaMetadataProvider: MediaMetadataProvider = + private val defaultMediaMetadataProvider: MediaMetadataProvider = DefaultMediaMetadataProvider() - val defaultVoiceSelector: (Language?, Set) -> AndroidTtsEngine.Voice? = + private val defaultVoiceSelector: (Language?, Set) -> AndroidTtsEngine.Voice? = { _, _ -> null } } From dcc884142fe0c2c1dc97f269c15f95dd959ca9b2 Mon Sep 17 00:00:00 2001 From: Quentin Gliosca Date: Wed, 22 Feb 2023 16:10:37 +0100 Subject: [PATCH 23/27] Move Tts preferences to UserPreferences.kt --- .../r2/testapp/reader/BaseReaderFragment.kt | 4 +- .../r2/testapp/reader/VisualReaderFragment.kt | 5 + .../reader/preferences/UserPreferences.kt | 61 +++++++- ...serPreferencesBottomSheetDialogFragment.kt | 14 +- .../preferences/UserPreferencesViewModel.kt | 15 +- .../r2/testapp/reader/tts/TtsControls.kt | 145 ++---------------- .../reader/tts/TtsPreferencesEditor.kt | 72 +++++++++ .../r2/testapp/reader/tts/TtsViewModel.kt | 28 ++-- 8 files changed, 185 insertions(+), 159 deletions(-) create mode 100644 test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsPreferencesEditor.kt diff --git a/test-app/src/main/java/org/readium/r2/testapp/reader/BaseReaderFragment.kt b/test-app/src/main/java/org/readium/r2/testapp/reader/BaseReaderFragment.kt index 403ba3ea26..3318834dba 100644 --- a/test-app/src/main/java/org/readium/r2/testapp/reader/BaseReaderFragment.kt +++ b/test-app/src/main/java/org/readium/r2/testapp/reader/BaseReaderFragment.kt @@ -77,7 +77,9 @@ abstract class BaseReaderFragment : Fragment() { model.insertBookmark(navigator.currentLocator.value) } R.id.settings -> { - UserPreferencesBottomSheetDialogFragment().show(childFragmentManager, "Settings") + val settingsModel = checkNotNull(model.settings) + UserPreferencesBottomSheetDialogFragment(settingsModel, "User Settings") + .show(childFragmentManager, "Settings") } R.id.drm -> { model.activityChannel.send(ReaderViewModel.Event.OpenDrmManagementRequested) diff --git a/test-app/src/main/java/org/readium/r2/testapp/reader/VisualReaderFragment.kt b/test-app/src/main/java/org/readium/r2/testapp/reader/VisualReaderFragment.kt index 5199a11f9d..b88905de3c 100644 --- a/test-app/src/main/java/org/readium/r2/testapp/reader/VisualReaderFragment.kt +++ b/test-app/src/main/java/org/readium/r2/testapp/reader/VisualReaderFragment.kt @@ -50,6 +50,7 @@ import org.readium.r2.shared.util.Language import org.readium.r2.testapp.R import org.readium.r2.testapp.databinding.FragmentReaderBinding import org.readium.r2.testapp.domain.model.Highlight +import org.readium.r2.testapp.reader.preferences.UserPreferencesBottomSheetDialogFragment import org.readium.r2.testapp.reader.tts.TtsControls import org.readium.r2.testapp.reader.tts.TtsViewModel import org.readium.r2.testapp.utils.* @@ -132,6 +133,10 @@ abstract class VisualReaderFragment : BaseReaderFragment(), VisualNavigator.List model.tts?.let { tts -> TtsControls( model = tts, + onPreferences = { + UserPreferencesBottomSheetDialogFragment(tts.preferencesModel, "TTS Settings") + .show(childFragmentManager, "TtsSettings") + }, modifier = Modifier .align(Alignment.BottomCenter) .padding(8.dp) diff --git a/test-app/src/main/java/org/readium/r2/testapp/reader/preferences/UserPreferences.kt b/test-app/src/main/java/org/readium/r2/testapp/reader/preferences/UserPreferences.kt index 234ae48d52..7e0b81089d 100644 --- a/test-app/src/main/java/org/readium/r2/testapp/reader/preferences/UserPreferences.kt +++ b/test-app/src/main/java/org/readium/r2/testapp/reader/preferences/UserPreferences.kt @@ -21,11 +21,16 @@ import org.readium.adapters.pdfium.navigator.PdfiumPreferencesEditor import org.readium.r2.navigator.epub.EpubPreferencesEditor import org.readium.r2.navigator.preferences.* import org.readium.r2.navigator.preferences.TextAlign as ReadiumTextAlign +import androidx.compose.ui.platform.LocalContext +import androidx.compose.ui.res.stringResource +import org.readium.r2.navigator.media3.tts.android.AndroidTtsEngine import org.readium.r2.shared.ExperimentalReadiumApi import org.readium.r2.shared.publication.epub.EpubLayout import org.readium.r2.shared.util.Language import org.readium.r2.testapp.LITERATA +import org.readium.r2.testapp.R import org.readium.r2.testapp.reader.ReaderViewModel +import org.readium.r2.testapp.reader.tts.TtsPreferencesEditor import org.readium.r2.testapp.shared.views.* import org.readium.r2.testapp.utils.compose.DropdownMenuButton @@ -33,25 +38,30 @@ import org.readium.r2.testapp.utils.compose.DropdownMenuButton * Stateful user settings component paired with a [ReaderViewModel]. */ @Composable -fun UserPreferences(model: UserPreferencesViewModel<*, *>) { +fun UserPreferences( + model: UserPreferencesViewModel<*, *>, + title: String +) { val editor by model.editor.collectAsState() UserPreferences( editor = editor, - commit = model::commit + commit = model::commit, + title = title ) } @Composable private fun

, E : PreferencesEditor

> UserPreferences( editor: E, - commit: () -> Unit + commit: () -> Unit, + title: String ) { Column( modifier = Modifier.padding(vertical = 24.dp) ) { Text( - text = "User settings", + text = title, textAlign = TextAlign.Center, style = MaterialTheme.typography.h6, modifier = Modifier @@ -124,10 +134,53 @@ private fun

, E : PreferencesEditor

> UserPref spread = editor.spread, ) } + is TtsPreferencesEditor -> + TtsUserPreferences( + commit = commit, + language = editor.language, + voice = editor.voice, + speed = editor.speed, + pitch = editor.pitch + ) } } } +@Composable +private fun ColumnScope.TtsUserPreferences( + commit: () -> Unit, + language: Preference, + voice: EnumPreference, + speed: RangePreference, + pitch: RangePreference +) { + Column { + StepperItem( + title = stringResource(R.string.speed_rate), + preference = speed, + commit = commit + ) + StepperItem( + title = stringResource(R.string.pitch_rate), + preference = pitch, + commit = commit + ) + LanguageItem( + preference = language, + commit = commit + ) + + val context = LocalContext.current + + MenuItem( + title = stringResource(R.string.tts_voice), + preference = voice, + formatValue = { it?.value ?: context.getString(R.string.defaultValue) }, + commit = commit + ) + } +} + /** * User settings for a publication with a fixed layout, such as fixed-layout EPUB, PDF or comic book. */ diff --git a/test-app/src/main/java/org/readium/r2/testapp/reader/preferences/UserPreferencesBottomSheetDialogFragment.kt b/test-app/src/main/java/org/readium/r2/testapp/reader/preferences/UserPreferencesBottomSheetDialogFragment.kt index 6d38c256a8..4cff03725d 100644 --- a/test-app/src/main/java/org/readium/r2/testapp/reader/preferences/UserPreferencesBottomSheetDialogFragment.kt +++ b/test-app/src/main/java/org/readium/r2/testapp/reader/preferences/UserPreferencesBottomSheetDialogFragment.kt @@ -9,18 +9,15 @@ package org.readium.r2.testapp.reader.preferences import android.app.Dialog import android.os.Bundle import androidx.compose.runtime.Composable -import androidx.fragment.app.activityViewModels import com.google.android.material.bottomsheet.BottomSheetDialog -import org.readium.r2.shared.ExperimentalReadiumApi -import org.readium.r2.testapp.reader.ReaderViewModel import org.readium.r2.testapp.utils.compose.ComposeBottomSheetDialogFragment -@OptIn(ExperimentalReadiumApi::class) -class UserPreferencesBottomSheetDialogFragment : ComposeBottomSheetDialogFragment( +class UserPreferencesBottomSheetDialogFragment( + private val model: UserPreferencesViewModel<*, *>, + private val title: String +) : ComposeBottomSheetDialogFragment( isScrollable = true ) { - private val model: ReaderViewModel by activityViewModels() - override fun onCreateDialog(savedInstanceState: Bundle?): Dialog = (super.onCreateDialog(savedInstanceState) as BottomSheetDialog).apply { // Reduce the dim to see the impact of the settings on the page. @@ -34,7 +31,6 @@ class UserPreferencesBottomSheetDialogFragment : ComposeBottomSheetDialogFragmen @Composable override fun Content() { - val settingsModel = checkNotNull(model.settings) - UserPreferences(settingsModel) + UserPreferences(model, title) } } diff --git a/test-app/src/main/java/org/readium/r2/testapp/reader/preferences/UserPreferencesViewModel.kt b/test-app/src/main/java/org/readium/r2/testapp/reader/preferences/UserPreferencesViewModel.kt index 3494faae2c..991639b320 100644 --- a/test-app/src/main/java/org/readium/r2/testapp/reader/preferences/UserPreferencesViewModel.kt +++ b/test-app/src/main/java/org/readium/r2/testapp/reader/preferences/UserPreferencesViewModel.kt @@ -10,14 +10,21 @@ import androidx.lifecycle.LifecycleOwner import androidx.lifecycle.flowWithLifecycle import androidx.lifecycle.lifecycleScope import kotlinx.coroutines.CoroutineScope -import kotlinx.coroutines.flow.* +import kotlinx.coroutines.flow.StateFlow +import kotlinx.coroutines.flow.launchIn +import kotlinx.coroutines.flow.onEach import kotlinx.coroutines.launch import org.readium.adapters.pdfium.navigator.PdfiumPreferences import org.readium.adapters.pdfium.navigator.PdfiumSettings -import org.readium.r2.navigator.epub.* -import org.readium.r2.navigator.preferences.* +import org.readium.r2.navigator.epub.EpubPreferences +import org.readium.r2.navigator.epub.EpubSettings +import org.readium.r2.navigator.preferences.Configurable +import org.readium.r2.navigator.preferences.PreferencesEditor import org.readium.r2.shared.ExperimentalReadiumApi -import org.readium.r2.testapp.reader.* +import org.readium.r2.testapp.reader.EpubReaderInitData +import org.readium.r2.testapp.reader.PdfReaderInitData +import org.readium.r2.testapp.reader.ReaderInitData +import org.readium.r2.testapp.reader.ReaderViewModel import org.readium.r2.testapp.utils.extensions.mapStateIn /** diff --git a/test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsControls.kt b/test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsControls.kt index a1fc051a5a..3f2a0282ae 100644 --- a/test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsControls.kt +++ b/test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsControls.kt @@ -8,77 +8,60 @@ package org.readium.r2.testapp.reader.tts -import androidx.compose.foundation.layout.* -import androidx.compose.material.* +import androidx.compose.foundation.layout.Arrangement +import androidx.compose.foundation.layout.Row +import androidx.compose.foundation.layout.Spacer +import androidx.compose.foundation.layout.size +import androidx.compose.material.Card +import androidx.compose.material.Icon +import androidx.compose.material.IconButton import androidx.compose.material.icons.Icons import androidx.compose.material.icons.filled.* -import androidx.compose.runtime.* +import androidx.compose.runtime.Composable +import androidx.compose.runtime.getValue import androidx.compose.ui.Alignment import androidx.compose.ui.Modifier -import androidx.compose.ui.platform.LocalContext import androidx.compose.ui.res.stringResource -import androidx.compose.ui.text.style.TextAlign import androidx.compose.ui.unit.dp -import org.readium.r2.navigator.media3.tts.android.AndroidTtsEngine -import org.readium.r2.navigator.media3.tts.android.AndroidTtsPreferencesEditor -import org.readium.r2.navigator.preferences.* import org.readium.r2.shared.ExperimentalReadiumApi -import org.readium.r2.shared.util.Language import org.readium.r2.testapp.R -import org.readium.r2.testapp.shared.views.LanguageItem -import org.readium.r2.testapp.shared.views.MenuItem import org.readium.r2.testapp.utils.extensions.asStateWhenStarted /** * TTS controls bar displayed at the bottom of the screen when speaking a publication. */ @Composable -fun TtsControls(model: TtsViewModel, modifier: Modifier = Modifier) { +fun TtsControls( + model: TtsViewModel, + onPreferences: () -> Unit, + modifier: Modifier = Modifier +) { val showControls by model.showControls.asStateWhenStarted() val isPlaying by model.isPlaying.asStateWhenStarted() - val editor by model.editor.asStateWhenStarted() if (showControls) { TtsControls( playing = isPlaying, - editor = editor, - availableVoices = model.voices, - commit = model::commit, onPlayPause = { if (isPlaying) model.pause() else model.play() }, onStop = model::stop, onPrevious = model::previous, onNext = model::next, + onPreferences = onPreferences, modifier = modifier ) } } -@OptIn(ExperimentalReadiumApi::class) @Composable fun TtsControls( playing: Boolean, - editor: AndroidTtsPreferencesEditor, - availableVoices: Set, - commit: () -> Unit, onPlayPause: () -> Unit, onStop: () -> Unit, onPrevious: () -> Unit, onNext: () -> Unit, + onPreferences: () -> Unit, modifier: Modifier = Modifier, ) { - var showSettings by remember { mutableStateOf(false) } - - if (showSettings) { - TtsPreferencesDialog( - speed = editor.speed, - pitch = editor.pitch, - language = editor.language, - voice = editor.voice(availableVoices), - commit = commit, - onDismiss = { showSettings = false } - ) - } - Card( modifier = modifier ) { @@ -126,7 +109,7 @@ fun TtsControls( Spacer(modifier = Modifier.size(8.dp)) - IconButton(onClick = { showSettings = true }) { + IconButton(onClick = onPreferences) { Icon( imageVector = Icons.Default.Settings, contentDescription = stringResource(R.string.tts_settings) @@ -136,97 +119,3 @@ fun TtsControls( } } -@Composable -private fun TtsPreferencesDialog( - speed: RangePreference, - pitch: RangePreference, - language: Preference, - voice: EnumPreference, - commit: () -> Unit, - onDismiss: () -> Unit -) { - AlertDialog( - onDismissRequest = onDismiss, - confirmButton = { - TextButton(onClick = onDismiss) { - Text(text = stringResource(R.string.close)) - } - }, - title = { - Text( - text = stringResource(R.string.tts_settings), - textAlign = TextAlign.Center, - style = MaterialTheme.typography.h6, - modifier = Modifier - .fillMaxWidth() - ) - }, - text = { - Column { - MenuItem( - title = stringResource(R.string.speed_rate), - preference = speed.withSupportedValues(0.25, 0.5, 0.75, 1.0, 1.25, 1.5, 2.0), - formatValue = speed::formatValue, - commit = commit - ) - MenuItem( - title = stringResource(R.string.pitch_rate), - preference = pitch.withSupportedValues(0.25, 0.5, 0.75, 1.0, 1.25, 1.5, 2.0), - formatValue = pitch::formatValue, - commit = commit - ) - LanguageItem( - preference = language, - commit = commit - ) - - val context = LocalContext.current - - MenuItem( - title = stringResource(R.string.tts_voice), - preference = voice, - formatValue = { it?.value ?: context.getString(R.string.defaultValue) }, - commit = commit - ) - } - } - ) -} - -/** - * [AndroidTtsPreferencesEditor] supports choosing voices for any language or region. - * For this test app, we've chosen to present to the user only the voice for the - * TTS default language and to ignore regions. - */ -private fun AndroidTtsPreferencesEditor.voice( - availableVoices: Set -): EnumPreference { - - // Recomposition will be triggered higher if the value changes. - val currentLanguage = language.effectiveValue?.removeRegion() - - return voices.map( - from = { voices -> - currentLanguage?.let { voices[it] } - }, - to = { voice -> - currentLanguage - ?.let { voices.value.orEmpty().update(it, voice) } - ?: voices.value.orEmpty() - } - ).withSupportedValues( - availableVoices - .filter { it.language.removeRegion() == currentLanguage } - .map { it.id } - ) -} - -private fun Map.update(key: K, value: V?): Map = - buildMap { - putAll(this@update) - if (value == null) { - remove(key) - } else { - put(key, value) - } - } diff --git a/test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsPreferencesEditor.kt b/test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsPreferencesEditor.kt new file mode 100644 index 0000000000..303743b977 --- /dev/null +++ b/test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsPreferencesEditor.kt @@ -0,0 +1,72 @@ +/* + * Copyright 2023 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + +package org.readium.r2.testapp.reader.tts + +import org.readium.r2.navigator.media3.tts.android.AndroidTtsEngine +import org.readium.r2.navigator.media3.tts.android.AndroidTtsPreferences +import org.readium.r2.navigator.media3.tts.android.AndroidTtsPreferencesEditor +import org.readium.r2.navigator.preferences.* +import org.readium.r2.shared.ExperimentalReadiumApi +import org.readium.r2.shared.util.Language + +@OptIn(ExperimentalReadiumApi::class) +class TtsPreferencesEditor( + private val editor: AndroidTtsPreferencesEditor, + private val availableVoices: Set +) : PreferencesEditor { + + override val preferences: AndroidTtsPreferences + get() = editor.preferences + + override fun clear() { + editor.clear() + } + + val language: Preference = + editor.language + + val pitch: RangePreference = + editor.pitch + + val speed: RangePreference = + editor.speed + + /** + * [AndroidTtsPreferencesEditor] supports choosing voices for any language or region. + * For this test app, we've chosen to present to the user only the voice for the + * TTS default language and to ignore regions. + */ + val voice: EnumPreference = run { + // Recomposition will be triggered higher if the value changes. + val currentLanguage = language.effectiveValue?.removeRegion() + + editor.voices.map( + from = { voices -> + currentLanguage?.let { voices[it] } + }, + to = { voice -> + currentLanguage + ?.let { editor.voices.value.orEmpty().update(it, voice) } + ?: editor.voices.value.orEmpty() + } + ).withSupportedValues( + availableVoices + .filter { it.language.removeRegion() == currentLanguage } + .map { it.id } + ) + } + + private fun Map.update(key: K, value: V?): Map = + buildMap { + putAll(this@update) + if (value == null) { + remove(key) + } else { + put(key, value) + } + } +} diff --git a/test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsViewModel.kt b/test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsViewModel.kt index f20fbce003..eac63df43a 100644 --- a/test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsViewModel.kt +++ b/test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsViewModel.kt @@ -6,9 +6,11 @@ package org.readium.r2.testapp.reader.tts -import kotlinx.coroutines.* +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.ExperimentalCoroutinesApi import kotlinx.coroutines.channels.Channel import kotlinx.coroutines.flow.* +import kotlinx.coroutines.launch import org.readium.r2.navigator.Navigator import org.readium.r2.navigator.VisualNavigator import org.readium.r2.navigator.media3.api.MediaNavigator @@ -17,7 +19,7 @@ import org.readium.r2.navigator.media3.tts.AndroidTtsNavigatorFactory import org.readium.r2.navigator.media3.tts.TtsNavigator import org.readium.r2.navigator.media3.tts.android.AndroidTtsEngine import org.readium.r2.navigator.media3.tts.android.AndroidTtsPreferences -import org.readium.r2.navigator.media3.tts.android.AndroidTtsPreferencesEditor +import org.readium.r2.navigator.media3.tts.android.AndroidTtsSettings import org.readium.r2.shared.ExperimentalReadiumApi import org.readium.r2.shared.UserException import org.readium.r2.shared.publication.Locator @@ -27,6 +29,7 @@ import org.readium.r2.testapp.R import org.readium.r2.testapp.reader.ReaderInitData import org.readium.r2.testapp.reader.VisualReaderInitData import org.readium.r2.testapp.reader.preferences.PreferencesManager +import org.readium.r2.testapp.reader.preferences.UserPreferencesViewModel import org.readium.r2.testapp.utils.extensions.mapStateIn /** @@ -42,7 +45,6 @@ class TtsViewModel private constructor( private val ttsNavigatorFactory: AndroidTtsNavigatorFactory, private val ttsServiceFacade: TtsServiceFacade, private val preferencesManager: PreferencesManager, - private val createPreferencesEditor: (AndroidTtsPreferences) -> AndroidTtsPreferencesEditor ) : TtsNavigator.Listener { companion object { @@ -64,8 +66,7 @@ class TtsViewModel private constructor( publication = readerInitData.publication, ttsNavigatorFactory = readerInitData.ttsInitData.ttsNavigatorFactory, ttsServiceFacade = readerInitData.ttsInitData.ttsServiceFacade, - preferencesManager = readerInitData.ttsInitData.preferencesManager, - createPreferencesEditor = readerInitData.ttsInitData.ttsNavigatorFactory::createTtsPreferencesEditor + preferencesManager = readerInitData.ttsInitData.preferencesManager ) } } @@ -91,8 +92,15 @@ class TtsViewModel private constructor( val events: Flow = _events.receiveAsFlow() - val editor: StateFlow = preferencesManager.preferences - .mapStateIn(viewModelScope, createPreferencesEditor) + val preferencesModel: UserPreferencesViewModel + get() = UserPreferencesViewModel( + viewModelScope = viewModelScope, + bookId = bookId, + preferencesManager = preferencesManager + ) { preferences -> + val baseEditor = ttsNavigatorFactory.createTtsPreferencesEditor(preferences) + TtsPreferencesEditor(baseEditor, voices) + } val showControls: StateFlow = ttsServiceFacade.session.mapStateIn(viewModelScope) { @@ -195,12 +203,6 @@ class TtsViewModel private constructor( navigatorNow?.goForward() } - fun commit() { - viewModelScope.launch { - preferencesManager.setPreferences(editor.value.preferences) - } - } - override fun onStopRequested() { stop() } From 940fc6cc9caf745fe0b0c9253f17bed4df8e0ae9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mickae=CC=88l=20Menu?= Date: Mon, 27 Feb 2023 17:54:57 +0100 Subject: [PATCH 24/27] Add `workflow_dispatch` event for the Checks GH workflow --- .github/workflows/checks.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 5a9630a536..2b10348505 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -1,6 +1,7 @@ name: Checks on: + workflow_dispatch: push: branches: [ main, develop ] pull_request: From 1da03ec07681fd152f7773908fe812242b4bd61c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mickae=CC=88l=20Menu?= Date: Fri, 3 Mar 2023 10:29:38 +0100 Subject: [PATCH 25/27] Fix content iterator --- .../iterators/HtmlResourceContentIterator.kt | 43 +++++++++++-------- .../HtmlResourceContentIteratorTest.kt | 14 +++++- 2 files changed, 37 insertions(+), 20 deletions(-) diff --git a/readium/shared/src/main/java/org/readium/r2/shared/publication/services/content/iterators/HtmlResourceContentIterator.kt b/readium/shared/src/main/java/org/readium/r2/shared/publication/services/content/iterators/HtmlResourceContentIterator.kt index 71cddbc6b7..4b4419fb8d 100644 --- a/readium/shared/src/main/java/org/readium/r2/shared/publication/services/content/iterators/HtmlResourceContentIterator.kt +++ b/readium/shared/src/main/java/org/readium/r2/shared/publication/services/content/iterators/HtmlResourceContentIterator.kt @@ -65,45 +65,50 @@ class HtmlResourceContentIterator( val delta: Int ) - private var requestedElement: ElementWithDelta? = null + private var currentElement: ElementWithDelta? = null override suspend fun hasPrevious(): Boolean { - if (requestedElement?.delta == -1) return true + if (currentElement?.delta == -1) return true + + val elements = elements() + val index = (currentIndex ?: elements.startIndex) - 1 + + val content = elements.elements.getOrNull(index) + ?: return false - val index = currentIndex() - 1 - val element = elements().elements.getOrNull(index) ?: return false currentIndex = index - requestedElement = ElementWithDelta(element, -1) + currentElement = ElementWithDelta(content, -1) return true } override fun previous(): Content.Element = - requestedElement + currentElement ?.takeIf { it.delta == -1 }?.element - ?.also { requestedElement = null } + ?.also { currentElement = null } ?: throw IllegalStateException("Called previous() without a successful call to hasPrevious() first") override suspend fun hasNext(): Boolean { - if (requestedElement?.delta == 1) return true + if (currentElement?.delta == +1) return true + + val elements = elements() + val index = (currentIndex ?: (elements.startIndex - 1)) + 1 - val index = currentIndex() - val element = elements().elements.getOrNull(index) ?: return false - currentIndex = index + 1 - requestedElement = ElementWithDelta(element, +1) + val content = elements.elements.getOrNull(index) + ?: return false + + currentIndex = index + currentElement = ElementWithDelta(content, +1) return true } override fun next(): Content.Element = - requestedElement - ?.takeIf { it.delta == 1 }?.element - ?.also { requestedElement = null } + currentElement + ?.takeIf { it.delta == +1 }?.element + ?.also { currentElement = null } ?: throw IllegalStateException("Called next() without a successful call to hasNext() first") private var currentIndex: Int? = null - private suspend fun currentIndex(): Int = - currentIndex ?: elements().startIndex - private suspend fun elements(): ParsedElements = parsedElements ?: parseElements().also { parsedElements = it } @@ -148,7 +153,7 @@ class HtmlResourceContentIterator( fun result() = ParsedElements( elements = elements, - startIndex = if (baseLocator.locations.progression == 1.0) elements.size - 1 + startIndex = if (baseLocator.locations.progression == 1.0) elements.size else startIndex ) diff --git a/readium/shared/src/test/java/org/readium/r2/shared/publication/services/content/iterators/HtmlResourceContentIteratorTest.kt b/readium/shared/src/test/java/org/readium/r2/shared/publication/services/content/iterators/HtmlResourceContentIteratorTest.kt index e923ad2594..77dc599581 100644 --- a/readium/shared/src/test/java/org/readium/r2/shared/publication/services/content/iterators/HtmlResourceContentIteratorTest.kt +++ b/readium/shared/src/test/java/org/readium/r2/shared/publication/services/content/iterators/HtmlResourceContentIteratorTest.kt @@ -175,6 +175,7 @@ class HtmlResourceContentIteratorTest { fun `cannot call previous() without first hasPrevious()`() = runTest { val iter = iterator(html) iter.hasNext(); iter.next() + iter.hasNext(); iter.next() assertThrows(IllegalStateException::class.java) { iter.previous() } iter.hasPrevious() @@ -208,10 +209,20 @@ class HtmlResourceContentIteratorTest { } @Test - fun `next() then previous() returns the first element`() = runTest { + fun `next() then previous() returns null`() = runTest { + val iter = iterator(html) + assertTrue(iter.hasNext()) + assertEquals(elements[0], iter.next()) + assertFalse(iter.hasPrevious()) + } + + @Test + fun `next() twice then previous() returns the first element`() = runTest { val iter = iterator(html) assertTrue(iter.hasNext()) assertEquals(elements[0], iter.next()) + assertTrue(iter.hasNext()) + assertEquals(elements[1], iter.next()) assertTrue(iter.hasPrevious()) assertEquals(elements[0], iter.previous()) } @@ -220,6 +231,7 @@ class HtmlResourceContentIteratorTest { fun `calling hasPrevious() several times doesn't move the index`() = runTest { val iter = iterator(html) iter.hasNext(); iter.next() + iter.hasNext(); iter.next() assertTrue(iter.hasPrevious()) assertTrue(iter.hasPrevious()) assertTrue(iter.hasPrevious()) From 60213fbbef6d84e1ff4ae569a5e6075a986185eb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mickae=CC=88l=20Menu?= Date: Fri, 3 Mar 2023 11:19:57 +0100 Subject: [PATCH 26/27] Fix CSS selector from `:root` --- .../iterators/HtmlResourceContentIterator.kt | 10 ++--- .../HtmlResourceContentIteratorTest.kt | 39 +++++++++++++++++++ 2 files changed, 43 insertions(+), 6 deletions(-) diff --git a/readium/shared/src/main/java/org/readium/r2/shared/publication/services/content/iterators/HtmlResourceContentIterator.kt b/readium/shared/src/main/java/org/readium/r2/shared/publication/services/content/iterators/HtmlResourceContentIterator.kt index 4b4419fb8d..6bccdccf7e 100644 --- a/readium/shared/src/main/java/org/readium/r2/shared/publication/services/content/iterators/HtmlResourceContentIterator.kt +++ b/readium/shared/src/main/java/org/readium/r2/shared/publication/services/content/iterators/HtmlResourceContentIterator.kt @@ -116,21 +116,19 @@ class HtmlResourceContentIterator( private var parsedElements: ParsedElements? = null private suspend fun parseElements(): ParsedElements { - val body = resource.use { res -> + val document = resource.use { res -> val html = res.readAsString().getOrThrow() Jsoup.parse(html) - }.body() + } val contentParser = ContentParser( baseLocator = locator, startElement = locator.locations.cssSelector?.let { - // The JS third-party library used to generate the CSS Selector sometimes adds - // :root >, which doesn't work with JSoup. - tryOrNull { body.selectFirst(it.removePrefix(":root > ")) } + tryOrNull { document.selectFirst(it) } }, beforeMaxLength = beforeMaxLength ) - NodeTraversor.traverse(contentParser, body) + NodeTraversor.traverse(contentParser, document.body()) return contentParser.result() } diff --git a/readium/shared/src/test/java/org/readium/r2/shared/publication/services/content/iterators/HtmlResourceContentIteratorTest.kt b/readium/shared/src/test/java/org/readium/r2/shared/publication/services/content/iterators/HtmlResourceContentIteratorTest.kt index 77dc599581..da0aa11365 100644 --- a/readium/shared/src/test/java/org/readium/r2/shared/publication/services/content/iterators/HtmlResourceContentIteratorTest.kt +++ b/readium/shared/src/test/java/org/readium/r2/shared/publication/services/content/iterators/HtmlResourceContentIteratorTest.kt @@ -298,6 +298,45 @@ class HtmlResourceContentIteratorTest { ) } + @Test + fun `starting from a CSS selector using the root selector`() = runTest { + val nbspHtml = """ + + + + +

Tout au loin sur la chaussée, aussi loin qu’on pouvait voir

+

Lui, notre colonel, savait peut-être pourquoi ces deux gens-là tiraient [...] On buvait de la bière sucrée.

+ + + """ + + val iter = iterator(nbspHtml, locator(selector = ":root > :nth-child(2) > :nth-child(2)")) + assertTrue(iter.hasNext()) + assertEquals( + TextElement( + locator = locator( + selector = "html > body > p:nth-child(2)", + before = "oin sur la chaussée, aussi loin qu’on pouvait voir", + highlight = "Lui, notre colonel, savait peut-être pourquoi ces deux gens-là tiraient [...] On buvait de la bière sucrée." + ), + role = TextElement.Role.Body, + segments = listOf( + Segment( + locator = locator( + selector = "html > body > p:nth-child(2)", + before = "oin sur la chaussée, aussi loin qu’on pouvait voir", + highlight = "Lui, notre colonel, savait peut-être pourquoi ces deux gens-là tiraient [...] On buvait de la bière sucrée.", + ), + text = "Lui, notre colonel, savait peut-être pourquoi ces deux gens-là tiraient [...] On buvait de la bière sucrée.", + attributes = listOf(Attribute(LANGUAGE, Language("fr"))) + ) + ) + ), + iter.next() + ) + } + @Test fun `iterating over image elements`() = runTest { val html = """ From 62a57db0b959b6070d52fafd6d247d9a19561d7a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mickae=CC=88l=20Menu?= Date: Fri, 3 Mar 2023 11:23:45 +0100 Subject: [PATCH 27/27] Formatting --- .../r2/testapp/reader/preferences/UserPreferences.kt | 6 +++--- .../java/org/readium/r2/testapp/reader/tts/TtsControls.kt | 1 - 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/test-app/src/main/java/org/readium/r2/testapp/reader/preferences/UserPreferences.kt b/test-app/src/main/java/org/readium/r2/testapp/reader/preferences/UserPreferences.kt index 7e0b81089d..88af6ccda2 100644 --- a/test-app/src/main/java/org/readium/r2/testapp/reader/preferences/UserPreferences.kt +++ b/test-app/src/main/java/org/readium/r2/testapp/reader/preferences/UserPreferences.kt @@ -15,15 +15,15 @@ import androidx.compose.runtime.collectAsState import androidx.compose.runtime.getValue import androidx.compose.ui.Alignment import androidx.compose.ui.Modifier +import androidx.compose.ui.platform.LocalContext +import androidx.compose.ui.res.stringResource import androidx.compose.ui.text.style.TextAlign import androidx.compose.ui.unit.dp import org.readium.adapters.pdfium.navigator.PdfiumPreferencesEditor import org.readium.r2.navigator.epub.EpubPreferencesEditor +import org.readium.r2.navigator.media3.tts.android.AndroidTtsEngine import org.readium.r2.navigator.preferences.* import org.readium.r2.navigator.preferences.TextAlign as ReadiumTextAlign -import androidx.compose.ui.platform.LocalContext -import androidx.compose.ui.res.stringResource -import org.readium.r2.navigator.media3.tts.android.AndroidTtsEngine import org.readium.r2.shared.ExperimentalReadiumApi import org.readium.r2.shared.publication.epub.EpubLayout import org.readium.r2.shared.util.Language diff --git a/test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsControls.kt b/test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsControls.kt index 3f2a0282ae..a5459d8ba0 100644 --- a/test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsControls.kt +++ b/test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsControls.kt @@ -118,4 +118,3 @@ fun TtsControls( } } } -