diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 5a9630a536..2b10348505 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -1,6 +1,7 @@ name: Checks on: + workflow_dispatch: push: branches: [ main, develop ] pull_request: diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 7a19bb7765..a32be7e185 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -23,6 +23,7 @@ androidx-lifecycle = "2.5.1" androidx-lifecycle-extensions = "2.2.0" androidx-media = "1.6.0" androidx-media2 = "1.2.1" +androidx-media3 = "1.0.0-rc01" androidx-navigation = "2.5.2" androidx-paging = "3.1.1" androidx-recyclerview = "1.2.1" @@ -87,6 +88,9 @@ androidx-lifecycle-vmsavedstate = { group = "androidx.lifecycle", name = "lifecy androidx-media = { group = "androidx.media", name = "media", version.ref = "androidx-media" } androidx-media2-session = { group = "androidx.media2", name = "media2-session", version.ref = "androidx-media2" } androidx-media2-player = { group = "androidx.media2", name = "media2-player", version.ref = "androidx-media2" } +androidx-media3-common = { group = "androidx.media3", name = "media3-common", version.ref = "androidx-media3" } +androidx-media3-session = { group = "androidx.media3", name = "media3-session", version.ref = "androidx-media3" } +androidx-media3-exoplayer = { group = "androidx.media3", name = "media3-exoplayer", version.ref = "androidx-media3" } androidx-navigation-fragment = { group = "androidx.navigation", name = "navigation-fragment-ktx", version.ref = "androidx-navigation" } androidx-navigation-ui = { group = "androidx.navigation", name = "navigation-ui-ktx", version.ref = "androidx-navigation" } androidx-paging = { group = "androidx.paging", name = "paging-runtime-ktx", version.ref = "androidx-paging" } @@ -138,6 +142,7 @@ coroutines = ["kotlinx-coroutines-core", "kotlinx-coroutines-android"] exoplayer = ["google-exoplayer-core", "google-exoplayer-ui", "google-exoplayer-mediasession", "google-exoplayer-workmanager", "google-exoplayer-extension-media2"] lifecycle = ["androidx-lifecycle-common", "androidx-lifecycle-extensions", "androidx-lifecycle-livedata", "androidx-lifecycle-runtime", "androidx-lifecycle-viewmodel", "androidx-lifecycle-vmsavedstate", "androidx-lifecycle-viewmodel-compose"] media2 = ["androidx-media2-session", "androidx-media2-player"] +media3 = ["androidx-media3-session", "androidx-media3-common", "androidx-media3-exoplayer"] navigation = ["androidx-navigation-fragment", "androidx-navigation-ui"] room = ["androidx-room-runtime", "androidx-room-ktx"] test-frameworks = ["junit", "androidx-ext-junit", "androidx-expresso-core", "robolectric", "kotlin-junit", "assertj", "kotlinx-coroutines-test"] diff --git a/readium/navigator/build.gradle.kts b/readium/navigator/build.gradle.kts index c4f7562363..e357ad38a6 100644 --- a/readium/navigator/build.gradle.kts +++ b/readium/navigator/build.gradle.kts @@ -67,6 +67,7 @@ dependencies { implementation(libs.bundles.lifecycle) implementation(libs.androidx.recyclerview) implementation(libs.androidx.media) + implementation(libs.bundles.media3) implementation(libs.androidx.viewpager2) implementation(libs.androidx.webkit) // Needed to avoid a crash with API 31, see https://stackoverflow.com/a/69152986/1474476 diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/api/DefaultMediaMetadataFactory.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/api/DefaultMediaMetadataFactory.kt new file mode 100644 index 0000000000..292e585273 --- /dev/null +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/api/DefaultMediaMetadataFactory.kt @@ -0,0 +1,73 @@ +/* + * Copyright 2022 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + +package org.readium.r2.navigator.media3.api + +import androidx.media3.common.MediaMetadata +import androidx.media3.common.MediaMetadata.PICTURE_TYPE_FRONT_COVER +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Deferred +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.async +import org.readium.r2.shared.publication.Publication + +/** + * Builds media metadata using the given title, author and cover, + * and fall back on what is in the publication. + */ +@androidx.annotation.OptIn(androidx.media3.common.util.UnstableApi::class) +internal class DefaultMediaMetadataFactory( + private val publication: Publication, + title: String? = null, + author: String? = null, + cover: ByteArray? = null +) : MediaMetadataFactory { + + private val coroutineScope = + CoroutineScope(Dispatchers.Default) + + private val title: String = + title ?: publication.metadata.title + + private val authors: String? = + author ?: publication.metadata.authors + .firstOrNull { it.name.isNotBlank() }?.name + + private val cover: Deferred = coroutineScope.async { + cover ?: publication.linkWithRel("cover") + ?.let { publication.get(it) } + ?.read() + ?.getOrNull() + } + + override suspend fun publicationMetadata(): MediaMetadata { + val builder = MediaMetadata.Builder() + .setTitle(title) + .setTotalTrackCount(publication.readingOrder.size) + + authors + ?.let { builder.setArtist(it) } + + cover.await() + ?.let { builder.maybeSetArtworkData(it, PICTURE_TYPE_FRONT_COVER) } + + return builder.build() + } + + override suspend fun resourceMetadata(index: Int): MediaMetadata { + val builder = MediaMetadata.Builder() + .setTrackNumber(index) + .setTitle(title) + + authors + ?.let { builder.setArtist(it) } + + cover.await() + ?.let { builder.maybeSetArtworkData(it, PICTURE_TYPE_FRONT_COVER) } + + return builder.build() + } +} diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/api/DefaultMediaMetadataProvider.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/api/DefaultMediaMetadataProvider.kt new file mode 100644 index 0000000000..573bbc2725 --- /dev/null +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/api/DefaultMediaMetadataProvider.kt @@ -0,0 +1,24 @@ +/* + * Copyright 2022 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + +package org.readium.r2.navigator.media3.api + +import org.readium.r2.shared.publication.Publication + +/** + * Builds a [MediaMetadataFactory] which will use the given title, author and cover, + * and fall back on what is in the publication. + */ +class DefaultMediaMetadataProvider( + private val title: String? = null, + private val author: String? = null, + private val cover: ByteArray? = null +) : MediaMetadataProvider { + + override fun createMetadataFactory(publication: Publication): MediaMetadataFactory { + return DefaultMediaMetadataFactory(publication, title, author, cover) + } +} diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/api/MediaMetadataFactory.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/api/MediaMetadataFactory.kt new file mode 100644 index 0000000000..0e705f354b --- /dev/null +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/api/MediaMetadataFactory.kt @@ -0,0 +1,27 @@ +/* + * Copyright 2022 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + +package org.readium.r2.navigator.media3.api + +import androidx.media3.common.MediaMetadata + +/** + * Factory for the [MediaMetadata] associated with the publication and its resources. + * + * The metadata are used for example in the media-style Android notification. + */ +interface MediaMetadataFactory { + + /** + * Creates the [MediaMetadata] for the whole publication. + */ + suspend fun publicationMetadata(): MediaMetadata + + /** + * Creates the [MediaMetadata] for the reading order resource at the given [index]. + */ + suspend fun resourceMetadata(index: Int): MediaMetadata +} diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/api/MediaMetadataProvider.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/api/MediaMetadataProvider.kt new file mode 100644 index 0000000000..fb1a45d8d1 --- /dev/null +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/api/MediaMetadataProvider.kt @@ -0,0 +1,17 @@ +/* + * Copyright 2022 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + +package org.readium.r2.navigator.media3.api + +import org.readium.r2.shared.publication.Publication + +/** + * To be implemented to use a custom [MediaMetadataFactory]. + */ +fun interface MediaMetadataProvider { + + fun createMetadataFactory(publication: Publication): MediaMetadataFactory +} diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/api/MediaNavigator.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/api/MediaNavigator.kt new file mode 100644 index 0000000000..0947ede120 --- /dev/null +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/api/MediaNavigator.kt @@ -0,0 +1,81 @@ +/* + * Copyright 2022 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + +package org.readium.r2.navigator.media3.api + +import androidx.media3.common.Player +import kotlinx.coroutines.flow.StateFlow +import org.readium.r2.navigator.Navigator +import org.readium.r2.shared.ExperimentalReadiumApi +import org.readium.r2.shared.util.Closeable + +@ExperimentalReadiumApi +interface MediaNavigator

: Navigator, Closeable { + + /** + * Marker interface for the [position] flow. + */ + interface Position + + /** + * State of the player. + */ + sealed interface State { + + /** + * The navigator is ready to play. + */ + interface Ready : State + + /** + * The end of the media has been reached. + */ + interface Ended : State + + /** + * The navigator cannot play because the buffer is starved. + */ + interface Buffering : State + + /** + * The navigator cannot play because an error occurred. + */ + interface Error : State + } + + /** + * State of the playback. + * + * @param state The current state. + * @param playWhenReady If the navigator should play as soon as the state is Ready. + */ + data class Playback( + val state: State, + val playWhenReady: Boolean + ) + + /** + * Indicates the current state of the playback. + */ + val playback: StateFlow + + val position: StateFlow

+ + /** + * Resumes the playback at the current location. + */ + fun play() + + /** + * Pauses the playback. + */ + fun pause() + + /** + * Adapts this navigator to the media3 [Player] interface. + */ + fun asPlayer(): Player +} diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/api/SynchronizedMediaNavigator.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/api/SynchronizedMediaNavigator.kt new file mode 100644 index 0000000000..6b5bbcb8c8 --- /dev/null +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/api/SynchronizedMediaNavigator.kt @@ -0,0 +1,33 @@ +/* + * Copyright 2022 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + +package org.readium.r2.navigator.media3.api + +import kotlinx.coroutines.flow.StateFlow +import org.readium.r2.shared.ExperimentalReadiumApi +import org.readium.r2.shared.publication.Locator + +/** + * A [MediaNavigator] aware of the utterances that are being read aloud. + */ +@ExperimentalReadiumApi +interface SynchronizedMediaNavigator

: + MediaNavigator

{ + + interface Utterance

{ + val text: String + + val position: P + + val range: IntRange? + + val utteranceLocator: Locator + + val tokenLocator: Locator? + } + + val utterance: StateFlow> +} diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/audio/AudioEngine.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/audio/AudioEngine.kt new file mode 100644 index 0000000000..9c6465b1bc --- /dev/null +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/audio/AudioEngine.kt @@ -0,0 +1,46 @@ +/* + * Copyright 2022 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + +package org.readium.r2.navigator.media3.audio + +import androidx.media3.common.Player +import kotlin.time.Duration +import kotlinx.coroutines.flow.StateFlow +import org.readium.r2.navigator.media3.api.MediaNavigator +import org.readium.r2.navigator.preferences.Configurable +import org.readium.r2.shared.ExperimentalReadiumApi + +@ExperimentalReadiumApi +interface AudioEngine, E : AudioEngine.Error> : + Configurable { + + interface Error + + data class Playback( + val state: MediaNavigator.State, + val playWhenReady: Boolean, + val error: E? + ) + + data class Position( + val index: Int, + val duration: Duration + ) + + val playback: StateFlow> + + val position: StateFlow + + fun play() + + fun pause() + + fun seek(index: Long, position: Duration) + + fun close() + + fun asPlayer(): Player +} diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/audio/AudioEngineProvider.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/audio/AudioEngineProvider.kt new file mode 100644 index 0000000000..4413448bea --- /dev/null +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/audio/AudioEngineProvider.kt @@ -0,0 +1,35 @@ +/* + * Copyright 2022 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + +package org.readium.r2.navigator.media3.audio + +import org.readium.r2.navigator.preferences.Configurable +import org.readium.r2.navigator.preferences.PreferencesEditor +import org.readium.r2.shared.ExperimentalReadiumApi +import org.readium.r2.shared.publication.Metadata +import org.readium.r2.shared.publication.Publication + +@ExperimentalReadiumApi +interface AudioEngineProvider, + E : PreferencesEditor

, F : AudioEngine.Error> { + + suspend fun createEngine(publication: Publication): AudioEngine + + /** + * Creates settings for [metadata] and [preferences]. + */ + fun computeSettings(metadata: Metadata, preferences: P): S + + /** + * Creates a preferences editor for [publication] and [initialPreferences]. + */ + fun createPreferenceEditor(publication: Publication, initialPreferences: P): E + + /** + * Creates an empty set of preferences of this TTS engine provider. + */ + fun createEmptyPreferences(): P +} diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/audio/AudioNavigator.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/audio/AudioNavigator.kt new file mode 100644 index 0000000000..69fda4d58d --- /dev/null +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/audio/AudioNavigator.kt @@ -0,0 +1,70 @@ +/* + * Copyright 2022 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + +package org.readium.r2.navigator.media3.audio + +import androidx.media3.common.Player +import kotlinx.coroutines.flow.StateFlow +import org.readium.r2.navigator.media3.api.MediaNavigator +import org.readium.r2.navigator.preferences.Configurable +import org.readium.r2.shared.ExperimentalReadiumApi +import org.readium.r2.shared.publication.Link +import org.readium.r2.shared.publication.Locator +import org.readium.r2.shared.publication.Publication + +@ExperimentalReadiumApi +class AudioNavigator, E : AudioEngine.Error>( + private val mediaEngine: AudioEngine +) : MediaNavigator, Configurable by mediaEngine { + + class Position : MediaNavigator.Position + + class Error : MediaNavigator.State.Error + + override val publication: Publication + get() = TODO("Not yet implemented") + + override val currentLocator: StateFlow + get() = TODO("Not yet implemented") + + override fun go(locator: Locator, animated: Boolean, completion: () -> Unit): Boolean { + TODO("Not yet implemented") + } + + override fun go(link: Link, animated: Boolean, completion: () -> Unit): Boolean { + TODO("Not yet implemented") + } + + override fun goForward(animated: Boolean, completion: () -> Unit): Boolean { + TODO("Not yet implemented") + } + + override fun goBackward(animated: Boolean, completion: () -> Unit): Boolean { + TODO("Not yet implemented") + } + + override fun close() { + TODO("Not yet implemented") + } + + override val playback: StateFlow + get() = TODO("Not yet implemented") + + override val position: StateFlow + get() = TODO("Not yet implemented") + + override fun play() { + TODO("Not yet implemented") + } + + override fun pause() { + TODO("Not yet implemented") + } + + override fun asPlayer(): Player { + TODO("Not yet implemented") + } +} diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/audio/DurationSerializer.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/audio/DurationSerializer.kt new file mode 100644 index 0000000000..ea9836096b --- /dev/null +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/audio/DurationSerializer.kt @@ -0,0 +1,28 @@ +/* + * Copyright 2022 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + +package org.readium.r2.navigator.media3.audio + +import kotlin.time.Duration +import kotlinx.serialization.KSerializer +import kotlinx.serialization.builtins.serializer +import kotlinx.serialization.descriptors.SerialDescriptor +import kotlinx.serialization.encoding.Decoder +import kotlinx.serialization.encoding.Encoder + +object DurationSerializer : KSerializer { + + private val serializer = Duration.serializer() + + override val descriptor: SerialDescriptor = serializer.descriptor + + override fun deserialize(decoder: Decoder): Duration = + decoder.decodeSerializableValue(serializer) + + override fun serialize(encoder: Encoder, value: Duration) { + encoder.encodeSerializableValue(serializer, value) + } +} diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/exoplayer/ExoPlayerDataSource.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/exoplayer/ExoPlayerDataSource.kt new file mode 100644 index 0000000000..c3411199fd --- /dev/null +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/exoplayer/ExoPlayerDataSource.kt @@ -0,0 +1,148 @@ +/* + * Copyright 2022 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + +package org.readium.r2.navigator.media3.exoplayer + +import android.net.Uri +import androidx.media3.common.C.LENGTH_UNSET +import androidx.media3.common.C.RESULT_END_OF_INPUT +import androidx.media3.datasource.BaseDataSource +import androidx.media3.datasource.DataSource +import androidx.media3.datasource.DataSpec +import androidx.media3.datasource.TransferListener +import java.io.IOException +import kotlinx.coroutines.runBlocking +import org.readium.r2.shared.fetcher.Resource +import org.readium.r2.shared.fetcher.buffered +import org.readium.r2.shared.publication.Publication + +sealed class ExoPlayerDataSourceException(message: String, cause: Throwable?) : IOException(message, cause) { + class NotOpened(message: String) : ExoPlayerDataSourceException(message, null) + class NotFound(message: String) : ExoPlayerDataSourceException(message, null) + class ReadFailed(uri: Uri, offset: Int, readLength: Int, cause: Throwable) : ExoPlayerDataSourceException("Failed to read $readLength bytes of URI $uri at offset $offset.", cause) +} + +/** + * An ExoPlayer's [DataSource] which retrieves resources from a [Publication]. + */ +@androidx.annotation.OptIn(androidx.media3.common.util.UnstableApi::class) +internal class ExoPlayerDataSource internal constructor(private val publication: Publication) : BaseDataSource(/* isNetwork = */ true) { + + class Factory( + private val publication: Publication, + private val transferListener: TransferListener? = null + ) : DataSource.Factory { + + override fun createDataSource(): DataSource = + ExoPlayerDataSource(publication).apply { + if (transferListener != null) { + addTransferListener(transferListener) + } + } + } + + private data class OpenedResource( + val resource: Resource, + val uri: Uri, + var position: Long, + ) + + private var openedResource: OpenedResource? = null + + override fun open(dataSpec: DataSpec): Long { + val link = publication.linkWithHref(dataSpec.uri.toString()) + ?: throw ExoPlayerDataSourceException.NotFound("Can't find a [Link] for URI: ${dataSpec.uri}. Make sure you only request resources declared in the manifest.") + + val resource = publication.get(link) + // Significantly improves performances, in particular with deflated ZIP entries. + .buffered(resourceLength = cachedLengths[dataSpec.uri.toString()]) + + openedResource = OpenedResource( + resource = resource, + uri = dataSpec.uri, + position = dataSpec.position, + ) + + val bytesToRead = + if (dataSpec.length != LENGTH_UNSET.toLong()) { + dataSpec.length + } else { + val contentLength = contentLengthOf(dataSpec.uri, resource) + ?: return dataSpec.length + contentLength - dataSpec.position + } + + return bytesToRead + } + + /** Cached content lengths indexed by their URL. */ + private var cachedLengths: MutableMap = mutableMapOf() + + private fun contentLengthOf(uri: Uri, resource: Resource): Long? { + cachedLengths[uri.toString()]?.let { return it } + + val length = runBlocking { resource.length() }.getOrNull() + ?: return null + + cachedLengths[uri.toString()] = length + return length + } + + override fun read(target: ByteArray, offset: Int, length: Int): Int { + if (length <= 0) { + return 0 + } + + val openedResource = openedResource ?: throw ExoPlayerDataSourceException.NotOpened("No opened resource to read from. Did you call open()?") + + try { + val data = runBlocking { + openedResource.resource + .read(range = openedResource.position until (openedResource.position + length)) + .getOrThrow() + } + + if (data.isEmpty()) { + return RESULT_END_OF_INPUT + } + + data.copyInto( + destination = target, + destinationOffset = offset, + startIndex = 0, + endIndex = data.size + ) + + openedResource.position += data.count() + return data.count() + } catch (e: Exception) { + if (e is InterruptedException) { + return 0 + } + throw ExoPlayerDataSourceException.ReadFailed( + uri = openedResource.uri, + offset = offset, + readLength = length, + cause = e + ) + } + } + + override fun getUri(): Uri? = openedResource?.uri + + override fun close() { + openedResource?.run { + try { + runBlocking { resource.close() } + } catch (e: Exception) { + if (e !is InterruptedException) { + throw e + } + } + } + openedResource = null + } +} diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/exoplayer/ExoPlayerEngine.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/exoplayer/ExoPlayerEngine.kt new file mode 100644 index 0000000000..3022884ce5 --- /dev/null +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/exoplayer/ExoPlayerEngine.kt @@ -0,0 +1,85 @@ +/* + * Copyright 2022 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + +package org.readium.r2.navigator.media3.exoplayer + +import android.app.Application +import androidx.media3.common.AudioAttributes +import androidx.media3.common.C +import androidx.media3.common.Player +import androidx.media3.datasource.DataSource +import androidx.media3.exoplayer.ExoPlayer +import androidx.media3.exoplayer.source.DefaultMediaSourceFactory +import kotlin.time.Duration +import kotlinx.coroutines.flow.StateFlow +import org.readium.r2.navigator.media3.audio.AudioEngine +import org.readium.r2.shared.ExperimentalReadiumApi +import org.readium.r2.shared.publication.Publication + +@ExperimentalReadiumApi +@androidx.annotation.OptIn(androidx.media3.common.util.UnstableApi::class) +class ExoPlayerEngine( + private val application: Application, + private val publication: Publication, + private val exoPlayer: ExoPlayer, +) : AudioEngine { + + companion object { + + private fun createExoPlayer( + application: Application, + publication: Publication, + ): ExoPlayer { + val dataSourceFactory: DataSource.Factory = ExoPlayerDataSource.Factory(publication) + return ExoPlayer.Builder(application) + .setMediaSourceFactory(DefaultMediaSourceFactory(dataSourceFactory)) + .setAudioAttributes( + AudioAttributes.Builder() + .setContentType(C.AUDIO_CONTENT_TYPE_MUSIC) + .setUsage(C.USAGE_MEDIA) + .build(), + true + ) + .setHandleAudioBecomingNoisy(true) + .build() + } + } + + class Error : AudioEngine.Error + + override val playback: StateFlow> + get() = TODO("Not yet implemented") + + override val position: StateFlow + get() = TODO("Not yet implemented") + + override val settings: StateFlow + get() = TODO("Not yet implemented") + + override fun play() { + TODO("Not yet implemented") + } + + override fun pause() { + TODO("Not yet implemented") + } + + override fun seek(index: Long, position: Duration) { + TODO("Not yet implemented") + } + + override fun close() { + TODO("Not yet implemented") + } + + override fun asPlayer(): Player { + return exoPlayer + } + + override fun submitPreferences(preferences: ExoPlayerPreferences) { + TODO("Not yet implemented") + } +} diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/exoplayer/ExoPlayerEngineProvider.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/exoplayer/ExoPlayerEngineProvider.kt new file mode 100644 index 0000000000..bbfa3d9211 --- /dev/null +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/exoplayer/ExoPlayerEngineProvider.kt @@ -0,0 +1,36 @@ +/* + * Copyright 2022 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + +package org.readium.r2.navigator.media3.exoplayer + +import org.readium.r2.navigator.media3.audio.AudioEngineProvider +import org.readium.r2.shared.ExperimentalReadiumApi +import org.readium.r2.shared.publication.Metadata +import org.readium.r2.shared.publication.Publication + +@ExperimentalReadiumApi +@androidx.annotation.OptIn(androidx.media3.common.util.UnstableApi::class) +class ExoPlayerEngineProvider() : AudioEngineProvider { + + override suspend fun createEngine(publication: Publication): ExoPlayerEngine { + TODO("Not yet implemented") + } + + override fun computeSettings( + metadata: Metadata, + preferences: ExoPlayerPreferences + ): ExoPlayerSettings = + ExoPlayerSettingsResolver(metadata).settings(preferences) + + override fun createPreferenceEditor( + publication: Publication, + initialPreferences: ExoPlayerPreferences + ): ExoPlayerPreferencesEditor = + ExoPlayerPreferencesEditor() + + override fun createEmptyPreferences(): ExoPlayerPreferences = + ExoPlayerPreferences() +} diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/exoplayer/ExoPlayerPreferences.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/exoplayer/ExoPlayerPreferences.kt new file mode 100644 index 0000000000..c77c551475 --- /dev/null +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/exoplayer/ExoPlayerPreferences.kt @@ -0,0 +1,22 @@ +/* + * Copyright 2022 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + +package org.readium.r2.navigator.media3.exoplayer + +import org.readium.r2.navigator.preferences.Configurable +import org.readium.r2.shared.ExperimentalReadiumApi + +@ExperimentalReadiumApi +@kotlinx.serialization.Serializable +data class ExoPlayerPreferences( + val rateMultiplier: Double? = null, +) : Configurable.Preferences { + + override fun plus(other: ExoPlayerPreferences): ExoPlayerPreferences = + ExoPlayerPreferences( + rateMultiplier = other.rateMultiplier ?: rateMultiplier, + ) +} diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/exoplayer/ExoPlayerPreferencesEditor.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/exoplayer/ExoPlayerPreferencesEditor.kt new file mode 100644 index 0000000000..6cc4efe150 --- /dev/null +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/exoplayer/ExoPlayerPreferencesEditor.kt @@ -0,0 +1,21 @@ +/* + * Copyright 2022 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + +package org.readium.r2.navigator.media3.exoplayer + +import org.readium.r2.navigator.preferences.PreferencesEditor +import org.readium.r2.shared.ExperimentalReadiumApi + +@ExperimentalReadiumApi +class ExoPlayerPreferencesEditor : PreferencesEditor { + + override val preferences: ExoPlayerPreferences + get() = TODO("Not yet implemented") + + override fun clear() { + TODO("Not yet implemented") + } +} diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/exoplayer/ExoPlayerPreferencesFilters.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/exoplayer/ExoPlayerPreferencesFilters.kt new file mode 100644 index 0000000000..db3f3b7aad --- /dev/null +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/exoplayer/ExoPlayerPreferencesFilters.kt @@ -0,0 +1,30 @@ +/* + * Copyright 2022 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + +package org.readium.r2.navigator.media3.exoplayer + +import org.readium.r2.navigator.preferences.PreferencesFilter +import org.readium.r2.shared.ExperimentalReadiumApi + +/** + * Suggested filter to keep only shared [ExoPlayerPreferences]. + */ +@ExperimentalReadiumApi +object ExoPlayerSharedPreferencesFilter : PreferencesFilter { + + override fun filter(preferences: ExoPlayerPreferences): ExoPlayerPreferences = + preferences.copy() +} + +/** + * Suggested filter to keep only publication-specific [ExoPlayerPreferences]. + */ +@ExperimentalReadiumApi +object ExoPlayerPublicationPreferencesFilter : PreferencesFilter { + + override fun filter(preferences: ExoPlayerPreferences): ExoPlayerPreferences = + ExoPlayerPreferences() +} diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/exoplayer/ExoPlayerPreferencesSerializer.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/exoplayer/ExoPlayerPreferencesSerializer.kt new file mode 100644 index 0000000000..aada34c11e --- /dev/null +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/exoplayer/ExoPlayerPreferencesSerializer.kt @@ -0,0 +1,24 @@ +/* + * Copyright 2022 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + +package org.readium.r2.navigator.media3.exoplayer + +import kotlinx.serialization.json.Json +import org.readium.r2.navigator.preferences.PreferencesSerializer +import org.readium.r2.shared.ExperimentalReadiumApi + +/** + * JSON serializer of [ExoPlayerPreferences]. + */ +@ExperimentalReadiumApi +class ExoPlayerPreferencesSerializer : PreferencesSerializer { + + override fun serialize(preferences: ExoPlayerPreferences): String = + Json.encodeToString(ExoPlayerPreferences.serializer(), preferences) + + override fun deserialize(preferences: String): ExoPlayerPreferences = + Json.decodeFromString(ExoPlayerPreferences.serializer(), preferences) +} diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/exoplayer/ExoPlayerSettings.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/exoplayer/ExoPlayerSettings.kt new file mode 100644 index 0000000000..58bc54b1e5 --- /dev/null +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/exoplayer/ExoPlayerSettings.kt @@ -0,0 +1,15 @@ +/* + * Copyright 2022 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + +package org.readium.r2.navigator.media3.exoplayer + +import org.readium.r2.navigator.preferences.Configurable +import org.readium.r2.shared.ExperimentalReadiumApi + +@ExperimentalReadiumApi +data class ExoPlayerSettings( + val rateMultiplier: Double +) : Configurable.Settings diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/exoplayer/ExoPlayerSettingsResolver.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/exoplayer/ExoPlayerSettingsResolver.kt new file mode 100644 index 0000000000..b940630a3e --- /dev/null +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/exoplayer/ExoPlayerSettingsResolver.kt @@ -0,0 +1,23 @@ +/* + * Copyright 2022 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + +package org.readium.r2.navigator.media3.exoplayer + +import org.readium.r2.shared.ExperimentalReadiumApi +import org.readium.r2.shared.publication.Metadata + +@ExperimentalReadiumApi +internal class ExoPlayerSettingsResolver( + private val metadata: Metadata, +) { + + fun settings(preferences: ExoPlayerPreferences): ExoPlayerSettings { + + return ExoPlayerSettings( + rateMultiplier = preferences.rateMultiplier ?: 1.0, + ) + } +} diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsAliases.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsAliases.kt new file mode 100644 index 0000000000..717a2e74c0 --- /dev/null +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsAliases.kt @@ -0,0 +1,19 @@ +/* + * Copyright 2022 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + +package org.readium.r2.navigator.media3.tts + +import org.readium.r2.navigator.media3.tts.android.AndroidTtsEngine +import org.readium.r2.navigator.media3.tts.android.AndroidTtsPreferences +import org.readium.r2.navigator.media3.tts.android.AndroidTtsPreferencesEditor +import org.readium.r2.navigator.media3.tts.android.AndroidTtsSettings +import org.readium.r2.shared.ExperimentalReadiumApi + +@OptIn(ExperimentalReadiumApi::class) +typealias AndroidTtsNavigatorFactory = TtsNavigatorFactory + +@OptIn(ExperimentalReadiumApi::class) +typealias AndroidTtsNavigator = TtsNavigator diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsContentIterator.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsContentIterator.kt new file mode 100644 index 0000000000..04e738242b --- /dev/null +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsContentIterator.kt @@ -0,0 +1,236 @@ +/* + * Copyright 2022 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + +package org.readium.r2.navigator.media3.tts + +import org.readium.r2.shared.ExperimentalReadiumApi +import org.readium.r2.shared.publication.Locator +import org.readium.r2.shared.publication.Publication +import org.readium.r2.shared.publication.html.cssSelector +import org.readium.r2.shared.publication.indexOfFirstWithHref +import org.readium.r2.shared.publication.services.content.Content +import org.readium.r2.shared.publication.services.content.ContentService +import org.readium.r2.shared.publication.services.content.TextContentTokenizer +import org.readium.r2.shared.util.CursorList +import org.readium.r2.shared.util.Language +import org.readium.r2.shared.util.tokenizer.TextTokenizer + +@ExperimentalReadiumApi + +/** + * A Content Iterator able to provide short utterances. + * + * Not thread-safe. + */ +internal class TtsContentIterator( + private val publication: Publication, + private val tokenizerFactory: (language: Language?) -> TextTokenizer, + initialLocator: Locator? +) { + data class Utterance( + val resourceIndex: Int, + val cssSelector: String, + val text: String, + val textBefore: String?, + val textAfter: String?, + val language: Language? + ) + + private val contentService: ContentService = + publication.findService(ContentService::class) + ?: throw IllegalStateException("No ContentService.") + + /** + * Current subset of utterances with a cursor. + */ + private var utterances: CursorList = + CursorList() + + /** + * [Content.Iterator] used to iterate through the [publication]. + */ + private var publicationIterator: Content.Iterator = createIterator(initialLocator) + set(value) { + field = value + utterances = CursorList() + } + + /** + * The tokenizer language. + * + * Modifying this property is not immediate, the new value will be applied as soon as possible. + */ + var language: Language? = + null + + /** + * Whether language information in content should be superseded by [language] while tokenizing. + * + * Modifying this property is not immediate, the new value will be applied as soon as possible. + */ + var overrideContentLanguage: Boolean = + false + + val resourceCount: Int = + publication.readingOrder.size + + /** + * Moves the iterator to the position provided in [locator]. + */ + fun seek(locator: Locator) { + publicationIterator = createIterator(locator) + } + + /** + * Moves the iterator to the beginning of the publication. + */ + fun seekToBeginning() { + publicationIterator = createIterator(locator = null) + } + + /** + * Moves the iterator to the resource with the given [index] in the publication reading order. + */ + fun seekToResource(index: Int) { + val link = publication.readingOrder.getOrNull(index) ?: return + val locator = publication.locatorFromLink(link) + publicationIterator = createIterator(locator) + } + + /** + * Creates a fresh content iterator for the publication starting from [Locator]. + */ + + private fun createIterator(locator: Locator?): Content.Iterator = + contentService.content(locator).iterator() + + /** + * Advances to the previous item and returns it, or null if we reached the beginning. + */ + suspend fun previousUtterance(): Utterance? = + nextUtterance(Direction.Backward) + + /** + * Advances to the next item and returns it, or null if we reached the end. + */ + suspend fun nextUtterance(): Utterance? = + nextUtterance(Direction.Forward) + + private enum class Direction { + Forward, Backward; + } + + /** + * Gets the next utterance in the given [direction], or null when reaching the beginning or the + * end. + */ + private suspend fun nextUtterance(direction: Direction): Utterance? { + val utterance = utterances.nextIn(direction) + if (utterance == null && loadNextUtterances(direction)) { + return nextUtterance(direction) + } + return utterance + } + + /** + * Loads the utterances for the next publication [Content.Element] item in the given [direction]. + */ + private suspend fun loadNextUtterances(direction: Direction): Boolean { + val content = publicationIterator.nextIn(direction) + ?: return false + + val nextUtterances = content + .tokenize() + .flatMap { it.utterances() } + + if (nextUtterances.isEmpty()) { + return loadNextUtterances(direction) + } + + utterances = CursorList( + list = nextUtterances, + index = when (direction) { + Direction.Forward -> -1 + Direction.Backward -> nextUtterances.size + } + ) + + return true + } + + /** + * Splits a publication [Content.Element] item into smaller chunks using the provided tokenizer. + * + * This is used to split a paragraph into sentences, for example. + */ + private fun Content.Element.tokenize(): List { + val contentTokenizer = TextContentTokenizer( + language = this@TtsContentIterator.language, + textTokenizerFactory = tokenizerFactory, + overrideContentLanguage = overrideContentLanguage + ) + return contentTokenizer.tokenize(this) + } + + /** + * Splits a publication [Content.Element] item into the utterances to be spoken. + */ + private fun Content.Element.utterances(): List { + fun utterance(text: String, locator: Locator, language: Language? = null): Utterance? { + if (!text.any { it.isLetterOrDigit() }) + return null + + val resourceIndex = publication.readingOrder.indexOfFirstWithHref(locator.href) + ?: throw IllegalStateException("Content Element cannot be found in readingOrder.") + + val cssSelector = locator.locations.cssSelector + ?: throw IllegalStateException("Css selectors are expected in iterator locators.") + + return Utterance( + text = text, + language = language, + resourceIndex = resourceIndex, + textBefore = locator.text.before, + textAfter = locator.text.after, + cssSelector = cssSelector, + ) + } + + return when (this) { + is Content.TextElement -> { + segments.mapNotNull { segment -> + utterance( + text = segment.text, + locator = segment.locator, + language = segment.language + ) + } + } + + is Content.TextualElement -> { + listOfNotNull( + text + ?.takeIf { it.isNotBlank() } + ?.let { utterance(text = it, locator = locator) } + ) + } + + else -> emptyList() + } + } + + private fun CursorList.nextIn(direction: Direction): E? = + when (direction) { + Direction.Forward -> if (hasNext()) next() else null + Direction.Backward -> if (hasPrevious()) previous() else null + } + + private suspend fun Content.Iterator.nextIn(direction: Direction): Content.Element? = + when (direction) { + Direction.Forward -> nextOrNull() + Direction.Backward -> previousOrNull() + } +} diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsEngine.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsEngine.kt new file mode 100644 index 0000000000..36ec182284 --- /dev/null +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsEngine.kt @@ -0,0 +1,121 @@ +/* + * Copyright 2022 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + +package org.readium.r2.navigator.media3.tts + +import org.readium.r2.navigator.preferences.Configurable +import org.readium.r2.shared.ExperimentalReadiumApi +import org.readium.r2.shared.util.Closeable +import org.readium.r2.shared.util.Language + +/** + * A text-to-speech engine synthesizes text utterances (e.g. sentence). + */ +@ExperimentalReadiumApi +interface TtsEngine, + E : TtsEngine.Error, V : TtsEngine.Voice> : Configurable, Closeable { + + interface Preferences

> : Configurable.Preferences

{ + + /** + * The default language to use when no language information is passed to [speak]. + */ + val language: Language? + } + + interface Settings : Configurable.Settings { + + /** + * The default language to use when no language information is passed to [speak]. + */ + val language: Language? + + /** + * Whether language information in content should be superseded by [language]. + */ + val overrideContentLanguage: Boolean + } + + interface Voice { + + /** + * The voice's language. + */ + val language: Language + } + + /** + * Marker interface for the errors that the [TtsEngine] returns. + */ + interface Error + + /** + * An id to identify a request to speak. + */ + @JvmInline + value class RequestId(val id: String) + + /** + * TTS engine callbacks. + */ + interface Listener { + + /** + * Called when the utterance with the given id starts as perceived by the caller. + */ + fun onStart(requestId: RequestId) + + /** + * Called when the [TtsEngine] is about to speak the specified [range] of the utterance with + * the given id. + * + * This callback may not be called if the [TtsEngine] does not provide range information. + */ + fun onRange(requestId: RequestId, range: IntRange) + + /** + * Called if the utterance with the given id has been stopped while in progress + * by a call to [stop]. + */ + fun onInterrupted(requestId: RequestId) + + /** + * Called when the utterance with the given id has been flushed from the synthesis queue + * by a call to [stop]. + */ + fun onFlushed(requestId: RequestId) + + /** + * Called when the utterance with the given id has successfully completed processing. + */ + fun onDone(requestId: RequestId) + + /** + * Called when an error has occurred during processing of the utterance with the given id. + */ + fun onError(requestId: RequestId, error: E) + } + + /** + * Sets of voices available with this [TtsEngine]. + */ + val voices: Set + + /** + * Enqueues a new speak request. + */ + fun speak(requestId: RequestId, text: String, language: Language?) + + /** + * Stops the [TtsEngine]. + */ + fun stop() + + /** + * Sets a new listener or removes the current one. + */ + fun setListener(listener: Listener?) +} diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsEngineFacade.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsEngineFacade.kt new file mode 100644 index 0000000000..5f5151ca28 --- /dev/null +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsEngineFacade.kt @@ -0,0 +1,98 @@ +/* + * Copyright 2022 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + +package org.readium.r2.navigator.media3.tts + +import java.util.* +import kotlinx.coroutines.CancellableContinuation +import kotlinx.coroutines.ExperimentalCoroutinesApi +import kotlinx.coroutines.suspendCancellableCoroutine +import org.readium.r2.navigator.preferences.Configurable +import org.readium.r2.shared.ExperimentalReadiumApi +import org.readium.r2.shared.util.Language + +@ExperimentalReadiumApi +@OptIn(ExperimentalCoroutinesApi::class) +internal class TtsEngineFacade, + E : TtsEngine.Error, V : TtsEngine.Voice>( + private val engine: TtsEngine +) : Configurable by engine { + + init { + val listener = EngineListener() + engine.setListener(listener) + } + + private var currentTask: UtteranceTask? = null + + val voices: Set + get() = engine.voices + + suspend fun speak(text: String, language: Language?, onRange: (IntRange) -> Unit): E? = + suspendCancellableCoroutine { continuation -> + continuation.invokeOnCancellation { engine.stop() } + currentTask?.continuation?.cancel() + val id = TtsEngine.RequestId(UUID.randomUUID().toString()) + currentTask = UtteranceTask(id, continuation, onRange) + engine.speak(id, text, language) + } + + fun close() { + currentTask?.continuation?.cancel() + engine.close() + } + + private data class UtteranceTask( + val requestId: TtsEngine.RequestId, + val continuation: CancellableContinuation, + val onRange: (IntRange) -> Unit + ) + + private inner class EngineListener : TtsEngine.Listener { + + override fun onStart(requestId: TtsEngine.RequestId) { + } + + override fun onRange(requestId: TtsEngine.RequestId, range: IntRange) { + currentTask + ?.takeIf { it.requestId == requestId } + ?.onRange + ?.invoke(range) + } + + override fun onInterrupted(requestId: TtsEngine.RequestId) { + currentTask + ?.takeIf { it.requestId == requestId } + ?.continuation + ?.cancel() + currentTask = null + } + + override fun onFlushed(requestId: TtsEngine.RequestId) { + currentTask + ?.takeIf { it.requestId == requestId } + ?.continuation + ?.cancel() + currentTask = null + } + + override fun onDone(requestId: TtsEngine.RequestId) { + currentTask + ?.takeIf { it.requestId == requestId } + ?.continuation + ?.resume(null) {} + currentTask = null + } + + override fun onError(requestId: TtsEngine.RequestId, error: E) { + currentTask + ?.takeIf { it.requestId == requestId } + ?.continuation + ?.resume(error) {} + currentTask = null + } + } +} diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsEngineProvider.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsEngineProvider.kt new file mode 100644 index 0000000000..5e1d26e580 --- /dev/null +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsEngineProvider.kt @@ -0,0 +1,51 @@ +/* + * Copyright 2022 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + +package org.readium.r2.navigator.media3.tts + +import androidx.media3.common.PlaybackException +import androidx.media3.common.PlaybackParameters +import org.readium.r2.navigator.preferences.PreferencesEditor +import org.readium.r2.shared.ExperimentalReadiumApi +import org.readium.r2.shared.publication.Publication + +/** + * To be implemented by adapters for third-party TTS engines which can be used with [TtsNavigator]. + */ +@ExperimentalReadiumApi +interface TtsEngineProvider, E : PreferencesEditor

, + F : TtsEngine.Error, V : TtsEngine.Voice> { + + /** + * Creates a [TtsEngine] for [publication] and [initialPreferences]. + */ + suspend fun createEngine(publication: Publication, initialPreferences: P): TtsEngine? + + /** + * Creates a preferences editor for [publication] and [initialPreferences]. + */ + fun createPreferencesEditor(publication: Publication, initialPreferences: P): E + + /** + * Creates an empty set of preferences of this TTS engine provider. + */ + fun createEmptyPreferences(): P + + /** + * Computes Media3 [PlaybackParameters] from the given [settings]. + */ + fun getPlaybackParameters(settings: S): PlaybackParameters + + /** + * Updates [previousPreferences] to honor the given Media3 [playbackParameters]. + */ + fun updatePlaybackParameters(previousPreferences: P, playbackParameters: PlaybackParameters): P + + /** + * Maps an engine-specific error to Media3 [PlaybackException]. + */ + fun mapEngineError(error: F): PlaybackException +} diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsNavigator.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsNavigator.kt new file mode 100644 index 0000000000..8240d0573e --- /dev/null +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsNavigator.kt @@ -0,0 +1,274 @@ +/* + * Copyright 2022 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + +package org.readium.r2.navigator.media3.tts + +import android.app.Application +import androidx.media3.common.MediaItem +import androidx.media3.common.PlaybackParameters +import androidx.media3.common.Player +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.MainScope +import kotlinx.coroutines.flow.StateFlow +import org.readium.r2.navigator.media3.api.MediaMetadataProvider +import org.readium.r2.navigator.media3.api.MediaNavigator +import org.readium.r2.navigator.media3.api.SynchronizedMediaNavigator +import org.readium.r2.navigator.media3.tts.session.TtsSessionAdapter +import org.readium.r2.navigator.preferences.Configurable +import org.readium.r2.shared.ExperimentalReadiumApi +import org.readium.r2.shared.extensions.mapStateIn +import org.readium.r2.shared.publication.Link +import org.readium.r2.shared.publication.Locator +import org.readium.r2.shared.publication.Publication +import org.readium.r2.shared.publication.services.content.ContentService +import org.readium.r2.shared.util.Language +import org.readium.r2.shared.util.tokenizer.TextTokenizer + +/** + * A navigator to read aloud a [Publication] with a TTS engine. + */ +@ExperimentalReadiumApi +class TtsNavigator, + E : TtsEngine.Error, V : TtsEngine.Voice> private constructor( + coroutineScope: CoroutineScope, + override val publication: Publication, + private val player: TtsPlayer, + private val sessionAdapter: TtsSessionAdapter, +) : SynchronizedMediaNavigator, Configurable by player { + + companion object { + + suspend operator fun , + E : TtsEngine.Error, V : TtsEngine.Voice> invoke( + application: Application, + publication: Publication, + ttsEngineProvider: TtsEngineProvider, + tokenizerFactory: (language: Language?) -> TextTokenizer, + metadataProvider: MediaMetadataProvider, + listener: Listener, + initialPreferences: P? = null, + initialLocator: Locator? = null, + ): TtsNavigator? { + + if (publication.findService(ContentService::class) == null) { + return null + } + + val actualInitialPreferences = + initialPreferences + ?: ttsEngineProvider.createEmptyPreferences() + + val contentIterator = + TtsContentIterator(publication, tokenizerFactory, initialLocator) + + val ttsEngine = + ttsEngineProvider.createEngine(publication, actualInitialPreferences) + ?: return null + + val metadataFactory = + metadataProvider.createMetadataFactory(publication) + + val playlistMetadata = + metadataFactory.publicationMetadata() + + val mediaItems = + publication.readingOrder.indices.map { index -> + val metadata = metadataFactory.resourceMetadata(index) + MediaItem.Builder() + .setMediaMetadata(metadata) + .build() + } + + val ttsPlayer = + TtsPlayer(ttsEngine, contentIterator, actualInitialPreferences) + ?: return null + + val coroutineScope = + MainScope() + + val playbackParameters = + ttsPlayer.settings.mapStateIn(coroutineScope) { + ttsEngineProvider.getPlaybackParameters(it) + } + + val onSetPlaybackParameters = { parameters: PlaybackParameters -> + val newPreferences = ttsEngineProvider.updatePlaybackParameters( + ttsPlayer.lastPreferences, + parameters + ) + ttsPlayer.submitPreferences(newPreferences) + } + + val sessionAdapter = + TtsSessionAdapter( + application, + ttsPlayer, + playlistMetadata, + mediaItems, + listener::onStopRequested, + playbackParameters, + onSetPlaybackParameters, + ttsEngineProvider::mapEngineError + ) + + return TtsNavigator(coroutineScope, publication, ttsPlayer, sessionAdapter) + } + } + + interface Listener { + + fun onStopRequested() + } + + data class Position( + val resourceIndex: Int, + val cssSelector: String, + val textBefore: String?, + val textAfter: String?, + ) : MediaNavigator.Position + + data class Utterance( + override val text: String, + override val position: Position, + override val range: IntRange?, + override val utteranceLocator: Locator, + override val tokenLocator: Locator? + ) : SynchronizedMediaNavigator.Utterance + + sealed class State { + + object Ready : MediaNavigator.State.Ready + + object Ended : MediaNavigator.State.Ended + + sealed class Error : MediaNavigator.State.Error { + + data class EngineError (val error: E) : Error() + + data class ContentError(val exception: Exception) : Error() + } + } + + val voices: Set get() = + player.voices + + override val playback: StateFlow = + player.playback.mapStateIn(coroutineScope) { it.toPlayback() } + + override val utterance: StateFlow = + player.utterance.mapStateIn(coroutineScope) { it.toUtterance() } + + override val position: StateFlow = + utterance.mapStateIn(coroutineScope) { utterance -> + utterance.position.copy(textAfter = utterance.text + utterance.position.textAfter) + } + + override fun play() { + player.play() + } + + override fun pause() { + player.pause() + } + + fun go(locator: Locator) { + player.go(locator) + } + + fun previousUtterance() { + player.previousUtterance() + } + + fun nextUtterance() { + player.nextUtterance() + } + + override fun asPlayer(): Player = + sessionAdapter + + override fun close() { + player.close() + } + + override val currentLocator: StateFlow = + utterance.mapStateIn(coroutineScope) { it.tokenLocator ?: it.utteranceLocator } + + override fun go(locator: Locator, animated: Boolean, completion: () -> Unit): Boolean { + player.go(locator) + return true + } + + override fun go(link: Link, animated: Boolean, completion: () -> Unit): Boolean { + val locator = publication.locatorFromLink(link) ?: return false + return go(locator, animated, completion) + } + + override fun goForward(animated: Boolean, completion: () -> Unit): Boolean { + player.nextUtterance() + return true + } + + override fun goBackward(animated: Boolean, completion: () -> Unit): Boolean { + player.previousUtterance() + return true + } + + private fun TtsPlayer.Playback.toPlayback() = + MediaNavigator.Playback( + state = state.toState(), + playWhenReady = playWhenReady, + ) + + private fun TtsPlayer.State.toState() = + when (this) { + TtsPlayer.State.Ready -> State.Ready + TtsPlayer.State.Ended -> State.Ended + is TtsPlayer.State.Error -> this.toError() + } + + private fun TtsPlayer.State.Error.toError(): State.Error = + when (this) { + is TtsPlayer.State.Error.ContentError -> State.Error.ContentError(exception) + is TtsPlayer.State.Error.EngineError<*> -> State.Error.EngineError(error) + } + + private fun TtsPlayer.Utterance.Position.toPosition(): Position = + Position( + resourceIndex = resourceIndex, + cssSelector = cssSelector, + textBefore = textBefore, + textAfter = textAfter + ) + + private fun TtsPlayer.Utterance.toUtterance(): Utterance { + val utteranceHighlight = publication + .locatorFromLink(publication.readingOrder[position.resourceIndex])!! + .copyWithLocations( + progression = null, + otherLocations = buildMap { + put("cssSelector", position.cssSelector) + } + ).copy( + text = + Locator.Text( + highlight = text, + before = position.textBefore, + after = position.textAfter + ) + ) + + val tokenHighlight = range + ?.let { utteranceHighlight.copy(text = utteranceHighlight.text.substring(it)) } + + return Utterance( + text = text, + position = position.toPosition(), + range = range, + utteranceLocator = utteranceHighlight, + tokenLocator = tokenHighlight, + ) + } +} diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsNavigatorFactory.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsNavigatorFactory.kt new file mode 100644 index 0000000000..48201ee251 --- /dev/null +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsNavigatorFactory.kt @@ -0,0 +1,138 @@ +/* + * Copyright 2022 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + +package org.readium.r2.navigator.media3.tts + +import android.app.Application +import org.readium.r2.navigator.media3.api.DefaultMediaMetadataProvider +import org.readium.r2.navigator.media3.api.MediaMetadataProvider +import org.readium.r2.navigator.media3.tts.android.AndroidTtsDefaults +import org.readium.r2.navigator.media3.tts.android.AndroidTtsEngine +import org.readium.r2.navigator.media3.tts.android.AndroidTtsEngineProvider +import org.readium.r2.navigator.preferences.PreferencesEditor +import org.readium.r2.shared.ExperimentalReadiumApi +import org.readium.r2.shared.publication.Locator +import org.readium.r2.shared.publication.Publication +import org.readium.r2.shared.publication.services.content.Content +import org.readium.r2.shared.publication.services.content.content +import org.readium.r2.shared.util.Language +import org.readium.r2.shared.util.tokenizer.DefaultTextContentTokenizer +import org.readium.r2.shared.util.tokenizer.TextTokenizer +import org.readium.r2.shared.util.tokenizer.TextUnit + +@ExperimentalReadiumApi +class TtsNavigatorFactory, E : PreferencesEditor

, + F : TtsEngine.Error, V : TtsEngine.Voice> private constructor( + private val application: Application, + private val publication: Publication, + private val ttsEngineProvider: TtsEngineProvider, + private val tokenizerFactory: (language: Language?) -> TextTokenizer, + private val metadataProvider: MediaMetadataProvider +) { + companion object { + + suspend operator fun invoke( + application: Application, + publication: Publication, + tokenizerFactory: (language: Language?) -> TextTokenizer = defaultTokenizerFactory, + metadataProvider: MediaMetadataProvider = defaultMediaMetadataProvider, + defaults: AndroidTtsDefaults = AndroidTtsDefaults(), + voiceSelector: (Language?, Set) -> AndroidTtsEngine.Voice? = defaultVoiceSelector, + listener: AndroidTtsEngine.Listener? = null + ): AndroidTtsNavigatorFactory? { + + val engineProvider = AndroidTtsEngineProvider( + context = application, + defaults = defaults, + voiceSelector = voiceSelector, + listener = listener + ) + + return createNavigatorFactory( + application, + publication, + engineProvider, + tokenizerFactory, + metadataProvider + ) + } + + suspend operator fun , E : PreferencesEditor

, + F : TtsEngine.Error, V : TtsEngine.Voice> invoke( + application: Application, + publication: Publication, + ttsEngineProvider: TtsEngineProvider, + tokenizerFactory: (language: Language?) -> TextTokenizer = defaultTokenizerFactory, + metadataProvider: MediaMetadataProvider = defaultMediaMetadataProvider + ): TtsNavigatorFactory? { + + return createNavigatorFactory( + application, + publication, + ttsEngineProvider, + tokenizerFactory, + metadataProvider + ) + } + + private suspend fun , E : PreferencesEditor

, + F : TtsEngine.Error, V : TtsEngine.Voice> createNavigatorFactory( + application: Application, + publication: Publication, + ttsEngineProvider: TtsEngineProvider, + tokenizerFactory: (language: Language?) -> TextTokenizer, + metadataProvider: MediaMetadataProvider + ): TtsNavigatorFactory? { + + publication.content() + ?.iterator() + ?.takeIf { it.hasNext() } + ?: return null + + return TtsNavigatorFactory( + application, + publication, + ttsEngineProvider, + tokenizerFactory, + metadataProvider + ) + } + + /** + * The default content tokenizer will split the [Content.Element] items into individual sentences. + */ + private val defaultTokenizerFactory: (Language?) -> TextTokenizer = { language -> + DefaultTextContentTokenizer(TextUnit.Sentence, language) + } + + private val defaultMediaMetadataProvider: MediaMetadataProvider = + DefaultMediaMetadataProvider() + + private val defaultVoiceSelector: (Language?, Set) -> AndroidTtsEngine.Voice? = + { _, _ -> null } + } + + suspend fun createNavigator( + listener: TtsNavigator.Listener, + initialPreferences: P? = null, + initialLocator: Locator? = null + ): TtsNavigator? { + return TtsNavigator( + application, + publication, + ttsEngineProvider, + tokenizerFactory, + metadataProvider, + listener, + initialPreferences, + initialLocator + ) + } + + fun createTtsPreferencesEditor( + currentPreferences: P, + ): E = ttsEngineProvider.createPreferencesEditor(publication, currentPreferences) +} diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsPlayer.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsPlayer.kt new file mode 100644 index 0000000000..d98a450d62 --- /dev/null +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsPlayer.kt @@ -0,0 +1,507 @@ +/* + * Copyright 2022 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + +package org.readium.r2.navigator.media3.tts + +import kotlin.coroutines.coroutineContext +import kotlinx.coroutines.* +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.StateFlow +import kotlinx.coroutines.flow.asStateFlow +import kotlinx.coroutines.sync.Mutex +import kotlinx.coroutines.sync.withLock +import org.readium.r2.navigator.preferences.Configurable +import org.readium.r2.shared.ExperimentalReadiumApi +import org.readium.r2.shared.extensions.tryOrNull +import org.readium.r2.shared.publication.Locator + +/** + * Plays the content from a [TtsContentIterator] with a [TtsEngine]. + */ +@ExperimentalReadiumApi +internal class TtsPlayer, + E : TtsEngine.Error, V : TtsEngine.Voice> private constructor( + private val engineFacade: TtsEngineFacade, + private val contentIterator: TtsContentIterator, + initialWindow: UtteranceWindow, + initialPreferences: P +) : Configurable { + + companion object { + + suspend operator fun , + E : TtsEngine.Error, V : TtsEngine.Voice> invoke( + engine: TtsEngine, + contentIterator: TtsContentIterator, + initialPreferences: P, + ): TtsPlayer? { + + val initialContext = tryOrNull { contentIterator.startContext() } + ?: return null + + val ttsEngineFacade = + TtsEngineFacade( + engine + ) + + return TtsPlayer( + ttsEngineFacade, + contentIterator, + initialContext, + initialPreferences + ) + } + + private suspend fun TtsContentIterator.startContext(): UtteranceWindow? { + val previousUtterance = previousUtterance() + val currentUtterance = nextUtterance() + + val startWindow = if (currentUtterance != null) { + UtteranceWindow( + previousUtterance = previousUtterance, + currentUtterance = currentUtterance, + nextUtterance = nextUtterance(), + ended = false + ) + } else { + val actualCurrentUtterance = previousUtterance ?: return null + val actualPreviousUtterance = previousUtterance() + + // Go back to the end of the iterator. + nextUtterance() + + UtteranceWindow( + previousUtterance = actualPreviousUtterance, + currentUtterance = actualCurrentUtterance, + nextUtterance = null, + ended = true + ) + } + + return startWindow + } + } + + /** + * State of the player. + */ + sealed interface State { + + /** + * The player is ready to play. + */ + object Ready : State + + /** + * The end of the media has been reached. + */ + object Ended : State + + /** + * The player cannot play because an error occurred. + */ + sealed class Error : State { + + data class EngineError (val error: E) : Error() + + data class ContentError(val exception: Exception) : Error() + } + } + + data class Playback( + val state: State, + val playWhenReady: Boolean, + ) + + data class Utterance( + val text: String, + val position: Position, + val range: IntRange? + ) { + + data class Position( + val resourceIndex: Int, + val cssSelector: String, + val textBefore: String?, + val textAfter: String?, + ) + } + + private data class UtteranceWindow( + val previousUtterance: TtsContentIterator.Utterance?, + val currentUtterance: TtsContentIterator.Utterance, + val nextUtterance: TtsContentIterator.Utterance?, + val ended: Boolean = false + ) + + private val coroutineScope: CoroutineScope = + MainScope() + + private var utteranceWindow: UtteranceWindow = + initialWindow + + private var playbackJob: Job? = + null + + private val mutex: Mutex = + Mutex() + + private val playbackMutable: MutableStateFlow = + MutableStateFlow( + Playback( + state = if (initialWindow.ended) State.Ended else State.Ready, + playWhenReady = false + ) + ) + + private val utteranceMutable: MutableStateFlow = + MutableStateFlow(initialWindow.currentUtterance.ttsPlayerUtterance()) + + override val settings: StateFlow = + engineFacade.settings + + val voices: Set = + engineFacade.voices + + val playback: StateFlow = + playbackMutable.asStateFlow() + + val utterance: StateFlow = + utteranceMutable.asStateFlow() + + /** + * We need to keep the last submitted preferences because TtsSessionAdapter deals with + * preferences, not settings. + */ + var lastPreferences: P = + initialPreferences + + init { + submitPreferences(initialPreferences) + } + + fun play() { + coroutineScope.launch { + playAsync() + } + } + + private suspend fun playAsync() = mutex.withLock { + if (isPlaying()) { + return + } + + playbackMutable.value = playbackMutable.value.copy(playWhenReady = true) + playIfReadyAndNotPaused() + } + + fun pause() { + coroutineScope.launch { + pauseAsync() + } + } + + private suspend fun pauseAsync() = mutex.withLock { + if (!playbackMutable.value.playWhenReady) { + return + } + + playbackMutable.value = playbackMutable.value.copy(playWhenReady = false) + utteranceMutable.value = utteranceMutable.value.copy(range = null) + playbackJob?.cancelAndJoin() + Unit + } + + fun tryRecover() { + coroutineScope.launch { + tryRecoverAsync() + } + } + + private suspend fun tryRecoverAsync() = mutex.withLock { + playbackMutable.value = playbackMutable.value.copy(state = State.Ready) + utteranceMutable.value = utteranceMutable.value.copy(range = null) + playbackJob?.join() + playIfReadyAndNotPaused() + } + + fun go(locator: Locator) { + coroutineScope.launch { + goAsync(locator) + } + } + + private suspend fun goAsync(locator: Locator) = mutex.withLock { + playbackJob?.cancel() + contentIterator.seek(locator) + resetContext() + playbackJob?.join() + playIfReadyAndNotPaused() + } + + fun go(resourceIndex: Int) { + coroutineScope.launch { + goAsync(resourceIndex) + } + } + + private suspend fun goAsync(resourceIndex: Int) = mutex.withLock { + playbackJob?.cancel() + contentIterator.seekToResource(resourceIndex) + resetContext() + playbackJob?.join() + playIfReadyAndNotPaused() + } + + fun restartUtterance() { + coroutineScope.launch { + restartUtteranceAsync() + } + } + + private suspend fun restartUtteranceAsync() = mutex.withLock { + playbackJob?.cancel() + if (playbackMutable.value.state == State.Ended) { + playbackMutable.value = playbackMutable.value.copy(state = State.Ready) + } + utteranceMutable.value = utteranceMutable.value.copy(range = null) + playbackJob?.join() + playIfReadyAndNotPaused() + } + + fun hasNextUtterance() = + utteranceWindow.nextUtterance != null + + fun nextUtterance() { + coroutineScope.launch { + nextUtteranceAsync() + } + } + + private suspend fun nextUtteranceAsync() = mutex.withLock { + if (utteranceWindow.nextUtterance == null) { + return + } + + playbackJob?.cancel() + tryLoadNextContext() + playbackJob?.join() + playIfReadyAndNotPaused() + } + + fun hasPreviousUtterance() = + utteranceWindow.previousUtterance != null + + fun previousUtterance() { + coroutineScope.launch { + previousUtteranceAsync() + } + } + + private suspend fun previousUtteranceAsync() = mutex.withLock { + if (utteranceWindow.previousUtterance == null) { + return + } + playbackJob?.cancel() + tryLoadPreviousContext() + playbackJob?.join() + playIfReadyAndNotPaused() + } + + fun hasNextResource(): Boolean = + utteranceMutable.value.position.resourceIndex + 1 < contentIterator.resourceCount + + fun nextResource() { + coroutineScope.launch { + nextResourceAsync() + } + } + + private suspend fun nextResourceAsync() = mutex.withLock { + if (!hasNextUtterance()) { + return + } + + playbackJob?.cancel() + val currentIndex = utteranceMutable.value.position.resourceIndex + contentIterator.seekToResource(currentIndex + 1) + resetContext() + playbackJob?.join() + playIfReadyAndNotPaused() + } + + fun hasPreviousResource(): Boolean = + utteranceMutable.value.position.resourceIndex > 0 + + fun previousResource() { + coroutineScope.launch { + previousResourceAsync() + } + } + + private suspend fun previousResourceAsync() = mutex.withLock { + if (!hasPreviousResource()) { + return + } + playbackJob?.cancel() + val currentIndex = utteranceMutable.value.position.resourceIndex + contentIterator.seekToResource(currentIndex - 1) + resetContext() + playbackJob?.join() + playIfReadyAndNotPaused() + } + + private fun playIfReadyAndNotPaused() { + check(playbackJob?.isCompleted ?: true) + if (playback.value.playWhenReady && playback.value.state == State.Ready) { + playbackJob = coroutineScope.launch { + playContinuous() + } + } + } + + private suspend fun tryLoadPreviousContext() { + val contextNow = utteranceWindow + + val previousUtterance = + try { + // Get previously currentUtterance once more + contentIterator.previousUtterance() + + // Get previously previousUtterance once more + contentIterator.previousUtterance() + + // Get new previous utterance + val previousUtterance = contentIterator.previousUtterance() + + // Go to currentUtterance position + contentIterator.nextUtterance() + + // Go to nextUtterance position + contentIterator.nextUtterance() + + previousUtterance + } catch (e: Exception) { + onContentError(e) + return + } + + utteranceWindow = UtteranceWindow( + previousUtterance = previousUtterance, + currentUtterance = checkNotNull(contextNow.previousUtterance), + nextUtterance = contextNow.currentUtterance + ) + utteranceMutable.value = utteranceWindow.currentUtterance.ttsPlayerUtterance() + } + + private suspend fun tryLoadNextContext() { + val contextNow = utteranceWindow + + if (contextNow.nextUtterance == null) { + onEndReached() + return + } + + val nextUtterance = try { + contentIterator.nextUtterance() + } catch (e: Exception) { + onContentError(e) + return + } + + utteranceWindow = UtteranceWindow( + previousUtterance = contextNow.currentUtterance, + currentUtterance = contextNow.nextUtterance, + nextUtterance = nextUtterance + ) + utteranceMutable.value = utteranceWindow.currentUtterance.ttsPlayerUtterance() + if (playbackMutable.value.state == State.Ended) { + playbackMutable.value = playbackMutable.value.copy(state = State.Ready) + } + } + + private suspend fun resetContext() { + val startContext = try { + contentIterator.startContext() + } catch (e: Exception) { + onContentError(e) + return + } + utteranceWindow = checkNotNull(startContext) + if (utteranceWindow.nextUtterance == null && utteranceWindow.ended) { + onEndReached() + } + } + + private fun onEndReached() { + playbackMutable.value = playbackMutable.value.copy( + state = State.Ended, + ) + } + + private suspend fun playContinuous() { + if (!coroutineContext.isActive) { + return + } + + val error = speakUtterance(utteranceWindow.currentUtterance) + + mutex.withLock { + error?.let { exception -> onEngineError(exception) } + tryLoadNextContext() + } + playContinuous() + } + + private suspend fun speakUtterance(utterance: TtsContentIterator.Utterance): E? = + engineFacade.speak(utterance.text, utterance.language, ::onRangeChanged) + + private fun onEngineError(error: E) { + playbackMutable.value = playbackMutable.value.copy( + state = State.Error.EngineError(error) + ) + playbackJob?.cancel() + } + + private fun onContentError(exception: Exception) { + playbackMutable.value = playbackMutable.value.copy( + state = State.Error.ContentError(exception) + ) + playbackJob?.cancel() + } + + private fun onRangeChanged(range: IntRange) { + val newUtterance = utteranceMutable.value.copy(range = range) + utteranceMutable.value = newUtterance + } + + fun close() { + coroutineScope.cancel() + engineFacade.close() + } + + override fun submitPreferences(preferences: P) { + lastPreferences = preferences + engineFacade.submitPreferences(preferences) + contentIterator.language = engineFacade.settings.value.language + contentIterator.overrideContentLanguage = engineFacade.settings.value.overrideContentLanguage + } + + private fun isPlaying() = + playbackMutable.value.playWhenReady && playback.value.state == State.Ready + + private fun TtsContentIterator.Utterance.ttsPlayerUtterance(): Utterance = + Utterance( + text = text, + range = null, + position = Utterance.Position( + resourceIndex = resourceIndex, + cssSelector = cssSelector, + textAfter = textAfter, + textBefore = textBefore + ) + ) +} diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/android/AndroidTtsDefaults.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/android/AndroidTtsDefaults.kt new file mode 100644 index 0000000000..a46e75eb7b --- /dev/null +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/android/AndroidTtsDefaults.kt @@ -0,0 +1,30 @@ +/* + * Copyright 2022 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + +package org.readium.r2.navigator.media3.tts.android + +import org.readium.r2.shared.ExperimentalReadiumApi +import org.readium.r2.shared.util.Language + +/** + * Default values for the Android TTS engine. + * + * These values will be used as a last resort by [AndroidTtsSettingsResolver] + * when no user preference takes precedence. + * + * @see AndroidTtsPreferences + */ +@ExperimentalReadiumApi +data class AndroidTtsDefaults( + val language: Language? = null, + val pitch: Double? = null, + val speed: Double? = null +) { + init { + require(pitch == null || pitch > 0) + require(speed == null || speed > 0) + } +} diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/android/AndroidTtsEngine.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/android/AndroidTtsEngine.kt new file mode 100644 index 0000000000..289f46c9ae --- /dev/null +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/android/AndroidTtsEngine.kt @@ -0,0 +1,322 @@ +/* + * Copyright 2022 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + +package org.readium.r2.navigator.media3.tts.android + +import android.content.Context +import android.content.Intent +import android.content.pm.PackageManager +import android.os.Build +import android.speech.tts.TextToSpeech +import android.speech.tts.TextToSpeech.* +import android.speech.tts.UtteranceProgressListener +import android.speech.tts.Voice as AndroidVoice +import android.speech.tts.Voice.* +import kotlinx.coroutines.CompletableDeferred +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.StateFlow +import kotlinx.coroutines.flow.asStateFlow +import org.readium.r2.navigator.media3.tts.TtsEngine +import org.readium.r2.shared.ExperimentalReadiumApi +import org.readium.r2.shared.util.Language + +/** + * Default [TtsEngine] implementation using Android's native text to speech engine. + */ +@ExperimentalReadiumApi +class AndroidTtsEngine private constructor( + private val engine: TextToSpeech, + private val settingsResolver: SettingsResolver, + private val voiceSelector: VoiceSelector, + private val listener: Listener?, + initialPreferences: AndroidTtsPreferences +) : TtsEngine { + + companion object { + + suspend operator fun invoke( + context: Context, + settingsResolver: SettingsResolver, + voiceSelector: VoiceSelector, + listener: Listener?, + initialPreferences: AndroidTtsPreferences + ): AndroidTtsEngine? { + + val init = CompletableDeferred() + + val initListener = OnInitListener { status -> + init.complete(status == SUCCESS) + } + val engine = TextToSpeech(context, initListener) + + return if (init.await()) + AndroidTtsEngine(engine, settingsResolver, voiceSelector, listener, initialPreferences) + else + null + } + + /** + * Starts the activity to install additional voice data. + */ + fun requestInstallVoice(context: Context) { + val intent = Intent() + .setAction(Engine.ACTION_INSTALL_TTS_DATA) + .setFlags(Intent.FLAG_ACTIVITY_NEW_TASK) + + val availableActivities = + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.TIRAMISU) { + context.packageManager.queryIntentActivities( + intent, + PackageManager.ResolveInfoFlags.of(0) + ) + } else { + @Suppress("Deprecation") + context.packageManager.queryIntentActivities(intent, 0) + } + + if (availableActivities.isNotEmpty()) { + context.startActivity(intent) + } + } + } + + fun interface SettingsResolver { + + /** + * Computes a set of engine settings from the engine preferences. + */ + fun settings(preferences: AndroidTtsPreferences): AndroidTtsSettings + } + + fun interface VoiceSelector { + + /** + * Selects a voice for the given [language]. + */ + fun voice(language: Language?, availableVoices: Set): Voice? + } + + class Error(code: Int) : TtsEngine.Error { + + val kind: Kind = + Kind.getOrDefault(code) + + /** + * Android's TTS error code. + * See https://developer.android.com/reference/android/speech/tts/TextToSpeech#ERROR + */ + enum class Kind(val code: Int) { + /** Denotes a generic operation failure. */ + Unknown(-1), + /** Denotes a failure caused by an invalid request. */ + InvalidRequest(-8), + /** Denotes a failure caused by a network connectivity problems. */ + Network(-6), + /** Denotes a failure caused by network timeout. */ + NetworkTimeout(-7), + /** Denotes a failure caused by an unfinished download of the voice data. */ + NotInstalledYet(-9), + /** Denotes a failure related to the output (audio device or a file). */ + Output(-5), + /** Denotes a failure of a TTS service. */ + Service(-4), + /** Denotes a failure of a TTS engine to synthesize the given input. */ + Synthesis(-3); + + companion object { + + fun getOrDefault(key: Int): Kind = + values() + .firstOrNull { it.code == key } + ?: Unknown + } + } + } + + /** + * Represents a voice provided by the TTS engine which can speak an utterance. + * + * @param id Unique and stable identifier for this voice + * @param language Language (and region) this voice belongs to. + * @param quality Voice quality. + * @param requiresNetwork Indicates whether using this voice requires an Internet connection. + */ + data class Voice( + val id: Id, + override val language: Language, + val quality: Quality = Quality.Normal, + val requiresNetwork: Boolean = false, + ) : TtsEngine.Voice { + + @kotlinx.serialization.Serializable + @JvmInline + value class Id(val value: String) + + enum class Quality { + Lowest, Low, Normal, High, Highest + } + } + + interface Listener { + + fun onMissingData(language: Language) + + fun onLanguageNotSupported(language: Language) + } + + private val _settings: MutableStateFlow = + MutableStateFlow(settingsResolver.settings(initialPreferences)) + + private var utteranceListener: TtsEngine.Listener? = + null + + override val voices: Set get() = + engine.voices + ?.map { it.toTtsEngineVoice() } + ?.toSet() + .orEmpty() + + override fun setListener( + listener: TtsEngine.Listener? + ) { + if (listener == null) { + engine.setOnUtteranceProgressListener(null) + this@AndroidTtsEngine.utteranceListener = null + } else { + this@AndroidTtsEngine.utteranceListener = listener + engine.setOnUtteranceProgressListener(UtteranceListener(listener)) + } + } + + override fun speak( + requestId: TtsEngine.RequestId, + text: String, + language: Language? + ) { + engine.setupVoice(settings.value, language, voices) + val queued = engine.speak(text, QUEUE_ADD, null, requestId.id) + if (queued == ERROR) { + utteranceListener?.onError(requestId, Error(Error.Kind.Unknown.code)) + } + } + + override fun stop() { + engine.stop() + } + + override fun close() { + engine.shutdown() + } + + override val settings: StateFlow = + _settings.asStateFlow() + + override fun submitPreferences(preferences: AndroidTtsPreferences) { + val newSettings = settingsResolver.settings(preferences) + engine.setupPitchAndSpeed(newSettings) + _settings.value = newSettings + } + + private fun TextToSpeech.setupPitchAndSpeed(settings: AndroidTtsSettings) { + setSpeechRate(settings.speed.toFloat()) + setPitch(settings.pitch.toFloat()) + } + + private fun TextToSpeech.setupVoice( + settings: AndroidTtsSettings, + utteranceLanguage: Language?, + voices: Set + ) { + val language = utteranceLanguage + .takeUnless { settings.overrideContentLanguage } + ?: settings.language + + when (engine.isLanguageAvailable(language.locale)) { + LANG_MISSING_DATA -> listener?.onMissingData(language) + LANG_NOT_SUPPORTED -> listener?.onLanguageNotSupported(language) + } + + val preferredVoiceWithRegion = + settings.voices[language] + ?.let { voiceForName(it.value) } + + val preferredVoiceWithoutRegion = + settings.voices[language.removeRegion()] + ?.let { voiceForName(it.value) } + + val voice = preferredVoiceWithRegion + ?: preferredVoiceWithoutRegion + ?: defaultVoice(language, voices) + + voice + ?.let { engine.voice = it } + ?: run { engine.language = language.locale } + } + + private fun defaultVoice(language: Language?, voices: Set): AndroidVoice? = + voiceSelector + .voice(language, voices) + ?.let { voiceForName(it.id.value) } + + private fun voiceForName(name: String) = + engine.voices + .firstOrNull { it.name == name } + + private fun AndroidVoice.toTtsEngineVoice() = + Voice( + id = Voice.Id(name), + language = Language(locale), + quality = when (quality) { + QUALITY_VERY_HIGH -> Voice.Quality.Highest + QUALITY_HIGH -> Voice.Quality.High + QUALITY_NORMAL -> Voice.Quality.Normal + QUALITY_LOW -> Voice.Quality.Low + QUALITY_VERY_LOW -> Voice.Quality.Lowest + else -> throw IllegalStateException("Unexpected voice quality.") + }, + requiresNetwork = isNetworkConnectionRequired + ) + + class UtteranceListener( + private val listener: TtsEngine.Listener? + ) : UtteranceProgressListener() { + override fun onStart(utteranceId: String) { + listener?.onStart(TtsEngine.RequestId(utteranceId)) + } + + override fun onStop(utteranceId: String, interrupted: Boolean) { + listener?.let { + val requestId = TtsEngine.RequestId(utteranceId) + if (interrupted) { + it.onInterrupted(requestId) + } else { + it.onFlushed(requestId) + } + } + } + + override fun onDone(utteranceId: String) { + listener?.onDone(TtsEngine.RequestId(utteranceId)) + } + + @Deprecated("Deprecated in the interface", ReplaceWith("onError(utteranceId, -1)")) + override fun onError(utteranceId: String) { + onError(utteranceId, -1) + } + + override fun onError(utteranceId: String, errorCode: Int) { + listener?.onError( + TtsEngine.RequestId(utteranceId), + Error(errorCode) + ) + } + + override fun onRangeStart(utteranceId: String, start: Int, end: Int, frame: Int) { + listener?.onRange(TtsEngine.RequestId(utteranceId), start until end) + } + } +} diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/android/AndroidTtsEngineProvider.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/android/AndroidTtsEngineProvider.kt new file mode 100644 index 0000000000..6b27277e1a --- /dev/null +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/android/AndroidTtsEngineProvider.kt @@ -0,0 +1,99 @@ +/* + * Copyright 2022 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + +package org.readium.r2.navigator.media3.tts.android + +import android.content.Context +import androidx.media3.common.PlaybackException +import androidx.media3.common.PlaybackException.* +import androidx.media3.common.PlaybackParameters +import org.readium.r2.navigator.media3.tts.TtsEngineProvider +import org.readium.r2.shared.ExperimentalReadiumApi +import org.readium.r2.shared.publication.Metadata +import org.readium.r2.shared.publication.Publication + +@ExperimentalReadiumApi +@androidx.annotation.OptIn(androidx.media3.common.util.UnstableApi::class) +class AndroidTtsEngineProvider( + private val context: Context, + private val defaults: AndroidTtsDefaults = AndroidTtsDefaults(), + private val listener: AndroidTtsEngine.Listener? = null, + private val voiceSelector: AndroidTtsEngine.VoiceSelector = AndroidTtsEngine.VoiceSelector { _, _ -> null } +) : TtsEngineProvider { + + override suspend fun createEngine( + publication: Publication, + initialPreferences: AndroidTtsPreferences + ): AndroidTtsEngine? { + val settingsResolver = + AndroidTtsSettingsResolver(publication.metadata, defaults) + + return AndroidTtsEngine( + context, + settingsResolver, + voiceSelector, + listener, + initialPreferences + ) + } + + fun computeSettings( + metadata: Metadata, + preferences: AndroidTtsPreferences + ): AndroidTtsSettings = + AndroidTtsSettingsResolver(metadata, defaults).settings(preferences) + + override fun createPreferencesEditor( + publication: Publication, + initialPreferences: AndroidTtsPreferences + ): AndroidTtsPreferencesEditor = + AndroidTtsPreferencesEditor(initialPreferences, publication.metadata, defaults) + + override fun createEmptyPreferences(): AndroidTtsPreferences = + AndroidTtsPreferences() + + override fun getPlaybackParameters( + settings: AndroidTtsSettings + ): PlaybackParameters { + return PlaybackParameters(settings.speed.toFloat(), settings.pitch.toFloat()) + } + + override fun updatePlaybackParameters( + previousPreferences: AndroidTtsPreferences, + playbackParameters: PlaybackParameters + ): AndroidTtsPreferences { + return previousPreferences.copy( + speed = playbackParameters.speed.toDouble(), + pitch = playbackParameters.pitch.toDouble() + ) + } + + override fun mapEngineError(error: AndroidTtsEngine.Error): PlaybackException { + val errorCode = when (error.kind) { + AndroidTtsEngine.Error.Kind.Unknown -> + ERROR_CODE_UNSPECIFIED + AndroidTtsEngine.Error.Kind.InvalidRequest -> + ERROR_CODE_IO_BAD_HTTP_STATUS + AndroidTtsEngine.Error.Kind.Network -> + ERROR_CODE_IO_NETWORK_CONNECTION_FAILED + AndroidTtsEngine.Error.Kind.NetworkTimeout -> + ERROR_CODE_IO_NETWORK_CONNECTION_TIMEOUT + AndroidTtsEngine.Error.Kind.NotInstalledYet -> + ERROR_CODE_UNSPECIFIED + AndroidTtsEngine.Error.Kind.Output -> + ERROR_CODE_UNSPECIFIED + AndroidTtsEngine.Error.Kind.Service -> + ERROR_CODE_UNSPECIFIED + AndroidTtsEngine.Error.Kind.Synthesis -> + ERROR_CODE_UNSPECIFIED + } + + val message = "Android TTS engine error: ${error.kind.code}" + + return PlaybackException(message, null, errorCode) + } +} diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/android/AndroidTtsPreferences.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/android/AndroidTtsPreferences.kt new file mode 100644 index 0000000000..b397d3712c --- /dev/null +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/android/AndroidTtsPreferences.kt @@ -0,0 +1,43 @@ +/* + * Copyright 2022 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + +package org.readium.r2.navigator.media3.tts.android + +import kotlinx.serialization.Serializable +import org.readium.r2.navigator.media3.tts.TtsEngine +import org.readium.r2.shared.ExperimentalReadiumApi +import org.readium.r2.shared.util.Language + +/** + * Preferences for the the Android built-in TTS engine. + * + * @param language Language of the publication content. + * @param pitch Playback pitch rate. + * @param speed Playback speed rate. + * @param voices Map of preferred voices for specific languages. + */ +@ExperimentalReadiumApi +@Serializable +data class AndroidTtsPreferences( + override val language: Language? = null, + val pitch: Double? = null, + val speed: Double? = null, + val voices: Map? = null, +) : TtsEngine.Preferences { + + init { + require(pitch == null || pitch > 0) + require(speed == null || speed > 0) + } + + override fun plus(other: AndroidTtsPreferences): AndroidTtsPreferences = + AndroidTtsPreferences( + language = other.language ?: language, + pitch = other.pitch ?: pitch, + speed = other.speed ?: speed, + voices = other.voices ?: voices, + ) +} diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/android/AndroidTtsPreferencesEditor.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/android/AndroidTtsPreferencesEditor.kt new file mode 100644 index 0000000000..b8a47e159e --- /dev/null +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/android/AndroidTtsPreferencesEditor.kt @@ -0,0 +1,92 @@ +/* + * Copyright 2022 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + +package org.readium.r2.navigator.media3.tts.android + +import org.readium.r2.navigator.extensions.format +import org.readium.r2.navigator.preferences.* +import org.readium.r2.shared.ExperimentalReadiumApi +import org.readium.r2.shared.publication.Metadata +import org.readium.r2.shared.util.Language + +/** + * Editor for a set of [AndroidTtsPreferences]. + * + * Use [AndroidTtsPreferencesEditor] to assist you in building a preferences user interface or modifying + * existing preferences. It includes rules for adjusting preferences, such as the supported values + * or ranges. + */ +@ExperimentalReadiumApi +class AndroidTtsPreferencesEditor( + initialPreferences: AndroidTtsPreferences, + publicationMetadata: Metadata, + defaults: AndroidTtsDefaults, +) : PreferencesEditor { + + private data class State( + val preferences: AndroidTtsPreferences, + val settings: AndroidTtsSettings + ) + + private val settingsResolver: AndroidTtsSettingsResolver = + AndroidTtsSettingsResolver(publicationMetadata, defaults) + + private var state: State = + initialPreferences.toState() + + override val preferences: AndroidTtsPreferences + get() = state.preferences + + override fun clear() { + updateValues { AndroidTtsPreferences() } + } + + val language: Preference = + PreferenceDelegate( + getValue = { preferences.language }, + getEffectiveValue = { state.settings.language }, + getIsEffective = { true }, + updateValue = { value -> updateValues { it.copy(language = value) } }, + ) + + val pitch: RangePreference = + RangePreferenceDelegate( + getValue = { preferences.pitch }, + getEffectiveValue = { state.settings.pitch }, + getIsEffective = { true }, + updateValue = { value -> updateValues { it.copy(pitch = value) } }, + supportedRange = 0.1..Double.MAX_VALUE, + progressionStrategy = DoubleIncrement(0.1), + valueFormatter = { "${it.format(2)}x" }, + ) + + val speed: RangePreference = + RangePreferenceDelegate( + getValue = { preferences.speed }, + getEffectiveValue = { state.settings.speed }, + getIsEffective = { true }, + updateValue = { value -> updateValues { it.copy(speed = value) } }, + supportedRange = 0.1..Double.MAX_VALUE, + progressionStrategy = DoubleIncrement(0.1), + valueFormatter = { "${it.format(2)}x" }, + ) + + val voices: Preference> = + PreferenceDelegate( + getValue = { preferences.voices }, + getEffectiveValue = { state.settings.voices }, + getIsEffective = { true }, + updateValue = { value -> updateValues { it.copy(voices = value) } }, + ) + + private fun updateValues(updater: (AndroidTtsPreferences) -> AndroidTtsPreferences) { + val newPreferences = updater(preferences) + state = newPreferences.toState() + } + + private fun AndroidTtsPreferences.toState() = + State(preferences = this, settings = settingsResolver.settings(this)) +} diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/android/AndroidTtsPreferencesFilters.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/android/AndroidTtsPreferencesFilters.kt new file mode 100644 index 0000000000..971bcd6f52 --- /dev/null +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/android/AndroidTtsPreferencesFilters.kt @@ -0,0 +1,34 @@ +/* + * Copyright 2022 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + +package org.readium.r2.navigator.media3.tts.android + +import org.readium.r2.navigator.preferences.PreferencesFilter +import org.readium.r2.shared.ExperimentalReadiumApi + +/** + * Suggested filter to keep only shared [AndroidTtsPreferences]. + */ +@ExperimentalReadiumApi +object AndroidTtsSharedPreferencesFilter : PreferencesFilter { + + override fun filter(preferences: AndroidTtsPreferences): AndroidTtsPreferences = + preferences.copy( + language = null + ) +} + +/** + * Suggested filter to keep only publication-specific [AndroidTtsPreferences]. + */ +@ExperimentalReadiumApi +object AndroidTtsPublicationPreferencesFilter : PreferencesFilter { + + override fun filter(preferences: AndroidTtsPreferences): AndroidTtsPreferences = + AndroidTtsPreferences( + language = preferences.language + ) +} diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/android/AndroidTtsPreferencesSerializer.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/android/AndroidTtsPreferencesSerializer.kt new file mode 100644 index 0000000000..e6e0b948b2 --- /dev/null +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/android/AndroidTtsPreferencesSerializer.kt @@ -0,0 +1,24 @@ +/* + * Copyright 2022 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + +package org.readium.r2.navigator.media3.tts.android + +import kotlinx.serialization.json.Json +import org.readium.r2.navigator.preferences.PreferencesSerializer +import org.readium.r2.shared.ExperimentalReadiumApi + +/** + * JSON serializer of [AndroidTtsPreferences]. + */ +@ExperimentalReadiumApi +class AndroidTtsPreferencesSerializer : PreferencesSerializer { + + override fun serialize(preferences: AndroidTtsPreferences): String = + Json.encodeToString(AndroidTtsPreferences.serializer(), preferences) + + override fun deserialize(preferences: String): AndroidTtsPreferences = + Json.decodeFromString(AndroidTtsPreferences.serializer(), preferences) +} diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/android/AndroidTtsSettings.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/android/AndroidTtsSettings.kt new file mode 100644 index 0000000000..4f0542fd04 --- /dev/null +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/android/AndroidTtsSettings.kt @@ -0,0 +1,25 @@ +/* + * Copyright 2022 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + +package org.readium.r2.navigator.media3.tts.android + +import org.readium.r2.navigator.media3.tts.TtsEngine +import org.readium.r2.shared.ExperimentalReadiumApi +import org.readium.r2.shared.util.Language + +/** + * Settings values of the Android built-in TTS engine. + * + * @see AndroidTtsPreferences + */ +@ExperimentalReadiumApi +data class AndroidTtsSettings( + override val language: Language, + override val overrideContentLanguage: Boolean, + val pitch: Double, + val speed: Double, + val voices: Map, +) : TtsEngine.Settings diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/android/AndroidTtsSettingsResolver.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/android/AndroidTtsSettingsResolver.kt new file mode 100644 index 0000000000..5a17cefd8f --- /dev/null +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/android/AndroidTtsSettingsResolver.kt @@ -0,0 +1,34 @@ +/* + * Copyright 2022 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + +package org.readium.r2.navigator.media3.tts.android + +import androidx.compose.ui.text.intl.Locale +import org.readium.r2.shared.ExperimentalReadiumApi +import org.readium.r2.shared.publication.Metadata +import org.readium.r2.shared.util.Language + +@ExperimentalReadiumApi +internal class AndroidTtsSettingsResolver( + private val metadata: Metadata, + private val defaults: AndroidTtsDefaults +) : AndroidTtsEngine.SettingsResolver { + + override fun settings(preferences: AndroidTtsPreferences): AndroidTtsSettings { + val language = preferences.language + ?: metadata.language + ?: defaults.language + ?: Language(Locale.current.toLanguageTag()) + + return AndroidTtsSettings( + language = language, + voices = preferences.voices ?: emptyMap(), + pitch = preferences.pitch ?: defaults.pitch ?: 1.0, + speed = preferences.speed ?: defaults.speed ?: 1.0, + overrideContentLanguage = preferences.language != null + ) + } +} diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/session/AudioBecomingNoisyManager.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/session/AudioBecomingNoisyManager.kt new file mode 100644 index 0000000000..57df1b096d --- /dev/null +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/session/AudioBecomingNoisyManager.kt @@ -0,0 +1,79 @@ +/* + * Copyright (C) 2019 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.readium.r2.navigator.media3.tts.session + +import android.content.BroadcastReceiver +import android.content.Context +import android.content.Intent +import android.content.IntentFilter +import android.media.AudioManager +import android.os.Handler + +internal class AudioBecomingNoisyManager( + context: Context, + eventHandler: Handler, + listener: EventListener +) { + private val context: Context + private val receiver: AudioBecomingNoisyReceiver + private var receiverRegistered = false + + interface EventListener { + fun onAudioBecomingNoisy() + } + + init { + this.context = context.applicationContext + receiver = AudioBecomingNoisyReceiver(eventHandler, listener) + } + + /** + * Enables the [AudioBecomingNoisyManager] which calls [ ][EventListener.onAudioBecomingNoisy] + * upon receiving an intent of [ ][AudioManager.ACTION_AUDIO_BECOMING_NOISY]. + * + * @param enabled True if the listener should be notified when audio is becoming noisy. + */ + fun setEnabled(enabled: Boolean) { + if (enabled && !receiverRegistered) { + context.registerReceiver( + receiver, IntentFilter(AudioManager.ACTION_AUDIO_BECOMING_NOISY) + ) + receiverRegistered = true + } else if (!enabled && receiverRegistered) { + context.unregisterReceiver(receiver) + receiverRegistered = false + } + } + + private inner class AudioBecomingNoisyReceiver( + private val eventHandler: Handler, + private val listener: EventListener + ) : + BroadcastReceiver(), Runnable { + override fun onReceive(context: Context, intent: Intent) { + if (AudioManager.ACTION_AUDIO_BECOMING_NOISY == intent.action) { + eventHandler.post(this) + } + } + + override fun run() { + if (receiverRegistered) { + listener.onAudioBecomingNoisy() + } + } + } +} diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/session/AudioFocusManager.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/session/AudioFocusManager.kt new file mode 100644 index 0000000000..1532391955 --- /dev/null +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/session/AudioFocusManager.kt @@ -0,0 +1,421 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.readium.r2.navigator.media3.tts.session + +import android.content.Context +import android.media.AudioFocusRequest +import android.media.AudioManager +import android.media.AudioManager.OnAudioFocusChangeListener +import android.os.Handler +import androidx.annotation.IntDef +import androidx.annotation.RequiresApi +import androidx.annotation.VisibleForTesting +import androidx.media3.common.AudioAttributes +import androidx.media3.common.C +import androidx.media3.common.Player +import androidx.media3.common.util.Log +import androidx.media3.common.util.Util +import org.readium.r2.navigator.media3.tts.session.AudioFocusManager.PlayerControl + +@androidx.annotation.OptIn(androidx.media3.common.util.UnstableApi::class) +/** Manages requesting and responding to changes in audio focus. + * + * @param context The current context. + * @param eventHandler A [Handler] to for the thread on which the player is used. + * @param playerControl A [PlayerControl] to handle commands from this instance. + */ +internal class AudioFocusManager( + context: Context, + eventHandler: Handler, + playerControl: PlayerControl +) { + /** Interface to allow AudioFocusManager to give commands to a player. */ + interface PlayerControl { + /** + * Called when the volume multiplier on the player should be changed. + * + * @param volumeMultiplier The new volume multiplier. + */ + fun setVolumeMultiplier(volumeMultiplier: Float) + + /** + * Called when a command must be executed on the player. + * + * @param playerCommand The command that must be executed. + */ + fun executePlayerCommand(playerCommand: @PlayerCommand Int) + } + + /** + * Player commands. One of [.PLAYER_COMMAND_DO_NOT_PLAY], [ ][.PLAYER_COMMAND_WAIT_FOR_CALLBACK] + * or [.PLAYER_COMMAND_PLAY_WHEN_READY]. + */ + @Target(AnnotationTarget.TYPE) + @MustBeDocumented + @Retention(AnnotationRetention.SOURCE) + @IntDef( + PLAYER_COMMAND_DO_NOT_PLAY, PLAYER_COMMAND_WAIT_FOR_CALLBACK, PLAYER_COMMAND_PLAY_WHEN_READY + ) + annotation class PlayerCommand + + /** Audio focus state. */ + @Target(AnnotationTarget.TYPE) + @MustBeDocumented + @Retention(AnnotationRetention.SOURCE) + @IntDef( + AUDIO_FOCUS_STATE_NO_FOCUS, + AUDIO_FOCUS_STATE_HAVE_FOCUS, + AUDIO_FOCUS_STATE_LOSS_TRANSIENT, + AUDIO_FOCUS_STATE_LOSS_TRANSIENT_DUCK + ) + private annotation class AudioFocusState + + /** + * Audio focus types. One of [.AUDIOFOCUS_NONE], [.AUDIOFOCUS_GAIN], [ ][.AUDIOFOCUS_GAIN_TRANSIENT], [.AUDIOFOCUS_GAIN_TRANSIENT_MAY_DUCK] or [ ][.AUDIOFOCUS_GAIN_TRANSIENT_EXCLUSIVE]. + */ + @Target(AnnotationTarget.TYPE) + @MustBeDocumented + @Retention(AnnotationRetention.SOURCE) + @IntDef( + AUDIOFOCUS_NONE, + AUDIOFOCUS_GAIN, + AUDIOFOCUS_GAIN_TRANSIENT, + AUDIOFOCUS_GAIN_TRANSIENT_MAY_DUCK, + AUDIOFOCUS_GAIN_TRANSIENT_EXCLUSIVE + ) + private annotation class AudioFocusGain + + private val audioManager: AudioManager + private val focusListener: AudioFocusListener + private var playerControl: PlayerControl? + private var audioAttributes: AudioAttributes? = null + private var audioFocusState: @AudioFocusState Int + private var focusGainToRequest = 0 + + /** Gets the current player volume multiplier. */ + var volumeMultiplier = VOLUME_MULTIPLIER_DEFAULT + private set + private lateinit var audioFocusRequest: AudioFocusRequest + private var rebuildAudioFocusRequest = false + + init { + audioManager = checkNotNull( + context.applicationContext.getSystemService(Context.AUDIO_SERVICE) as AudioManager + ) + this.playerControl = playerControl + focusListener = AudioFocusListener(eventHandler) + audioFocusState = AUDIO_FOCUS_STATE_NO_FOCUS + } + + /** + * Sets audio attributes that should be used to manage audio focus. + * + * + * Call [.updateAudioFocus] to update the audio focus based on these + * attributes. + * + * @param audioAttributes The audio attributes or `null` if audio focus should not be + * managed automatically. + */ + fun setAudioAttributes(audioAttributes: AudioAttributes?) { + if (!Util.areEqual(this.audioAttributes, audioAttributes)) { + this.audioAttributes = audioAttributes + focusGainToRequest = convertAudioAttributesToFocusGain(audioAttributes) + require( + focusGainToRequest == AUDIOFOCUS_GAIN || focusGainToRequest == AUDIOFOCUS_NONE + ) { "Automatic handling of audio focus is only available for USAGE_MEDIA and USAGE_GAME." } + } + } + + /** + * Called by the player to abandon or request audio focus based on the desired player state. + * + * @param playWhenReady The desired value of playWhenReady. + * @param playbackState The desired playback state. + * @return A [PlayerCommand] to execute on the player. + */ + fun updateAudioFocus( + playWhenReady: Boolean, + playbackState: @Player.State Int + ): @PlayerCommand Int { + if (shouldAbandonAudioFocusIfHeld(playbackState)) { + abandonAudioFocusIfHeld() + return if (playWhenReady) PLAYER_COMMAND_PLAY_WHEN_READY else PLAYER_COMMAND_DO_NOT_PLAY + } + return if (playWhenReady) requestAudioFocus() else PLAYER_COMMAND_DO_NOT_PLAY + } + + /** + * Called when the manager is no longer required. Audio focus will be released without making any + * calls to the [PlayerControl]. + */ + fun release() { + playerControl = null + abandonAudioFocusIfHeld() + } + + // Internal methods. + @VisibleForTesting + fun /* package */getFocusListener(): OnAudioFocusChangeListener { + return focusListener + } + + private fun shouldAbandonAudioFocusIfHeld(playbackState: @Player.State Int): Boolean { + return playbackState == Player.STATE_IDLE || focusGainToRequest != AUDIOFOCUS_GAIN + } + + private fun requestAudioFocus(): @PlayerCommand Int { + if (audioFocusState == AUDIO_FOCUS_STATE_HAVE_FOCUS) { + return PLAYER_COMMAND_PLAY_WHEN_READY + } + val requestResult = + if (Util.SDK_INT >= 26) requestAudioFocusV26() else requestAudioFocusDefault() + return if (requestResult == AudioManager.AUDIOFOCUS_REQUEST_GRANTED) { + setAudioFocusState(AUDIO_FOCUS_STATE_HAVE_FOCUS) + PLAYER_COMMAND_PLAY_WHEN_READY + } else { + setAudioFocusState(AUDIO_FOCUS_STATE_NO_FOCUS) + PLAYER_COMMAND_DO_NOT_PLAY + } + } + + private fun abandonAudioFocusIfHeld() { + if (audioFocusState == AUDIO_FOCUS_STATE_NO_FOCUS) { + return + } + if (Util.SDK_INT >= 26) { + abandonAudioFocusV26() + } else { + abandonAudioFocusDefault() + } + setAudioFocusState(AUDIO_FOCUS_STATE_NO_FOCUS) + } + + @Suppress("Deprecation") + private fun requestAudioFocusDefault(): Int { + return audioManager.requestAudioFocus( + focusListener, + Util.getStreamTypeForAudioUsage(checkNotNull(audioAttributes).usage), + focusGainToRequest + ) + } + + @RequiresApi(26) + private fun requestAudioFocusV26(): Int { + if (!::audioFocusRequest.isInitialized || rebuildAudioFocusRequest) { + val builder = + if (!::audioFocusRequest.isInitialized) + AudioFocusRequest.Builder(focusGainToRequest) + else + AudioFocusRequest.Builder( + audioFocusRequest + ) + val willPauseWhenDucked = willPauseWhenDucked() + audioFocusRequest = builder + .setAudioAttributes( + checkNotNull(audioAttributes).audioAttributesV21.audioAttributes + ) + .setWillPauseWhenDucked(willPauseWhenDucked) + .setOnAudioFocusChangeListener(focusListener) + .build() + rebuildAudioFocusRequest = false + } + return audioManager.requestAudioFocus(audioFocusRequest) + } + + @Suppress("Deprecation") + private fun abandonAudioFocusDefault() { + audioManager.abandonAudioFocus(focusListener) + } + + @RequiresApi(26) + private fun abandonAudioFocusV26() { + if (::audioFocusRequest.isInitialized) { + audioManager.abandonAudioFocusRequest(audioFocusRequest) + } + } + + private fun willPauseWhenDucked(): Boolean { + return audioAttributes != null && audioAttributes!!.contentType == C.AUDIO_CONTENT_TYPE_SPEECH + } + + private fun setAudioFocusState(audioFocusState: @AudioFocusState Int) { + if (this.audioFocusState == audioFocusState) { + return + } + this.audioFocusState = audioFocusState + val volumeMultiplier = + if (audioFocusState == AUDIO_FOCUS_STATE_LOSS_TRANSIENT_DUCK) + VOLUME_MULTIPLIER_DUCK + else + VOLUME_MULTIPLIER_DEFAULT + if (this.volumeMultiplier == volumeMultiplier) { + return + } + this.volumeMultiplier = volumeMultiplier + if (playerControl != null) { + playerControl!!.setVolumeMultiplier(volumeMultiplier) + } + } + + private fun handlePlatformAudioFocusChange(focusChange: Int) { + when (focusChange) { + AudioManager.AUDIOFOCUS_GAIN -> { + setAudioFocusState(AUDIO_FOCUS_STATE_HAVE_FOCUS) + executePlayerCommand(PLAYER_COMMAND_PLAY_WHEN_READY) + return + } + AudioManager.AUDIOFOCUS_LOSS -> { + executePlayerCommand(PLAYER_COMMAND_DO_NOT_PLAY) + abandonAudioFocusIfHeld() + return + } + AudioManager.AUDIOFOCUS_LOSS_TRANSIENT, AudioManager.AUDIOFOCUS_LOSS_TRANSIENT_CAN_DUCK -> { + if (focusChange == AudioManager.AUDIOFOCUS_LOSS_TRANSIENT || willPauseWhenDucked()) { + executePlayerCommand(PLAYER_COMMAND_WAIT_FOR_CALLBACK) + setAudioFocusState(AUDIO_FOCUS_STATE_LOSS_TRANSIENT) + } else { + setAudioFocusState(AUDIO_FOCUS_STATE_LOSS_TRANSIENT_DUCK) + } + return + } + else -> Log.w( + TAG, + "Unknown focus change type: $focusChange" + ) + } + } + + private fun executePlayerCommand(playerCommand: @PlayerCommand Int) { + if (playerControl != null) { + playerControl!!.executePlayerCommand(playerCommand) + } + } + + // Internal audio focus listener. + private inner class AudioFocusListener(private val eventHandler: Handler) : + OnAudioFocusChangeListener { + override fun onAudioFocusChange(focusChange: Int) { + eventHandler.post { handlePlatformAudioFocusChange(focusChange) } + } + } + + companion object { + /** Do not play. */ + const val PLAYER_COMMAND_DO_NOT_PLAY = -1 + + /** Do not play now. Wait for callback to play. */ + const val PLAYER_COMMAND_WAIT_FOR_CALLBACK = 0 + + /** Play freely. */ + const val PLAYER_COMMAND_PLAY_WHEN_READY = 1 + + /** No audio focus is currently being held. */ + private const val AUDIO_FOCUS_STATE_NO_FOCUS = 0 + + /** The requested audio focus is currently held. */ + private const val AUDIO_FOCUS_STATE_HAVE_FOCUS = 1 + + /** Audio focus has been temporarily lost. */ + private const val AUDIO_FOCUS_STATE_LOSS_TRANSIENT = 2 + + /** Audio focus has been temporarily lost, but playback may continue with reduced volume. */ + private const val AUDIO_FOCUS_STATE_LOSS_TRANSIENT_DUCK = 3 + + /** + * @see AudioManager.AUDIOFOCUS_NONE + */ + private const val AUDIOFOCUS_NONE = AudioManager.AUDIOFOCUS_NONE + + /** + * @see AudioManager.AUDIOFOCUS_GAIN + */ + private const val AUDIOFOCUS_GAIN = AudioManager.AUDIOFOCUS_GAIN + + /** + * @see AudioManager.AUDIOFOCUS_GAIN_TRANSIENT + */ + private const val AUDIOFOCUS_GAIN_TRANSIENT = AudioManager.AUDIOFOCUS_GAIN_TRANSIENT + + /** + * @see AudioManager.AUDIOFOCUS_GAIN_TRANSIENT_MAY_DUCK + */ + private const val AUDIOFOCUS_GAIN_TRANSIENT_MAY_DUCK = + AudioManager.AUDIOFOCUS_GAIN_TRANSIENT_MAY_DUCK + + /** + * @see AudioManager.AUDIOFOCUS_GAIN_TRANSIENT_EXCLUSIVE + */ + private const val AUDIOFOCUS_GAIN_TRANSIENT_EXCLUSIVE = + AudioManager.AUDIOFOCUS_GAIN_TRANSIENT_EXCLUSIVE + private const val TAG = "AudioFocusManager" + private const val VOLUME_MULTIPLIER_DUCK = 0.2f + private const val VOLUME_MULTIPLIER_DEFAULT = 1.0f + + /** + * Converts [AudioAttributes] to one of the audio focus request. + * + * + * This follows the class Javadoc of [AudioFocusRequest]. + * + * @param audioAttributes The audio attributes associated with this focus request. + * @return The type of audio focus gain that should be requested. + */ + private fun convertAudioAttributesToFocusGain( + audioAttributes: AudioAttributes? + ): @AudioFocusGain Int { + return if (audioAttributes == null) { + // Don't handle audio focus. It may be either video only contents or developers + // want to have more finer grained control. (e.g. adding audio focus listener) + AUDIOFOCUS_NONE + } else when (audioAttributes.usage) { + C.USAGE_VOICE_COMMUNICATION_SIGNALLING -> AUDIOFOCUS_NONE + C.USAGE_GAME, C.USAGE_MEDIA -> AUDIOFOCUS_GAIN + C.USAGE_UNKNOWN -> { + Log.w( + TAG, + "Specify a proper usage in the audio attributes for audio focus" + + " handling. Using AUDIOFOCUS_GAIN by default." + ) + AUDIOFOCUS_GAIN + } + C.USAGE_ALARM, C.USAGE_VOICE_COMMUNICATION -> AUDIOFOCUS_GAIN_TRANSIENT + C.USAGE_ASSISTANCE_NAVIGATION_GUIDANCE, C.USAGE_ASSISTANCE_SONIFICATION, + C.USAGE_NOTIFICATION, C.USAGE_NOTIFICATION_COMMUNICATION_DELAYED, + C.USAGE_NOTIFICATION_COMMUNICATION_INSTANT, C.USAGE_NOTIFICATION_COMMUNICATION_REQUEST, + C.USAGE_NOTIFICATION_EVENT, C.USAGE_NOTIFICATION_RINGTONE -> + AUDIOFOCUS_GAIN_TRANSIENT_MAY_DUCK + C.USAGE_ASSISTANT -> + if (Util.SDK_INT >= 19) { + AUDIOFOCUS_GAIN_TRANSIENT_EXCLUSIVE + } else { + AUDIOFOCUS_GAIN_TRANSIENT + } + C.USAGE_ASSISTANCE_ACCESSIBILITY -> { + if (audioAttributes.contentType == C.AUDIO_CONTENT_TYPE_SPEECH) { + // Voice shouldn't be interrupted by other playback. + AUDIOFOCUS_GAIN_TRANSIENT + } else AUDIOFOCUS_GAIN_TRANSIENT_MAY_DUCK + } + else -> { + Log.w(TAG, "Unidentified audio usage: " + audioAttributes.usage) + AUDIOFOCUS_NONE + } + } + } + } +} diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/session/StreamVolumeManager.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/session/StreamVolumeManager.kt new file mode 100644 index 0000000000..1190643074 --- /dev/null +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/session/StreamVolumeManager.kt @@ -0,0 +1,219 @@ +/* + * Copyright 2020 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.readium.r2.navigator.media3.tts.session + +import android.content.BroadcastReceiver +import android.content.Context +import android.content.Intent +import android.content.IntentFilter +import android.media.AudioManager +import android.os.Handler +import androidx.media3.common.C +import androidx.media3.common.util.Assertions +import androidx.media3.common.util.Log +import androidx.media3.common.util.Util + +@androidx.annotation.OptIn(androidx.media3.common.util.UnstableApi::class) +/** A manager that wraps [AudioManager] to control/listen audio stream volume. */ /* package */ +internal class StreamVolumeManager(context: Context, eventHandler: Handler, listener: Listener) { + /** A listener for changes in the manager. */ + interface Listener { + /** Called when the audio stream type is changed. */ + fun onStreamTypeChanged(streamType: @C.StreamType Int) + + /** Called when the audio stream volume or mute state is changed. */ + fun onStreamVolumeChanged(streamVolume: Int, streamMuted: Boolean) + } + + private val applicationContext: Context + private val eventHandler: Handler + private val listener: Listener + private val audioManager: AudioManager + private var receiver: VolumeChangeReceiver? = null + private var streamType: @C.StreamType Int + private var volume: Int + private var muted: Boolean + + /** Creates a manager. */ + init { + applicationContext = context.applicationContext + this.eventHandler = eventHandler + this.listener = listener + audioManager = Assertions.checkStateNotNull( + applicationContext.getSystemService(Context.AUDIO_SERVICE) as AudioManager + ) + streamType = C.STREAM_TYPE_DEFAULT + volume = getVolumeFromManager(audioManager, streamType) + muted = getMutedFromManager(audioManager, streamType) + val receiver = VolumeChangeReceiver() + val filter = IntentFilter(VOLUME_CHANGED_ACTION) + try { + applicationContext.registerReceiver(receiver, filter) + this.receiver = receiver + } catch (e: RuntimeException) { + Log.w(TAG, "Error registering stream volume receiver", e) + } + } + + /** Sets the audio stream type. */ + fun setStreamType(streamType: @C.StreamType Int) { + if (this.streamType == streamType) { + return + } + this.streamType = streamType + updateVolumeAndNotifyIfChanged() + listener.onStreamTypeChanged(streamType) + } + + /** + * Gets the minimum volume for the current audio stream. It can be changed if [ ][.setStreamType] is called. + */ + val minVolume: Int + get() = if (Util.SDK_INT >= 28) audioManager.getStreamMinVolume(streamType) else 0 + + /** + * Gets the maximum volume for the current audio stream. It can be changed if [ ][.setStreamType] is called. + */ + val maxVolume: Int + get() = audioManager.getStreamMaxVolume(streamType) + + /** Gets the current volume for the current audio stream. */ + fun getVolume(): Int { + return volume + } + + /** Gets whether the current audio stream is muted or not. */ + fun isMuted(): Boolean { + return muted + } + + /** + * Sets the volume with the given value for the current audio stream. The value should be between + * [.getMinVolume] and [.getMaxVolume], otherwise it will be ignored. + */ + fun setVolume(volume: Int) { + if (volume < minVolume || volume > maxVolume) { + return + } + audioManager.setStreamVolume(streamType, volume, VOLUME_FLAGS) + updateVolumeAndNotifyIfChanged() + } + + /** + * Increases the volume by one for the current audio stream. It will be ignored if the current + * volume is equal to [.getMaxVolume]. + */ + fun increaseVolume() { + if (volume >= maxVolume) { + return + } + audioManager.adjustStreamVolume(streamType, AudioManager.ADJUST_RAISE, VOLUME_FLAGS) + updateVolumeAndNotifyIfChanged() + } + + /** + * Decreases the volume by one for the current audio stream. It will be ignored if the current + * volume is equal to [.getMinVolume]. + */ + fun decreaseVolume() { + if (volume <= minVolume) { + return + } + audioManager.adjustStreamVolume(streamType, AudioManager.ADJUST_LOWER, VOLUME_FLAGS) + updateVolumeAndNotifyIfChanged() + } + + /** Sets the mute state of the current audio stream. */ + fun setMuted(muted: Boolean) { + if (Util.SDK_INT >= 23) { + audioManager.adjustStreamVolume( + streamType, + if (muted) AudioManager.ADJUST_MUTE else AudioManager.ADJUST_UNMUTE, + VOLUME_FLAGS + ) + } else { + @Suppress("Deprecation") + audioManager.setStreamMute(streamType, muted) + } + updateVolumeAndNotifyIfChanged() + } + + /** Releases the manager. It must be called when the manager is no longer required. */ + fun release() { + if (receiver != null) { + try { + applicationContext.unregisterReceiver(receiver) + } catch (e: RuntimeException) { + Log.w(TAG, "Error unregistering stream volume receiver", e) + } + receiver = null + } + } + + private fun updateVolumeAndNotifyIfChanged() { + val newVolume = getVolumeFromManager(audioManager, streamType) + val newMuted = getMutedFromManager(audioManager, streamType) + if (volume != newVolume || muted != newMuted) { + volume = newVolume + muted = newMuted + listener.onStreamVolumeChanged(newVolume, newMuted) + } + } + + private inner class VolumeChangeReceiver : BroadcastReceiver() { + override fun onReceive(context: Context, intent: Intent) { + eventHandler.post { updateVolumeAndNotifyIfChanged() } + } + } + + companion object { + private const val TAG = "StreamVolumeManager" + + // TODO(b/151280453): Replace the hidden intent action with an official one. + // Copied from AudioManager#VOLUME_CHANGED_ACTION + private const val VOLUME_CHANGED_ACTION = "android.media.VOLUME_CHANGED_ACTION" + + // TODO(b/153317944): Allow users to override these flags. + private const val VOLUME_FLAGS = AudioManager.FLAG_SHOW_UI + private fun getVolumeFromManager( + audioManager: AudioManager, + streamType: @C.StreamType Int + ): Int { + // AudioManager#getStreamVolume(int) throws an exception on some devices. See + // https://github.com/google/ExoPlayer/issues/8191. + return try { + audioManager.getStreamVolume(streamType) + } catch (e: RuntimeException) { + Log.w( + TAG, + "Could not retrieve stream volume for stream type $streamType", e + ) + audioManager.getStreamMaxVolume(streamType) + } + } + + private fun getMutedFromManager( + audioManager: AudioManager, + streamType: @C.StreamType Int + ): Boolean { + return if (Util.SDK_INT >= 23) { + audioManager.isStreamMute(streamType) + } else { + getVolumeFromManager(audioManager, streamType) == 0 + } + } + } +} diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/session/TtsSessionAdapter.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/session/TtsSessionAdapter.kt new file mode 100644 index 0000000000..a50c7ac9f2 --- /dev/null +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/session/TtsSessionAdapter.kt @@ -0,0 +1,890 @@ +/* + * Copyright 2022 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + +package org.readium.r2.navigator.media3.tts.session + +import android.app.Application +import android.os.Handler +import android.os.Looper +import android.view.Surface +import android.view.SurfaceHolder +import android.view.SurfaceView +import android.view.TextureView +import androidx.media3.common.* +import androidx.media3.common.C.* +import androidx.media3.common.PlaybackException.* +import androidx.media3.common.Player.* +import androidx.media3.common.text.CueGroup +import androidx.media3.common.util.Clock +import androidx.media3.common.util.ListenerSet +import androidx.media3.common.util.Size +import androidx.media3.common.util.Util +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.MainScope +import kotlinx.coroutines.flow.StateFlow +import kotlinx.coroutines.flow.launchIn +import kotlinx.coroutines.flow.onEach +import org.readium.r2.navigator.media3.tts.TtsEngine +import org.readium.r2.navigator.media3.tts.TtsPlayer +import org.readium.r2.shared.ExperimentalReadiumApi +import org.readium.r2.shared.fetcher.Resource + +/** + * Adapts the [TtsPlayer] to media3 [Player] interface. + */ +@ExperimentalReadiumApi +@androidx.annotation.OptIn(androidx.media3.common.util.UnstableApi::class) +internal class TtsSessionAdapter( + private val application: Application, + private val ttsPlayer: TtsPlayer<*, *, E, *>, + private val playlistMetadata: MediaMetadata, + private val mediaItems: List, + private val onStop: () -> Unit, + private val playbackParametersState: StateFlow, + private val updatePlaybackParameters: (PlaybackParameters) -> Unit, + private val mapEngineError: (E) -> PlaybackException +) : Player { + + private val coroutineScope: CoroutineScope = + MainScope() + + private val eventHandler: Handler = + Handler(applicationLooper) + + private val window: Timeline.Window = + Timeline.Window() + + private var lastPlayback: TtsPlayer.Playback = + ttsPlayer.playback.value + + private var lastPlaybackParameters: PlaybackParameters = + playbackParametersState.value + + private val streamVolumeManager = StreamVolumeManager( + application, + Handler(applicationLooper), + StreamVolumeManagerListener() + ) + + init { + streamVolumeManager.setStreamType(Util.getStreamTypeForAudioUsage(audioAttributes.usage)) + } + + private val audioFocusManager = AudioFocusManager( + application, + eventHandler, + AudioFocusManagerListener() + ) + + init { + audioFocusManager.setAudioAttributes(audioAttributes) + } + + private val audioBecomingNoisyManager = AudioBecomingNoisyManager( + application, + eventHandler, + AudioBecomingNoisyManagerListener() + ) + + init { + audioBecomingNoisyManager.setEnabled(true) + } + + private var deviceInfo: DeviceInfo = + createDeviceInfo(streamVolumeManager) + + init { + ttsPlayer.playback + .onEach { playback -> + notifyListenersPlaybackChanged(lastPlayback, playback) + lastPlayback = playback + audioFocusManager.updateAudioFocus(playback.playWhenReady, playback.state.playerCode) + }.launchIn(coroutineScope) + + playbackParametersState + .onEach { playbackParameters -> + notifyListenersPlaybackParametersChanged(lastPlaybackParameters, playbackParameters) + lastPlaybackParameters = playbackParameters + } + } + + private var listeners: ListenerSet = + ListenerSet( + applicationLooper, + Clock.DEFAULT, + ) { listener: Listener, flags: FlagSet? -> + listener.onEvents(this, Events(flags!!)) + } + + private val permanentAvailableCommands = + Commands.Builder() + .addAll( + COMMAND_PLAY_PAUSE, + COMMAND_STOP, + + // COMMAND_SEEK_BACK, + // COMMAND_SEEK_FORWARD, + + // COMMAND_SEEK_TO_NEXT_MEDIA_ITEM, + // COMMAND_SEEK_TO_PREVIOUS_MEDIA_ITEM, + + COMMAND_GET_AUDIO_ATTRIBUTES, + COMMAND_GET_DEVICE_VOLUME, + COMMAND_SET_DEVICE_VOLUME, + + COMMAND_SET_SPEED_AND_PITCH, + COMMAND_GET_CURRENT_MEDIA_ITEM, + COMMAND_GET_MEDIA_ITEMS_METADATA, + COMMAND_GET_TEXT + ).build() + + override fun getApplicationLooper(): Looper { + return Looper.getMainLooper() + } + + override fun addListener(listener: Listener) { + listeners.add(listener) + } + + override fun removeListener(listener: Listener) { + listeners.remove(listener) + } + + override fun setMediaItems(mediaItems: MutableList) { + } + + override fun setMediaItems(mediaItems: MutableList, resetPosition: Boolean) { + } + + override fun setMediaItems( + mediaItems: MutableList, + startIndex: Int, + startPositionMs: Long + ) { + } + + override fun setMediaItem(mediaItem: MediaItem) { + } + + override fun setMediaItem(mediaItem: MediaItem, startPositionMs: Long) { + } + + override fun setMediaItem(mediaItem: MediaItem, resetPosition: Boolean) { + } + + override fun addMediaItem(mediaItem: MediaItem) { + } + + override fun addMediaItem(index: Int, mediaItem: MediaItem) { + } + + override fun addMediaItems(mediaItems: MutableList) { + } + + override fun addMediaItems(index: Int, mediaItems: MutableList) { + } + + override fun moveMediaItem(currentIndex: Int, newIndex: Int) { + } + + override fun moveMediaItems(fromIndex: Int, toIndex: Int, newIndex: Int) { + } + + override fun removeMediaItem(index: Int) { + } + + override fun removeMediaItems(fromIndex: Int, toIndex: Int) { + } + + override fun clearMediaItems() { + } + + override fun isCommandAvailable(command: Int): Boolean { + return command in availableCommands + } + + override fun canAdvertiseSession(): Boolean { + return true + } + + override fun getAvailableCommands(): Commands { + return Commands.Builder() + .addAll(permanentAvailableCommands) + .build() + } + + override fun prepare() { + ttsPlayer.tryRecover() + } + + override fun getPlaybackState(): Int { + return ttsPlayer.playback.value.state.playerCode + } + + override fun getPlaybackSuppressionReason(): Int { + return PLAYBACK_SUPPRESSION_REASON_NONE + } + + override fun isPlaying(): Boolean { + return ( + playbackState == STATE_READY && playWhenReady && + playbackSuppressionReason == PLAYBACK_SUPPRESSION_REASON_NONE + ) + } + + override fun getPlayerError(): PlaybackException? { + return (lastPlayback.state as? TtsPlayer.State.Error)?.toPlaybackException() + } + + override fun play() { + playWhenReady = true + } + + override fun pause() { + playWhenReady = false + } + + override fun setPlayWhenReady(playWhenReady: Boolean) { + if (playWhenReady) { + ttsPlayer.play() + } else { + ttsPlayer.pause() + } + } + + override fun getPlayWhenReady(): Boolean { + return ttsPlayer.playback.value.playWhenReady + } + + override fun setRepeatMode(repeatMode: Int) { + } + + override fun getRepeatMode(): Int { + return REPEAT_MODE_OFF + } + + override fun setShuffleModeEnabled(shuffleModeEnabled: Boolean) { + } + + override fun getShuffleModeEnabled(): Boolean { + return false + } + + override fun isLoading(): Boolean { + return false + } + + override fun seekToDefaultPosition() { + seekToDefaultPosition(currentMediaItemIndex) + } + + override fun seekToDefaultPosition(mediaItemIndex: Int) { + seekTo(mediaItemIndex, 0L) + } + + override fun seekTo(positionMs: Long) { + } + + override fun seekTo(mediaItemIndex: Int, positionMs: Long) { + val timeline: Timeline = currentTimeline + if (mediaItemIndex < 0 || !timeline.isEmpty && mediaItemIndex >= timeline.windowCount + ) { + throw IllegalSeekPositionException(timeline, mediaItemIndex, positionMs) + } + + ttsPlayer.go(mediaItemIndex) + } + + override fun getSeekBackIncrement(): Long { + return 0 + } + + override fun seekBack() { + ttsPlayer.previousUtterance() + } + + override fun getSeekForwardIncrement(): Long { + return 0 + } + + override fun seekForward() { + ttsPlayer.nextUtterance() + } + + @Deprecated("Deprecated in Java", ReplaceWith("hasPreviousMediaItem()")) + override fun hasPrevious(): Boolean { + return hasPreviousMediaItem() + } + + @Deprecated("Deprecated in Java", ReplaceWith("hasPreviousMediaItem()")) + override fun hasPreviousWindow(): Boolean { + return hasPreviousMediaItem() + } + + override fun hasPreviousMediaItem(): Boolean { + return previousMediaItemIndex != INDEX_UNSET + } + + @Deprecated("Deprecated in Java", ReplaceWith("TODO(\"Not yet implemented\")")) + override fun previous() { + seekToPreviousMediaItem() + } + + @Deprecated("Deprecated in Java", ReplaceWith("TODO(\"Not yet implemented\")")) + override fun seekToPreviousWindow() { + seekToPreviousMediaItem() + } + + override fun seekToPreviousMediaItem() { + val previousMediaItemIndex = previousMediaItemIndex + if (previousMediaItemIndex != INDEX_UNSET) { + seekToDefaultPosition(previousMediaItemIndex) + } + } + + override fun getMaxSeekToPreviousPosition(): Long { + return 0 + } + + override fun seekToPrevious() { + val timeline = currentTimeline + if (timeline.isEmpty || isPlayingAd) { + return + } + val hasPreviousMediaItem = hasPreviousMediaItem() + if (isCurrentMediaItemLive && !isCurrentMediaItemSeekable) { + if (hasPreviousMediaItem) { + seekToPreviousMediaItem() + } + } else if (hasPreviousMediaItem && currentPosition <= maxSeekToPreviousPosition) { + seekToPreviousMediaItem() + } else { + seekTo( /* positionMs= */0) + } + } + + @Deprecated("Deprecated in Java", ReplaceWith("hasNextMediaItem()")) + override fun hasNext(): Boolean { + return hasNextMediaItem() + } + + @Deprecated("Deprecated in Java", ReplaceWith("hasNextMediaItem()")) + override fun hasNextWindow(): Boolean { + return hasNextMediaItem() + } + + override fun hasNextMediaItem(): Boolean { + return nextMediaItemIndex != INDEX_UNSET + } + + @Deprecated("Deprecated in Java", ReplaceWith("seekToNextMediaItem()")) + override fun next() { + seekToNextMediaItem() + } + + @Deprecated("Deprecated in Java", ReplaceWith("seekToNextMediaItem()")) + override fun seekToNextWindow() { + seekToNextMediaItem() + } + + override fun seekToNextMediaItem() { + val nextMediaItemIndex = nextMediaItemIndex + if (nextMediaItemIndex != INDEX_UNSET) { + seekToDefaultPosition(nextMediaItemIndex) + } + } + + override fun seekToNext() { + val timeline = currentTimeline + if (timeline.isEmpty || isPlayingAd) { + return + } + if (hasNextMediaItem()) { + seekToNextMediaItem() + } else if (isCurrentMediaItemLive && isCurrentMediaItemDynamic) { + seekToDefaultPosition() + } + } + + override fun setPlaybackParameters(playbackParameters: PlaybackParameters) { + updatePlaybackParameters(playbackParameters) + } + + override fun setPlaybackSpeed(speed: Float) { + updatePlaybackParameters(playbackParametersState.value.withSpeed(speed)) + } + + override fun getPlaybackParameters(): PlaybackParameters { + return playbackParametersState.value + } + + override fun stop() { + onStop() + } + + @Deprecated("Deprecated in Java") + override fun stop(reset: Boolean) {} + + override fun release() { + streamVolumeManager.release() + audioFocusManager.release() + audioBecomingNoisyManager.setEnabled(false) + eventHandler.removeCallbacksAndMessages(null) + // This object does not own the TtsPlayer instance, do not close it. + } + + override fun getCurrentTracks(): Tracks { + return Tracks.EMPTY + } + + override fun getTrackSelectionParameters(): TrackSelectionParameters { + return TrackSelectionParameters.Builder(application) + .build() + } + + override fun setTrackSelectionParameters(parameters: TrackSelectionParameters) { + } + + override fun getMediaMetadata(): MediaMetadata { + return currentTimeline.getWindow(currentMediaItemIndex, window) + .mediaItem.mediaMetadata + } + + override fun getPlaylistMetadata(): MediaMetadata { + return playlistMetadata + } + + override fun setPlaylistMetadata(mediaMetadata: MediaMetadata) { + throw NotImplementedError() + } + + override fun getCurrentManifest(): Any? { + val timeline = currentTimeline + return if (timeline.isEmpty) + null + else + timeline.getWindow(currentMediaItemIndex, window).manifest + } + + override fun getCurrentTimeline(): Timeline { + // MediaNotificationManager requires a non-empty timeline to start foreground playing. + return TtsTimeline(mediaItems) + } + + override fun getCurrentPeriodIndex(): Int { + return ttsPlayer.utterance.value.position.resourceIndex + } + + @Deprecated("Deprecated in Java", ReplaceWith("currentMediaItemIndex")) + override fun getCurrentWindowIndex(): Int { + return currentMediaItemIndex + } + + override fun getCurrentMediaItemIndex(): Int { + return ttsPlayer.utterance.value.position.resourceIndex + } + + @Deprecated("Deprecated in Java", ReplaceWith("nextMediaItemIndex")) + override fun getNextWindowIndex(): Int { + return nextMediaItemIndex + } + + override fun getNextMediaItemIndex(): Int { + val timeline = currentTimeline + return if (timeline.isEmpty) + INDEX_UNSET + else + timeline.getNextWindowIndex( + currentMediaItemIndex, + getRepeatModeForNavigation(), + shuffleModeEnabled + ) + } + + @Deprecated("Deprecated in Java", ReplaceWith("previousMediaItemIndex")) + override fun getPreviousWindowIndex(): Int { + return previousMediaItemIndex + } + + override fun getPreviousMediaItemIndex(): Int { + val timeline = currentTimeline + return if (timeline.isEmpty) + INDEX_UNSET + else + timeline.getPreviousWindowIndex( + currentMediaItemIndex, getRepeatModeForNavigation(), shuffleModeEnabled + ) + } + + override fun getCurrentMediaItem(): MediaItem? { + val timeline = currentTimeline + return if (timeline.isEmpty) + null + else + timeline.getWindow(currentMediaItemIndex, window).mediaItem + } + + override fun getMediaItemCount(): Int { + return currentTimeline.windowCount + } + + override fun getMediaItemAt(index: Int): MediaItem { + return currentTimeline.getWindow(index, window).mediaItem + } + + override fun getDuration(): Long { + return TIME_UNSET + } + + override fun getCurrentPosition(): Long { + return 0 + } + + override fun getBufferedPosition(): Long { + return 0 + } + + override fun getBufferedPercentage(): Int { + val position = bufferedPosition + val duration = duration + return if (position == TIME_UNSET || duration == TIME_UNSET) + 0 + else if (duration == 0L) + 100 + else Util.constrainValue( + (position * 100 / duration).toInt(), + 0, + 100 + ) + } + + override fun getTotalBufferedDuration(): Long { + return 0 + } + + @Deprecated("Deprecated in Java", ReplaceWith("isCurrentMediaItemDynamic")) + override fun isCurrentWindowDynamic(): Boolean { + return isCurrentMediaItemDynamic + } + + override fun isCurrentMediaItemDynamic(): Boolean { + val timeline = currentTimeline + return !timeline.isEmpty && timeline.getWindow(currentMediaItemIndex, window).isDynamic + } + + @Deprecated("Deprecated in Java", ReplaceWith("isCurrentMediaItemLive")) + override fun isCurrentWindowLive(): Boolean { + return isCurrentMediaItemLive + } + + override fun isCurrentMediaItemLive(): Boolean { + val timeline = currentTimeline + return !timeline.isEmpty && timeline.getWindow(currentMediaItemIndex, window).isLive() + } + + override fun getCurrentLiveOffset(): Long { + val timeline = currentTimeline + if (timeline.isEmpty) { + return TIME_UNSET + } + val windowStartTimeMs = timeline.getWindow(currentMediaItemIndex, window).windowStartTimeMs + return if (windowStartTimeMs == TIME_UNSET) { + TIME_UNSET + } else window.currentUnixTimeMs - window.windowStartTimeMs - contentPosition + } + + @Deprecated("Deprecated in Java", ReplaceWith("isCurrentMediaItemSeekable")) + override fun isCurrentWindowSeekable(): Boolean { + return isCurrentMediaItemSeekable + } + + override fun isCurrentMediaItemSeekable(): Boolean { + val timeline = currentTimeline + return !timeline.isEmpty && timeline.getWindow(currentMediaItemIndex, window).isSeekable + } + + override fun isPlayingAd(): Boolean { + return false + } + + override fun getCurrentAdGroupIndex(): Int { + return INDEX_UNSET + } + + override fun getCurrentAdIndexInAdGroup(): Int { + return INDEX_UNSET + } + + override fun getContentDuration(): Long { + val timeline = currentTimeline + return if (timeline.isEmpty) + TIME_UNSET + else + timeline.getWindow(currentMediaItemIndex, window).durationMs + } + + override fun getContentPosition(): Long { + return 0 + } + + override fun getContentBufferedPosition(): Long { + return 0 + } + + override fun getAudioAttributes(): AudioAttributes { + return AudioAttributes.Builder() + .setUsage(USAGE_MEDIA) + .setContentType(AUDIO_CONTENT_TYPE_SPEECH) + .setAllowedCapturePolicy(ALLOW_CAPTURE_BY_SYSTEM) + .build() + } + + override fun setVolume(volume: Float) { + } + + override fun getVolume(): Float { + return 1.0f + } + + override fun clearVideoSurface() { + } + + override fun clearVideoSurface(surface: Surface?) { + } + + override fun setVideoSurface(surface: Surface?) { + } + + override fun setVideoSurfaceHolder(surfaceHolder: SurfaceHolder?) { + } + + override fun clearVideoSurfaceHolder(surfaceHolder: SurfaceHolder?) { + } + + override fun setVideoSurfaceView(surfaceView: SurfaceView?) { + } + + override fun clearVideoSurfaceView(surfaceView: SurfaceView?) { + } + + override fun setVideoTextureView(textureView: TextureView?) { + } + + override fun clearVideoTextureView(textureView: TextureView?) { + } + + override fun getVideoSize(): VideoSize { + return VideoSize.UNKNOWN + } + + override fun getSurfaceSize(): Size { + return Size.UNKNOWN + } + + override fun getCurrentCues(): CueGroup { + return CueGroup.EMPTY_TIME_ZERO + } + + override fun getDeviceInfo(): DeviceInfo { + return deviceInfo + } + + override fun getDeviceVolume(): Int { + return streamVolumeManager.getVolume() + } + + override fun isDeviceMuted(): Boolean { + return streamVolumeManager.isMuted() + } + + override fun setDeviceVolume(volume: Int) { + streamVolumeManager.setVolume(volume) + } + + override fun increaseDeviceVolume() { + streamVolumeManager.increaseVolume() + } + + override fun decreaseDeviceVolume() { + streamVolumeManager.decreaseVolume() + } + + override fun setDeviceMuted(muted: Boolean) { + streamVolumeManager.setMuted(muted) + } + + private fun notifyListenersPlaybackChanged( + previousPlaybackInfo: TtsPlayer.Playback, + playbackInfo: TtsPlayer.Playback, + // playWhenReadyChangeReason: @Player.PlayWhenReadyChangeReason Int, + ) { + if (previousPlaybackInfo.state as? TtsPlayer.State.Error != playbackInfo.state as? Error) { + listeners.queueEvent( + EVENT_PLAYER_ERROR + ) { listener: Listener -> + listener.onPlayerErrorChanged( + (playbackInfo.state as? TtsPlayer.State.Error)?.toPlaybackException() + ) + } + if (playbackInfo.state is TtsPlayer.State.Error) { + listeners.queueEvent( + EVENT_PLAYER_ERROR + ) { listener: Listener -> + listener.onPlayerError( + playbackInfo.state.toPlaybackException() + ) + } + } + } + + if (previousPlaybackInfo.state != playbackInfo.state) { + listeners.queueEvent( + EVENT_PLAYBACK_STATE_CHANGED + ) { listener: Listener -> + listener.onPlaybackStateChanged( + playbackInfo.state.playerCode + ) + } + } + + if (previousPlaybackInfo.playWhenReady != playbackInfo.playWhenReady) { + listeners.queueEvent( + EVENT_PLAY_WHEN_READY_CHANGED + ) { listener: Listener -> + listener.onPlayWhenReadyChanged( + playbackInfo.playWhenReady, + if (playbackInfo.state == TtsPlayer.State.Ended) + PLAY_WHEN_READY_CHANGE_REASON_END_OF_MEDIA_ITEM + else + PLAY_WHEN_READY_CHANGE_REASON_USER_REQUEST + // PLAYBACK_SUPPRESSION_REASON_NONE + // playWhenReadyChangeReason + ) + } + } + + if (isPlaying(previousPlaybackInfo) != isPlaying(playbackInfo)) { + listeners.queueEvent( + EVENT_IS_PLAYING_CHANGED + ) { listener: Listener -> + listener.onIsPlayingChanged(isPlaying(playbackInfo)) + } + } + + listeners.flushEvents() + } + + private fun notifyListenersPlaybackParametersChanged( + previousPlaybackParameters: PlaybackParameters, + playbackParameters: PlaybackParameters + ) { + if (previousPlaybackParameters != playbackParameters) { + listeners.sendEvent( + EVENT_PLAYBACK_PARAMETERS_CHANGED + ) { listener: Listener -> + listener.onPlaybackParametersChanged( + playbackParameters + ) + } + } + } + + private fun createDeviceInfo(streamVolumeManager: StreamVolumeManager): DeviceInfo { + val newDeviceInfo = DeviceInfo( + DeviceInfo.PLAYBACK_TYPE_LOCAL, + streamVolumeManager.minVolume, + streamVolumeManager.maxVolume + ) + deviceInfo = newDeviceInfo + return newDeviceInfo + } + + private fun isPlaying(playbackInfo: TtsPlayer.Playback): Boolean { + return (playbackInfo.state == TtsPlayer.State.Ready && playbackInfo.playWhenReady) + } + + private fun getRepeatModeForNavigation(): @RepeatMode Int { + val repeatMode = repeatMode + return if (repeatMode == REPEAT_MODE_ONE) REPEAT_MODE_OFF else repeatMode + } + + private inner class StreamVolumeManagerListener : StreamVolumeManager.Listener { + + override fun onStreamTypeChanged(streamType: @StreamType Int) { + val newDeviceInfo = createDeviceInfo(streamVolumeManager) + if (newDeviceInfo != deviceInfo) { + listeners.sendEvent( + EVENT_DEVICE_INFO_CHANGED + ) { listener: Listener -> + listener.onDeviceInfoChanged( + newDeviceInfo + ) + } + } + } + + override fun onStreamVolumeChanged(streamVolume: Int, streamMuted: Boolean) { + listeners.sendEvent( + EVENT_DEVICE_VOLUME_CHANGED + ) { listener: Listener -> + listener.onDeviceVolumeChanged( + streamVolume, + streamMuted + ) + } + } + } + + private inner class AudioFocusManagerListener : AudioFocusManager.PlayerControl { + + override fun setVolumeMultiplier(volumeMultiplier: Float) { + // Do nothing as we're not supposed to duck volume with + // contentType == C.AUDIO_CONTENT_TYPE_SPEECH + } + + override fun executePlayerCommand(playerCommand: Int) { + playWhenReady = playWhenReady && playerCommand != AudioFocusManager.PLAYER_COMMAND_DO_NOT_PLAY + } + } + + private inner class AudioBecomingNoisyManagerListener : AudioBecomingNoisyManager.EventListener { + + override fun onAudioBecomingNoisy() { + playWhenReady = false + } + } + + private val TtsPlayer.State.playerCode get() = when (this) { + TtsPlayer.State.Ready -> STATE_READY + TtsPlayer.State.Ended -> STATE_ENDED + is TtsPlayer.State.Error -> STATE_IDLE + } + + @Suppress("Unchecked_cast") + private fun TtsPlayer.State.Error.toPlaybackException(): PlaybackException = when (this) { + is TtsPlayer.State.Error.EngineError<*> -> mapEngineError(error as E) + is TtsPlayer.State.Error.ContentError -> when (exception) { + is Resource.Exception.BadRequest -> + PlaybackException(exception.message, exception.cause, ERROR_CODE_IO_BAD_HTTP_STATUS) + is Resource.Exception.NotFound -> + PlaybackException(exception.message, exception.cause, ERROR_CODE_IO_BAD_HTTP_STATUS) + is Resource.Exception.Forbidden -> + PlaybackException(exception.message, exception.cause, ERROR_CODE_DRM_DISALLOWED_OPERATION) + is Resource.Exception.Unavailable -> + PlaybackException(exception.message, exception.cause, ERROR_CODE_IO_NETWORK_CONNECTION_FAILED) + is Resource.Exception.Offline -> + PlaybackException(exception.message, exception.cause, ERROR_CODE_IO_NETWORK_CONNECTION_FAILED) + is Resource.Exception.OutOfMemory -> + PlaybackException(exception.message, exception.cause, ERROR_CODE_UNSPECIFIED) + is Resource.Exception.Cancelled -> + PlaybackException(exception.message, exception.cause, ERROR_CODE_IO_UNSPECIFIED) + is Resource.Exception.Other -> + PlaybackException(exception.message, exception.cause, ERROR_CODE_UNSPECIFIED) + else -> + PlaybackException(exception.message, exception.cause, ERROR_CODE_UNSPECIFIED) + } + } +} diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/session/TtsTimeline.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/session/TtsTimeline.kt new file mode 100644 index 0000000000..8cfdacad23 --- /dev/null +++ b/readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/session/TtsTimeline.kt @@ -0,0 +1,57 @@ +/* + * Copyright 2022 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + +package org.readium.r2.navigator.media3.tts.session + +import androidx.media3.common.MediaItem +import androidx.media3.common.Timeline +import java.util.* + +@androidx.annotation.OptIn(androidx.media3.common.util.UnstableApi::class) +internal class TtsTimeline( + private val mediaItems: List, +) : Timeline() { + + private val uuids = (0 until windowCount) + .map { UUID.randomUUID() } + + override fun getWindowCount(): Int { + return mediaItems.size + } + + override fun getWindow( + windowIndex: Int, + window: Window, + defaultPositionProjectionUs: Long + ): Window { + window.uid = uuids[windowIndex] + window.firstPeriodIndex = windowIndex + window.lastPeriodIndex = windowIndex + window.mediaItem = mediaItems[windowIndex] + window.isSeekable = false + return window + } + + override fun getPeriodCount(): Int { + return mediaItems.size + } + + override fun getPeriod(periodIndex: Int, period: Period, setIds: Boolean): Period { + period.windowIndex += periodIndex + if (setIds) { + period.uid = uuids[periodIndex] + } + return period + } + + override fun getIndexOfPeriod(uid: Any): Int { + return uuids.indexOfFirst { it == uid } + } + + override fun getUidOfPeriod(periodIndex: Int): Any { + return uuids[periodIndex] + } +} diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/tts/AndroidTtsEngine.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/tts/AndroidTtsEngine.kt deleted file mode 100644 index cfe696fb59..0000000000 --- a/readium/navigator/src/main/java/org/readium/r2/navigator/tts/AndroidTtsEngine.kt +++ /dev/null @@ -1,288 +0,0 @@ -/* - * Copyright 2022 Readium Foundation. All rights reserved. - * Use of this source code is governed by the BSD-style license - * available in the top-level LICENSE file of the project. - */ - -package org.readium.r2.navigator.tts - -import android.content.Context -import android.content.Intent -import android.speech.tts.TextToSpeech -import android.speech.tts.UtteranceProgressListener -import android.speech.tts.Voice as AndroidVoice -import java.util.* -import kotlin.Exception -import kotlin.coroutines.resume -import kotlinx.coroutines.* -import kotlinx.coroutines.channels.Channel -import kotlinx.coroutines.channels.onFailure -import org.readium.r2.navigator.tts.TtsEngine.Voice -import org.readium.r2.shared.ExperimentalReadiumApi -import org.readium.r2.shared.extensions.tryOrLog -import org.readium.r2.shared.util.Language -import org.readium.r2.shared.util.Try - -/** - * Default [TtsEngine] implementation using Android's native text to speech engine. - */ -@ExperimentalReadiumApi -class AndroidTtsEngine( - context: Context, - private val listener: TtsEngine.Listener -) : TtsEngine { - - /** - * Android's TTS error code. - * See https://developer.android.com/reference/android/speech/tts/TextToSpeech#ERROR - */ - enum class EngineError(val code: Int) { - /** Denotes a generic operation failure. */ - Unknown(-1), - /** Denotes a failure caused by an invalid request. */ - InvalidRequest(-8), - /** Denotes a failure caused by a network connectivity problems. */ - Network(-6), - /** Denotes a failure caused by network timeout. */ - NetworkTimeout(-7), - /** Denotes a failure caused by an unfinished download of the voice data. */ - NotInstalledYet(-9), - /** Denotes a failure related to the output (audio device or a file). */ - Output(-5), - /** Denotes a failure of a TTS service. */ - Service(-4), - /** Denotes a failure of a TTS engine to synthesize the given input. */ - Synthesis(-3); - - companion object { - fun getOrDefault(key: Int): EngineError = - values() - .firstOrNull { it.code == key } - ?: Unknown - } - } - - class EngineException(code: Int) : Exception("Android TTS engine error: $code") { - val error: EngineError = - EngineError.getOrDefault(code) - } - - private val scope = CoroutineScope(SupervisorJob() + Dispatchers.Default) - - /** - * Utterances to be synthesized, in order of [speak] calls. - */ - private val tasks = Channel(Channel.BUFFERED) - - /** Future completed when the [engine] is fully initialized. */ - private val init = CompletableDeferred() - - init { - scope.launch { - init.await() - - for (task in tasks) { - ensureActive() - task.run() - } - } - } - - override val rateMultiplierRange: ClosedRange = 0.1..4.0 - - override var availableVoices: List = emptyList() - private set(value) { - field = value - listener.onAvailableVoicesChange(value) - } - - override suspend fun close() { - scope.cancel() - tasks.cancel() - engine.shutdown() - } - - override suspend fun speak( - utterance: TtsEngine.Utterance, - onSpeakRange: (IntRange) -> Unit - ): TtsTry = - suspendCancellableCoroutine { cont -> - val result = tasks.trySend( - UtteranceTask( - utterance = utterance, - continuation = cont, - onSpeakRange = onSpeakRange - ) - ) - - result.onFailure { - listener.onEngineError( - TtsEngine.Exception.Other(IllegalStateException("Failed to schedule a new utterance task")) - ) - } - } - - /** - * Start the activity to install additional language data. - * To be called if you receive a [TtsEngine.Exception.LanguageSupportIncomplete] error. - * - * Returns whether the request was successful. - * - * See https://android-developers.googleblog.com/2009/09/introduction-to-text-to-speech-in.html - */ - fun requestInstallMissingVoice( - context: Context, - intentFlags: Int = Intent.FLAG_ACTIVITY_NEW_TASK - ): Boolean { - val intent = Intent() - .setAction(TextToSpeech.Engine.ACTION_INSTALL_TTS_DATA) - .setFlags(intentFlags) - - if (context.packageManager.queryIntentActivities(intent, 0).isEmpty()) { - return false - } - - context.startActivity(intent) - return true - } - - // Engine - - /** Underlying Android [TextToSpeech] engine. */ - private val engine = TextToSpeech(context, EngineInitListener()) - - private inner class EngineInitListener : TextToSpeech.OnInitListener { - override fun onInit(status: Int) { - if (status == TextToSpeech.SUCCESS) { - scope.launch { - tryOrLog { - availableVoices = engine.voices.map { it.toVoice() } - } - init.complete(Unit) - } - } else { - listener.onEngineError(TtsEngine.Exception.InitializationFailed()) - } - } - } - - /** - * Holds a single utterance to be synthesized and the continuation for the [speak] call. - */ - private inner class UtteranceTask( - val utterance: TtsEngine.Utterance, - val continuation: CancellableContinuation>, - val onSpeakRange: (IntRange) -> Unit, - ) { - fun run() { - if (!continuation.isActive) return - - // Interrupt the engine when the task is cancelled. - continuation.invokeOnCancellation { - tryOrLog { - engine.stop() - engine.setOnUtteranceProgressListener(null) - } - } - - try { - val id = UUID.randomUUID().toString() - engine.setup() - engine.setOnUtteranceProgressListener(Listener(id)) - engine.speak(utterance.text, TextToSpeech.QUEUE_FLUSH, null, id) - } catch (e: Exception) { - finish(TtsEngine.Exception.wrap(e)) - } - } - - /** - * Terminates this task. - */ - private fun finish(error: TtsEngine.Exception? = null) { - continuation.resume( - error?.let { Try.failure(error) } - ?: Try.success(Unit) - ) - } - - /** - * Setups the [engine] using the [utterance]'s configuration. - */ - private fun TextToSpeech.setup() { - setSpeechRate(utterance.rateMultiplier.toFloat()) - - utterance.voiceOrLanguage - .onLeft { voice -> - // Setup the user selected voice. - engine.voice = engine.voices - .firstOrNull { it.name == voice.id } - ?: throw IllegalStateException("Unknown Android voice: ${voice.id}") - } - .onRight { language -> - // Or fallback on the language. - val localeResult = engine.setLanguage(language.locale) - if (localeResult < TextToSpeech.LANG_AVAILABLE) { - if (localeResult == TextToSpeech.LANG_MISSING_DATA) - throw TtsEngine.Exception.LanguageSupportIncomplete(language) - else - throw TtsEngine.Exception.LanguageNotSupported(language) - } - } - } - - inner class Listener(val id: String) : UtteranceProgressListener() { - override fun onStart(utteranceId: String?) {} - - override fun onStop(utteranceId: String?, interrupted: Boolean) { - require(utteranceId == id) - finish() - } - - override fun onDone(utteranceId: String?) { - require(utteranceId == id) - finish() - } - - @Deprecated("Deprecated in the interface", ReplaceWith("onError(utteranceId, -1)")) - override fun onError(utteranceId: String?) { - onError(utteranceId, -1) - } - - override fun onError(utteranceId: String?, errorCode: Int) { - require(utteranceId == id) - - val error = EngineException(errorCode) - finish( - when (error.error) { - EngineError.Network, EngineError.NetworkTimeout -> - TtsEngine.Exception.Network(error) - EngineError.NotInstalledYet -> - TtsEngine.Exception.LanguageSupportIncomplete(utterance.language, cause = error) - else -> TtsEngine.Exception.Other(error) - } - ) - } - - override fun onRangeStart(utteranceId: String?, start: Int, end: Int, frame: Int) { - require(utteranceId == id) - onSpeakRange(start until end) - } - } - } -} - -@OptIn(ExperimentalReadiumApi::class) -private fun AndroidVoice.toVoice(): Voice = - Voice( - id = name, - name = null, - language = Language(locale), - quality = when (quality) { - AndroidVoice.QUALITY_VERY_HIGH -> Voice.Quality.Highest - AndroidVoice.QUALITY_HIGH -> Voice.Quality.High - AndroidVoice.QUALITY_LOW -> Voice.Quality.Low - AndroidVoice.QUALITY_VERY_LOW -> Voice.Quality.Lowest - else -> Voice.Quality.Normal - }, - requiresNetwork = isNetworkConnectionRequired - ) diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/tts/PublicationSpeechSynthesizer.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/tts/PublicationSpeechSynthesizer.kt deleted file mode 100644 index a04cffe01f..0000000000 --- a/readium/navigator/src/main/java/org/readium/r2/navigator/tts/PublicationSpeechSynthesizer.kt +++ /dev/null @@ -1,556 +0,0 @@ -/* - * Copyright 2022 Readium Foundation. All rights reserved. - * Use of this source code is governed by the BSD-style license - * available in the top-level LICENSE file of the project. - */ - -package org.readium.r2.navigator.tts - -import android.content.Context -import java.util.* -import kotlinx.coroutines.* -import kotlinx.coroutines.flow.MutableStateFlow -import kotlinx.coroutines.flow.StateFlow -import kotlinx.coroutines.flow.asStateFlow -import kotlinx.coroutines.flow.update -import org.readium.r2.shared.DelicateReadiumApi -import org.readium.r2.shared.ExperimentalReadiumApi -import org.readium.r2.shared.extensions.tryOrLog -import org.readium.r2.shared.publication.Locator -import org.readium.r2.shared.publication.Publication -import org.readium.r2.shared.publication.services.content.Content -import org.readium.r2.shared.publication.services.content.ContentTokenizer -import org.readium.r2.shared.publication.services.content.TextContentTokenizer -import org.readium.r2.shared.publication.services.content.content -import org.readium.r2.shared.util.* -import org.readium.r2.shared.util.tokenizer.TextUnit - -/** - * [PublicationSpeechSynthesizer] orchestrates the rendition of a [publication] by iterating through - * its content, splitting it into individual utterances using a [ContentTokenizer], then using a - * [TtsEngine] to read them aloud. - * - * Don't forget to call [close] when you are done using the [PublicationSpeechSynthesizer]. - */ -@OptIn(DelicateReadiumApi::class) -@ExperimentalReadiumApi -@Deprecated("The API described in this guide will be changed in the next version of the Kotlin toolkit to support background TTS playback and media notifications. It is recommended that you wait before integrating it in your app.") -class PublicationSpeechSynthesizer private constructor( - private val publication: Publication, - config: Configuration, - engineFactory: (listener: TtsEngine.Listener) -> E, - private val tokenizerFactory: (defaultLanguage: Language?) -> ContentTokenizer, - var listener: Listener? = null, -) : SuspendingCloseable { - - companion object { - - /** - * Creates a [PublicationSpeechSynthesizer] using the default native [AndroidTtsEngine]. - * - * @param publication Publication which will be iterated through and synthesized. - * @param config Initial TTS configuration. - * @param tokenizerFactory Factory to create a [ContentTokenizer] which will be used to - * split each [Content.Element] item into smaller chunks. Splits by sentences by default. - * @param listener Optional callbacks listener. - */ - operator fun invoke( - context: Context, - publication: Publication, - config: Configuration = Configuration(), - tokenizerFactory: (defaultLanguage: Language?) -> ContentTokenizer = defaultTokenizerFactory, - listener: Listener? = null, - ): PublicationSpeechSynthesizer? = invoke( - publication, - config = config, - engineFactory = { AndroidTtsEngine(context, listener = it) }, - tokenizerFactory = tokenizerFactory, - listener = listener - ) - - /** - * Creates a [PublicationSpeechSynthesizer] using a custom [TtsEngine]. - * - * @param publication Publication which will be iterated through and synthesized. - * @param config Initial TTS configuration. - * @param engineFactory Factory to create an instance of [TtsEngine]. - * @param tokenizerFactory Factory to create a [ContentTokenizer] which will be used to - * split each [Content.Element] item into smaller chunks. Splits by sentences by default. - * @param listener Optional callbacks listener. - */ - operator fun invoke( - publication: Publication, - config: Configuration = Configuration(), - engineFactory: (TtsEngine.Listener) -> E, - tokenizerFactory: (defaultLanguage: Language?) -> ContentTokenizer = defaultTokenizerFactory, - listener: Listener? = null, - ): PublicationSpeechSynthesizer? { - if (!canSpeak(publication)) return null - - return PublicationSpeechSynthesizer(publication, config, engineFactory, tokenizerFactory, listener) - } - - /** - * The default content tokenizer will split the [Content.Element] items into individual sentences. - */ - val defaultTokenizerFactory: (Language?) -> ContentTokenizer = { language -> - TextContentTokenizer( - unit = TextUnit.Sentence, - defaultLanguage = language - ) - } - - /** - * Returns whether the [publication] can be played with a [PublicationSpeechSynthesizer]. - */ - fun canSpeak(publication: Publication): Boolean = - publication.content() != null - } - - @ExperimentalReadiumApi - interface Listener { - /** Called when an [error] occurs while speaking [utterance]. */ - fun onUtteranceError(utterance: Utterance, error: Exception) - - /** Called when a global [error] occurs. */ - fun onError(error: Exception) - } - - @ExperimentalReadiumApi - sealed class Exception private constructor( - override val message: String, - cause: Throwable? = null - ) : kotlin.Exception(message, cause) { - - /** Underlying [TtsEngine] error. */ - class Engine(val error: TtsEngine.Exception) : - Exception(error.message, error) - } - - /** - * An utterance is an arbitrary text (e.g. sentence) extracted from the [publication], that can - * be synthesized by the TTS [engine]. - * - * @param text Text to be spoken. - * @param locator Locator to the utterance in the [publication]. - * @param language Language of this utterance, if it differs from the default publication - * language. - */ - @ExperimentalReadiumApi - data class Utterance( - val text: String, - val locator: Locator, - val language: Language?, - ) - - /** - * Represents a state of the [PublicationSpeechSynthesizer]. - */ - sealed class State { - /** The [PublicationSpeechSynthesizer] is completely stopped and must be (re)started from a given locator. */ - object Stopped : State() - - /** The [PublicationSpeechSynthesizer] is paused at the given utterance. */ - data class Paused(val utterance: Utterance) : State() - - /** - * The TTS engine is synthesizing [utterance]. - * - * [range] will be regularly updated while the [utterance] is being played. - */ - data class Playing(val utterance: Utterance, val range: Locator? = null) : State() - } - - private val _state = MutableStateFlow(State.Stopped) - - /** - * Current state of the [PublicationSpeechSynthesizer]. - */ - val state: StateFlow = _state.asStateFlow() - - private val scope = MainScope() - - init { - require(canSpeak(publication)) { - "The content of the publication cannot be synthesized, as it is not iterable" - } - } - - /** - * Underlying [TtsEngine] instance. - * - * WARNING: Don't control the playback or set the config directly with the engine. Use the - * [PublicationSpeechSynthesizer] APIs instead. This property is used to access engine-specific APIs such as - * [AndroidTtsEngine.requestInstallMissingVoice]. - */ - @DelicateReadiumApi - val engine: E by lazy { - engineFactory(object : TtsEngine.Listener { - override fun onEngineError(error: TtsEngine.Exception) { - listener?.onError(Exception.Engine(error)) - stop() - } - - override fun onAvailableVoicesChange(voices: List) { - _availableVoices.value = voices - } - }) - } - - /** - * Interrupts the [TtsEngine] and closes this [PublicationSpeechSynthesizer]. - */ - override suspend fun close() { - tryOrLog { - scope.cancel() - if (::engine.isLazyInitialized) { - engine.close() - } - } - } - - /** - * User configuration for the text-to-speech engine. - * - * @param defaultLanguage Language overriding the publication one. - * @param voiceId Identifier for the voice used to speak the utterances. - * @param rateMultiplier Multiplier for the voice speech rate. Normal is 1.0. See [rateMultiplierRange] - * for the range of values supported by the [TtsEngine]. - * @param extras Extensibility for custom TTS engines. - */ - @ExperimentalReadiumApi - data class Configuration( - val defaultLanguage: Language? = null, - val voiceId: String? = null, - val rateMultiplier: Double = 1.0, - val extras: Any? = null - ) - - private val _config = MutableStateFlow(config) - - /** - * Current user configuration. - */ - val config: StateFlow = _config.asStateFlow() - - /** - * Updates the user configuration. - * - * The change is not immediate, it will be applied for the next utterance. - */ - fun setConfig(config: Configuration) { - _config.value = config.copy( - rateMultiplier = config.rateMultiplier.coerceIn(engine.rateMultiplierRange), - ) - } - - /** - * Range for the speech rate multiplier. Normal is 1.0. - */ - val rateMultiplierRange: ClosedRange - get() = engine.rateMultiplierRange - - private val _availableVoices = MutableStateFlow>(emptyList()) - - /** - * List of synthesizer voices supported by the TTS [engine]. - */ - val availableVoices: StateFlow> = _availableVoices.asStateFlow() - - /** - * Returns the first voice with the given [id] supported by the TTS [engine]. - * - * This can be used to restore the user selected voice after storing it in the shared - * preferences. - */ - fun voiceWithId(id: String): TtsEngine.Voice? { - val voice = lastUsedVoice?.takeIf { it.id == id } - ?: engine.voiceWithId(id) - ?: return null - - lastUsedVoice = voice - return voice - } - - /** - * Cache for the last requested voice, for performance. - */ - private var lastUsedVoice: TtsEngine.Voice? = null - - /** - * (Re)starts the TTS from the given locator or the beginning of the publication. - */ - fun start(fromLocator: Locator? = null) { - replacePlaybackJob { - publicationIterator = publication.content(fromLocator)?.iterator() - playNextUtterance(Direction.Forward) - } - } - - /** - * Stops the synthesizer. - * - * Use [start] to restart it. - */ - fun stop() { - replacePlaybackJob { - _state.value = State.Stopped - publicationIterator = null - } - } - - /** - * Interrupts a played utterance. - * - * Use [resume] to restart the playback from the same utterance. - */ - fun pause() { - replacePlaybackJob { - _state.update { state -> - when (state) { - is State.Playing -> State.Paused(state.utterance) - else -> state - } - } - } - } - - /** - * Resumes an utterance interrupted with [pause]. - */ - fun resume() { - replacePlaybackJob { - (state.value as? State.Paused)?.let { paused -> - play(paused.utterance) - } - } - } - - /** - * Pauses or resumes the playback of the current utterance. - */ - fun pauseOrResume() { - when (state.value) { - is State.Stopped -> return - is State.Playing -> pause() - is State.Paused -> resume() - } - } - - /** - * Skips to the previous utterance. - */ - fun previous() { - replacePlaybackJob { - playNextUtterance(Direction.Backward) - } - } - - /** - * Skips to the next utterance. - */ - fun next() { - replacePlaybackJob { - playNextUtterance(Direction.Forward) - } - } - - /** - * [Content.Iterator] used to iterate through the [publication]. - */ - private var publicationIterator: Content.Iterator? = null - set(value) { - field = value - utterances = CursorList() - } - - /** - * Utterances for the current publication [Content.Element] item. - */ - private var utterances: CursorList = CursorList() - - /** - * Plays the next utterance in the given [direction]. - */ - private suspend fun playNextUtterance(direction: Direction) { - val utterance = nextUtterance(direction) - if (utterance == null) { - _state.value = State.Stopped - return - } - play(utterance) - } - - /** - * Plays the given [utterance] with the TTS [engine]. - */ - private suspend fun play(utterance: Utterance) { - _state.value = State.Playing(utterance) - - engine - .speak( - utterance = TtsEngine.Utterance( - text = utterance.text, - rateMultiplier = config.value.rateMultiplier, - voiceOrLanguage = utterance.voiceOrLanguage() - ), - onSpeakRange = { range -> - _state.value = State.Playing( - utterance = utterance, - range = utterance.locator.copy( - text = utterance.locator.text.substring(range) - ) - ) - } - ) - .onSuccess { - playNextUtterance(Direction.Forward) - } - .onFailure { - _state.value = State.Paused(utterance) - listener?.onUtteranceError(utterance, Exception.Engine(it)) - } - } - - /** - * Returns the user selected voice if it's compatible with the utterance language. Otherwise, - * falls back on the languages. - */ - private fun Utterance.voiceOrLanguage(): Either { - // User selected voice, if it's compatible with the utterance language. - // Or fallback on the languages. - val voice = config.value.voiceId - ?.let { voiceWithId(it) } - ?.takeIf { language == null || it.language.removeRegion() == language.removeRegion() } - - return Either( - voice - ?: language - ?: config.value.defaultLanguage - ?: publication.metadata.language - ?: Language(Locale.getDefault()) - ) - } - - /** - * Gets the next utterance in the given [direction], or null when reaching the beginning or the - * end. - */ - private suspend fun nextUtterance(direction: Direction): Utterance? { - val utterance = utterances.nextIn(direction) - if (utterance == null && loadNextUtterances(direction)) { - return nextUtterance(direction) - } - return utterance - } - - /** - * Loads the utterances for the next publication [Content.Element] item in the given [direction]. - */ - private suspend fun loadNextUtterances(direction: Direction): Boolean { - val content = publicationIterator?.nextIn(direction) - ?: return false - - val nextUtterances = content - .tokenize() - .flatMap { it.utterances() } - - if (nextUtterances.isEmpty()) { - return loadNextUtterances(direction) - } - - utterances = CursorList( - list = nextUtterances, - startIndex = when (direction) { - Direction.Forward -> 0 - Direction.Backward -> nextUtterances.size - 1 - } - ) - - return true - } - - /** - * Splits a publication [Content.Element] item into smaller chunks using the provided tokenizer. - * - * This is used to split a paragraph into sentences, for example. - */ - private fun Content.Element.tokenize(): List = - tokenizerFactory(config.value.defaultLanguage ?: publication.metadata.language) - .tokenize(this) - - /** - * Splits a publication [Content.Element] item into the utterances to be spoken. - */ - private fun Content.Element.utterances(): List { - fun utterance(text: String, locator: Locator, language: Language? = null): Utterance? { - if (!text.any { it.isLetterOrDigit() }) - return null - - return Utterance( - text = text, - locator = locator, - language = language - // If the language is the same as the one declared globally in the publication, - // we omit it. This way, the app can customize the default language used in the - // configuration. - ?.takeIf { it != publication.metadata.language } - ) - } - - return when (this) { - is Content.TextElement -> { - segments.mapNotNull { segment -> - utterance( - text = segment.text, - locator = segment.locator, - language = segment.language - ) - } - } - - is Content.TextualElement -> { - listOfNotNull( - text - ?.takeIf { it.isNotBlank() } - ?.let { utterance(text = it, locator = locator) } - ) - } - - else -> emptyList() - } - } - - /** - * Cancels the previous playback-related job and starts a new one with the given suspending - * [block]. - * - * This is used to interrupt on-going commands. - */ - private fun replacePlaybackJob(block: suspend CoroutineScope.() -> Unit) { - scope.launch { - playbackJob?.cancelAndJoin() - playbackJob = launch { - block() - } - } - } - - private var playbackJob: Job? = null - - private enum class Direction { - Forward, Backward; - } - - private fun CursorList.nextIn(direction: Direction): E? = - when (direction) { - Direction.Forward -> next() - Direction.Backward -> previous() - } - - private suspend fun Content.Iterator.nextIn(direction: Direction): Content.Element? = - when (direction) { - Direction.Forward -> nextOrNull() - Direction.Backward -> previousOrNull() - } -} diff --git a/readium/navigator/src/main/java/org/readium/r2/navigator/tts/TtsEngine.kt b/readium/navigator/src/main/java/org/readium/r2/navigator/tts/TtsEngine.kt deleted file mode 100644 index ae3a454f97..0000000000 --- a/readium/navigator/src/main/java/org/readium/r2/navigator/tts/TtsEngine.kt +++ /dev/null @@ -1,149 +0,0 @@ -/* - * Copyright 2022 Readium Foundation. All rights reserved. - * Use of this source code is governed by the BSD-style license - * available in the top-level LICENSE file of the project. - */ - -package org.readium.r2.navigator.tts - -import org.readium.r2.shared.ExperimentalReadiumApi -import org.readium.r2.shared.util.Either -import org.readium.r2.shared.util.Language -import org.readium.r2.shared.util.SuspendingCloseable -import org.readium.r2.shared.util.Try - -@ExperimentalReadiumApi -typealias TtsTry = Try - -/** - * A text-to-speech engine synthesizes text utterances (e.g. sentence). - * - * Implement this interface to support third-party engines with [PublicationSpeechSynthesizer]. - */ -@ExperimentalReadiumApi -interface TtsEngine : SuspendingCloseable { - - @ExperimentalReadiumApi - sealed class Exception private constructor( - override val message: String, - cause: Throwable? = null - ) : kotlin.Exception(message, cause) { - /** Failed to initialize the TTS engine. */ - class InitializationFailed(cause: Throwable? = null) : - Exception("The TTS engine failed to initialize", cause) - - /** Tried to synthesize an utterance with an unsupported language. */ - class LanguageNotSupported(val language: Language, cause: Throwable? = null) : - Exception("The language ${language.code} is not supported by the TTS engine", cause) - - /** The selected language is missing downloadable data. */ - class LanguageSupportIncomplete(val language: Language, cause: Throwable? = null) : - Exception("The language ${language.code} requires additional files by the TTS engine", cause) - - /** Error during network calls. */ - class Network(cause: Throwable? = null) : - Exception("A network error occurred", cause) - - /** Other engine-specific errors. */ - class Other(override val cause: Throwable) : - Exception(cause.message ?: "An unknown error occurred", cause) - - companion object { - fun wrap(e: Throwable): Exception = when (e) { - is Exception -> e - else -> Other(e) - } - } - } - - /** - * TTS engine callbacks. - */ - @ExperimentalReadiumApi - interface Listener { - /** - * Called when a general engine error occurred. - */ - fun onEngineError(error: Exception) - - /** - * Called when the list of available voices is updated. - */ - fun onAvailableVoicesChange(voices: List) - } - - /** - * An utterance is an arbitrary text (e.g. sentence) that can be synthesized by the TTS engine. - * - * @param text Text to be spoken. - * @param rateMultiplier Multiplier for the speech rate. - * @param voiceOrLanguage Either an explicit voice or the language of the text. If a language - * is provided, the default voice for this language will be used. - */ - @ExperimentalReadiumApi - data class Utterance( - val text: String, - val rateMultiplier: Double, - val voiceOrLanguage: Either - ) { - val language: Language = - when (val vl = voiceOrLanguage) { - is Either.Left -> vl.value.language - is Either.Right -> vl.value - } - } - - /** - * Represents a voice provided by the TTS engine which can speak an utterance. - * - * @param id Unique and stable identifier for this voice. Can be used to store and retrieve the - * voice from the user preferences. - * @param name Human-friendly name for this voice, when available. - * @param language Language (and region) this voice belongs to. - * @param quality Voice quality. - * @param requiresNetwork Indicates whether using this voice requires an Internet connection. - */ - @ExperimentalReadiumApi - data class Voice( - val id: String, - val name: String? = null, - val language: Language, - val quality: Quality = Quality.Normal, - val requiresNetwork: Boolean = false, - ) { - enum class Quality { - Lowest, Low, Normal, High, Highest - } - } - - /** - * Synthesizes the given [utterance] and returns its status. - * - * [onSpeakRange] is called repeatedly while the engine plays portions (e.g. words) of the - * utterance. - * - * To interrupt the utterance, cancel the parent coroutine job. - */ - suspend fun speak( - utterance: Utterance, - onSpeakRange: (IntRange) -> Unit = { _ -> } - ): TtsTry - - /** - * Supported range for the speech rate multiplier. - */ - val rateMultiplierRange: ClosedRange - - /** - * List of available synthesizer voices. - * - * Implement [Listener.onAvailableVoicesChange] to be aware of changes in the available voices. - */ - val availableVoices: List - - /** - * Returns the voice with given identifier, if it exists. - */ - fun voiceWithId(id: String): Voice? = - availableVoices.firstOrNull { it.id == id } -} diff --git a/readium/shared/src/main/java/org/readium/r2/shared/publication/services/content/Content.kt b/readium/shared/src/main/java/org/readium/r2/shared/publication/services/content/Content.kt index baafea867d..d28ce9328d 100644 --- a/readium/shared/src/main/java/org/readium/r2/shared/publication/services/content/Content.kt +++ b/readium/shared/src/main/java/org/readium/r2/shared/publication/services/content/Content.kt @@ -197,6 +197,9 @@ interface Content { /** * Iterates through a list of [Element] items asynchronously. + * + * [hasNext] and [hasPrevious] refer to the last element computed by a previous call + * to any of both methods. */ @ExperimentalReadiumApi interface Iterator { diff --git a/readium/shared/src/main/java/org/readium/r2/shared/publication/services/content/ContentTokenizer.kt b/readium/shared/src/main/java/org/readium/r2/shared/publication/services/content/ContentTokenizer.kt index 48472caf60..0603894ba8 100644 --- a/readium/shared/src/main/java/org/readium/r2/shared/publication/services/content/ContentTokenizer.kt +++ b/readium/shared/src/main/java/org/readium/r2/shared/publication/services/content/ContentTokenizer.kt @@ -23,10 +23,13 @@ fun interface ContentTokenizer : Tokenizer * portions. * * @param contextSnippetLength Length of `before` and `after` snippets in the produced [Locator]s. + * @param overrideContentLanguage If true, let [language] override language information that could be available in + * content. If false, [language] will be used only as a default when there is no data-specific information. */ @ExperimentalReadiumApi class TextContentTokenizer( - private val defaultLanguage: Language?, + private val language: Language?, + private val overrideContentLanguage: Boolean = false, private val contextSnippetLength: Int = 50, private val textTokenizerFactory: (Language?) -> TextTokenizer ) : ContentTokenizer { @@ -34,9 +37,10 @@ class TextContentTokenizer( /** * A [ContentTokenizer] using the default [TextTokenizer] to split the text of the [Content.Element]. */ - constructor(defaultLanguage: Language?, unit: TextUnit) : this( - defaultLanguage = defaultLanguage, - textTokenizerFactory = { language -> DefaultTextContentTokenizer(unit, language) } + constructor(language: Language?, unit: TextUnit, overrideContentLanguage: Boolean = false) : this( + language = language, + textTokenizerFactory = { contentLanguage -> DefaultTextContentTokenizer(unit, contentLanguage) }, + overrideContentLanguage = overrideContentLanguage ) override fun tokenize(data: Content.Element): List = listOf( @@ -50,7 +54,7 @@ class TextContentTokenizer( ) private fun tokenize(segment: Content.TextElement.Segment): List = - textTokenizerFactory(segment.language ?: defaultLanguage).tokenize(segment.text) + textTokenizerFactory(resolveSegmentLanguage(segment)).tokenize(segment.text) .map { range -> segment.copy( locator = segment.locator.copy(text = extractTextContextIn(segment.text, range)), @@ -58,6 +62,9 @@ class TextContentTokenizer( ) } + private fun resolveSegmentLanguage(segment: Content.TextElement.Segment): Language? = + segment.language.takeUnless { overrideContentLanguage } ?: language + private fun extractTextContextIn(string: String, range: IntRange): Locator.Text { val after = string.substring(range.last, (range.last + contextSnippetLength).coerceAtMost(string.length)) val before = string.substring((range.first - contextSnippetLength).coerceAtLeast(0), range.first) diff --git a/readium/shared/src/main/java/org/readium/r2/shared/publication/services/content/iterators/HtmlResourceContentIterator.kt b/readium/shared/src/main/java/org/readium/r2/shared/publication/services/content/iterators/HtmlResourceContentIterator.kt index 33007b08ac..6bccdccf7e 100644 --- a/readium/shared/src/main/java/org/readium/r2/shared/publication/services/content/iterators/HtmlResourceContentIterator.kt +++ b/readium/shared/src/main/java/org/readium/r2/shared/publication/services/content/iterators/HtmlResourceContentIterator.kt @@ -65,45 +65,50 @@ class HtmlResourceContentIterator( val delta: Int ) - private var requestedElement: ElementWithDelta? = null + private var currentElement: ElementWithDelta? = null override suspend fun hasPrevious(): Boolean { - if (requestedElement?.delta == -1) return true + if (currentElement?.delta == -1) return true + + val elements = elements() + val index = (currentIndex ?: elements.startIndex) - 1 + + val content = elements.elements.getOrNull(index) + ?: return false - val index = currentIndex() - 1 - val element = elements().elements.getOrNull(index) ?: return false currentIndex = index - requestedElement = ElementWithDelta(element, -1) + currentElement = ElementWithDelta(content, -1) return true } override fun previous(): Content.Element = - requestedElement + currentElement ?.takeIf { it.delta == -1 }?.element - ?.also { requestedElement = null } + ?.also { currentElement = null } ?: throw IllegalStateException("Called previous() without a successful call to hasPrevious() first") override suspend fun hasNext(): Boolean { - if (requestedElement?.delta == 1) return true + if (currentElement?.delta == +1) return true + + val elements = elements() + val index = (currentIndex ?: (elements.startIndex - 1)) + 1 + + val content = elements.elements.getOrNull(index) + ?: return false - val index = currentIndex() - val element = elements().elements.getOrNull(index) ?: return false - currentIndex = index + 1 - requestedElement = ElementWithDelta(element, +1) + currentIndex = index + currentElement = ElementWithDelta(content, +1) return true } override fun next(): Content.Element = - requestedElement - ?.takeIf { it.delta == 1 }?.element - ?.also { requestedElement = null } + currentElement + ?.takeIf { it.delta == +1 }?.element + ?.also { currentElement = null } ?: throw IllegalStateException("Called next() without a successful call to hasNext() first") private var currentIndex: Int? = null - private suspend fun currentIndex(): Int = - currentIndex ?: elements().startIndex - private suspend fun elements(): ParsedElements = parsedElements ?: parseElements().also { parsedElements = it } @@ -111,26 +116,24 @@ class HtmlResourceContentIterator( private var parsedElements: ParsedElements? = null private suspend fun parseElements(): ParsedElements { - val body = resource.use { res -> + val document = resource.use { res -> val html = res.readAsString().getOrThrow() Jsoup.parse(html) - }.body() + } val contentParser = ContentParser( baseLocator = locator, startElement = locator.locations.cssSelector?.let { - // The JS third-party library used to generate the CSS Selector sometimes adds - // :root >, which doesn't work with JSoup. - tryOrNull { body.selectFirst(it.removePrefix(":root > ")) } + tryOrNull { document.selectFirst(it) } }, beforeMaxLength = beforeMaxLength ) - NodeTraversor.traverse(contentParser, body) + NodeTraversor.traverse(contentParser, document.body()) return contentParser.result() } /** - * Holds the result of parsing the HTML resource into a list of `ContentElement`. + * Holds the result of parsing the HTML resource into a list of [Content.Element]. * * The [startIndex] will be calculated from the element matched by the base [locator], if * possible. Defaults to 0. @@ -148,7 +151,7 @@ class HtmlResourceContentIterator( fun result() = ParsedElements( elements = elements, - startIndex = if (baseLocator.locations.progression == 1.0) elements.size - 1 + startIndex = if (baseLocator.locations.progression == 1.0) elements.size else startIndex ) diff --git a/readium/shared/src/main/java/org/readium/r2/shared/util/CursorList.kt b/readium/shared/src/main/java/org/readium/r2/shared/util/CursorList.kt index cd23198885..41ebeaa398 100644 --- a/readium/shared/src/main/java/org/readium/r2/shared/util/CursorList.kt +++ b/readium/shared/src/main/java/org/readium/r2/shared/util/CursorList.kt @@ -4,44 +4,51 @@ import org.readium.r2.shared.InternalReadiumApi /** * A [List] with a mutable cursor index. + * + * [next] and [previous] refer to the last element returned by a previous call + * to any of both methods. + * + * @param list the content of the [CursorList] + * @param index the index of the element that will initially be considered + * as the last returned element. May be -1 or the size of the list as well. */ @InternalReadiumApi class CursorList( private val list: List = emptyList(), - private val startIndex: Int = 0 + private var index: Int = -1 ) : List by list { - private var index: Int? = null - /** - * Returns the current element. - */ - fun current(): E? = - moveAndGet(index ?: startIndex) + init { + check(index in -1..list.size) + } + + fun hasPrevious(): Boolean { + return index > 0 + } /** * Moves the cursor backward and returns the element, or null when reaching the beginning. */ - fun previous(): E? = - moveAndGet( - index - ?.let { it - 1 } - ?: startIndex - ) + fun previous(): E? { + if (!hasPrevious()) + return null + + index-- + return list[index] + } + + fun hasNext(): Boolean { + return index + 1 < list.size + } /** * Moves the cursor forward and returns the element, or null when reaching the end. */ - fun next(): E? = - moveAndGet( - index?.let { it + 1 } - ?: startIndex - ) - - private fun moveAndGet(index: Int): E? { - if (!list.indices.contains(index)) { + fun next(): E? { + if (!hasNext()) return null - } - this.index = index - return get(index) + + index++ + return list[index] } } diff --git a/readium/shared/src/test/java/org/readium/r2/shared/publication/services/content/iterators/HtmlResourceContentIteratorTest.kt b/readium/shared/src/test/java/org/readium/r2/shared/publication/services/content/iterators/HtmlResourceContentIteratorTest.kt index e923ad2594..da0aa11365 100644 --- a/readium/shared/src/test/java/org/readium/r2/shared/publication/services/content/iterators/HtmlResourceContentIteratorTest.kt +++ b/readium/shared/src/test/java/org/readium/r2/shared/publication/services/content/iterators/HtmlResourceContentIteratorTest.kt @@ -175,6 +175,7 @@ class HtmlResourceContentIteratorTest { fun `cannot call previous() without first hasPrevious()`() = runTest { val iter = iterator(html) iter.hasNext(); iter.next() + iter.hasNext(); iter.next() assertThrows(IllegalStateException::class.java) { iter.previous() } iter.hasPrevious() @@ -208,10 +209,20 @@ class HtmlResourceContentIteratorTest { } @Test - fun `next() then previous() returns the first element`() = runTest { + fun `next() then previous() returns null`() = runTest { + val iter = iterator(html) + assertTrue(iter.hasNext()) + assertEquals(elements[0], iter.next()) + assertFalse(iter.hasPrevious()) + } + + @Test + fun `next() twice then previous() returns the first element`() = runTest { val iter = iterator(html) assertTrue(iter.hasNext()) assertEquals(elements[0], iter.next()) + assertTrue(iter.hasNext()) + assertEquals(elements[1], iter.next()) assertTrue(iter.hasPrevious()) assertEquals(elements[0], iter.previous()) } @@ -220,6 +231,7 @@ class HtmlResourceContentIteratorTest { fun `calling hasPrevious() several times doesn't move the index`() = runTest { val iter = iterator(html) iter.hasNext(); iter.next() + iter.hasNext(); iter.next() assertTrue(iter.hasPrevious()) assertTrue(iter.hasPrevious()) assertTrue(iter.hasPrevious()) @@ -286,6 +298,45 @@ class HtmlResourceContentIteratorTest { ) } + @Test + fun `starting from a CSS selector using the root selector`() = runTest { + val nbspHtml = """ + + + + +

Tout au loin sur la chaussée, aussi loin qu’on pouvait voir

+

Lui, notre colonel, savait peut-être pourquoi ces deux gens-là tiraient [...] On buvait de la bière sucrée.

+ + + """ + + val iter = iterator(nbspHtml, locator(selector = ":root > :nth-child(2) > :nth-child(2)")) + assertTrue(iter.hasNext()) + assertEquals( + TextElement( + locator = locator( + selector = "html > body > p:nth-child(2)", + before = "oin sur la chaussée, aussi loin qu’on pouvait voir", + highlight = "Lui, notre colonel, savait peut-être pourquoi ces deux gens-là tiraient [...] On buvait de la bière sucrée." + ), + role = TextElement.Role.Body, + segments = listOf( + Segment( + locator = locator( + selector = "html > body > p:nth-child(2)", + before = "oin sur la chaussée, aussi loin qu’on pouvait voir", + highlight = "Lui, notre colonel, savait peut-être pourquoi ces deux gens-là tiraient [...] On buvait de la bière sucrée.", + ), + text = "Lui, notre colonel, savait peut-être pourquoi ces deux gens-là tiraient [...] On buvait de la bière sucrée.", + attributes = listOf(Attribute(LANGUAGE, Language("fr"))) + ) + ) + ), + iter.next() + ) + } + @Test fun `iterating over image elements`() = runTest { val html = """ diff --git a/test-app/build.gradle.kts b/test-app/build.gradle.kts index 53670c41b8..6fb8969723 100644 --- a/test-app/build.gradle.kts +++ b/test-app/build.gradle.kts @@ -103,6 +103,7 @@ dependencies { implementation(libs.jsoup) implementation(libs.bundles.media2) + implementation(libs.bundles.media3) // Room database implementation(libs.bundles.room) diff --git a/test-app/src/main/AndroidManifest.xml b/test-app/src/main/AndroidManifest.xml index 63344a09e2..129d540698 100644 --- a/test-app/src/main/AndroidManifest.xml +++ b/test-app/src/main/AndroidManifest.xml @@ -189,16 +189,27 @@ android:name=".reader.ReaderActivity" android:label="Reader" /> - - + + + + + + + + diff --git a/test-app/src/main/java/org/readium/r2/testapp/Application.kt b/test-app/src/main/java/org/readium/r2/testapp/Application.kt index 576ee32e52..57aaa248c0 100755 --- a/test-app/src/main/java/org/readium/r2/testapp/Application.kt +++ b/test-app/src/main/java/org/readium/r2/testapp/Application.kt @@ -6,11 +6,7 @@ package org.readium.r2.testapp -import android.content.ComponentName -import android.content.Context -import android.content.Intent -import android.content.ServiceConnection -import android.os.IBinder +import android.content.* import androidx.datastore.core.DataStore import androidx.datastore.preferences.core.Preferences import androidx.datastore.preferences.preferencesDataStore @@ -43,27 +39,6 @@ class Application : android.app.Application() { private val Context.navigatorPreferences: DataStore by preferencesDataStore(name = "navigator-preferences") - private val mediaServiceBinder: CompletableDeferred = - CompletableDeferred() - - private val mediaServiceConnection = object : ServiceConnection { - - override fun onServiceConnected(name: ComponentName?, service: IBinder) { - Timber.d("MediaService bound.") - mediaServiceBinder.complete(service as MediaService.Binder) - } - - override fun onServiceDisconnected(name: ComponentName) { - Timber.d("MediaService disconnected.") - // Should not happen, do nothing. - } - - override fun onNullBinding(name: ComponentName) { - Timber.d("Failed to bind to MediaService.") - // Should not happen, do nothing. - } - } - override fun onCreate() { super.onCreate() DynamicColors.applyToActivitiesIfAvailable(this) @@ -73,31 +48,28 @@ class Application : android.app.Application() { storageDir = computeStorageDir() - /* - * Starting media service. - */ - - // MediaSessionService.onBind requires the intent to have a non-null action. - val intent = Intent(MediaService.SERVICE_INTERFACE) - .apply { setClass(applicationContext, MediaService::class.java) } - startService(intent) - bindService(intent, mediaServiceConnection, 0) - /* * Initializing repositories */ bookRepository = BookDatabase.getDatabase(this).booksDao() - .let { BookRepository(this, it, storageDir, readium.lcpService, readium.streamer) } + .let { dao -> + BookRepository( + applicationContext, + dao, + storageDir, + readium.lcpService, + readium.streamer + ) + } readerRepository = coroutineScope.async { ReaderRepository( this@Application, readium, - mediaServiceBinder.await(), bookRepository, - navigatorPreferences + navigatorPreferences, ) } } diff --git a/test-app/src/main/java/org/readium/r2/testapp/bookshelf/BookshelfFragment.kt b/test-app/src/main/java/org/readium/r2/testapp/bookshelf/BookshelfFragment.kt index 6e136d6dbe..d40da78c98 100644 --- a/test-app/src/main/java/org/readium/r2/testapp/bookshelf/BookshelfFragment.kt +++ b/test-app/src/main/java/org/readium/r2/testapp/bookshelf/BookshelfFragment.kt @@ -23,7 +23,6 @@ import androidx.fragment.app.activityViewModels import androidx.recyclerview.widget.RecyclerView import com.google.android.material.dialog.MaterialAlertDialogBuilder import com.google.android.material.snackbar.Snackbar -import org.readium.r2.shared.extensions.tryOrLog import org.readium.r2.testapp.R import org.readium.r2.testapp.databinding.FragmentBookshelfBinding import org.readium.r2.testapp.domain.model.Book @@ -36,7 +35,6 @@ class BookshelfFragment : Fragment() { private val bookshelfViewModel: BookshelfViewModel by activityViewModels() private lateinit var bookshelfAdapter: BookshelfAdapter private lateinit var documentPickerLauncher: ActivityResultLauncher - private lateinit var readerLauncher: ActivityResultLauncher private var binding: FragmentBookshelfBinding by viewLifecycle() override fun onCreateView( @@ -66,11 +64,6 @@ class BookshelfFragment : Fragment() { } } - readerLauncher = - registerForActivityResult(ReaderActivityContract()) { input -> - input?.let { tryOrLog { bookshelfViewModel.closePublication(input.bookId) } } - } - binding.bookshelfBookList.apply { setHasFixedSize(true) layoutManager = GridAutoFitLayoutManager(requireContext(), 120) @@ -147,7 +140,8 @@ class BookshelfFragment : Fragment() { } is BookshelfViewModel.Event.LaunchReader -> { - readerLauncher.launch(event.arguments) + val intent = ReaderActivityContract().createIntent(requireContext(), event.arguments) + startActivity(intent) null } } diff --git a/test-app/src/main/java/org/readium/r2/testapp/bookshelf/BookshelfViewModel.kt b/test-app/src/main/java/org/readium/r2/testapp/bookshelf/BookshelfViewModel.kt index 676869ab44..ec97f4a1ed 100644 --- a/test-app/src/main/java/org/readium/r2/testapp/bookshelf/BookshelfViewModel.kt +++ b/test-app/src/main/java/org/readium/r2/testapp/bookshelf/BookshelfViewModel.kt @@ -110,11 +110,6 @@ class BookshelfViewModel(application: Application) : AndroidViewModel(applicatio } } - fun closePublication(bookId: Long) = viewModelScope.launch { - val readerRepository = app.readerRepository.await() - readerRepository.close(bookId) - } - sealed class Event { object ImportPublicationSuccess : Event() diff --git a/test-app/src/main/java/org/readium/r2/testapp/reader/BaseReaderFragment.kt b/test-app/src/main/java/org/readium/r2/testapp/reader/BaseReaderFragment.kt index 403ba3ea26..3318834dba 100644 --- a/test-app/src/main/java/org/readium/r2/testapp/reader/BaseReaderFragment.kt +++ b/test-app/src/main/java/org/readium/r2/testapp/reader/BaseReaderFragment.kt @@ -77,7 +77,9 @@ abstract class BaseReaderFragment : Fragment() { model.insertBookmark(navigator.currentLocator.value) } R.id.settings -> { - UserPreferencesBottomSheetDialogFragment().show(childFragmentManager, "Settings") + val settingsModel = checkNotNull(model.settings) + UserPreferencesBottomSheetDialogFragment(settingsModel, "User Settings") + .show(childFragmentManager, "Settings") } R.id.drm -> { model.activityChannel.send(ReaderViewModel.Event.OpenDrmManagementRequested) diff --git a/test-app/src/main/java/org/readium/r2/testapp/MediaService.kt b/test-app/src/main/java/org/readium/r2/testapp/reader/MediaService.kt similarity index 57% rename from test-app/src/main/java/org/readium/r2/testapp/MediaService.kt rename to test-app/src/main/java/org/readium/r2/testapp/reader/MediaService.kt index ba121a94c1..8e420630b4 100644 --- a/test-app/src/main/java/org/readium/r2/testapp/MediaService.kt +++ b/test-app/src/main/java/org/readium/r2/testapp/reader/MediaService.kt @@ -4,16 +4,18 @@ * available in the top-level LICENSE file of the project. */ -package org.readium.r2.testapp +package org.readium.r2.testapp.reader +import android.app.Application import android.app.PendingIntent +import android.content.ComponentName import android.content.Intent +import android.content.ServiceConnection import android.os.Build import android.os.IBinder import androidx.lifecycle.lifecycleScope import androidx.media2.session.MediaSession -import kotlin.time.ExperimentalTime -import kotlinx.coroutines.ExperimentalCoroutinesApi +import kotlinx.coroutines.CompletableDeferred import kotlinx.coroutines.FlowPreview import kotlinx.coroutines.Job import kotlinx.coroutines.flow.launchIn @@ -21,20 +23,19 @@ import kotlinx.coroutines.flow.onEach import kotlinx.coroutines.flow.sample import org.readium.navigator.media2.ExperimentalMedia2 import org.readium.navigator.media2.MediaNavigator -import org.readium.r2.testapp.reader.ReaderActivityContract -import org.readium.r2.testapp.utils.LifecycleMediaSessionService +import org.readium.r2.testapp.utils.LifecycleMedia2SessionService import timber.log.Timber -@OptIn(ExperimentalTime::class, ExperimentalMedia2::class, ExperimentalCoroutinesApi::class) -class MediaService : LifecycleMediaSessionService() { +@OptIn(ExperimentalMedia2::class) +class MediaService : LifecycleMedia2SessionService() { /** * The service interface to be used by the app. */ inner class Binder : android.os.Binder() { - private val app: Application - get() = application as Application + private val app: org.readium.r2.testapp.Application + get() = application as org.readium.r2.testapp.Application private var saveLocationJob: Job? = null @@ -55,7 +56,7 @@ class MediaService : LifecycleMediaSessionService() { @OptIn(FlowPreview::class) fun bindNavigator(navigator: MediaNavigator, bookId: Long) { - val activityIntent = createSessionActivityIntent(bookId) + val activityIntent = createSessionActivityIntent() mediaNavigator = navigator mediaSession = navigator.session(applicationContext, activityIntent) .also { addSession(it) } @@ -70,20 +71,14 @@ class MediaService : LifecycleMediaSessionService() { .launchIn(lifecycleScope) } - private fun createSessionActivityIntent(bookId: Long): PendingIntent { + private fun createSessionActivityIntent(): PendingIntent { // This intent will be triggered when the notification is clicked. var flags = PendingIntent.FLAG_UPDATE_CURRENT if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) { flags = flags or PendingIntent.FLAG_IMMUTABLE } - val intent = - ReaderActivityContract().createIntent( - applicationContext, - ReaderActivityContract.Arguments(bookId) - ) - intent.addFlags(Intent.FLAG_ACTIVITY_CLEAR_TOP) - + val intent = application.packageManager.getLaunchIntentForPackage(application.packageName) return PendingIntent.getActivity(applicationContext, 0, intent, flags) } } @@ -130,6 +125,54 @@ class MediaService : LifecycleMediaSessionService() { } companion object { - const val SERVICE_INTERFACE = "org.readium.r2.testapp.MediaService" + + const val SERVICE_INTERFACE = "org.readium.r2.testapp.reader.MediaService" + + fun start(application: Application) { + val intent = intent(application) + application.startService(intent) + } + + suspend fun bind(application: Application): MediaService.Binder { + val mediaServiceBinder: CompletableDeferred = + CompletableDeferred() + + val mediaServiceConnection = object : ServiceConnection { + + override fun onServiceConnected(name: ComponentName?, service: IBinder) { + Timber.d("MediaService bound.") + mediaServiceBinder.complete(service as MediaService.Binder) + } + + override fun onServiceDisconnected(name: ComponentName) { + Timber.e("MediaService disconnected.") + + // Should not happen, do nothing. + } + + override fun onNullBinding(name: ComponentName) { + val errorMessage = "Failed to bind to MediaService." + Timber.e(errorMessage) + val exception = IllegalStateException(errorMessage) + mediaServiceBinder.completeExceptionally(exception) + // Should not happen, do nothing. + } + } + + val intent = intent(application) + application.bindService(intent, mediaServiceConnection, 0) + + return mediaServiceBinder.await() + } + + fun stop(application: Application) { + val intent = intent(application) + application.stopService(intent) + } + + private fun intent(application: Application) = + Intent(SERVICE_INTERFACE) + // MediaSessionService.onBind requires the intent to have a non-null action + .apply { setClass(application, MediaService::class.java) } } } diff --git a/test-app/src/main/java/org/readium/r2/testapp/reader/ReaderActivity.kt b/test-app/src/main/java/org/readium/r2/testapp/reader/ReaderActivity.kt index cc8867dc7b..42354a09fe 100644 --- a/test-app/src/main/java/org/readium/r2/testapp/reader/ReaderActivity.kt +++ b/test-app/src/main/java/org/readium/r2/testapp/reader/ReaderActivity.kt @@ -6,8 +6,6 @@ package org.readium.r2.testapp.reader -import android.app.Activity -import android.content.Intent import android.os.Build import android.os.Bundle import android.view.MenuItem @@ -150,7 +148,7 @@ open class ReaderActivity : AppCompatActivity() { } override fun finish() { - setResult(Activity.RESULT_OK, Intent().putExtras(intent)) + model.close() super.finish() } diff --git a/test-app/src/main/java/org/readium/r2/testapp/reader/ReaderInitData.kt b/test-app/src/main/java/org/readium/r2/testapp/reader/ReaderInitData.kt index 21086934e1..cf042237f6 100644 --- a/test-app/src/main/java/org/readium/r2/testapp/reader/ReaderInitData.kt +++ b/test-app/src/main/java/org/readium/r2/testapp/reader/ReaderInitData.kt @@ -15,10 +15,13 @@ import org.readium.navigator.media2.ExperimentalMedia2 import org.readium.navigator.media2.MediaNavigator import org.readium.r2.navigator.epub.EpubNavigatorFactory import org.readium.r2.navigator.epub.EpubPreferences +import org.readium.r2.navigator.media3.tts.AndroidTtsNavigatorFactory +import org.readium.r2.navigator.media3.tts.android.AndroidTtsPreferences import org.readium.r2.navigator.pdf.PdfNavigatorFactory import org.readium.r2.shared.ExperimentalReadiumApi import org.readium.r2.shared.publication.* import org.readium.r2.testapp.reader.preferences.PreferencesManager +import org.readium.r2.testapp.reader.tts.TtsServiceFacade sealed class ReaderInitData { abstract val bookId: Long @@ -28,36 +31,43 @@ sealed class ReaderInitData { sealed class VisualReaderInitData( override val bookId: Long, override val publication: Publication, - val initialLocation: Locator?, + var initialLocation: Locator?, + val ttsInitData: TtsInitData?, ) : ReaderInitData() class ImageReaderInitData( bookId: Long, publication: Publication, - initialLocation: Locator? -) : VisualReaderInitData(bookId, publication, initialLocation) + initialLocation: Locator?, + ttsInitData: TtsInitData?, +) : VisualReaderInitData(bookId, publication, initialLocation, ttsInitData) class EpubReaderInitData( bookId: Long, publication: Publication, initialLocation: Locator?, val preferencesManager: PreferencesManager, - val navigatorFactory: EpubNavigatorFactory -) : VisualReaderInitData(bookId, publication, initialLocation) + val navigatorFactory: EpubNavigatorFactory, + ttsInitData: TtsInitData?, +) : VisualReaderInitData(bookId, publication, initialLocation, ttsInitData) class PdfReaderInitData( bookId: Long, publication: Publication, initialLocation: Locator?, val preferencesManager: PreferencesManager, - val navigatorFactory: PdfNavigatorFactory -) : VisualReaderInitData(bookId, publication, initialLocation) + val navigatorFactory: PdfNavigatorFactory, + ttsInitData: TtsInitData?, +) : VisualReaderInitData(bookId, publication, initialLocation, ttsInitData) -@ExperimentalMedia2 +@OptIn(ExperimentalMedia2::class) class MediaReaderInitData( override val bookId: Long, override val publication: Publication, val mediaNavigator: MediaNavigator, + val sessionBinder: MediaService.Binder + // val preferencesManager: PreferencesManager, + // val navigatorFactory: PlayerNavigatorFactory ) : ReaderInitData() class DummyReaderInitData( @@ -69,3 +79,9 @@ class DummyReaderInitData( ) ) } + +class TtsInitData( + val ttsServiceFacade: TtsServiceFacade, + val ttsNavigatorFactory: AndroidTtsNavigatorFactory, + val preferencesManager: PreferencesManager, +) diff --git a/test-app/src/main/java/org/readium/r2/testapp/reader/ReaderRepository.kt b/test-app/src/main/java/org/readium/r2/testapp/reader/ReaderRepository.kt index 5af2a13c6c..db187d7d25 100644 --- a/test-app/src/main/java/org/readium/r2/testapp/reader/ReaderRepository.kt +++ b/test-app/src/main/java/org/readium/r2/testapp/reader/ReaderRepository.kt @@ -16,6 +16,7 @@ import org.readium.adapters.pdfium.navigator.PdfiumEngineProvider import org.readium.navigator.media2.ExperimentalMedia2 import org.readium.navigator.media2.MediaNavigator import org.readium.r2.navigator.epub.EpubNavigatorFactory +import org.readium.r2.navigator.media3.tts.TtsNavigatorFactory import org.readium.r2.navigator.pdf.PdfNavigatorFactory import org.readium.r2.shared.ExperimentalReadiumApi import org.readium.r2.shared.publication.Locator @@ -25,24 +26,25 @@ import org.readium.r2.shared.publication.services.isRestricted import org.readium.r2.shared.publication.services.protectionError import org.readium.r2.shared.util.Try import org.readium.r2.shared.util.getOrElse -import org.readium.r2.testapp.MediaService import org.readium.r2.testapp.Readium import org.readium.r2.testapp.bookshelf.BookRepository +import org.readium.r2.testapp.reader.preferences.AndroidTtsPreferencesManagerFactory import org.readium.r2.testapp.reader.preferences.EpubPreferencesManagerFactory import org.readium.r2.testapp.reader.preferences.PdfiumPreferencesManagerFactory +import org.readium.r2.testapp.reader.tts.TtsServiceFacade +import timber.log.Timber /** * Open and store publications in order for them to be listened or read. * * Ensure you call [open] before any attempt to start a [ReaderActivity]. * Pass the method result to the activity to enable it to know which current publication it must - * retrieve from this rep+ository - media or visual. + * retrieve from this repository - media or visual. */ -@OptIn(ExperimentalMedia2::class, ExperimentalReadiumApi::class) +@OptIn(ExperimentalReadiumApi::class) class ReaderRepository( private val application: Application, private val readium: Readium, - private val mediaBinder: MediaService.Binder, private val bookRepository: BookRepository, private val preferencesDataStore: DataStore, ) { @@ -51,6 +53,9 @@ class ReaderRepository( private val repository: MutableMap = mutableMapOf() + private val ttsServiceFacade: TtsServiceFacade = + TtsServiceFacade(application) + operator fun get(bookId: Long): ReaderInitData? = repository[bookId] @@ -108,16 +113,53 @@ class ReaderRepository( publication: Publication, initialLocator: Locator? ): MediaReaderInitData { + val navigator = MediaNavigator.create( application, publication, initialLocator ).getOrElse { throw Exception("Cannot open audiobook.") } + MediaService.start(application) + val mediaBinder = MediaService.bind(application) mediaBinder.bindNavigator(navigator, bookId) - return MediaReaderInitData(bookId, publication, navigator) + return MediaReaderInitData(bookId, publication, navigator, mediaBinder) } + /* private suspend fun openAudio( + bookId: Long, + publication: Publication, + initialLocator: Locator? + ): MediaReaderInitData { + + val preferencesManager = ExoPlayerPreferencesManagerFactory(preferencesDataStore) + .createPreferenceManager(bookId) + val mediaEngine = ExoPlayerEngineProvider(application) + val initialPreferences = preferencesManager.preferences.value + val actualInitialLocator = initialLocator + ?: publication.locatorFromLink(publication.readingOrder[0])!! + + val navigatorFactory = PlayerNavigatorFactory( + publication, + mediaEngine, + DefaultMetadataProvider(), + initialPreferences, + actualInitialLocator, + ) + + val navigator = navigatorFactory.getMediaNavigator() + .getOrElse { throw Exception("Cannot open audiobook.") } + + val navigator = MediaNavigator.create( + application, + publication, + initialLocator + ).getOrElse { throw Exception("Cannot open audiobook.") } + + mediaBinder.bindNavigator(navigator, bookId) + return MediaReaderInitData(bookId, publication,, preferencesManager, navigatorFactory) + } */ + private suspend fun openEpub( bookId: Long, publication: Publication, @@ -127,10 +169,11 @@ class ReaderRepository( val preferencesManager = EpubPreferencesManagerFactory(preferencesDataStore) .createPreferenceManager(bookId) val navigatorFactory = EpubNavigatorFactory(publication) + val ttsInitData = getTtsInitData(bookId, publication) return EpubReaderInitData( bookId, publication, initialLocator, - preferencesManager, navigatorFactory + preferencesManager, navigatorFactory, ttsInitData ) } @@ -144,14 +187,16 @@ class ReaderRepository( .createPreferenceManager(bookId) val pdfEngine = PdfiumEngineProvider() val navigatorFactory = PdfNavigatorFactory(publication, pdfEngine) + val ttsInitData = getTtsInitData(bookId, publication) return PdfReaderInitData( bookId, publication, initialLocator, - preferencesManager, navigatorFactory + preferencesManager, navigatorFactory, + ttsInitData ) } - private fun openImage( + private suspend fun openImage( bookId: Long, publication: Publication, initialLocator: Locator? @@ -159,16 +204,31 @@ class ReaderRepository( return ImageReaderInitData( bookId = bookId, publication = publication, - initialLocation = initialLocator + initialLocation = initialLocator, + ttsInitData = getTtsInitData(bookId, publication) ) } - fun close(bookId: Long) { + private suspend fun getTtsInitData( + bookId: Long, + publication: Publication, + ): TtsInitData? { + val preferencesManager = AndroidTtsPreferencesManagerFactory(preferencesDataStore) + .createPreferenceManager(bookId) + val navigatorFactory = TtsNavigatorFactory(application, publication) ?: return null + return TtsInitData(ttsServiceFacade, navigatorFactory, preferencesManager) + } + + suspend fun close(bookId: Long) { + Timber.d("Closing Publication") when (val initData = repository.remove(bookId)) { is MediaReaderInitData -> { - mediaBinder.closeNavigator() + initData.sessionBinder.closeNavigator() + MediaService.stop(application) + initData.publication.close() } is VisualReaderInitData -> { + initData.ttsInitData?.ttsServiceFacade?.closeSession() initData.publication.close() } null, is DummyReaderInitData -> { diff --git a/test-app/src/main/java/org/readium/r2/testapp/reader/ReaderViewModel.kt b/test-app/src/main/java/org/readium/r2/testapp/reader/ReaderViewModel.kt index b42551c0b9..2ddf3868ef 100644 --- a/test-app/src/main/java/org/readium/r2/testapp/reader/ReaderViewModel.kt +++ b/test-app/src/main/java/org/readium/r2/testapp/reader/ReaderViewModel.kt @@ -37,20 +37,26 @@ import org.readium.r2.testapp.reader.tts.TtsViewModel import org.readium.r2.testapp.search.SearchPagingSource import org.readium.r2.testapp.utils.EventChannel import org.readium.r2.testapp.utils.createViewModelFactory +import timber.log.Timber @OptIn(Search::class, ExperimentalDecorator::class, ExperimentalCoroutinesApi::class) class ReaderViewModel( - application: Application, - val readerInitData: ReaderInitData, + private val bookId: Long, + private val readerRepository: ReaderRepository, private val bookRepository: BookRepository, ) : ViewModel() { + val readerInitData = + try { + checkNotNull(readerRepository[bookId]) + } catch (e: Exception) { + // Fallbacks on a dummy Publication to avoid crashing the app until the Activity finishes. + DummyReaderInitData(bookId) + } + val publication: Publication = readerInitData.publication - val bookId: Long = - readerInitData.bookId - val activityChannel: EventChannel = EventChannel(Channel(Channel.BUFFERED), viewModelScope) @@ -58,9 +64,8 @@ class ReaderViewModel( EventChannel(Channel(Channel.BUFFERED), viewModelScope) val tts: TtsViewModel? = TtsViewModel( - context = application, - publication = readerInitData.publication, - scope = viewModelScope + viewModelScope = viewModelScope, + readerInitData = readerInitData ) val settings: UserPreferencesViewModel<*, *>? = UserPreferencesViewModel( @@ -68,12 +73,15 @@ class ReaderViewModel( readerInitData = readerInitData ) - override fun onCleared() { - super.onCleared() - tts?.onCleared() + fun close() { + viewModelScope.launch { + tts?.stop() + readerRepository.close(bookId) + } } fun saveProgression(locator: Locator) = viewModelScope.launch { + Timber.v("Saving locator for book $bookId: $locator.") bookRepository.saveProgression(locator, bookId) } @@ -255,16 +263,10 @@ class ReaderViewModel( companion object { fun createFactory(application: Application, arguments: ReaderActivityContract.Arguments) = createViewModelFactory { - val readerInitData = - try { - val readerRepository = application.readerRepository.getCompleted() - checkNotNull(readerRepository[arguments.bookId]) - } catch (e: Exception) { - // Fallbacks on a dummy Publication to avoid crashing the app until the Activity finishes. - DummyReaderInitData(arguments.bookId) - } - - ReaderViewModel(application, readerInitData, application.bookRepository) + val readerRepository = + application.readerRepository.getCompleted() + + ReaderViewModel(arguments.bookId, readerRepository, application.bookRepository) } } } diff --git a/test-app/src/main/java/org/readium/r2/testapp/reader/VisualReaderFragment.kt b/test-app/src/main/java/org/readium/r2/testapp/reader/VisualReaderFragment.kt index 9bfbdaddb6..b88905de3c 100644 --- a/test-app/src/main/java/org/readium/r2/testapp/reader/VisualReaderFragment.kt +++ b/test-app/src/main/java/org/readium/r2/testapp/reader/VisualReaderFragment.kt @@ -35,19 +35,22 @@ import androidx.fragment.app.Fragment import androidx.lifecycle.Lifecycle import androidx.lifecycle.lifecycleScope import androidx.lifecycle.repeatOnLifecycle -import kotlin.time.Duration.Companion.milliseconds import kotlin.time.Duration.Companion.seconds import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.flow.* import kotlinx.coroutines.launch import kotlinx.parcelize.Parcelize import org.readium.r2.navigator.* +import org.readium.r2.navigator.media3.tts.android.AndroidTtsEngine import org.readium.r2.navigator.util.BaseActionModeCallback import org.readium.r2.navigator.util.EdgeTapNavigation +import org.readium.r2.shared.ExperimentalReadiumApi +import org.readium.r2.shared.publication.Locator import org.readium.r2.shared.util.Language import org.readium.r2.testapp.R import org.readium.r2.testapp.databinding.FragmentReaderBinding import org.readium.r2.testapp.domain.model.Highlight +import org.readium.r2.testapp.reader.preferences.UserPreferencesBottomSheetDialogFragment import org.readium.r2.testapp.reader.tts.TtsControls import org.readium.r2.testapp.reader.tts.TtsViewModel import org.readium.r2.testapp.utils.* @@ -59,7 +62,7 @@ import org.readium.r2.testapp.utils.extensions.throttleLatest * * Provides common menu items and saves last location on stop. */ -@OptIn(ExperimentalDecorator::class) +@OptIn(ExperimentalDecorator::class, ExperimentalReadiumApi::class) abstract class VisualReaderFragment : BaseReaderFragment(), VisualNavigator.Listener { protected var binding: FragmentReaderBinding by viewLifecycle() @@ -80,6 +83,13 @@ abstract class VisualReaderFragment : BaseReaderFragment(), VisualNavigator.List */ private var disableTouches by mutableStateOf(false) + /** + * When true, the fragment won't save progression. + * This is useful in the case where the TTS is on and a service is saving progression + * in background. + */ + private var preventProgressionSaving: Boolean = false + override fun onViewCreated(view: View, savedInstanceState: Bundle?) { super.onViewCreated(view, savedInstanceState) @@ -123,6 +133,10 @@ abstract class VisualReaderFragment : BaseReaderFragment(), VisualNavigator.List model.tts?.let { tts -> TtsControls( model = tts, + onPreferences = { + UserPreferencesBottomSheetDialogFragment(tts.preferencesModel, "TTS Settings") + .show(childFragmentManager, "TtsSettings") + }, modifier = Modifier .align(Alignment.BottomCenter) .padding(8.dp) @@ -134,7 +148,11 @@ abstract class VisualReaderFragment : BaseReaderFragment(), VisualNavigator.List viewLifecycleOwner.lifecycleScope.launch { viewLifecycleOwner.repeatOnLifecycle(Lifecycle.State.STARTED) { navigator.currentLocator - .onEach { model.saveProgression(it) } + .onEach { + if (!preventProgressionSaving) { + model.saveProgression(it) + } + } .launchIn(this) setupHighlights(this) @@ -179,19 +197,9 @@ abstract class VisualReaderFragment : BaseReaderFragment(), VisualNavigator.List } .launchIn(scope) - // Navigate to the currently spoken utterance. - state.map { it.playingUtterance } - .filterNotNull() - // Prevent jumping to many locations when the user skips repeatedly forward/backward. - .throttleLatest(500.milliseconds) - .onEach { locator -> - navigator.go(locator, animated = false) - } - .launchIn(scope) - // Navigate to the currently spoken word. // This will automatically turn pages when needed. - state.map { it.playingWordRange } + position .filterNotNull() // Improve performances by throttling the moves to maximum one per second. .throttleLatest(1.seconds) @@ -202,8 +210,7 @@ abstract class VisualReaderFragment : BaseReaderFragment(), VisualNavigator.List // Prevent interacting with the publication (including page turns) while the TTS is // playing. - state.map { it.isPlaying } - .distinctUntilChanged() + isPlaying .onEach { isPlaying -> disableTouches = isPlaying } @@ -211,8 +218,7 @@ abstract class VisualReaderFragment : BaseReaderFragment(), VisualNavigator.List // Highlight the currently spoken utterance. (navigator as? DecorableNavigator)?.let { navigator -> - state.map { it.playingUtterance } - .distinctUntilChanged() + highlight .onEach { locator -> val decoration = locator?.let { Decoration( @@ -225,6 +231,12 @@ abstract class VisualReaderFragment : BaseReaderFragment(), VisualNavigator.List } .launchIn(scope) } + + showControls + .onEach { showControls -> + preventProgressionSaving = showControls + } + .launchIn(scope) } } @@ -240,21 +252,20 @@ abstract class VisualReaderFragment : BaseReaderFragment(), VisualNavigator.List getString(R.string.tts_error_language_support_incomplete, language.locale.displayLanguage) ) ) { - tts.requestInstallVoice(activity) + AndroidTtsEngine.requestInstallVoice(activity) } } + override fun go(locator: Locator, animated: Boolean) { + model.tts?.stop() + super.go(locator, animated) + } + override fun onDestroyView() { (navigator as? DecorableNavigator)?.removeDecorationListener(decorationListener) super.onDestroyView() } - override fun onStop() { - super.onStop() - - model.tts?.pause() - } - override fun onHiddenChanged(hidden: Boolean) { super.onHiddenChanged(hidden) setMenuVisibility(!hidden) diff --git a/test-app/src/main/java/org/readium/r2/testapp/reader/preferences/PreferencesManagers.kt b/test-app/src/main/java/org/readium/r2/testapp/reader/preferences/PreferencesManagers.kt index e9b575e304..9cd54fb689 100644 --- a/test-app/src/main/java/org/readium/r2/testapp/reader/preferences/PreferencesManagers.kt +++ b/test-app/src/main/java/org/readium/r2/testapp/reader/preferences/PreferencesManagers.kt @@ -27,6 +27,14 @@ import org.readium.r2.navigator.epub.EpubPreferences import org.readium.r2.navigator.epub.EpubPreferencesSerializer import org.readium.r2.navigator.epub.EpubPublicationPreferencesFilter import org.readium.r2.navigator.epub.EpubSharedPreferencesFilter +import org.readium.r2.navigator.media3.exoplayer.ExoPlayerPreferences +import org.readium.r2.navigator.media3.exoplayer.ExoPlayerPreferencesSerializer +import org.readium.r2.navigator.media3.exoplayer.ExoPlayerPublicationPreferencesFilter +import org.readium.r2.navigator.media3.exoplayer.ExoPlayerSharedPreferencesFilter +import org.readium.r2.navigator.media3.tts.android.AndroidTtsPreferences +import org.readium.r2.navigator.media3.tts.android.AndroidTtsPreferencesSerializer +import org.readium.r2.navigator.media3.tts.android.AndroidTtsPublicationPreferencesFilter +import org.readium.r2.navigator.media3.tts.android.AndroidTtsSharedPreferencesFilter import org.readium.r2.navigator.preferences.Configurable import org.readium.r2.navigator.preferences.PreferencesFilter import org.readium.r2.navigator.preferences.PreferencesSerializer @@ -130,3 +138,25 @@ class PdfiumPreferencesManagerFactory( preferencesSerializer = PdfiumPreferencesSerializer(), emptyPreferences = PdfiumPreferences() ) + +class ExoPlayerPreferencesManagerFactory( + dataStore: DataStore +) : PreferencesManagerFactory( + dataStore = dataStore, + klass = ExoPlayerPreferences::class, + sharedPreferencesFilter = ExoPlayerSharedPreferencesFilter, + publicationPreferencesFilter = ExoPlayerPublicationPreferencesFilter, + preferencesSerializer = ExoPlayerPreferencesSerializer(), + emptyPreferences = ExoPlayerPreferences() +) + +class AndroidTtsPreferencesManagerFactory( + dataStore: DataStore +) : PreferencesManagerFactory( + dataStore = dataStore, + klass = AndroidTtsPreferences::class, + sharedPreferencesFilter = AndroidTtsSharedPreferencesFilter, + publicationPreferencesFilter = AndroidTtsPublicationPreferencesFilter, + preferencesSerializer = AndroidTtsPreferencesSerializer(), + emptyPreferences = AndroidTtsPreferences() +) diff --git a/test-app/src/main/java/org/readium/r2/testapp/reader/preferences/UserPreferences.kt b/test-app/src/main/java/org/readium/r2/testapp/reader/preferences/UserPreferences.kt index c44be733b5..88af6ccda2 100644 --- a/test-app/src/main/java/org/readium/r2/testapp/reader/preferences/UserPreferences.kt +++ b/test-app/src/main/java/org/readium/r2/testapp/reader/preferences/UserPreferences.kt @@ -8,57 +8,60 @@ package org.readium.r2.testapp.reader.preferences -import androidx.compose.foundation.clickable import androidx.compose.foundation.layout.* import androidx.compose.material.* -import androidx.compose.material.icons.Icons -import androidx.compose.material.icons.filled.* -import androidx.compose.runtime.* +import androidx.compose.runtime.Composable +import androidx.compose.runtime.collectAsState +import androidx.compose.runtime.getValue import androidx.compose.ui.Alignment import androidx.compose.ui.Modifier -import androidx.compose.ui.graphics.Color -import androidx.compose.ui.graphics.luminance +import androidx.compose.ui.platform.LocalContext +import androidx.compose.ui.res.stringResource import androidx.compose.ui.text.style.TextAlign import androidx.compose.ui.unit.dp -import androidx.compose.ui.window.Dialog -import java.util.* import org.readium.adapters.pdfium.navigator.PdfiumPreferencesEditor import org.readium.r2.navigator.epub.EpubPreferencesEditor +import org.readium.r2.navigator.media3.tts.android.AndroidTtsEngine import org.readium.r2.navigator.preferences.* -import org.readium.r2.navigator.preferences.Color as ReadiumColor import org.readium.r2.navigator.preferences.TextAlign as ReadiumTextAlign import org.readium.r2.shared.ExperimentalReadiumApi import org.readium.r2.shared.publication.epub.EpubLayout import org.readium.r2.shared.util.Language import org.readium.r2.testapp.LITERATA +import org.readium.r2.testapp.R import org.readium.r2.testapp.reader.ReaderViewModel -import org.readium.r2.testapp.utils.compose.ColorPicker +import org.readium.r2.testapp.reader.tts.TtsPreferencesEditor +import org.readium.r2.testapp.shared.views.* import org.readium.r2.testapp.utils.compose.DropdownMenuButton -import org.readium.r2.testapp.utils.compose.ToggleButtonGroup /** * Stateful user settings component paired with a [ReaderViewModel]. */ @Composable -fun UserPreferences(model: UserPreferencesViewModel<*, *>) { +fun UserPreferences( + model: UserPreferencesViewModel<*, *>, + title: String +) { val editor by model.editor.collectAsState() UserPreferences( editor = editor, - commit = model::commit + commit = model::commit, + title = title ) } @Composable private fun

, E : PreferencesEditor

> UserPreferences( editor: E, - commit: () -> Unit + commit: () -> Unit, + title: String ) { Column( modifier = Modifier.padding(vertical = 24.dp) ) { Text( - text = "User settings", + text = title, textAlign = TextAlign.Center, style = MaterialTheme.typography.h6, modifier = Modifier @@ -131,10 +134,53 @@ private fun

, E : PreferencesEditor

> UserPref spread = editor.spread, ) } + is TtsPreferencesEditor -> + TtsUserPreferences( + commit = commit, + language = editor.language, + voice = editor.voice, + speed = editor.speed, + pitch = editor.pitch + ) } } } +@Composable +private fun ColumnScope.TtsUserPreferences( + commit: () -> Unit, + language: Preference, + voice: EnumPreference, + speed: RangePreference, + pitch: RangePreference +) { + Column { + StepperItem( + title = stringResource(R.string.speed_rate), + preference = speed, + commit = commit + ) + StepperItem( + title = stringResource(R.string.pitch_rate), + preference = pitch, + commit = commit + ) + LanguageItem( + preference = language, + commit = commit + ) + + val context = LocalContext.current + + MenuItem( + title = stringResource(R.string.tts_voice), + preference = voice, + formatValue = { it?.value ?: context.getString(R.string.defaultValue) }, + commit = commit + ) + } +} + /** * User settings for a publication with a fixed layout, such as fixed-layout EPUB, PDF or comic book. */ @@ -143,7 +189,7 @@ private fun ColumnScope.FixedLayoutUserPreferences( commit: () -> Unit, language: Preference? = null, readingProgression: EnumPreference? = null, - backgroundColor: Preference? = null, + backgroundColor: Preference? = null, scroll: Preference? = null, scrollAxis: EnumPreference? = null, fit: EnumPreference? = null, @@ -261,7 +307,7 @@ private fun ColumnScope.FixedLayoutUserPreferences( @Composable private fun ColumnScope.ReflowableUserPreferences( commit: () -> Unit, - backgroundColor: Preference? = null, + backgroundColor: Preference? = null, columnCount: EnumPreference? = null, fontFamily: Preference? = null, fontSize: RangePreference? = null, @@ -279,7 +325,7 @@ private fun ColumnScope.ReflowableUserPreferences( readingProgression: EnumPreference? = null, scroll: Preference? = null, textAlign: EnumPreference? = null, - textColor: Preference? = null, + textColor: Preference? = null, textNormalization: Preference? = null, theme: EnumPreference? = null, typeScale: RangePreference? = null, @@ -411,6 +457,7 @@ private fun ColumnScope.ReflowableUserPreferences( title = "Typeface", preference = fontFamily .withSupportedValues( + null, FontFamily.LITERATA, FontFamily.SANS_SERIF, FontFamily.IA_WRITER_DUOSPACE, @@ -547,378 +594,6 @@ private fun ColumnScope.ReflowableUserPreferences( } } -/** - * Component for an [EnumPreference] displayed as a group of mutually exclusive buttons. - * This works best with a small number of enum values. - */ -@Composable -private fun ButtonGroupItem( - title: String, - preference: EnumPreference, - commit: () -> Unit, - formatValue: (T) -> String -) { - ButtonGroupItem( - title = title, - options = preference.supportedValues, - isActive = preference.isEffective, - activeOption = preference.effectiveValue, - selectedOption = preference.value, - formatValue = formatValue, - onClear = { preference.clear(); commit() } - .takeIf { preference.value != null }, - onSelectedOptionChanged = { newValue -> - if (newValue == preference.value) { - preference.clear() - } else { - preference.set(newValue) - } - commit() - } - ) -} - -/** - * Group of mutually exclusive buttons. - */ -@Composable -private fun ButtonGroupItem( - title: String, - options: List, - isActive: Boolean, - activeOption: T, - selectedOption: T?, - formatValue: (T) -> String, - onClear: (() -> Unit)?, - onSelectedOptionChanged: (T) -> Unit, -) { - Item(title, isActive = isActive, onClear = onClear) { - ToggleButtonGroup( - options = options, - activeOption = activeOption, - selectedOption = selectedOption, - onSelectOption = { option -> onSelectedOptionChanged(option) } - ) { option -> - Text( - text = formatValue(option), - style = MaterialTheme.typography.caption - ) - } - } -} - -/** - * Component for an [EnumPreference] displayed as a dropdown menu. - */ -@Composable -private fun MenuItem( - title: String, - preference: EnumPreference, - commit: () -> Unit, - formatValue: (T?) -> String -) { - MenuItem( - title = title, - value = preference.value ?: preference.effectiveValue, - values = listOf(null) + preference.supportedValues, - isActive = preference.isEffective, - formatValue = formatValue, - onClear = { preference.clear(); commit() } - .takeIf { preference.value != null }, - onValueChanged = { value -> - preference.set(value) - commit() - } - ) -} - -/** - * Dropdown menu. - */ -@Composable -private fun MenuItem( - title: String, - value: T, - values: List, - isActive: Boolean, - formatValue: (T) -> String, - onValueChanged: (T) -> Unit, - onClear: (() -> Unit)? -) { - Item(title, isActive = isActive, onClear = onClear) { - DropdownMenuButton( - text = { - Text( - text = formatValue(value), - style = MaterialTheme.typography.caption - ) - } - ) { dismiss -> - for (value in values) { - DropdownMenuItem( - onClick = { - dismiss() - onValueChanged(value) - } - ) { - Text(formatValue(value)) - } - } - } - } -} - -/** - * Component for a [RangePreference] with decrement and increment buttons. - */ -@Composable -private fun > StepperItem( - title: String, - preference: RangePreference, - commit: () -> Unit -) { - StepperItem( - title = title, - isActive = preference.isEffective, - value = preference.value ?: preference.effectiveValue, - formatValue = preference::formatValue, - onDecrement = { preference.decrement(); commit() }, - onIncrement = { preference.increment(); commit() }, - onClear = { preference.clear(); commit() } - .takeIf { preference.value != null }, - ) -} - -/** - * Component for a [RangePreference] with decrement and increment buttons. - */ -@Composable -private fun StepperItem( - title: String, - isActive: Boolean, - value: T, - formatValue: (T) -> String, - onDecrement: () -> Unit, - onIncrement: () -> Unit, - onClear: (() -> Unit)? -) { - Item(title, isActive = isActive, onClear = onClear) { - Row( - verticalAlignment = Alignment.CenterVertically, - horizontalArrangement = Arrangement.spacedBy(16.dp) - ) { - IconButton( - onClick = onDecrement, - content = { - Icon(Icons.Default.Remove, contentDescription = "Less") - } - ) - - Text( - text = formatValue(value), - modifier = Modifier.widthIn(min = 30.dp), - textAlign = TextAlign.Center - ) - - IconButton( - onClick = onIncrement, - content = { - Icon(Icons.Default.Add, contentDescription = "More") - } - ) - } - } -} - -/** - * Component for a boolean [Preference]. - */ -@Composable -private fun SwitchItem( - title: String, - preference: Preference, - commit: () -> Unit -) { - SwitchItem( - title = title, - value = preference.value ?: preference.effectiveValue, - isActive = preference.isEffective, - onCheckedChange = { preference.set(it); commit() }, - onToggle = { preference.toggle(); commit() }, - onClear = { preference.clear(); commit() } - .takeIf { preference.value != null }, - ) -} - -/** - * Switch - */ -@Composable -private fun SwitchItem( - title: String, - value: Boolean, - isActive: Boolean, - onCheckedChange: (Boolean) -> Unit, - onToggle: () -> Unit, - onClear: (() -> Unit)? -) { - Item( - title = title, - isActive = isActive, - onClick = onToggle, - onClear = onClear - ) { - Switch( - checked = value, - onCheckedChange = onCheckedChange - ) - } -} - -/** - * Component for a [Preference]. - */ -@Composable -private fun ColorItem( - title: String, - preference: Preference, - commit: () -> Unit -) { - ColorItem( - title = title, - isActive = preference.isEffective, - value = preference.value ?: preference.effectiveValue, - noValueSelected = preference.value == null, - onColorChanged = { preference.set(it); commit() }, - onClear = { preference.clear(); commit() } - .takeIf { preference.value != null } - ) -} - -/** - * Color picker - */ -@Composable -private fun ColorItem( - title: String, - isActive: Boolean, - value: ReadiumColor, - noValueSelected: Boolean, - onColorChanged: (ReadiumColor?) -> Unit, - onClear: (() -> Unit)? -) { - var isPicking by remember { mutableStateOf(false) } - - Item( - title = title, - isActive = isActive, - onClick = { isPicking = true }, - onClear = onClear - ) { - val color = Color(value.int) - - OutlinedButton( - onClick = { isPicking = true }, - colors = ButtonDefaults.buttonColors(backgroundColor = color) - ) { - if (noValueSelected) { - Icon( - imageVector = Icons.Default.Palette, - contentDescription = "Change color", - tint = if (color.luminance() > 0.5) Color.Black else Color.White - ) - } - } - - if (isPicking) { - Dialog( - onDismissRequest = { isPicking = false } - ) { - Column( - horizontalAlignment = Alignment.End - ) { - ColorPicker { color -> - isPicking = false - onColorChanged(ReadiumColor(color)) - } - Button( - onClick = { - isPicking = false - onColorChanged(null) - } - ) { - Text("Clear") - } - } - } - } - } -} - -/** - * Component for a [Preference]`. - */ -@Composable -fun LanguageItem( - preference: Preference, - commit: () -> Unit -) { - val languages = remember { - Locale.getAvailableLocales() - .map { Language(it).removeRegion() } - .distinct() - .sortedBy { it.locale.displayName } - } - - MenuItem( - title = "Language", - isActive = preference.isEffective, - value = preference.value ?: preference.effectiveValue, - values = languages, - formatValue = { it?.locale?.displayName ?: "Unknown" }, - onClear = { preference.clear(); commit() } - .takeIf { preference.value != null }, - onValueChanged = { value -> - preference.set(value) - commit() - } - ) -} - -@OptIn(ExperimentalMaterialApi::class) -@Composable -private fun Item( - title: String, - isActive: Boolean = true, - onClick: (() -> Unit)? = null, - onClear: (() -> Unit)? = null, - content: @Composable () -> Unit -) { - ListItem( - modifier = - if (onClick != null) Modifier.clickable(onClick = onClick) - else Modifier, - text = { - val alpha = if (isActive) 1.0f else ContentAlpha.disabled - CompositionLocalProvider(LocalContentAlpha provides alpha) { - Text(title) - } - }, - trailing = { - Row { - content() - - IconButton(onClick = onClear ?: {}, enabled = onClear != null) { - Icon( - Icons.Default.Backspace, - contentDescription = "Clear" - ) - } - } - } - ) -} - @Composable private fun Divider() { Divider(modifier = Modifier.padding(vertical = 16.dp)) diff --git a/test-app/src/main/java/org/readium/r2/testapp/reader/preferences/UserPreferencesBottomSheetDialogFragment.kt b/test-app/src/main/java/org/readium/r2/testapp/reader/preferences/UserPreferencesBottomSheetDialogFragment.kt index 6d38c256a8..4cff03725d 100644 --- a/test-app/src/main/java/org/readium/r2/testapp/reader/preferences/UserPreferencesBottomSheetDialogFragment.kt +++ b/test-app/src/main/java/org/readium/r2/testapp/reader/preferences/UserPreferencesBottomSheetDialogFragment.kt @@ -9,18 +9,15 @@ package org.readium.r2.testapp.reader.preferences import android.app.Dialog import android.os.Bundle import androidx.compose.runtime.Composable -import androidx.fragment.app.activityViewModels import com.google.android.material.bottomsheet.BottomSheetDialog -import org.readium.r2.shared.ExperimentalReadiumApi -import org.readium.r2.testapp.reader.ReaderViewModel import org.readium.r2.testapp.utils.compose.ComposeBottomSheetDialogFragment -@OptIn(ExperimentalReadiumApi::class) -class UserPreferencesBottomSheetDialogFragment : ComposeBottomSheetDialogFragment( +class UserPreferencesBottomSheetDialogFragment( + private val model: UserPreferencesViewModel<*, *>, + private val title: String +) : ComposeBottomSheetDialogFragment( isScrollable = true ) { - private val model: ReaderViewModel by activityViewModels() - override fun onCreateDialog(savedInstanceState: Bundle?): Dialog = (super.onCreateDialog(savedInstanceState) as BottomSheetDialog).apply { // Reduce the dim to see the impact of the settings on the page. @@ -34,7 +31,6 @@ class UserPreferencesBottomSheetDialogFragment : ComposeBottomSheetDialogFragmen @Composable override fun Content() { - val settingsModel = checkNotNull(model.settings) - UserPreferences(settingsModel) + UserPreferences(model, title) } } diff --git a/test-app/src/main/java/org/readium/r2/testapp/reader/preferences/UserPreferencesViewModel.kt b/test-app/src/main/java/org/readium/r2/testapp/reader/preferences/UserPreferencesViewModel.kt index 3494faae2c..991639b320 100644 --- a/test-app/src/main/java/org/readium/r2/testapp/reader/preferences/UserPreferencesViewModel.kt +++ b/test-app/src/main/java/org/readium/r2/testapp/reader/preferences/UserPreferencesViewModel.kt @@ -10,14 +10,21 @@ import androidx.lifecycle.LifecycleOwner import androidx.lifecycle.flowWithLifecycle import androidx.lifecycle.lifecycleScope import kotlinx.coroutines.CoroutineScope -import kotlinx.coroutines.flow.* +import kotlinx.coroutines.flow.StateFlow +import kotlinx.coroutines.flow.launchIn +import kotlinx.coroutines.flow.onEach import kotlinx.coroutines.launch import org.readium.adapters.pdfium.navigator.PdfiumPreferences import org.readium.adapters.pdfium.navigator.PdfiumSettings -import org.readium.r2.navigator.epub.* -import org.readium.r2.navigator.preferences.* +import org.readium.r2.navigator.epub.EpubPreferences +import org.readium.r2.navigator.epub.EpubSettings +import org.readium.r2.navigator.preferences.Configurable +import org.readium.r2.navigator.preferences.PreferencesEditor import org.readium.r2.shared.ExperimentalReadiumApi -import org.readium.r2.testapp.reader.* +import org.readium.r2.testapp.reader.EpubReaderInitData +import org.readium.r2.testapp.reader.PdfReaderInitData +import org.readium.r2.testapp.reader.ReaderInitData +import org.readium.r2.testapp.reader.ReaderViewModel import org.readium.r2.testapp.utils.extensions.mapStateIn /** diff --git a/test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsControls.kt b/test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsControls.kt index 423e1dcd63..a5459d8ba0 100644 --- a/test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsControls.kt +++ b/test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsControls.kt @@ -4,82 +4,64 @@ * available in the top-level LICENSE file of the project. */ +@file:OptIn(ExperimentalReadiumApi::class) + package org.readium.r2.testapp.reader.tts -import androidx.compose.foundation.layout.* -import androidx.compose.material.* +import androidx.compose.foundation.layout.Arrangement +import androidx.compose.foundation.layout.Row +import androidx.compose.foundation.layout.Spacer +import androidx.compose.foundation.layout.size +import androidx.compose.material.Card +import androidx.compose.material.Icon +import androidx.compose.material.IconButton import androidx.compose.material.icons.Icons import androidx.compose.material.icons.filled.* -import androidx.compose.runtime.* +import androidx.compose.runtime.Composable +import androidx.compose.runtime.getValue import androidx.compose.ui.Alignment import androidx.compose.ui.Modifier import androidx.compose.ui.res.stringResource import androidx.compose.ui.unit.dp -import java.text.DecimalFormat -import org.readium.r2.navigator.tts.PublicationSpeechSynthesizer.Configuration -import org.readium.r2.navigator.tts.TtsEngine.Voice import org.readium.r2.shared.ExperimentalReadiumApi -import org.readium.r2.shared.util.Language import org.readium.r2.testapp.R -import org.readium.r2.testapp.shared.views.SelectorListItem import org.readium.r2.testapp.utils.extensions.asStateWhenStarted /** * TTS controls bar displayed at the bottom of the screen when speaking a publication. */ -@OptIn(ExperimentalReadiumApi::class) @Composable -fun TtsControls(model: TtsViewModel, modifier: Modifier = Modifier) { - val showControls by model.state.asStateWhenStarted { it.showControls } - val isPlaying by model.state.asStateWhenStarted { it.isPlaying } - val settings by model.state.asStateWhenStarted { it.settings } +fun TtsControls( + model: TtsViewModel, + onPreferences: () -> Unit, + modifier: Modifier = Modifier +) { + val showControls by model.showControls.asStateWhenStarted() + val isPlaying by model.isPlaying.asStateWhenStarted() if (showControls) { TtsControls( playing = isPlaying, - availableRates = listOf(0.25, 0.5, 0.75, 1.0, 1.25, 1.5, 2.0) - .filter { it in settings.rateRange }, - availableLanguages = settings.availableLanguages, - availableVoices = settings.availableVoices, - config = settings.config, - onConfigChange = model::setConfig, - onPlayPause = model::pauseOrResume, + onPlayPause = { if (isPlaying) model.pause() else model.play() }, onStop = model::stop, onPrevious = model::previous, onNext = model::next, + onPreferences = onPreferences, modifier = modifier ) } } -@OptIn(ExperimentalReadiumApi::class) @Composable fun TtsControls( playing: Boolean, - availableRates: List, - availableLanguages: List, - availableVoices: List, - config: Configuration?, - onConfigChange: (Configuration) -> Unit, onPlayPause: () -> Unit, onStop: () -> Unit, onPrevious: () -> Unit, onNext: () -> Unit, + onPreferences: () -> Unit, modifier: Modifier = Modifier, ) { - var showSettings by remember { mutableStateOf(false) } - - if (config != null && showSettings) { - TtsSettingsDialog( - availableRates = availableRates, - availableLanguages = availableLanguages, - availableVoices = availableVoices, - config = config, - onConfigChange = onConfigChange, - onDismiss = { showSettings = false } - ) - } - Card( modifier = modifier ) { @@ -127,7 +109,7 @@ fun TtsControls( Spacer(modifier = Modifier.size(8.dp)) - IconButton(onClick = { showSettings = true }) { + IconButton(onClick = onPreferences) { Icon( imageVector = Icons.Default.Settings, contentDescription = stringResource(R.string.tts_settings) @@ -136,60 +118,3 @@ fun TtsControls( } } } - -@OptIn(ExperimentalReadiumApi::class) -@Composable -private fun TtsSettingsDialog( - availableRates: List, - availableLanguages: List, - availableVoices: List, - config: Configuration, - onConfigChange: (Configuration) -> Unit, - onDismiss: () -> Unit -) { - AlertDialog( - onDismissRequest = onDismiss, - confirmButton = { - TextButton(onClick = onDismiss) { - Text(text = stringResource(R.string.close)) - } - }, - title = { Text(stringResource(R.string.tts_settings)) }, - text = { - Column { - if (availableRates.size > 1) { - SelectorListItem( - label = stringResource(R.string.tts_rate), - values = availableRates, - selection = config.rateMultiplier, - titleForValue = { rate -> - DecimalFormat("x#.##").format(rate) - }, - onSelected = { - onConfigChange(config.copy(rateMultiplier = it)) - } - ) - } - - SelectorListItem( - label = stringResource(R.string.language), - values = availableLanguages, - selection = config.defaultLanguage, - titleForValue = { language -> - language?.locale?.displayName - ?: stringResource(R.string.auto) - }, - onSelected = { onConfigChange(config.copy(defaultLanguage = it, voiceId = null)) } - ) - - SelectorListItem( - label = stringResource(R.string.tts_voice), - values = availableVoices, - selection = availableVoices.firstOrNull { it.id == config.voiceId }, - titleForValue = { it?.name ?: it?.id ?: stringResource(R.string.auto) }, - onSelected = { onConfigChange(config.copy(voiceId = it?.id)) } - ) - } - } - ) -} diff --git a/test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsPreferencesEditor.kt b/test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsPreferencesEditor.kt new file mode 100644 index 0000000000..303743b977 --- /dev/null +++ b/test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsPreferencesEditor.kt @@ -0,0 +1,72 @@ +/* + * Copyright 2023 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + +package org.readium.r2.testapp.reader.tts + +import org.readium.r2.navigator.media3.tts.android.AndroidTtsEngine +import org.readium.r2.navigator.media3.tts.android.AndroidTtsPreferences +import org.readium.r2.navigator.media3.tts.android.AndroidTtsPreferencesEditor +import org.readium.r2.navigator.preferences.* +import org.readium.r2.shared.ExperimentalReadiumApi +import org.readium.r2.shared.util.Language + +@OptIn(ExperimentalReadiumApi::class) +class TtsPreferencesEditor( + private val editor: AndroidTtsPreferencesEditor, + private val availableVoices: Set +) : PreferencesEditor { + + override val preferences: AndroidTtsPreferences + get() = editor.preferences + + override fun clear() { + editor.clear() + } + + val language: Preference = + editor.language + + val pitch: RangePreference = + editor.pitch + + val speed: RangePreference = + editor.speed + + /** + * [AndroidTtsPreferencesEditor] supports choosing voices for any language or region. + * For this test app, we've chosen to present to the user only the voice for the + * TTS default language and to ignore regions. + */ + val voice: EnumPreference = run { + // Recomposition will be triggered higher if the value changes. + val currentLanguage = language.effectiveValue?.removeRegion() + + editor.voices.map( + from = { voices -> + currentLanguage?.let { voices[it] } + }, + to = { voice -> + currentLanguage + ?.let { editor.voices.value.orEmpty().update(it, voice) } + ?: editor.voices.value.orEmpty() + } + ).withSupportedValues( + availableVoices + .filter { it.language.removeRegion() == currentLanguage } + .map { it.id } + ) + } + + private fun Map.update(key: K, value: V?): Map = + buildMap { + putAll(this@update) + if (value == null) { + remove(key) + } else { + put(key, value) + } + } +} diff --git a/test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsService.kt b/test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsService.kt new file mode 100644 index 0000000000..b759bfc0e0 --- /dev/null +++ b/test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsService.kt @@ -0,0 +1,185 @@ +/* + * Copyright 2022 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + +package org.readium.r2.testapp.reader.tts + +import android.app.Application +import android.app.PendingIntent +import android.content.ComponentName +import android.content.Intent +import android.content.ServiceConnection +import android.os.Build +import android.os.IBinder +import androidx.core.content.ContextCompat +import androidx.media3.session.MediaSession +import androidx.media3.session.MediaSessionService +import kotlinx.coroutines.* +import kotlinx.coroutines.flow.* +import org.readium.r2.navigator.media3.tts.AndroidTtsNavigator +import org.readium.r2.shared.ExperimentalReadiumApi +import timber.log.Timber + +@OptIn(ExperimentalReadiumApi::class) +@androidx.annotation.OptIn(androidx.media3.common.util.UnstableApi::class) +class TtsService : MediaSessionService() { + + class Session( + val bookId: Long, + val navigator: AndroidTtsNavigator, + val mediaSession: MediaSession, + ) { + val coroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.Main) + } + + /** + * The service interface to be used by the app. + */ + inner class Binder : android.os.Binder() { + + private val app: org.readium.r2.testapp.Application + get() = application as org.readium.r2.testapp.Application + + private val sessionMutable: MutableStateFlow = + MutableStateFlow(null) + + val session: StateFlow = + sessionMutable.asStateFlow() + + fun closeSession() { + stopForeground(true) + session.value?.mediaSession?.release() + session.value?.navigator?.close() + session.value?.coroutineScope?.cancel() + sessionMutable.value = null + } + + @OptIn(FlowPreview::class) + fun openSession( + navigator: AndroidTtsNavigator, + bookId: Long + ) { + val activityIntent = createSessionActivityIntent() + val mediaSession = MediaSession.Builder(applicationContext, navigator.asPlayer()) + .setSessionActivity(activityIntent) + .setId(bookId.toString()) + .build() + + addSession(mediaSession) + + val session = Session( + bookId, + navigator, + mediaSession + ) + + sessionMutable.value = session + + /* + * Launch a job for saving progression even when playback is going on in the background + * with no ReaderActivity opened. + */ + navigator.currentLocator + .sample(3000) + .onEach { locator -> + Timber.d("Saving TTS progression $locator") + app.bookRepository.saveProgression(locator, bookId) + }.launchIn(session.coroutineScope) + } + + private fun createSessionActivityIntent(): PendingIntent { + // This intent will be triggered when the notification is clicked. + var flags = PendingIntent.FLAG_UPDATE_CURRENT + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) { + flags = flags or PendingIntent.FLAG_IMMUTABLE + } + + val intent = application.packageManager.getLaunchIntentForPackage(application.packageName) + + return PendingIntent.getActivity(applicationContext, 0, intent, flags) + } + } + + private val binder by lazy { + Binder() + } + + override fun onBind(intent: Intent?): IBinder? { + Timber.d("onBind called with $intent") + + return if (intent?.action == SERVICE_INTERFACE) { + super.onBind(intent) + // Readium-aware client. + Timber.d("Returning custom binder.") + binder + } else { + // External controller. + Timber.d("Returning MediaSessionService binder.") + super.onBind(intent) + } + } + + override fun onGetSession(controllerInfo: MediaSession.ControllerInfo): MediaSession? { + return binder.session.value?.mediaSession + } + + override fun onTaskRemoved(rootIntent: Intent) { + super.onTaskRemoved(rootIntent) + Timber.d("Task removed. Stopping session and service.") + // Close the navigator to allow the service to be stopped. + binder.closeSession() + stopSelf() + } + + companion object { + + const val SERVICE_INTERFACE = "org.readium.r2.testapp.reader.tts.TtsService" + + fun start(application: Application) { + val intent = intent(application) + ContextCompat.startForegroundService(application, intent) + } + + suspend fun bind(application: Application): TtsService.Binder { + val mediaServiceBinder: CompletableDeferred = + CompletableDeferred() + + val mediaServiceConnection = object : ServiceConnection { + + override fun onServiceConnected(name: ComponentName?, service: IBinder) { + Timber.d("MediaService bound.") + mediaServiceBinder.complete(service as Binder) + } + + override fun onServiceDisconnected(name: ComponentName) { + Timber.d("MediaService disconnected.") + // Should not happen, do nothing. + } + + override fun onNullBinding(name: ComponentName) { + val errorMessage = "Failed to bind to MediaService." + Timber.e(errorMessage) + val exception = IllegalStateException(errorMessage) + mediaServiceBinder.completeExceptionally(exception) + } + } + + val intent = intent(application) + application.bindService(intent, mediaServiceConnection, 0) + + return mediaServiceBinder.await() + } + + fun stop(application: Application) { + val intent = intent(application) + application.stopService(intent) + } + + private fun intent(application: Application) = + Intent(SERVICE_INTERFACE) + // MediaSessionService.onBind requires the intent to have a non-null action + .apply { setClass(application, TtsService::class.java) } + } +} diff --git a/test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsServiceFacade.kt b/test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsServiceFacade.kt new file mode 100644 index 0000000000..7bece6912e --- /dev/null +++ b/test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsServiceFacade.kt @@ -0,0 +1,74 @@ +package org.readium.r2.testapp.reader.tts + +import android.app.Application +import kotlinx.coroutines.* +import kotlinx.coroutines.flow.* +import kotlinx.coroutines.sync.Mutex +import kotlinx.coroutines.sync.withLock +import org.readium.r2.navigator.media3.tts.AndroidTtsNavigator +import org.readium.r2.shared.ExperimentalReadiumApi + +/** + * Enables to try to close a session without starting the [TtsService] if it is not started. + */ +@OptIn(ExperimentalReadiumApi::class) +class TtsServiceFacade( + private val application: Application +) { + private val coroutineScope: CoroutineScope = + MainScope() + + private val mutex: Mutex = + Mutex() + + private var binder: TtsService.Binder? = + null + + private var bindingJob: Job? = + null + + private val sessionMutable: MutableStateFlow = + MutableStateFlow(null) + + val session: StateFlow = + sessionMutable.asStateFlow() + + suspend fun openSession( + bookId: Long, + navigator: AndroidTtsNavigator + ) = mutex.withLock { + if (session.value != null) { + throw CancellationException("A session is already running.") + } + + try { + if (binder == null) { + TtsService.start(application) + val binder = TtsService.bind(application) + this.binder = binder + bindingJob = binder.session + .onEach { sessionMutable.value = it } + .launchIn(coroutineScope) + } + + binder!!.openSession(navigator, bookId) + } catch (e: CancellationException) { + TtsService.stop(application) + throw e + } + } + + suspend fun closeSession() = mutex.withLock { + if (session.value == null) { + throw CancellationException("No session to close.") + } + + withContext(NonCancellable) { + bindingJob!!.cancelAndJoin() + binder!!.closeSession() + sessionMutable.value = null + binder = null + TtsService.stop(application) + } + } +} diff --git a/test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsViewModel.kt b/test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsViewModel.kt index fca7ea5845..eac63df43a 100644 --- a/test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsViewModel.kt +++ b/test-app/src/main/java/org/readium/r2/testapp/reader/tts/TtsViewModel.kt @@ -6,38 +6,46 @@ package org.readium.r2.testapp.reader.tts -import android.content.Context import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.ExperimentalCoroutinesApi import kotlinx.coroutines.channels.Channel import kotlinx.coroutines.flow.* import kotlinx.coroutines.launch -import kotlinx.coroutines.runBlocking import org.readium.r2.navigator.Navigator import org.readium.r2.navigator.VisualNavigator -import org.readium.r2.navigator.tts.AndroidTtsEngine -import org.readium.r2.navigator.tts.PublicationSpeechSynthesizer -import org.readium.r2.navigator.tts.PublicationSpeechSynthesizer.Configuration -import org.readium.r2.navigator.tts.PublicationSpeechSynthesizer.State as TtsState -import org.readium.r2.navigator.tts.TtsEngine -import org.readium.r2.navigator.tts.TtsEngine.Voice -import org.readium.r2.shared.DelicateReadiumApi +import org.readium.r2.navigator.media3.api.MediaNavigator +import org.readium.r2.navigator.media3.tts.AndroidTtsNavigator +import org.readium.r2.navigator.media3.tts.AndroidTtsNavigatorFactory +import org.readium.r2.navigator.media3.tts.TtsNavigator +import org.readium.r2.navigator.media3.tts.android.AndroidTtsEngine +import org.readium.r2.navigator.media3.tts.android.AndroidTtsPreferences +import org.readium.r2.navigator.media3.tts.android.AndroidTtsSettings import org.readium.r2.shared.ExperimentalReadiumApi import org.readium.r2.shared.UserException import org.readium.r2.shared.publication.Locator import org.readium.r2.shared.publication.Publication import org.readium.r2.shared.util.Language import org.readium.r2.testapp.R +import org.readium.r2.testapp.reader.ReaderInitData +import org.readium.r2.testapp.reader.VisualReaderInitData +import org.readium.r2.testapp.reader.preferences.PreferencesManager +import org.readium.r2.testapp.reader.preferences.UserPreferencesViewModel +import org.readium.r2.testapp.utils.extensions.mapStateIn /** - * View model controlling a [PublicationSpeechSynthesizer] to read a publication aloud. + * View model controlling a [TtsNavigator] to read a publication aloud. * - * Note: This is not an Android [ViewModel], but it is a component of [ReaderViewModel]. + * Note: This is not an Android ViewModel, but it is a component of ReaderViewModel. */ -@OptIn(ExperimentalReadiumApi::class) +@OptIn(ExperimentalReadiumApi::class, ExperimentalCoroutinesApi::class) class TtsViewModel private constructor( - private val synthesizer: PublicationSpeechSynthesizer, - private val scope: CoroutineScope -) { + private val viewModelScope: CoroutineScope, + private val bookId: Long, + private val publication: Publication, + private val ttsNavigatorFactory: AndroidTtsNavigatorFactory, + private val ttsServiceFacade: TtsServiceFacade, + private val preferencesManager: PreferencesManager, +) : TtsNavigator.Listener { companion object { /** @@ -45,45 +53,27 @@ class TtsViewModel private constructor( * TTS engine. */ operator fun invoke( - context: Context, - publication: Publication, - scope: CoroutineScope - ): TtsViewModel? = - PublicationSpeechSynthesizer(context, publication) - ?.let { TtsViewModel(it, scope) } - } - - /** - * @param showControls Whether the TTS was enabled by the user. - * @param isPlaying Whether the TTS is currently speaking. - * @param playingWordRange Locator to the currently spoken word. - * @param playingUtterance Locator for the currently spoken utterance (e.g. sentence). - * @param settings Current user settings and their constraints. - */ - data class State( - val showControls: Boolean = false, - val isPlaying: Boolean = false, - val playingWordRange: Locator? = null, - val playingUtterance: Locator? = null, - val settings: Settings = Settings() - ) + viewModelScope: CoroutineScope, + readerInitData: ReaderInitData, + ): TtsViewModel? { + if (readerInitData !is VisualReaderInitData || readerInitData.ttsInitData == null) { + return null + } - /** - * @param config Currently selected user settings. - * @param rateRange Supported range for the rate setting. - * @param availableLanguages Languages supported by the TTS engine. - * @param availableVoices Voices supported by the TTS engine, for the selected language. - */ - data class Settings( - val config: Configuration = Configuration(), - val rateRange: ClosedRange = 1.0..1.0, - val availableLanguages: List = emptyList(), - val availableVoices: List = emptyList(), - ) + return TtsViewModel( + viewModelScope = viewModelScope, + bookId = readerInitData.bookId, + publication = readerInitData.publication, + ttsNavigatorFactory = readerInitData.ttsInitData.ttsNavigatorFactory, + ttsServiceFacade = readerInitData.ttsInitData.ttsServiceFacade, + preferencesManager = readerInitData.ttsInitData.preferencesManager + ) + } + } sealed class Event { /** - * Emitted when the [PublicationSpeechSynthesizer] fails with an error. + * Emitted when the [TtsNavigator] fails with an error. */ class OnError(val error: UserException) : Event() @@ -93,194 +83,147 @@ class TtsViewModel private constructor( class OnMissingVoiceData(val language: Language) : Event() } - /** - * Current state of the view model. - */ - val state: StateFlow - - private val _events: Channel = Channel(Channel.BUFFERED) - val events: Flow = _events.receiveAsFlow() + private val navigatorNow: AndroidTtsNavigator? get() = + ttsServiceFacade.session.value?.navigator - /** - * Indicates whether the TTS is in the Stopped state. - */ - private val isStopped: StateFlow + private val _events: Channel = + Channel(Channel.BUFFERED) - init { - synthesizer.listener = SynthesizerListener() + val events: Flow = + _events.receiveAsFlow() - // Automatically close the TTS when reaching the Stopped state. - isStopped = synthesizer.state - .map { it == PublicationSpeechSynthesizer.State.Stopped } - .stateIn(scope, SharingStarted.Lazily, initialValue = true) + val preferencesModel: UserPreferencesViewModel + get() = UserPreferencesViewModel( + viewModelScope = viewModelScope, + bookId = bookId, + preferencesManager = preferencesManager + ) { preferences -> + val baseEditor = ttsNavigatorFactory.createTtsPreferencesEditor(preferences) + TtsPreferencesEditor(baseEditor, voices) + } - // Supported voices grouped by their language. - val voicesByLanguage: Flow>> = - synthesizer.availableVoices - .map { voices -> voices.groupBy { it.language } } + val showControls: StateFlow = + ttsServiceFacade.session.mapStateIn(viewModelScope) { + it != null + } - // All supported languages. - val languages: Flow> = voicesByLanguage - .map { voices -> - voices.keys.sortedBy { it.locale.displayName } - } + val isPlaying: StateFlow = + ttsServiceFacade.session.flatMapLatest { session -> + session?.navigator?.playback?.map { playback -> playback.playWhenReady } + ?: MutableStateFlow(false) + }.stateIn(viewModelScope, SharingStarted.Eagerly, false) - // Supported voices for the language selected in the synthesizer configuration. - val voicesForSelectedLanguage: Flow> = - combine( - synthesizer.config.map { it.defaultLanguage }, - voicesByLanguage, - ) { language, voices -> - language - ?.let { voices[it] } - ?.sortedBy { it.name ?: it.id } - ?: emptyList() - } + val position: StateFlow = + ttsServiceFacade.session.flatMapLatest { session -> + session?.navigator?.currentLocator ?: MutableStateFlow(null) + }.stateIn(viewModelScope, SharingStarted.Eagerly, null) - // Settings model for the current configuration. - val settings: Flow = combine( - synthesizer.config, - languages, - voicesForSelectedLanguage, - ) { config, langs, voices -> - Settings( - config = config, - rateRange = synthesizer.rateMultiplierRange, - availableLanguages = langs, - availableVoices = voices - ) - } + val highlight: StateFlow = + ttsServiceFacade.session.flatMapLatest { session -> + session?.navigator?.utterance?.map { it.utteranceLocator } + ?: MutableStateFlow(null) + }.stateIn(viewModelScope, SharingStarted.Eagerly, null) - // Current view model state. - state = combine( - isStopped, - synthesizer.state, - settings - ) { isStopped, state, currentSettings -> - val playing = (state as? TtsState.Playing) - val paused = (state as? TtsState.Paused) + val voices: Set get() = + ttsServiceFacade.session.value?.navigator?.voices.orEmpty() - State( - showControls = !isStopped, - isPlaying = (playing != null), - playingWordRange = playing?.range, - playingUtterance = (playing?.utterance ?: paused?.utterance)?.locator, - settings = currentSettings - ) - }.stateIn(scope, SharingStarted.Eagerly, initialValue = State()) - } + init { + ttsServiceFacade.session + .flatMapLatest { it?.navigator?.playback ?: MutableStateFlow(null) } + .onEach { playback -> + when (playback?.state) { + null -> { + } + is MediaNavigator.State.Ended -> { + stop() + } + is MediaNavigator.State.Error -> { + onPlaybackError(playback.state as TtsNavigator.State.Error) + } + is MediaNavigator.State.Ready -> {} + is MediaNavigator.State.Buffering -> {} + } + }.launchIn(viewModelScope) - fun onCleared() { - runBlocking { - synthesizer.close() - } + preferencesManager.preferences + .onEach { ttsServiceFacade.session.value?.navigator?.submitPreferences(it) } + .launchIn(viewModelScope) } /** * Starts the TTS using the first visible locator in the given [navigator]. */ fun start(navigator: Navigator) { - if (!isStopped.value) return + viewModelScope.launch { + if (ttsServiceFacade.session.value != null) + return@launch - scope.launch { - val start = (navigator as? VisualNavigator)?.firstVisibleElementLocator() - synthesizer.start(fromLocator = start) + openSession(navigator) } } + private suspend fun openSession(navigator: Navigator) { + val start = (navigator as? VisualNavigator)?.firstVisibleElementLocator() + + val ttsNavigator = ttsNavigatorFactory.createNavigator( + this, + preferencesManager.preferences.value, + start + ) ?: run { + val exception = UserException(R.string.tts_error_initialization) + _events.send(Event.OnError(exception)) + return + } + + // playWhenReady must be true for the MediaSessionService to call Service.startForeground + // and prevent crashing + ttsNavigator.play() + ttsServiceFacade.openSession(bookId, ttsNavigator) + } + fun stop() { - if (isStopped.value) return - synthesizer.stop() + viewModelScope.launch { + ttsServiceFacade.closeSession() + } } - fun pauseOrResume() { - synthesizer.pauseOrResume() + fun play() { + navigatorNow?.play() } fun pause() { - synthesizer.pause() + navigatorNow?.pause() } fun previous() { - synthesizer.previous() + navigatorNow?.goBackward() } fun next() { - synthesizer.next() - } - - fun setConfig(config: Configuration) { - synthesizer.setConfig(config) + navigatorNow?.goForward() } - /** - * Starts the activity to install additional voice data. - */ - @OptIn(DelicateReadiumApi::class) - fun requestInstallVoice(context: Context) { - synthesizer.engine.requestInstallMissingVoice(context) + override fun onStopRequested() { + stop() } - private inner class SynthesizerListener : PublicationSpeechSynthesizer.Listener { - override fun onUtteranceError( - utterance: PublicationSpeechSynthesizer.Utterance, - error: PublicationSpeechSynthesizer.Exception - ) { - scope.launch { - // The synthesizer is paused when encountering an error while playing an - // utterance. Here we will skip to the next utterance unless the exception is - // recoverable. - val shouldContinuePlayback = !handleTtsException(error) - if (shouldContinuePlayback) { - next() + private fun onPlaybackError(error: TtsNavigator.State.Error) { + val exception = when (error) { + is TtsNavigator.State.Error.ContentError -> { + UserException(R.string.tts_error_other, cause = error.exception) + } + is TtsNavigator.State.Error.EngineError<*> -> { + when ((error.error as AndroidTtsEngine.Error).kind) { + AndroidTtsEngine.Error.Kind.Network -> + UserException(R.string.tts_error_network) + else -> + UserException(R.string.tts_error_other) } } } - override fun onError(error: PublicationSpeechSynthesizer.Exception) { - scope.launch { - handleTtsException(error) - } + viewModelScope.launch { + _events.send(Event.OnError(exception)) } - - /** - * Handles the given error and returns whether it was recovered from. - */ - private suspend fun handleTtsException(error: PublicationSpeechSynthesizer.Exception): Boolean = - when (error) { - is PublicationSpeechSynthesizer.Exception.Engine -> when (val err = error.error) { - // The `LanguageSupportIncomplete` exception is a special case. We can recover from - // it by asking the user to download the missing voice data. - is TtsEngine.Exception.LanguageSupportIncomplete -> { - _events.send(Event.OnMissingVoiceData(err.language)) - true - } - - else -> { - _events.send(Event.OnError(err.toUserException())) - false - } - } - } - - private fun TtsEngine.Exception.toUserException(): UserException = - when (this) { - is TtsEngine.Exception.InitializationFailed -> - UserException(R.string.tts_error_initialization) - is TtsEngine.Exception.LanguageNotSupported -> - UserException( - R.string.tts_error_language_not_supported, - language.locale.displayName - ) - is TtsEngine.Exception.LanguageSupportIncomplete -> - UserException( - R.string.tts_error_language_support_incomplete, - language.locale.displayName - ) - is TtsEngine.Exception.Network -> - UserException(R.string.tts_error_network) - is TtsEngine.Exception.Other -> - UserException(R.string.tts_error_other) - } } } diff --git a/test-app/src/main/java/org/readium/r2/testapp/shared/views/List.kt b/test-app/src/main/java/org/readium/r2/testapp/shared/views/List.kt deleted file mode 100644 index b2ad6b3ddd..0000000000 --- a/test-app/src/main/java/org/readium/r2/testapp/shared/views/List.kt +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Copyright 2022 Readium Foundation. All rights reserved. - * Use of this source code is governed by the BSD-style license - * available in the top-level LICENSE file of the project. - */ - -package org.readium.r2.testapp.shared.views - -import androidx.compose.foundation.clickable -import androidx.compose.material.* -import androidx.compose.runtime.* -import androidx.compose.ui.Modifier - -/** - * A Material [ListItem] displaying a dropdown menu to select a value. The current value is - * displayed on the right. - */ -@OptIn(ExperimentalMaterialApi::class) -@Composable -fun SelectorListItem( - label: String, - values: List, - selection: T, - titleForValue: @Composable (T) -> String, - onSelected: (T) -> Unit, - enabled: Boolean = values.isNotEmpty(), -) { - var isExpanded by remember { mutableStateOf(false) } - fun dismiss() { isExpanded = false } - - ListItem( - modifier = Modifier - .clickable(enabled = enabled) { - isExpanded = true - }, - text = { - Group(enabled = enabled) { - Text(label) - } - }, - trailing = { - Group(enabled = enabled) { - Text(titleForValue(selection)) - } - - DropdownMenu( - expanded = isExpanded, - onDismissRequest = { dismiss() } - ) { - for (value in values) { - DropdownMenuItem( - onClick = { - onSelected(value) - dismiss() - } - ) { - Text(titleForValue(value)) - } - } - } - } - ) -} diff --git a/test-app/src/main/java/org/readium/r2/testapp/shared/views/Preferences.kt b/test-app/src/main/java/org/readium/r2/testapp/shared/views/Preferences.kt new file mode 100644 index 0000000000..38bf618988 --- /dev/null +++ b/test-app/src/main/java/org/readium/r2/testapp/shared/views/Preferences.kt @@ -0,0 +1,472 @@ +/* + * Copyright 2022 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + +@file:OptIn(ExperimentalReadiumApi::class) + +package org.readium.r2.testapp.shared.views + +import androidx.compose.foundation.clickable +import androidx.compose.foundation.layout.Arrangement +import androidx.compose.foundation.layout.Column +import androidx.compose.foundation.layout.Row +import androidx.compose.foundation.layout.widthIn +import androidx.compose.material.* +import androidx.compose.material.icons.Icons +import androidx.compose.material.icons.filled.Add +import androidx.compose.material.icons.filled.Backspace +import androidx.compose.material.icons.filled.Palette +import androidx.compose.material.icons.filled.Remove +import androidx.compose.runtime.* +import androidx.compose.ui.Alignment +import androidx.compose.ui.Modifier +import androidx.compose.ui.graphics.Color +import androidx.compose.ui.graphics.luminance +import androidx.compose.ui.text.style.TextAlign +import androidx.compose.ui.unit.dp +import androidx.compose.ui.window.Dialog +import java.util.* +import org.readium.r2.navigator.preferences.* +import org.readium.r2.navigator.preferences.Color as ReadiumColor +import org.readium.r2.shared.ExperimentalReadiumApi +import org.readium.r2.shared.util.Language +import org.readium.r2.testapp.utils.compose.ColorPicker +import org.readium.r2.testapp.utils.compose.DropdownMenuButton +import org.readium.r2.testapp.utils.compose.ToggleButtonGroup + +/** + * Component for an [EnumPreference] displayed as a group of mutually exclusive buttons. + * This works best with a small number of enum values. + */ +@Composable +fun ButtonGroupItem( + title: String, + preference: EnumPreference, + commit: () -> Unit, + formatValue: (T) -> String +) { + ButtonGroupItem( + title = title, + options = preference.supportedValues, + isActive = preference.isEffective, + activeOption = preference.effectiveValue, + selectedOption = preference.value, + formatValue = formatValue, + onClear = { preference.clear(); commit() } + .takeIf { preference.value != null }, + onSelectedOptionChanged = { newValue -> + if (newValue == preference.value) { + preference.clear() + } else { + preference.set(newValue) + } + commit() + } + ) +} + +/** + * Group of mutually exclusive buttons. + */ +@Composable +private fun ButtonGroupItem( + title: String, + options: List, + isActive: Boolean, + activeOption: T, + selectedOption: T?, + formatValue: (T) -> String, + onClear: (() -> Unit)?, + onSelectedOptionChanged: (T) -> Unit, +) { + Item(title, isActive = isActive, onClear = onClear) { + ToggleButtonGroup( + options = options, + activeOption = activeOption, + selectedOption = selectedOption, + onSelectOption = { option -> onSelectedOptionChanged(option) } + ) { option -> + Text( + text = formatValue(option), + style = MaterialTheme.typography.caption + ) + } + } +} + +/** + * Component for an [EnumPreference] displayed as a dropdown menu. + */ +@Composable +fun MenuItem( + title: String, + preference: EnumPreference, + commit: () -> Unit, + formatValue: (T) -> String +) { + MenuItem( + title = title, + value = preference.value ?: preference.effectiveValue, + values = preference.supportedValues, + isActive = preference.isEffective, + formatValue = formatValue, + onClear = { preference.clear(); commit() } + .takeIf { preference.value != null }, + onValueChanged = { value -> + preference.set(value) + commit() + } + ) +} + +/** + * Dropdown menu. + */ +@Composable +private fun MenuItem( + title: String, + value: T, + values: List, + isActive: Boolean, + formatValue: (T) -> String, + onValueChanged: (T) -> Unit, + onClear: (() -> Unit)? +) { + Item(title, isActive = isActive, onClear = onClear) { + DropdownMenuButton( + text = { + Text( + text = formatValue(value), + style = MaterialTheme.typography.caption + ) + } + ) { dismiss -> + for (value in values) { + DropdownMenuItem( + onClick = { + dismiss() + onValueChanged(value) + } + ) { + Text(formatValue(value)) + } + } + } + } +} + +/** + * Component for a [RangePreference] with decrement and increment buttons. + */ +@Composable +fun > StepperItem( + title: String, + preference: RangePreference, + commit: () -> Unit +) { + StepperItem( + title = title, + isActive = preference.isEffective, + value = preference.value ?: preference.effectiveValue, + formatValue = preference::formatValue, + onDecrement = { preference.decrement(); commit() }, + onIncrement = { preference.increment(); commit() }, + onClear = { preference.clear(); commit() } + .takeIf { preference.value != null }, + ) +} + +/** + * Component for a [RangePreference] with decrement and increment buttons. + */ +@Composable +private fun StepperItem( + title: String, + isActive: Boolean, + value: T, + formatValue: (T) -> String, + onDecrement: () -> Unit, + onIncrement: () -> Unit, + onClear: (() -> Unit)? +) { + Item(title, isActive = isActive, onClear = onClear) { + Row( + verticalAlignment = Alignment.CenterVertically, + horizontalArrangement = Arrangement.spacedBy(16.dp) + ) { + IconButton( + onClick = onDecrement, + content = { + Icon(Icons.Default.Remove, contentDescription = "Less") + } + ) + + Text( + text = formatValue(value), + modifier = Modifier.widthIn(min = 30.dp), + textAlign = TextAlign.Center + ) + + IconButton( + onClick = onIncrement, + content = { + Icon(Icons.Default.Add, contentDescription = "More") + } + ) + } + } +} + +/** + * Component for a boolean [Preference]. + */ +@Composable +fun SwitchItem( + title: String, + preference: Preference, + commit: () -> Unit +) { + SwitchItem( + title = title, + value = preference.value ?: preference.effectiveValue, + isActive = preference.isEffective, + onCheckedChange = { preference.set(it); commit() }, + onToggle = { preference.toggle(); commit() }, + onClear = { preference.clear(); commit() } + .takeIf { preference.value != null }, + ) +} + +/** + * Switch + */ +@Composable +private fun SwitchItem( + title: String, + value: Boolean, + isActive: Boolean, + onCheckedChange: (Boolean) -> Unit, + onToggle: () -> Unit, + onClear: (() -> Unit)? +) { + Item( + title = title, + isActive = isActive, + onClick = onToggle, + onClear = onClear + ) { + Switch( + checked = value, + onCheckedChange = onCheckedChange + ) + } +} + +/** + * Component for a [Preference]. + */ +@Composable +fun ColorItem( + title: String, + preference: Preference, + commit: () -> Unit +) { + ColorItem( + title = title, + isActive = preference.isEffective, + value = preference.value ?: preference.effectiveValue, + noValueSelected = preference.value == null, + onColorChanged = { preference.set(it); commit() }, + onClear = { preference.clear(); commit() } + .takeIf { preference.value != null } + ) +} + +/** + * Color picker + */ +@Composable +private fun ColorItem( + title: String, + isActive: Boolean, + value: ReadiumColor, + noValueSelected: Boolean, + onColorChanged: (ReadiumColor?) -> Unit, + onClear: (() -> Unit)? +) { + var isPicking by remember { mutableStateOf(false) } + + Item( + title = title, + isActive = isActive, + onClick = { isPicking = true }, + onClear = onClear + ) { + val color = Color(value.int) + + OutlinedButton( + onClick = { isPicking = true }, + colors = ButtonDefaults.buttonColors(backgroundColor = color) + ) { + if (noValueSelected) { + Icon( + imageVector = Icons.Default.Palette, + contentDescription = "Change color", + tint = if (color.luminance() > 0.5) Color.Black else Color.White + ) + } + } + + if (isPicking) { + Dialog( + onDismissRequest = { isPicking = false } + ) { + Column( + horizontalAlignment = Alignment.End + ) { + ColorPicker { color -> + isPicking = false + onColorChanged(ReadiumColor(color)) + } + Button( + onClick = { + isPicking = false + onColorChanged(null) + } + ) { + Text("Clear") + } + } + } + } + } +} + +/** + * Component for a [Preference]`. + */ +@Composable +fun LanguageItem( + preference: Preference, + commit: () -> Unit +) { + val languages = remember { + Locale.getAvailableLocales() + .map { Language(it).removeRegion() } + .distinct() + .sortedBy { it.locale.displayName } + } + + MenuItem( + title = "Language", + preference = preference.withSupportedValues(languages + null), + formatValue = { it?.locale?.displayName ?: "Unknown" }, + commit = commit + ) +} + +@OptIn(ExperimentalMaterialApi::class) +@Composable +private fun Item( + title: String, + isActive: Boolean = true, + onClick: (() -> Unit)? = null, + onClear: (() -> Unit)? = null, + content: @Composable () -> Unit +) { + ListItem( + modifier = + if (onClick != null) Modifier.clickable(onClick = onClick) + else Modifier, + text = { + val alpha = if (isActive) 1.0f else ContentAlpha.disabled + CompositionLocalProvider(LocalContentAlpha provides alpha) { + Text(title) + } + }, + trailing = { + Row { + content() + + IconButton(onClick = onClear ?: {}, enabled = onClear != null) { + Icon( + Icons.Default.Backspace, + contentDescription = "Clear" + ) + } + } + } + ) +} + +@Composable +fun SelectorListItem( + title: String, + preference: EnumPreference, + formatValue: (T) -> String, + commit: () -> Unit +) { + SelectorListItem( + title = title, + values = preference.supportedValues, + selection = preference.value ?: preference.effectiveValue, + formatValue = formatValue, + onSelected = { + preference.set(it) + commit() + } + ) +} + +/** + * A Material [ListItem] displaying a dropdown menu to select a value. The current value is + * displayed on the right. + */ +@OptIn(ExperimentalMaterialApi::class) +@Composable +private fun SelectorListItem( + title: String, + values: List, + selection: T, + formatValue: (T) -> String, + onSelected: (T) -> Unit, + enabled: Boolean = values.isNotEmpty(), +) { + var isExpanded by remember { mutableStateOf(false) } + fun dismiss() { isExpanded = false } + + ListItem( + modifier = Modifier + .clickable(enabled = enabled) { + isExpanded = true + }, + text = { + Group(enabled = enabled) { + Text(title) + } + }, + trailing = { + Group(enabled = enabled) { + Text(formatValue(selection)) + } + + DropdownMenu( + expanded = isExpanded, + onDismissRequest = { dismiss() } + ) { + for (value in values) { + DropdownMenuItem( + onClick = { + onSelected(value) + dismiss() + } + ) { + Text(formatValue(value)) + } + } + } + } + ) +} diff --git a/test-app/src/main/java/org/readium/r2/testapp/utils/LifecycleMediaSessionService.kt b/test-app/src/main/java/org/readium/r2/testapp/utils/LifecycleMedia2SessionService.kt similarity index 86% rename from test-app/src/main/java/org/readium/r2/testapp/utils/LifecycleMediaSessionService.kt rename to test-app/src/main/java/org/readium/r2/testapp/utils/LifecycleMedia2SessionService.kt index 5dacff318b..7a26c0c4dc 100644 --- a/test-app/src/main/java/org/readium/r2/testapp/utils/LifecycleMediaSessionService.kt +++ b/test-app/src/main/java/org/readium/r2/testapp/utils/LifecycleMedia2SessionService.kt @@ -1,3 +1,9 @@ +/* + * Copyright 2022 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + package org.readium.r2.testapp.utils import android.content.Intent @@ -13,7 +19,7 @@ import androidx.media2.session.MediaSessionService * https://android.googlesource.com/platform/frameworks/support/+/refs/heads/androidx-main/lifecycle/lifecycle-service/src/main/java/androidx/lifecycle/LifecycleService.java */ -abstract class LifecycleMediaSessionService : MediaSessionService(), LifecycleOwner { +abstract class LifecycleMedia2SessionService : MediaSessionService(), LifecycleOwner { @Suppress("LeakingThis") private val lifecycleDispatcher = ServiceLifecycleDispatcher(this) diff --git a/test-app/src/main/res/values/strings.xml b/test-app/src/main/res/values/strings.xml index 6f18a55ec8..6a4bbec932 100644 --- a/test-app/src/main/res/values/strings.xml +++ b/test-app/src/main/res/values/strings.xml @@ -197,7 +197,7 @@ Select a font Close Language - Auto + Default Failed to initialize the TTS engine The language %s is not supported @@ -208,7 +208,8 @@ Pause Play Go backward - Rate + Speed + Pitch Speech settings Stop Voice