diff --git a/lib/src/main/java/com/otaliastudios/transcoder/internal/CustomSegments.kt b/lib/src/main/java/com/otaliastudios/transcoder/internal/CustomSegments.kt new file mode 100644 index 00000000..e2546335 --- /dev/null +++ b/lib/src/main/java/com/otaliastudios/transcoder/internal/CustomSegments.kt @@ -0,0 +1,96 @@ +@file:Suppress("ReturnCount") + +package com.otaliastudios.transcoder.internal + +import android.media.MediaFormat +import com.otaliastudios.transcoder.common.TrackStatus +import com.otaliastudios.transcoder.common.TrackType +import com.otaliastudios.transcoder.internal.pipeline.Pipeline +import com.otaliastudios.transcoder.internal.utils.Logger +import com.otaliastudios.transcoder.internal.utils.mutableTrackMapOf +import com.otaliastudios.transcoder.source.DataSource + +class CustomSegments( + private val sources: DataSources, + private val tracks: Tracks, + private val factory: (TrackType, Int, TrackStatus, MediaFormat) -> Pipeline, +) { + + private val log = Logger("Segments") + private var currentSegment: Segment? = null + val currentIndex = mutableTrackMapOf(-1, -1) + private val segmentMap = mutableMapOf() + + fun hasNext(type: TrackType): Boolean { + if (!sources.has(type)) return false + log.v( + "hasNext($type): segment=${currentSegment} lastIndex=${sources.getOrNull(type)?.lastIndex}" + + " canAdvance=${currentSegment?.canAdvance()}" + ) + val segment = currentSegment ?: return true // not started + val lastIndex = sources.getOrNull(type)?.lastIndex ?: return false // no track! + return segment.canAdvance() || segment.index < lastIndex + } + + fun hasNext() = hasNext(TrackType.VIDEO) + + + // it will be time dependent + // 1. make segments work for thumbnails as is + // 2. inject segments dynamically + // 3. seek to segment and destroy previous ones + // 4. destroy only if necessary, else reuse + + fun getSegment(id: String): Segment? { + return segmentMap.getOrPut(id) { + destroySegment(id) + tryCreateSegment(id).also { + currentSegment = it + } + } + } + + fun release() = destroySegment() + + private fun tryCreateSegment(id: String): Segment? { + val index = sources[TrackType.VIDEO].indexOfFirst { it.mediaId() == id } + // Return null if out of bounds, either because segments are over or because the + // source set does not have sources for this track type. + val source = sources[TrackType.VIDEO].getOrNull(index) ?: return null + source.init() + log.i("tryCreateSegment(${TrackType.VIDEO}, $index): created!") + if (tracks.active.has(TrackType.VIDEO)) { + source.selectTrack(TrackType.VIDEO) + } + // Update current index before pipeline creation, for other components + // who check it during pipeline init. + currentIndex[TrackType.VIDEO] = index + val pipeline = factory( + TrackType.VIDEO, + index, + tracks.all[TrackType.VIDEO], + tracks.outputFormats[TrackType.VIDEO] + ) + return Segment(TrackType.VIDEO, index, pipeline) + } + + private fun destroySegment(id: String? = null) { + currentSegment?.let { + it.release() + val source = sources[it.type][it.index] + if (tracks.active.has(it.type)) { + source.releaseTrack(it.type) + } + } + if (id == null) { + segmentMap.clear() + } + else { + segmentMap[id] = null + } + } + private fun DataSource.init() = if (!isInitialized) initialize() else Unit + + private fun DataSource.deinit() = if (isInitialized) deinitialize() else Unit + +} diff --git a/lib/src/main/java/com/otaliastudios/transcoder/internal/DataSources.kt b/lib/src/main/java/com/otaliastudios/transcoder/internal/DataSources.kt index 97caa630..5dea66d0 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/internal/DataSources.kt +++ b/lib/src/main/java/com/otaliastudios/transcoder/internal/DataSources.kt @@ -28,20 +28,21 @@ class DataSources constructor( audioSources.init() } - private val videoSources: List = run { - val valid = videoSources.count { it.getTrackFormat(TrackType.VIDEO) != null } - when (valid) { - 0 -> listOf().also { videoSources.deinit() } - videoSources.size -> videoSources - else -> videoSources // Tracks will crash + private var videoSources: List = updateVideoSources(videoSources) + set(value) { + field = updateVideoSources(value) + } + + private var audioSources: List = updateAudioSources(audioSources) + set(value) { + field = updateAudioSources(value) } - } - private val audioSources: List = run { - val valid = audioSources.count { it.getTrackFormat(TrackType.AUDIO) != null } - when (valid) { - 0 -> listOf().also { audioSources.deinit() } - audioSources.size -> audioSources + private fun updateAudioSources(sources: List) : List { + val valid = sources.count { it.getTrackFormat(TrackType.AUDIO) != null } + return when (valid) { + 0 -> listOf().also { sources.deinit() } + sources.size -> sources else -> { // Some tracks do not have audio, while some do. Replace with BlankAudio. audioSources.map { source -> @@ -52,6 +53,55 @@ class DataSources constructor( } } + private fun updateVideoSources(sources: List): List { + val valid = sources.count { it.getTrackFormat(TrackType.VIDEO) != null } + return when (valid) { + 0 -> listOf().also { sources.deinit() } + sources.size -> sources + else -> sources // Tracks will crash + } + } + + fun addDataSource(dataSource: DataSource) { + addVideoDataSource(dataSource) + addAudioDataSource(dataSource) + } + + fun addVideoDataSource(dataSource: DataSource) { + dataSource.init() + if (dataSource.getTrackFormat(TrackType.VIDEO) != null && dataSource !in videoSources) { + videoSources = videoSources + dataSource + } + } + fun addAudioDataSource(dataSource: DataSource) { + dataSource.init() + if (dataSource.getTrackFormat(TrackType.AUDIO) != null) { + audioSources = audioSources + dataSource + } + } + + fun removeDataSource(dataSourceId: String) { + removeAudioDataSource(dataSourceId) + removeVideoDataSource(dataSourceId) + } + + fun removeVideoDataSource(dataSourceId: String) { + val source = videoSources.find { it.mediaId() == dataSourceId } + if (source?.getTrackFormat(TrackType.VIDEO) != null) { + videoSources = videoSources - source + source.releaseTrack(TrackType.VIDEO) + } + source?.deinit() + } + + fun removeAudioDataSource(dataSourceId: String) { + val source = audioSources.find { it.mediaId() == dataSourceId } + if (source?.getTrackFormat(TrackType.AUDIO) != null) { + audioSources = audioSources - source + source.releaseTrack(TrackType.AUDIO) + } + source?.deinit() + } override fun get(type: TrackType) = when (type) { TrackType.AUDIO -> audioSources TrackType.VIDEO -> videoSources diff --git a/lib/src/main/java/com/otaliastudios/transcoder/internal/Tracks.kt b/lib/src/main/java/com/otaliastudios/transcoder/internal/Tracks.kt index 44fd586d..ea892023 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/internal/Tracks.kt +++ b/lib/src/main/java/com/otaliastudios/transcoder/internal/Tracks.kt @@ -11,19 +11,23 @@ import com.otaliastudios.transcoder.source.DataSource import com.otaliastudios.transcoder.strategy.TrackStrategy class Tracks( - strategies: TrackMap, - sources: DataSources, - videoRotation: Int, - forceCompression: Boolean + val strategies: TrackMap, + val sources: DataSources, + val videoRotation: Int, + val forceCompression: Boolean ) { private val log = Logger("Tracks") - val all: TrackMap - - val outputFormats: TrackMap + lateinit var all: TrackMap + lateinit var outputFormats: TrackMap + lateinit var active: TrackMap init { + updateTracksInfo() + } + + fun updateTracksInfo() { val (audioFormat, audioStatus) = resolveTrack(TrackType.AUDIO, strategies.audio, sources.audioOrNull()) val (videoFormat, videoStatus) = resolveTrack(TrackType.VIDEO, strategies.video, sources.videoOrNull()) all = trackMapOf( @@ -33,12 +37,13 @@ class Tracks( outputFormats = trackMapOf(video = videoFormat, audio = audioFormat) log.i("init: videoStatus=$videoStatus, resolvedVideoStatus=${all.video}, videoFormat=$videoFormat") log.i("init: audioStatus=$audioStatus, resolvedAudioStatus=${all.audio}, audioFormat=$audioFormat") + + active = trackMapOf( + video = all.video.takeIf { it.isTranscoding }, + audio = all.audio.takeIf { it.isTranscoding } + ) } - val active: TrackMap = trackMapOf( - video = all.video.takeIf { it.isTranscoding }, - audio = all.audio.takeIf { it.isTranscoding } - ) private fun resolveVideoStatus(status: TrackStatus, forceCompression: Boolean, rotation: Int): TrackStatus { val force = forceCompression || rotation != 0 diff --git a/lib/src/main/java/com/otaliastudios/transcoder/internal/thumbnails/DefaultThumbnailsEngine.kt b/lib/src/main/java/com/otaliastudios/transcoder/internal/thumbnails/DefaultThumbnailsEngine.kt index c2ae7066..32b2be69 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/internal/thumbnails/DefaultThumbnailsEngine.kt +++ b/lib/src/main/java/com/otaliastudios/transcoder/internal/thumbnails/DefaultThumbnailsEngine.kt @@ -5,8 +5,8 @@ package com.otaliastudios.transcoder.internal.thumbnails import android.media.MediaFormat import com.otaliastudios.transcoder.common.TrackStatus import com.otaliastudios.transcoder.common.TrackType +import com.otaliastudios.transcoder.internal.CustomSegments import com.otaliastudios.transcoder.internal.DataSources -import com.otaliastudios.transcoder.internal.Segments import com.otaliastudios.transcoder.internal.Timer import com.otaliastudios.transcoder.internal.Tracks import com.otaliastudios.transcoder.internal.codec.Decoder @@ -60,7 +60,7 @@ class DefaultThumbnailsEngine( true ) - private val segments = Segments(dataSources, tracks, ::createPipeline, false) + private val segments = CustomSegments(dataSources, tracks, ::createPipeline) private val timer = Timer(DefaultTimeInterpolator(), dataSources, tracks, segments.currentIndex) @@ -109,7 +109,9 @@ class DefaultThumbnailsEngine( outputFormat: MediaFormat ): Pipeline { val source = dataSources[type][index].ignoringEOS() - + if(VERBOSE) { + log.i("Creating pipeline #$index. absoluteUs=${stubs.joinToString { it.toString() }}") + } return Pipeline.build("Thumbnails") { Seeker(source) { var seek = false @@ -219,19 +221,57 @@ class DefaultThumbnailsEngine( return nextKeyFrameIndex } + override fun addDataSource(dataSource: DataSource) { + dataSources.addVideoDataSource(dataSource) + tracks.updateTracksInfo() + if (tracks.active.has(TrackType.VIDEO)) { + dataSource.selectTrack(TrackType.VIDEO) + } + } + + override fun removeDataSource(dataSourceId: String) { + dataSources.removeVideoDataSource(dataSourceId) + tracks.updateTracksInfo() + } + override suspend fun queueThumbnails(list: List) { - val segment = segments.next(TrackType.VIDEO) - segment?.let { - this.updatePositions(list, it.index) + + val map = list.groupBy { it.sourceId() } + + map.forEach { entry -> + val positions = entry.value.flatMap { request -> + val duration = timer.totalDurationUs + request.locate(duration).map { it to request } + }.sortedBy { it.first } + val index = dataSources[TrackType.VIDEO].indexOfFirst { it.mediaId() == entry.key } + if (index >= 0) { + stubs.addAll( + positions.map { (positionUs, request) -> + Stub(request, positionUs, positionUs) + }.toMutableList().reorder(dataSources[TrackType.VIDEO][index]) + ) + } + if (VERBOSE) { + log.i("Updating pipeline positions for segment Index#$index absoluteUs=${positions.joinToString { it.first.toString() }}, and stubs $stubs") + } } - while (currentCoroutineContext().isActive) { - val advanced = segments.next(TrackType.VIDEO)?.advance() ?: false - val completed = !advanced && !segments.hasNext() // avoid calling hasNext if we advanced. - if (completed || stubs.isEmpty()) { - log.i("loop broken $stubs") - break - } else if (!advanced) { - delay(WAIT_MS) + + if (stubs.isNotEmpty()) { + while (currentCoroutineContext().isActive) { + val segment = + stubs.firstOrNull()?.request?.sourceId()?.let { segments.getSegment(it) } + if (VERBOSE) { + log.i("loop advancing for $segment") + } + val advanced = segment?.advance() ?: false + // avoid calling hasNext if we advanced. + val completed = !advanced && !segments.hasNext() + if (completed || stubs.isEmpty()) { + log.i("loop broken $stubs") + break + } else if (!advanced) { + delay(WAIT_MS) + } } } } @@ -241,12 +281,12 @@ class DefaultThumbnailsEngine( segments.release() } - override suspend fun removePosition(positionUs: Long) { - if (positionUs == stubs.firstOrNull()?.positionUs) { + override suspend fun removePosition(source: String, positionUs: Long) { + if (stubs.firstOrNull()?.request?.sourceId() == source && positionUs == stubs.firstOrNull()?.positionUs) { return } val locatedTimestampUs = SingleThumbnailRequest(positionUs).locate(timer.durationUs.video)[0] - val stub = stubs.find { it.positionUs == locatedTimestampUs } + val stub = stubs.find {it.request.sourceId() == source && it.positionUs == locatedTimestampUs } if (stub != null) { log.i("removePosition Match: $positionUs :$stubs") stubs.remove(stub) @@ -254,22 +294,6 @@ class DefaultThumbnailsEngine( } } - private fun updatePositions(requests: List, index: Int) { - val positions = requests.flatMap { request -> - val duration = timer.totalDurationUs - request.locate(duration).map { it to request } - } - log.i("Creating pipeline #$index. absoluteUs=${positions.joinToString { it.first.toString() }}") - - stubs.addAll( - positions.mapNotNull { (positionUs, request) -> - val localizedUs = timer.localize(TrackType.VIDEO, index, positionUs) - localizedUs?.let { Stub(request, positionUs, localizedUs) } -// }.toMutableList().sortedBy { it.positionUs } - }.toMutableList().reorder(dataSources[TrackType.VIDEO][0]) - ) - } - private fun List.reorder(source: DataSource): Collection { val bucketListMap = LinkedHashMap>() val finalList = ArrayList() diff --git a/lib/src/main/java/com/otaliastudios/transcoder/internal/thumbnails/ThumbnailsEngine.kt b/lib/src/main/java/com/otaliastudios/transcoder/internal/thumbnails/ThumbnailsEngine.kt index 8d4c5c3d..dea669df 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/internal/thumbnails/ThumbnailsEngine.kt +++ b/lib/src/main/java/com/otaliastudios/transcoder/internal/thumbnails/ThumbnailsEngine.kt @@ -5,6 +5,7 @@ package com.otaliastudios.transcoder.internal.thumbnails import com.otaliastudios.transcoder.ThumbnailerOptions import com.otaliastudios.transcoder.internal.DataSources import com.otaliastudios.transcoder.internal.utils.Logger +import com.otaliastudios.transcoder.source.DataSource import com.otaliastudios.transcoder.thumbnail.Thumbnail import com.otaliastudios.transcoder.thumbnail.ThumbnailRequest import kotlinx.coroutines.flow.Flow @@ -13,9 +14,13 @@ abstract class ThumbnailsEngine { abstract val progressFlow: Flow + abstract fun addDataSource(dataSource: DataSource) + + abstract fun removeDataSource(dataSourceId: String) + abstract suspend fun queueThumbnails(list: List) - abstract suspend fun removePosition(positionUs: Long) + abstract suspend fun removePosition(source: String, positionUs: Long) abstract fun cleanup() diff --git a/lib/src/main/java/com/otaliastudios/transcoder/thumbnail/Thumbnail.kt b/lib/src/main/java/com/otaliastudios/transcoder/thumbnail/Thumbnail.kt index dcebe09c..fe96f706 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/thumbnail/Thumbnail.kt +++ b/lib/src/main/java/com/otaliastudios/transcoder/thumbnail/Thumbnail.kt @@ -2,7 +2,7 @@ package com.otaliastudios.transcoder.thumbnail import android.graphics.Bitmap -class Thumbnail internal constructor( +class Thumbnail constructor( val request: ThumbnailRequest, val positionUs: Long, val bitmap: Bitmap