Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -19,4 +19,5 @@ data class AudioStreamItem(
val audioTrackId: String?,
val audioTrackName: String?,
val audioLocale: String?,
val isOriginal: Boolean,
)
2 changes: 2 additions & 0 deletions src/main/kotlin/dev/typetype/server/models/StreamResponse.kt
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,8 @@ data class StreamResponse(
val dashMpdUrl: String,
val videoStreams: List<VideoStreamItem>,
val audioStreams: List<AudioStreamItem>,
val originalAudioTrackId: String?,
val preferredDefaultAudioTrackId: String?,
val videoOnlyStreams: List<VideoStreamItem>,
val subtitles: List<SubtitleItem>,
val previewFrames: List<PreviewFrameItem>,
Expand Down
10 changes: 5 additions & 5 deletions src/main/kotlin/dev/typetype/server/services/ManifestService.kt
Original file line number Diff line number Diff line change
Expand Up @@ -7,13 +7,12 @@ import java.net.URLEncoder
import java.nio.charset.StandardCharsets

class ManifestService(private val streamService: StreamService) {

suspend fun dashManifest(videoUrl: String): ExtractionResult<String> {
val result = streamService.getStreamInfo(videoUrl)
if (result !is ExtractionResult.Success) return result.recast()
val info = result.data
val videos = compatibleVideoStreams(info.videoOnlyStreams)
val audios = compatibleAudioStreams(info.audioStreams)
val audios = compatibleAudioStreams(info.audioStreams, info.preferredDefaultAudioTrackId)
if (videos.isEmpty() && audios.isEmpty())
return ExtractionResult.Failure("No compatible streams found for DASH manifest")
return ExtractionResult.Success(buildMpd(videos, audios, info.duration))
Expand All @@ -23,9 +22,10 @@ class ManifestService(private val streamService: StreamService) {
streams.filter { it.codec?.startsWith("av01") != true && it.url.isNotBlank() && !it.codec.isNullOrBlank() }
.sortedWith(compareBy({ codecPriority(it.codec ?: "") }, { -(it.bitrate ?: bwFromUrl(it.url) ?: 0) }))

private fun compatibleAudioStreams(streams: List<AudioStreamItem>): List<AudioStreamItem> =
private fun compatibleAudioStreams(streams: List<AudioStreamItem>, preferredTrackId: String?): List<AudioStreamItem> =
streams.filter { it.url.isNotBlank() && !it.codec.isNullOrBlank() }
.sortedByDescending { it.bitrate ?: 0 }
.sortedWith(compareBy<AudioStreamItem> { preferredTrackId != null && it.audioTrackId != preferredTrackId }
.thenByDescending { it.bitrate ?: 0 })

private fun codecPriority(codec: String): Int = when {
codec.startsWith("avc1") -> 0
Expand Down Expand Up @@ -117,4 +117,4 @@ class ManifestService(private val streamService: StreamService) {
is ExtractionResult.BadRequest -> ExtractionResult.BadRequest(message)
is ExtractionResult.Failure -> ExtractionResult.Failure(message)
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ import org.schabi.newpipe.extractor.stream.VideoStream

internal object NativeManifestBuilder {

fun build(videos: List<VideoStream>, audios: List<AudioStream>, duration: Long): String {
fun build(videos: List<VideoStream>, audios: List<AudioStream>, duration: Long, preferredAudioTrackId: String?): String {
val sb = StringBuilder()
sb.appendLine("<?xml version=\"1.0\" encoding=\"UTF-8\"?>")
sb.appendLine("<MPD xmlns=\"urn:mpeg:dash:schema:mpd:2011\"")
Expand All @@ -18,7 +18,7 @@ internal object NativeManifestBuilder {
sb.appendLine(" minBufferTime=\"PT4S\">")
sb.appendLine(" <Period>")
buildVideoAdaptationSets(sb, videos, duration)
buildAudioAdaptationSets(sb, audios, duration)
buildAudioAdaptationSets(sb, audios, duration, preferredAudioTrackId)
sb.appendLine(" </Period>")
sb.append("</MPD>")
return sb.toString()
Expand All @@ -34,10 +34,18 @@ internal object NativeManifestBuilder {
}
}

private fun buildAudioAdaptationSets(sb: StringBuilder, audios: List<AudioStream>, duration: Long) {
private fun buildAudioAdaptationSets(
sb: StringBuilder,
audios: List<AudioStream>,
duration: Long,
preferredAudioTrackId: String?,
) {
if (audios.isEmpty()) return
var id = 0
audios.groupBy { audioMimeType(it.getCodec() ?: "") }
val ordered = if (preferredAudioTrackId == null) audios else audios.sortedBy {
it.getAudioTrackId() != preferredAudioTrackId
}
ordered.groupBy { audioMimeType(it.getCodec() ?: "") }
.forEach { (mime, group) ->
sb.appendLine(" <AdaptationSet mimeType=\"$mime\">")
group.forEach { s -> appendAudioRepresentation(sb, s, id++, duration) }
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -31,11 +31,12 @@ class NativeManifestService {
private fun buildManifest(info: StreamInfo): ExtractionResult<String> {
val videos = compatibleVideoStreams(info.videoOnlyStreams)
val audios = compatibleAudioStreams(info.audioStreams)
val preferredAudioTrackId = resolvePreferredAudioTrackId(audios)
if (videos.isEmpty() && audios.isEmpty())
return ExtractionResult.Failure("No compatible streams found")
return runCatching {
ExtractionResult.Success(
NativeManifestBuilder.build(videos, audios, info.duration)
NativeManifestBuilder.build(videos, audios, info.duration, preferredAudioTrackId)
)
}.getOrElse {
ExtractionResult.Failure(it.message ?: "Manifest build failed")
Expand Down Expand Up @@ -73,4 +74,19 @@ class NativeManifestService {
else -> 2
}
}

private fun resolvePreferredAudioTrackId(audios: List<AudioStream>): String? {
val original = audios.firstNotNullOfOrNull { stream ->
val name = stream.getAudioTrackName()?.lowercase() ?: return@firstNotNullOfOrNull null
stream.getAudioTrackId()?.takeIf { "original" in name || "default" in name || "yokuqala" in name }
}
if (original != null) return original
val english = audios.firstNotNullOfOrNull { stream ->
stream.getAudioTrackId()?.takeIf {
stream.getAudioLocale() == "en" || it.substringBefore('.').substringBefore('-') == "en"
}
}
if (english != null) return english
return audios.firstNotNullOfOrNull { it.getAudioTrackId() }
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@ internal class PipePipeStreamService(
val segmentsDeferred = async { resolveSegments(extractor) }
val streamInfo = streamInfoDeferred.await()
streamInfo.setSponsorBlockSegments(segmentsDeferred.await())
val response = streamInfo.toStreamResponse()
val response = StreamAudioContractResolver.apply(streamInfo.toStreamResponse())
val withSubtitles = if (response.subtitles.isEmpty() && service.serviceId == 0) {
response.copy(subtitles = subtitleService.fetchSubtitles(streamInfo.id))
} else {
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@
package dev.typetype.server.services

import dev.typetype.server.models.AudioStreamItem
import dev.typetype.server.models.StreamResponse

object StreamAudioContractResolver {
fun apply(
response: StreamResponse,
fallbackLanguage: String = defaultFallbackLanguage(),
): StreamResponse {
val normalized = response.audioStreams.map { stream ->
stream.copy(isOriginal = isOriginalTrack(stream))
}
val originalTrackId = normalized.firstNotNullOfOrNull { stream ->
stream.audioTrackId?.takeIf { stream.isOriginal }
}
val preferredTrackId = resolvePreferredTrackId(normalized, fallbackLanguage, originalTrackId)
return response.copy(
audioStreams = normalized,
originalAudioTrackId = originalTrackId,
preferredDefaultAudioTrackId = preferredTrackId,
)
}

private fun resolvePreferredTrackId(
streams: List<AudioStreamItem>,
fallbackLanguage: String,
originalTrackId: String?,
): String? {
if (originalTrackId != null) return originalTrackId
val fallbackTrackId = streams.firstNotNullOfOrNull { stream ->
stream.audioTrackId?.takeIf { matchesFallback(stream, fallbackLanguage) }
}
if (fallbackTrackId != null) return fallbackTrackId
return streams.firstNotNullOfOrNull { it.audioTrackId }
}

private fun matchesFallback(stream: AudioStreamItem, fallbackLanguage: String): Boolean {
val wanted = normalizedLanguage(fallbackLanguage)
val locale = stream.audioLocale?.let(::normalizedLanguage)
if (locale != null && locale == wanted) return true
val trackLanguage = stream.audioTrackId?.substringBefore('.')?.let(::normalizedLanguage)
return trackLanguage != null && trackLanguage == wanted
}

private fun isOriginalTrack(stream: AudioStreamItem): Boolean {
val lowered = stream.audioTrackName?.lowercase() ?: return false
if ("original" in lowered) return true
if ("default" in lowered) return true
if ("yokuqala" in lowered) return true
return false
}

private fun normalizedLanguage(language: String): String = language.lowercase().substringBefore('-')

private fun defaultFallbackLanguage(): String =
System.getenv(AUDIO_FALLBACK_ENV)?.trim().orEmpty().ifBlank { DEFAULT_FALLBACK_LANGUAGE }

private const val AUDIO_FALLBACK_ENV = "DEFAULT_AUDIO_FALLBACK_LANGUAGE"
private const val DEFAULT_FALLBACK_LANGUAGE = "en"
}
Original file line number Diff line number Diff line change
@@ -1,5 +1,4 @@
package dev.typetype.server.services

import dev.typetype.server.models.AudioStreamItem
import dev.typetype.server.models.PreviewFrameItem
import dev.typetype.server.models.StreamResponse
Expand Down Expand Up @@ -46,14 +45,15 @@ internal fun StreamInfo.toStreamResponse(): StreamResponse {
dashMpdUrl = dashMpdUrl?.takeIf { it.startsWith("http") } ?: "",
videoStreams = videoStreams.map { it.toVideoStreamItem(false) },
audioStreams = audioStreams.mapNotNull { runCatching { it.toAudioStreamItem() }.getOrNull() },
originalAudioTrackId = null,
preferredDefaultAudioTrackId = null,
videoOnlyStreams = videoOnlyStreams.map { it.toVideoStreamItem(true) },
subtitles = subtitles.mapNotNull { runCatching { it.toSubtitleItem() }.getOrNull() },
previewFrames = previewFrames.mapNotNull { runCatching { it.toPreviewFrameItem() }.getOrNull() },
sponsorBlockSegments = runCatching { getSponsorBlockSegments().map { it.toSegmentItem() } }.getOrElse { emptyList() },
relatedStreams = relatedItems.filterIsInstance<StreamInfoItem>().mapNotNull { runCatching { it.toVideoItem() }.getOrNull() },
)
}

internal fun VideoStream.toVideoStreamItem(isVideoOnly: Boolean): VideoStreamItem =
VideoStreamItem(
url = getContent() ?: "",
Expand Down Expand Up @@ -90,6 +90,7 @@ internal fun AudioStream.toAudioStreamItem(): AudioStreamItem = AudioStreamItem(
audioTrackId = getAudioTrackId(),
audioTrackName = getAudioTrackName(),
audioLocale = getAudioLocale(),
isOriginal = false,
)

internal fun SubtitlesStream.toSubtitleItem(): SubtitleItem = SubtitleItem(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -78,7 +78,8 @@ class HomeRecommendationCandidateServiceTest {
uploaderSubscriberCount = 0, uploaderVerified = false, category = "", license = "", visibility = "",
tags = emptyList(), streamType = "video_stream", isShortFormContent = false, requiresMembership = false,
startPosition = 0, streamSegments = emptyList(), hlsUrl = "", dashMpdUrl = "", videoStreams = emptyList(),
audioStreams = emptyList(), videoOnlyStreams = emptyList(), subtitles = emptyList(), previewFrames = emptyList(),
audioStreams = emptyList(), originalAudioTrackId = null, preferredDefaultAudioTrackId = null,
videoOnlyStreams = emptyList(), subtitles = emptyList(), previewFrames = emptyList(),
sponsorBlockSegments = emptyList(), relatedStreams = related, publishedAt = 0,
)

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ class NativeManifestBuilderCoreTest {
every { video.getInitStart() } returns 0
every { video.getInitEnd() } returns 220

val manifest = NativeManifestBuilder.build(videos = listOf(video), audios = emptyList(), duration = 300)
val manifest = NativeManifestBuilder.build(videos = listOf(video), audios = emptyList(), duration = 300, preferredAudioTrackId = null)

assertTrue(manifest.contains("mediaPresentationDuration=\"PT300S\""))
assertTrue(manifest.contains("<AdaptationSet mimeType=\"video/mp4\""))
Expand All @@ -50,7 +50,7 @@ class NativeManifestBuilderCoreTest {
every { video.getInitStart() } returns 0
every { video.getInitEnd() } returns 0

val manifest = NativeManifestBuilder.build(videos = listOf(video), audios = emptyList(), duration = 120)
val manifest = NativeManifestBuilder.build(videos = listOf(video), audios = emptyList(), duration = 120, preferredAudioTrackId = null)

assertTrue(manifest.contains("<AdaptationSet mimeType=\"video/webm\""))
assertTrue(manifest.contains("<Representation id=\"v-0\""))
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,53 @@
package dev.typetype.server

import dev.typetype.server.services.StreamAudioContractResolver
import org.junit.jupiter.api.Assertions.assertEquals
import org.junit.jupiter.api.Assertions.assertFalse
import org.junit.jupiter.api.Assertions.assertNull
import org.junit.jupiter.api.Assertions.assertTrue
import org.junit.jupiter.api.Test

class StreamAudioContractResolverTest {
@Test
fun `original track id becomes preferred default`() {
val fr = testAudioStream(audioTrackId = "fr.0", audioTrackName = "fr", audioLocale = "fr")
val original = testAudioStream(
audioTrackId = "en-US.4",
audioTrackName = "en-US (original)",
audioLocale = "en",
)
val response = testStreamResponse(audioStreams = listOf(fr, original))

val resolved = StreamAudioContractResolver.apply(response, fallbackLanguage = "es")

assertEquals("en-US.4", resolved.originalAudioTrackId)
assertEquals("en-US.4", resolved.preferredDefaultAudioTrackId)
assertFalse(resolved.audioStreams.first { it.audioTrackId == "fr.0" }.isOriginal)
assertTrue(resolved.audioStreams.first { it.audioTrackId == "en-US.4" }.isOriginal)
}

@Test
fun `fallback language wins when no original track exists`() {
val fr = testAudioStream(audioTrackId = "fr.0", audioTrackName = "fr", audioLocale = "fr")
val en = testAudioStream(audioTrackId = "en.0", audioTrackName = "en", audioLocale = "en")
val response = testStreamResponse(audioStreams = listOf(fr, en))

val resolved = StreamAudioContractResolver.apply(response, fallbackLanguage = "en")

assertNull(resolved.originalAudioTrackId)
assertEquals("en.0", resolved.preferredDefaultAudioTrackId)
assertTrue(resolved.audioStreams.none { it.isOriginal })
}

@Test
fun `first valid track id is used when original and fallback are unavailable`() {
val unknown = testAudioStream(audioTrackId = null, audioTrackName = "und", audioLocale = null)
val de = testAudioStream(audioTrackId = "de.0", audioTrackName = "de", audioLocale = "de")
val response = testStreamResponse(audioStreams = listOf(unknown, de))

val resolved = StreamAudioContractResolver.apply(response, fallbackLanguage = "en")

assertNull(resolved.originalAudioTrackId)
assertEquals("de.0", resolved.preferredDefaultAudioTrackId)
}
}
12 changes: 10 additions & 2 deletions src/test/kotlin/dev/typetype/server/StreamFieldsTest.kt
Original file line number Diff line number Diff line change
Expand Up @@ -73,11 +73,19 @@ class StreamFieldsTest {

@Test
fun `GET streams serializes audioLocale in audioStreams`() = withApp {
val audio = testAudioStream().copy(audioLocale = "en")
val audio = testAudioStream().copy(audioLocale = "en", isOriginal = true)
coEvery { streamService.getStreamInfo(any()) } returns
ExtractionResult.Success(testStreamResponse(audioStreams = listOf(audio)))
ExtractionResult.Success(
testStreamResponse(audioStreams = listOf(audio)).copy(
originalAudioTrackId = "en.0",
preferredDefaultAudioTrackId = "en.0",
)
)
val body = client.get("/streams?url=https://youtube.com/watch?v=test").bodyAsText()
assertTrue(body.contains("\"audioLocale\":\"en\""))
assertTrue(body.contains("\"isOriginal\":true"))
assertTrue(body.contains("\"originalAudioTrackId\":\"en.0\""))
assertTrue(body.contains("\"preferredDefaultAudioTrackId\":\"en.0\""))
}

@Test
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,7 @@ class StreamInfoMappersCoreTest {
assertNull(mappedAudio.bitrate)
assertNull(mappedAudio.codec)
assertEquals("en", mappedAudio.audioLocale)
assertEquals(false, mappedAudio.isOriginal)
}

@Test
Expand Down
5 changes: 4 additions & 1 deletion src/test/kotlin/dev/typetype/server/TestFixtures.kt
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,7 @@ fun testAudioStream(
audioTrackId: String? = null,
audioTrackName: String? = null,
audioLocale: String? = null,
isOriginal: Boolean = false,
): AudioStreamItem = AudioStreamItem(
url = url,
mimeType = "audio/mp4",
Expand All @@ -64,6 +65,7 @@ fun testAudioStream(
audioTrackId = audioTrackId,
audioTrackName = audioTrackName,
audioLocale = audioLocale,
isOriginal = isOriginal,
)

fun testStreamResponse(
Expand Down Expand Up @@ -100,10 +102,11 @@ fun testStreamResponse(
dashMpdUrl = dashMpdUrl,
videoStreams = emptyList(),
audioStreams = audioStreams,
originalAudioTrackId = null,
preferredDefaultAudioTrackId = null,
videoOnlyStreams = videoOnlyStreams,
subtitles = emptyList(),
previewFrames = emptyList(),
sponsorBlockSegments = emptyList(),
relatedStreams = emptyList(),
)