diff --git a/build.gradle.kts b/build.gradle.kts index ae7b3e35..d4655639 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -38,7 +38,7 @@ dependencies { implementation("io.ktor:ktor-server-call-logging-jvm") implementation("io.ktor:ktor-server-rate-limit-jvm") implementation("ch.qos.logback:logback-classic:1.6.1") - implementation("com.github.Priveetee.PipePipeExtractor:extractor:21ab9b6ac415d23ef88df19c5ad126c7952bb543") + implementation("com.github.Priveetee.PipePipeExtractor:extractor:f156813dd4bbebf3b4dffe541fee6c27ae1dd294") compileOnly("com.github.TeamNewPipe:nanojson:1d9e1aea9049fc9f85e68b43ba39fe7be1c1f751") implementation("org.json:json:20260719") implementation("com.squareup.okhttp3:okhttp:5.4.0") diff --git a/gradle.properties b/gradle.properties index 6d84d772..99db84dc 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,5 +1,5 @@ org.gradle.jvmargs=-Xmx2g -XX:+UseG1GC kotlin.code.style=official -appVersion=1.3.1 +appVersion=1.4.0 systemProp.sun.net.client.defaultReadTimeout=180000 systemProp.sun.net.client.defaultConnectTimeout=60000 diff --git a/openapi.yaml b/openapi.yaml index ab456f14..00121b5c 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -25,6 +25,8 @@ paths: /streams/bilibili: { $ref: ./openapi/paths/streams.yaml#/BiliBiliStreams } /streams/audio-only: { $ref: ./openapi/paths/streams.yaml#/AudioOnly } /streams/audio-only/source: { $ref: ./openapi/paths/streams.yaml#/AudioOnlySource } + /subtitles/youtube/{videoId}: { $ref: ./openapi/paths/subtitles.yaml#/YoutubeSubtitle } + /proxy: { $ref: ./openapi/paths/proxy.yaml#/Proxy } /sabr/download/{videoId}: { $ref: ./openapi/paths/sabr-download.yaml#/SabrDownload } /sabr/playback/{sessionId}/position: { $ref: ./openapi/paths/sabr-playback.yaml#/SabrPlaybackPosition } /sabr/playback/{sessionId}/window: { $ref: ./openapi/paths/sabr-playback.yaml#/SabrPlaybackWindow } @@ -100,6 +102,8 @@ components: $ref: ./openapi/components/streams.yaml#/StreamResponse AudioOnlyStreamResponse: $ref: ./openapi/components/streams.yaml#/AudioOnlyStreamResponse + SubtitleItem: + $ref: ./openapi/components/streams.yaml#/SubtitleItem SabrPlaybackPositionRequest: $ref: ./openapi/components/sabr-playback.yaml#/SabrPlaybackPositionRequest SabrPlaybackWindowRequest: @@ -108,6 +112,8 @@ components: $ref: ./openapi/components/media.yaml#/SearchPageResponse SearchFilterOption: $ref: ./openapi/components/media.yaml#/SearchFilterOption + SearchFilterGroup: + $ref: ./openapi/components/media.yaml#/SearchFilterGroup SearchFiltersResponse: $ref: ./openapi/components/media.yaml#/SearchFiltersResponse ChannelResultItem: diff --git a/openapi/components/media.yaml b/openapi/components/media.yaml index 3ee436bc..8bc028b9 100644 --- a/openapi/components/media.yaml +++ b/openapi/components/media.yaml @@ -49,12 +49,22 @@ SearchFilterOption: properties: value: { type: string } label: { type: string } + isDefault: { type: boolean, default: false } +SearchFilterGroup: + type: object + required: [key, label, multiSelect, options] + properties: + key: { type: string } + label: { type: string } + multiSelect: { type: boolean } + options: { type: array, items: { $ref: '#/SearchFilterOption' } } SearchFiltersResponse: type: object - required: [contentFilters, sortFilters] + required: [contentFilters, sortFilters, filterGroups] properties: contentFilters: { type: array, items: { $ref: '#/SearchFilterOption' } } sortFilters: { type: array, items: { $ref: '#/SearchFilterOption' } } + filterGroups: { type: array, items: { $ref: '#/SearchFilterGroup' }, default: [] } ChannelResponse: type: object required: [name, description, avatarUrl, bannerUrl, subscriberCount, isVerified, videos, nextpage] diff --git a/openapi/components/streams.yaml b/openapi/components/streams.yaml index a7bbe17e..4ef4c5e2 100644 --- a/openapi/components/streams.yaml +++ b/openapi/components/streams.yaml @@ -48,6 +48,15 @@ AudioOnlyStreamResponse: bitrate: { type: integer, nullable: true } contentLength: { type: integer, format: int64, nullable: true } duration: { type: integer, format: int64, nullable: true } +SubtitleItem: + type: object + required: [url, mimeType, languageTag, displayLanguageName, isAutoGenerated] + properties: + url: { type: string, format: uri } + mimeType: { type: string } + languageTag: { type: string } + displayLanguageName: { type: string } + isAutoGenerated: { type: boolean } StreamResponse: type: object required: [id, title, uploaderName, uploaderUrl, thumbnailUrl, description, duration, viewCount, uploadDate, uploaded, streamType, isLive, isPostLive, isLiveContent, hasLiveManifest, videoStreams, audioStreams, videoOnlyStreams, subtitles, relatedStreams] @@ -77,6 +86,6 @@ StreamResponse: videoStreams: { type: array, items: { $ref: '#/VideoStreamItem' } } audioStreams: { type: array, items: { $ref: '#/AudioStreamItem' } } videoOnlyStreams: { type: array, items: { $ref: '#/VideoStreamItem' } } - subtitles: { type: array, items: { type: object }, default: [] } + subtitles: { type: array, items: { $ref: '#/SubtitleItem' }, default: [] } relatedStreams: { type: array, items: { $ref: './media.yaml#/VideoItem' } } sponsorBlockSegments: { type: array, items: { type: object }, default: [] } diff --git a/openapi/paths/proxy.yaml b/openapi/paths/proxy.yaml new file mode 100644 index 00000000..b1162a6a --- /dev/null +++ b/openapi/paths/proxy.yaml @@ -0,0 +1,50 @@ +Proxy: + get: + tags: [extraction] + summary: Retrieve proxied media + description: >- + Streams supported remote content. Existing clients that submit a YouTube timed-text URL are + routed through the dedicated subtitle resolver for compatibility; new clients should use + /subtitles/youtube/{videoId}. + parameters: + - name: url + in: query + required: true + schema: { type: string, format: uri } + - name: Range + in: header + required: false + schema: { type: string } + responses: + '200': + description: Complete proxied content. + headers: + X-Request-ID: + $ref: ../components/common.yaml#/RequestIdHeader + content: + text/vtt: + schema: { type: string } + application/octet-stream: + schema: { type: string, format: binary } + '206': + description: Partial generic proxy response. + headers: + X-Request-ID: + $ref: ../components/common.yaml#/RequestIdHeader + Content-Range: + schema: { type: string } + Accept-Ranges: + schema: { type: string } + content: + application/octet-stream: + schema: { type: string, format: binary } + '400': + $ref: ../components/common.yaml#/JsonError + '404': + $ref: ../components/common.yaml#/JsonError + '422': + $ref: ../components/common.yaml#/JsonError + '429': + $ref: ../components/common.yaml#/JsonError + '502': + $ref: ../components/common.yaml#/JsonError diff --git a/openapi/paths/search.yaml b/openapi/paths/search.yaml index 398ccc9d..815b8949 100644 --- a/openapi/paths/search.yaml +++ b/openapi/paths/search.yaml @@ -22,7 +22,18 @@ Search: - name: sortFilter in: query required: false + deprecated: true + description: Legacy single-filter parameter. Use `filter` for grouped selections. schema: { type: string } + - name: filter + in: query + required: false + description: Repeat for each selected search filter. + style: form + explode: true + schema: + type: array + items: { type: string } responses: '200': description: Search page. @@ -46,6 +57,11 @@ SearchFilters: in: query required: true schema: { type: integer, enum: [0, 3, 4, 5, 6] } + - name: contentFilter + in: query + required: false + description: Return filter groups applicable to this content type. + schema: { type: string } responses: '200': description: Search filter capabilities. diff --git a/openapi/paths/streams.yaml b/openapi/paths/streams.yaml index 743342b7..33aad372 100644 --- a/openapi/paths/streams.yaml +++ b/openapi/paths/streams.yaml @@ -19,6 +19,10 @@ YoutubeSabrStreams: $ref: ../components/streams.yaml#/StreamResponse '400': $ref: ../components/common.yaml#/JsonError + '401': + $ref: ../components/common.yaml#/JsonError + '403': + $ref: ../components/common.yaml#/JsonError '422': $ref: ../components/common.yaml#/JsonError YoutubeSabrBootstrap: @@ -42,6 +46,10 @@ YoutubeSabrBootstrap: $ref: ../components/streams.yaml#/StreamResponse '400': $ref: ../components/common.yaml#/JsonError + '401': + $ref: ../components/common.yaml#/JsonError + '403': + $ref: ../components/common.yaml#/JsonError '422': $ref: ../components/common.yaml#/JsonError NicoNicoStreams: @@ -62,6 +70,10 @@ NicoNicoStreams: $ref: ../components/streams.yaml#/StreamResponse '400': $ref: ../components/common.yaml#/JsonError + '401': + $ref: ../components/common.yaml#/JsonError + '403': + $ref: ../components/common.yaml#/JsonError '422': $ref: ../components/common.yaml#/JsonError BiliBiliStreams: @@ -82,6 +94,10 @@ BiliBiliStreams: $ref: ../components/streams.yaml#/StreamResponse '400': $ref: ../components/common.yaml#/JsonError + '401': + $ref: ../components/common.yaml#/JsonError + '403': + $ref: ../components/common.yaml#/JsonError '422': $ref: ../components/common.yaml#/JsonError AudioOnly: diff --git a/openapi/paths/subtitles.yaml b/openapi/paths/subtitles.yaml new file mode 100644 index 00000000..a7468276 --- /dev/null +++ b/openapi/paths/subtitles.yaml @@ -0,0 +1,62 @@ +YoutubeSubtitle: + get: + tags: [playback] + summary: Retrieve a YouTube subtitle track + description: >- + Resolves a fresh timed-text track through PipePipeExtractor and returns bounded subtitle + content through the configured YouTube egress. VOD responses are cached by selection and + simultaneous identical requests are coalesced. Active and post-live tracks use a separate + short-lived server cache and are not browser-cacheable. + parameters: + - name: videoId + in: path + required: true + schema: + type: string + pattern: '^[A-Za-z0-9_-]{11}$' + - name: language + in: query + required: true + schema: { type: string, minLength: 1, maxLength: 64 } + - name: variant + in: query + required: true + schema: { type: string, enum: [manual, auto] } + - name: format + in: query + required: false + schema: { type: string, enum: [vtt, ttml], default: vtt } + - name: sourceLanguage + in: query + required: false + schema: { type: string, minLength: 1, maxLength: 64 } + - name: translation + in: query + required: false + schema: { type: string, minLength: 1, maxLength: 64 } + - name: name + in: query + required: false + description: Distinguishes multiple tracks with the same language and generation variant. + schema: { type: string, minLength: 1, maxLength: 128 } + responses: + '200': + description: Selected subtitle content. + headers: + X-Request-ID: + $ref: ../components/common.yaml#/RequestIdHeader + Cache-Control: + schema: { type: string } + content: + text/vtt: + schema: { type: string } + application/ttml+xml: + schema: { type: string } + '400': + $ref: ../components/common.yaml#/JsonError + '404': + $ref: ../components/common.yaml#/JsonError + '429': + $ref: ../components/common.yaml#/JsonError + '502': + $ref: ../components/common.yaml#/JsonError diff --git a/src/main/kotlin/dev/typetype/server/Application.kt b/src/main/kotlin/dev/typetype/server/Application.kt index 498607fb..299d304c 100644 --- a/src/main/kotlin/dev/typetype/server/Application.kt +++ b/src/main/kotlin/dev/typetype/server/Application.kt @@ -4,6 +4,7 @@ import dev.typetype.server.db.DatabaseFactory import dev.typetype.server.downloader.YoutubeProxySelector import dev.typetype.server.services.ActiveSessionService import dev.typetype.server.services.AuthService +import dev.typetype.server.services.AuthSessionConfig import dev.typetype.server.services.AdminSettingsService import dev.typetype.server.services.AvatarService import dev.typetype.server.services.DownloaderGatewayService @@ -36,7 +37,8 @@ fun Application.module() { val dbPassword = System.getenv("DATABASE_PASSWORD") ?: "typetype" DatabaseFactory.init(dbUrl, dbUser, dbPassword) val jwtSecret = System.getenv("JWT_SECRET") ?: UUID.randomUUID().toString() - val authService = AuthService(jwtSecret) + val authSessionConfig = AuthSessionConfig.fromEnvironment() + val authService = AuthService(jwtSecret, sessionConfig = authSessionConfig) val oidcAuthService = OidcAuthService(OidcConfigLoader.fromEnvironment(), jwtSecret, authService) val userAdminService = UserAdminService() val passwordResetService = PasswordResetService() @@ -85,6 +87,7 @@ fun Application.module() { installApplicationRoutes( svc = svc, authService = authService, + authSessionConfig = authSessionConfig, adminSettingsService = adminSettingsService, activeSessionService = activeSessionService, downloaderGatewayService = downloaderGatewayService, diff --git a/src/main/kotlin/dev/typetype/server/ApplicationRoutes.kt b/src/main/kotlin/dev/typetype/server/ApplicationRoutes.kt index e1bef03f..44d80b01 100644 --- a/src/main/kotlin/dev/typetype/server/ApplicationRoutes.kt +++ b/src/main/kotlin/dev/typetype/server/ApplicationRoutes.kt @@ -27,6 +27,7 @@ import dev.typetype.server.routes.youtubeRemoteBrowserRoutes import dev.typetype.server.services.ActiveSessionService import dev.typetype.server.services.AdminSettingsService import dev.typetype.server.services.AuthService +import dev.typetype.server.services.AuthSessionConfig import dev.typetype.server.services.AvatarService import dev.typetype.server.services.DownloaderGatewayService import dev.typetype.server.services.GitHubIssueService @@ -46,6 +47,7 @@ import io.ktor.server.routing.routing internal fun Application.installApplicationRoutes( svc: ServiceRegistry, authService: AuthService, + authSessionConfig: AuthSessionConfig, adminSettingsService: AdminSettingsService, activeSessionService: ActiveSessionService, downloaderGatewayService: DownloaderGatewayService, @@ -90,8 +92,15 @@ internal fun Application.installApplicationRoutes( ) } downloaderGatewayRoutes(downloaderGatewayService) - oidcAuthRoutes(oidcAuthService, adminSettingsService) - authRoutes(authService, passwordResetService, profileService, adminSettingsService, svc.homeRecommendationWarmupService) + oidcAuthRoutes(oidcAuthService, adminSettingsService, authSessionConfig) + authRoutes( + authService, + passwordResetService, + profileService, + adminSettingsService, + svc.homeRecommendationWarmupService, + authSessionConfig, + ) adminRoutes(authService, userAdminService, passwordResetService, adminSettingsService) adminIdentityRoutes(svc.accountIdentityService, authService) adminAllowListRoutes(authService, userAdminService, svc.adminManagedAccessService, svc.adminUserLookupService, svc.allowedChannelsService, svc.allowedPlaylistsService) diff --git a/src/main/kotlin/dev/typetype/server/ApplicationStreamRoutes.kt b/src/main/kotlin/dev/typetype/server/ApplicationStreamRoutes.kt index b965446a..681158ef 100644 --- a/src/main/kotlin/dev/typetype/server/ApplicationStreamRoutes.kt +++ b/src/main/kotlin/dev/typetype/server/ApplicationStreamRoutes.kt @@ -8,6 +8,7 @@ import dev.typetype.server.routes.proxyRoutes import dev.typetype.server.routes.storyboardProxyRoutes import dev.typetype.server.routes.streamRoutes import dev.typetype.server.routes.withPlayableSabrStreams +import dev.typetype.server.routes.youtubeSubtitleRoutes import dev.typetype.server.services.AdminSettingsService import dev.typetype.server.services.AuthService import io.ktor.server.plugins.ratelimit.rateLimit @@ -27,6 +28,7 @@ internal fun Route.installStreamRoutes( authService = authService, accessControlService = svc.accessControlService, adminSettingsService = adminSettingsService, + blockedService = svc.blockedService, publicHlsManifestTokenService = svc.publicHlsManifestTokenService, sabrStreamContractFilter = { url, data -> data.withPlayableSabrStreams(url, svc.sabrSessionStore) }, ) @@ -56,7 +58,8 @@ internal fun Route.installStreamRoutes( internal fun Route.installProxyRoutes(svc: ServiceRegistry) { rateLimit(PROXY_ZONE) { - proxyRoutes(svc.proxyService) + proxyRoutes(svc.proxyService, svc.youtubeSubtitleDeliveryService) + youtubeSubtitleRoutes(svc.youtubeSubtitleDeliveryService) audioOnlySourceRoutes( streamService = svc.streamService, proxyService = svc.proxyService, diff --git a/src/main/kotlin/dev/typetype/server/CompressionConfig.kt b/src/main/kotlin/dev/typetype/server/CompressionConfig.kt index b0fc8920..18afbd5d 100644 --- a/src/main/kotlin/dev/typetype/server/CompressionConfig.kt +++ b/src/main/kotlin/dev/typetype/server/CompressionConfig.kt @@ -1,15 +1,18 @@ package dev.typetype.server import io.ktor.http.ContentType +import io.ktor.http.HttpStatusCode import io.ktor.server.application.Application import io.ktor.server.application.install import io.ktor.server.plugins.compression.Compression +import io.ktor.server.plugins.compression.condition import io.ktor.server.plugins.compression.excludeContentType import io.ktor.server.plugins.compression.gzip fun Application.configureCompression(): Unit { install(Compression) { gzip { + condition { _ -> response.status() != HttpStatusCode.TooManyRequests } excludeContentType(ContentType.parse("application/vnd.apple.mpegurl")) excludeContentType(ContentType.Image.Any) excludeContentType(ContentType.Video.Any) diff --git a/src/main/kotlin/dev/typetype/server/ExtractionServiceRegistry.kt b/src/main/kotlin/dev/typetype/server/ExtractionServiceRegistry.kt index 6182e354..1fe29a3e 100644 --- a/src/main/kotlin/dev/typetype/server/ExtractionServiceRegistry.kt +++ b/src/main/kotlin/dev/typetype/server/ExtractionServiceRegistry.kt @@ -34,6 +34,11 @@ import dev.typetype.server.services.SabrSessionStore import dev.typetype.server.services.SignedHlsManifestTokenService import dev.typetype.server.services.TypetypeTokenYoutubeSessionClient import dev.typetype.server.services.YouTubeSubtitleService +import dev.typetype.server.services.YouTubeSubtitleCache +import dev.typetype.server.services.YouTubeSubtitleDeliveryService +import dev.typetype.server.services.OkHttpYouTubeSubtitleContentFetcher +import dev.typetype.server.services.StreamYouTubeSubtitleResolver +import dev.typetype.server.services.TokenYouTubeSubtitleContentFetcher import dev.typetype.server.services.YoutubePlayerClient import dev.typetype.server.services.YoutubePlayerClientFallbackStreamService import dev.typetype.server.services.YoutubePlayerClientStreamService @@ -88,11 +93,11 @@ internal class ExtractionServiceRegistry( ) private val publicStreamService = YoutubePlayerClientStreamService( directPipePipeStreamService, - YoutubePlayerClient.WEB_SAFARI, + YoutubePlayerClient.VISIONOS, ) private val authenticatedStreamService = YoutubePlayerClientFallbackStreamService( directPipePipeStreamService, - listOf(YoutubePlayerClient.TV_DOWNGRADED, YoutubePlayerClient.WEB_SAFARI), + listOf(YoutubePlayerClient.TV_DOWNGRADED, YoutubePlayerClient.VISIONOS), ) private val sabrPublicStreamService = YoutubePlayerClientStreamService( sabrPipePipeStreamService, @@ -111,6 +116,15 @@ internal class ExtractionServiceRegistry( cache, "stream-youtube-sabr:v1", ) + val youtubeSubtitleDeliveryService = YouTubeSubtitleDeliveryService( + StreamYouTubeSubtitleResolver(youtubeSabrStreamService, youtubeSubtitleService::fetchSubtitleInventory), + TokenYouTubeSubtitleContentFetcher( + httpClient, + subtitleServiceUrl, + OkHttpYouTubeSubtitleContentFetcher(httpClient), + ), + YouTubeSubtitleCache(cache), + ) val youtubeSabrBootstrapStreamService = CachedStreamService( YoutubeScopedStreamService(SabrBootstrapStreamService(sabrSessionStore, tokenYoutubeSessionClient)), cache, diff --git a/src/main/kotlin/dev/typetype/server/Plugins.kt b/src/main/kotlin/dev/typetype/server/Plugins.kt index 8e575846..63c900d5 100644 --- a/src/main/kotlin/dev/typetype/server/Plugins.kt +++ b/src/main/kotlin/dev/typetype/server/Plugins.kt @@ -7,6 +7,7 @@ import io.ktor.http.HttpMethod import io.ktor.http.HttpStatusCode import io.ktor.serialization.kotlinx.json.json import io.ktor.server.application.Application +import io.ktor.server.application.ApplicationCall import io.ktor.server.application.install import io.ktor.server.plugins.calllogging.CallLogging import io.ktor.server.plugins.contentnegotiation.ContentNegotiation @@ -17,6 +18,7 @@ import io.ktor.server.plugins.statuspages.StatusPages import io.ktor.server.request.path import io.ktor.server.response.respond import io.ktor.server.websocket.WebSockets +import io.ktor.util.AttributeKey import kotlinx.serialization.json.Json import org.slf4j.LoggerFactory import kotlin.time.Duration.Companion.minutes @@ -30,6 +32,7 @@ private const val PROXY_STORYBOARD_RATE_LIMIT = 1200 private const val USER_DATA_RATE_LIMIT = 120 private const val MAX_WEBSOCKET_FRAME_BYTES = 64L * 1024L * 1024L private val RATE_LIMIT_WINDOW = 1.minutes +private val preserveTooManyRequestsBodyAttribute = AttributeKey("preserveTooManyRequestsBody") val EXTRACTION_ZONE = RateLimitName("extraction") val DEARROW_ZONE = RateLimitName("dearrow") @@ -40,7 +43,6 @@ val PROXY_STORYBOARD_ZONE = RateLimitName("proxy-storyboard") val USER_DATA_ZONE = RateLimitName("user-data") fun Application.configurePlugins(authService: AuthService) { - val log = LoggerFactory.getLogger("RequestLogger") installRequestObservability() install(CallLogging) { format(::requestLogLine) @@ -94,8 +96,18 @@ fun Application.configurePlugins(authService: AuthService) { requestKey { call -> userDataRateLimitKey(call, authService) } } } + configureStatusPages() +} + +internal fun ApplicationCall.preserveTooManyRequestsBody() { + attributes.put(preserveTooManyRequestsBodyAttribute, Unit) +} + +internal fun Application.configureStatusPages() { + val log = LoggerFactory.getLogger("RequestLogger") install(StatusPages) { status(HttpStatusCode.TooManyRequests) { call, status -> + if (call.attributes.contains(preserveTooManyRequestsBodyAttribute)) return@status if (!call.response.headers.contains(HttpHeaders.RetryAfter)) call.response.headers.append(HttpHeaders.RetryAfter, "60") call.respond(status, ErrorResponse("Too many requests", "rate_limited")) } diff --git a/src/main/kotlin/dev/typetype/server/ServiceRegistry.kt b/src/main/kotlin/dev/typetype/server/ServiceRegistry.kt index 0ec40261..b0cff887 100644 --- a/src/main/kotlin/dev/typetype/server/ServiceRegistry.kt +++ b/src/main/kotlin/dev/typetype/server/ServiceRegistry.kt @@ -69,6 +69,7 @@ internal class ServiceRegistry( val podcastService = extraction.podcastService val publicPlaylistService = extraction.publicPlaylistService val proxyService = extraction.proxyService + val youtubeSubtitleDeliveryService = extraction.youtubeSubtitleDeliveryService val nicoVideoProxyService = extraction.nicoVideoProxyService val manifestService = extraction.manifestService val nativeManifestService = extraction.nativeManifestService diff --git a/src/main/kotlin/dev/typetype/server/downloader/YoutubeProxySelector.kt b/src/main/kotlin/dev/typetype/server/downloader/YoutubeProxySelector.kt index 24bca222..d9016d4b 100644 --- a/src/main/kotlin/dev/typetype/server/downloader/YoutubeProxySelector.kt +++ b/src/main/kotlin/dev/typetype/server/downloader/YoutubeProxySelector.kt @@ -11,7 +11,7 @@ internal class YoutubeProxySelector private constructor( private val proxy: Proxy, ) : ProxySelector() { override fun select(uri: URI): List = - if (isYoutubeHost(uri.host)) listOf(proxy, Proxy.NO_PROXY) else DIRECT + if (isYoutubeHost(uri.host)) listOf(proxy) else DIRECT override fun connectFailed(uri: URI, socketAddress: SocketAddress, exception: IOException) = Unit diff --git a/src/main/kotlin/dev/typetype/server/models/SearchFilterGroup.kt b/src/main/kotlin/dev/typetype/server/models/SearchFilterGroup.kt new file mode 100644 index 00000000..c8b179c6 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/models/SearchFilterGroup.kt @@ -0,0 +1,11 @@ +package dev.typetype.server.models + +import kotlinx.serialization.Serializable + +@Serializable +data class SearchFilterGroup( + val key: String, + val label: String, + val multiSelect: Boolean, + val options: List, +) diff --git a/src/main/kotlin/dev/typetype/server/models/SearchFilterOption.kt b/src/main/kotlin/dev/typetype/server/models/SearchFilterOption.kt index 8c0b03e4..4ca1a82f 100644 --- a/src/main/kotlin/dev/typetype/server/models/SearchFilterOption.kt +++ b/src/main/kotlin/dev/typetype/server/models/SearchFilterOption.kt @@ -6,4 +6,5 @@ import kotlinx.serialization.Serializable data class SearchFilterOption( val value: String, val label: String, + val isDefault: Boolean = false, ) diff --git a/src/main/kotlin/dev/typetype/server/models/SearchFiltersResponse.kt b/src/main/kotlin/dev/typetype/server/models/SearchFiltersResponse.kt index cd96b2f9..ea492363 100644 --- a/src/main/kotlin/dev/typetype/server/models/SearchFiltersResponse.kt +++ b/src/main/kotlin/dev/typetype/server/models/SearchFiltersResponse.kt @@ -6,4 +6,5 @@ import kotlinx.serialization.Serializable data class SearchFiltersResponse( val contentFilters: List, val sortFilters: List, + val filterGroups: List, ) diff --git a/src/main/kotlin/dev/typetype/server/routes/AuthRoutes.kt b/src/main/kotlin/dev/typetype/server/routes/AuthRoutes.kt index 179ebd20..19bfda0f 100644 --- a/src/main/kotlin/dev/typetype/server/routes/AuthRoutes.kt +++ b/src/main/kotlin/dev/typetype/server/routes/AuthRoutes.kt @@ -5,6 +5,7 @@ import dev.typetype.server.models.UserProfileItem import dev.typetype.server.services.AdminSettingsService import dev.typetype.server.services.AuthCookieHelpers import dev.typetype.server.services.AuthService +import dev.typetype.server.services.AuthSessionConfig import dev.typetype.server.services.HomeRecommendationWarmup import dev.typetype.server.services.NoopHomeRecommendationWarmup import dev.typetype.server.services.PasswordResetService @@ -17,8 +18,15 @@ import io.ktor.server.routing.Route import io.ktor.server.routing.get import io.ktor.server.routing.post -fun Route.authRoutes(authService: AuthService, passwordResetService: PasswordResetService, profileService: ProfileService, adminSettingsService: AdminSettingsService, warmupService: HomeRecommendationWarmup = NoopHomeRecommendationWarmup) { - registerRoutes(authService, adminSettingsService, warmupService) +fun Route.authRoutes( + authService: AuthService, + passwordResetService: PasswordResetService, + profileService: ProfileService, + adminSettingsService: AdminSettingsService, + warmupService: HomeRecommendationWarmup = NoopHomeRecommendationWarmup, + sessionConfig: AuthSessionConfig = AuthSessionConfig(), +) { + registerRoutes(authService, adminSettingsService, warmupService, sessionConfig) post("/auth/login") { if (!adminSettingsService.get().localLoginEnabled) { @@ -32,7 +40,7 @@ fun Route.authRoutes(authService: AuthService, passwordResetService: PasswordRes call.respond(HttpStatusCode.Unauthorized, ErrorResponse("Invalid credentials")) return@post } - AuthCookieHelpers.setRefreshCookie(call.response, token.refreshToken) + AuthCookieHelpers.setRefreshCookie(call.response, token.refreshToken, sessionConfig) token.accessToken.warm(authService, warmupService) call.respond(SessionResponse(token.accessToken)) } @@ -44,14 +52,14 @@ fun Route.authRoutes(authService: AuthService, passwordResetService: PasswordRes call.respond(HttpStatusCode.Unauthorized, ErrorResponse("Invalid token")) return@post } - AuthCookieHelpers.setRefreshCookie(call.response, newToken.refreshToken) + AuthCookieHelpers.setRefreshCookie(call.response, newToken.refreshToken, sessionConfig) newToken.accessToken.warm(authService, warmupService) call.respond(SessionResponse(newToken.accessToken)) } post("/auth/logout") { val refreshToken = AuthCookieHelpers.extractRefreshToken(call) authService.logout(refreshToken) - AuthCookieHelpers.clearRefreshCookie(call.response) + AuthCookieHelpers.clearRefreshCookie(call.response, sessionConfig) call.respond(HttpStatusCode.NoContent) } diff --git a/src/main/kotlin/dev/typetype/server/routes/HomeRecommendationRoutes.kt b/src/main/kotlin/dev/typetype/server/routes/HomeRecommendationRoutes.kt index 4de28fc8..bc9f264e 100644 --- a/src/main/kotlin/dev/typetype/server/routes/HomeRecommendationRoutes.kt +++ b/src/main/kotlin/dev/typetype/server/routes/HomeRecommendationRoutes.kt @@ -3,6 +3,7 @@ package dev.typetype.server.routes import dev.typetype.server.models.ErrorResponse import dev.typetype.server.services.AccessControlService import dev.typetype.server.services.AuthService +import dev.typetype.server.services.BlockedService import dev.typetype.server.services.HomeRecommendationCursorCodec import dev.typetype.server.services.HomeRecommendationContext import dev.typetype.server.services.HomeRecommendationDeviceClass @@ -12,6 +13,7 @@ import dev.typetype.server.services.HomeRecommendationService import dev.typetype.server.services.VALID_SERVICE_IDS import dev.typetype.server.services.YOUTUBE_SERVICE_ID import dev.typetype.server.services.filterAllowed +import dev.typetype.server.services.filterBlocked import io.ktor.http.HttpStatusCode import io.ktor.server.response.respond import io.ktor.server.routing.Route @@ -22,6 +24,7 @@ private const val MAX_RECOMMENDATION_LIMIT = 60 fun Route.homeRecommendationRoutes( recommendationService: HomeRecommendationService, authService: AuthService, + blockedService: BlockedService, accessControlService: AccessControlService? = null, ) { get("/recommendations/home") { @@ -44,13 +47,14 @@ fun Route.homeRecommendationRoutes( ) val profile = accessControlService?.profileFor(userId, authService.getUserRole(userId)) ?: dev.typetype.server.services.AccessControlProfile.unrestricted + val blocked = blockedService.profileFor(userId) val response = recommendationService.getHome( userId = userId, serviceId = serviceId, limit = limit, cursor = cursor, context = HomeRecommendationContext(serviceId, sessionContext), - ).filterAllowed(profile) + ).filterAllowed(profile).filterBlocked(blocked) call.respond(response) } } diff --git a/src/main/kotlin/dev/typetype/server/routes/HomeRecommendationShortsRoutes.kt b/src/main/kotlin/dev/typetype/server/routes/HomeRecommendationShortsRoutes.kt index cdbdc796..a17b6524 100644 --- a/src/main/kotlin/dev/typetype/server/routes/HomeRecommendationShortsRoutes.kt +++ b/src/main/kotlin/dev/typetype/server/routes/HomeRecommendationShortsRoutes.kt @@ -3,6 +3,7 @@ package dev.typetype.server.routes import dev.typetype.server.models.ErrorResponse import dev.typetype.server.services.AccessControlService import dev.typetype.server.services.AuthService +import dev.typetype.server.services.BlockedService import dev.typetype.server.services.HomeRecommendationContext import dev.typetype.server.services.HomeRecommendationCursorCodec import dev.typetype.server.services.HomeRecommendationDeviceClass @@ -12,6 +13,7 @@ import dev.typetype.server.services.HomeRecommendationService import dev.typetype.server.services.VALID_SERVICE_IDS import dev.typetype.server.services.YOUTUBE_SERVICE_ID import dev.typetype.server.services.filterAllowed +import dev.typetype.server.services.filterBlocked import io.ktor.http.HttpStatusCode import io.ktor.server.response.respond import io.ktor.server.routing.Route @@ -22,6 +24,7 @@ private const val MAX_RECOMMENDATION_SHORTS_LIMIT = 60 fun Route.homeRecommendationShortsRoutes( recommendationService: HomeRecommendationService, authService: AuthService, + blockedService: BlockedService, accessControlService: AccessControlService? = null, ) { get("/recommendations/shorts") { @@ -45,6 +48,7 @@ fun Route.homeRecommendationShortsRoutes( ) val profile = accessControlService?.profileFor(userId, authService.getUserRole(userId)) ?: dev.typetype.server.services.AccessControlProfile.unrestricted + val blocked = blockedService.profileFor(userId) val response = recommendationService.getShorts( userId = userId, serviceId = serviceId, @@ -52,7 +56,7 @@ fun Route.homeRecommendationShortsRoutes( cursor = cursor, context = HomeRecommendationContext(serviceId, sessionContext), debug = debug, - ).filterAllowed(profile) + ).filterAllowed(profile).filterBlocked(blocked) call.respond(response) } } diff --git a/src/main/kotlin/dev/typetype/server/routes/OidcAuthRoutes.kt b/src/main/kotlin/dev/typetype/server/routes/OidcAuthRoutes.kt index ddc413e9..0421ca69 100644 --- a/src/main/kotlin/dev/typetype/server/routes/OidcAuthRoutes.kt +++ b/src/main/kotlin/dev/typetype/server/routes/OidcAuthRoutes.kt @@ -8,6 +8,7 @@ import dev.typetype.server.models.OidcStartResponse import dev.typetype.server.models.OidcStatusResponse import dev.typetype.server.services.AdminSettingsService import dev.typetype.server.services.AuthCookieHelpers +import dev.typetype.server.services.AuthSessionConfig import dev.typetype.server.services.OidcAuthService import dev.typetype.server.services.OidcCallbackSession import io.ktor.http.HttpStatusCode @@ -19,7 +20,11 @@ import io.ktor.server.routing.Route import io.ktor.server.routing.get import io.ktor.server.routing.post -fun Route.oidcAuthRoutes(oidcAuthService: OidcAuthService, adminSettingsService: AdminSettingsService) { +fun Route.oidcAuthRoutes( + oidcAuthService: OidcAuthService, + adminSettingsService: AdminSettingsService, + sessionConfig: AuthSessionConfig = AuthSessionConfig(), +) { get("/auth/oidc/status") { val settings = adminSettingsService.get() val config = oidcAuthService.publicConfig() @@ -41,7 +46,7 @@ fun Route.oidcAuthRoutes(oidcAuthService: OidcAuthService, adminSettingsService: val request = runCatching { call.receive() }.getOrElse { return@post call.respond(HttpStatusCode.BadRequest, ErrorResponse("Invalid request body")) } - call.respondOidcCallbackResult(oidcAuthService.callback(request)) + call.respondOidcCallbackResult(oidcAuthService.callback(request), sessionConfig) } } @@ -53,10 +58,13 @@ private suspend fun ApplicationCall.respondOidcStartResult(result: ExtractionRes } } -private suspend fun ApplicationCall.respondOidcCallbackResult(result: ExtractionResult) { +private suspend fun ApplicationCall.respondOidcCallbackResult( + result: ExtractionResult, + sessionConfig: AuthSessionConfig, +) { when (result) { is ExtractionResult.Success -> { - AuthCookieHelpers.setRefreshCookie(response, result.data.refreshToken) + AuthCookieHelpers.setRefreshCookie(response, result.data.refreshToken, sessionConfig) respond(OidcCallbackResponse(accessToken = result.data.accessToken, returnTo = result.data.returnTo)) } is ExtractionResult.BadRequest -> respond(HttpStatusCode.BadRequest, ErrorResponse(result.message)) diff --git a/src/main/kotlin/dev/typetype/server/routes/ProxyRoutes.kt b/src/main/kotlin/dev/typetype/server/routes/ProxyRoutes.kt index ad61981e..e5b2bf2e 100644 --- a/src/main/kotlin/dev/typetype/server/routes/ProxyRoutes.kt +++ b/src/main/kotlin/dev/typetype/server/routes/ProxyRoutes.kt @@ -2,16 +2,29 @@ package dev.typetype.server.routes import dev.typetype.server.models.ErrorResponse import dev.typetype.server.services.ProxyService +import dev.typetype.server.services.YouTubeSubtitleContentResult +import dev.typetype.server.services.YouTubeSubtitleDeliveryService +import dev.typetype.server.services.isYouTubeTimedTextUrl +import dev.typetype.server.services.subtitleSelectionFromTimedTextUrl import io.ktor.http.HttpStatusCode import io.ktor.server.response.respond import io.ktor.server.routing.Route import io.ktor.server.routing.get -fun Route.proxyRoutes(proxyService: ProxyService) { +internal fun Route.proxyRoutes( + proxyService: ProxyService, + youtubeSubtitleService: YouTubeSubtitleDeliveryService? = null, +) { get("/proxy") { val url = call.request.queryParameters["url"] ?: return@get call.respond(HttpStatusCode.BadRequest, ErrorResponse("Missing 'url' parameter")) + if (youtubeSubtitleService != null && isYouTubeTimedTextUrl(url)) { + val selection = subtitleSelectionFromTimedTextUrl(url) + ?: return@get call.respondYouTubeSubtitle(YouTubeSubtitleContentResult.InvalidRequest) + return@get call.respondYouTubeSubtitle(youtubeSubtitleService.fetch(selection)) + } + val rangeHeader = call.request.headers["Range"] val domandBid = call.request.queryParameters["domand_bid"] diff --git a/src/main/kotlin/dev/typetype/server/routes/RegisterRoutes.kt b/src/main/kotlin/dev/typetype/server/routes/RegisterRoutes.kt index 34e03c5e..ac9f1f4a 100644 --- a/src/main/kotlin/dev/typetype/server/routes/RegisterRoutes.kt +++ b/src/main/kotlin/dev/typetype/server/routes/RegisterRoutes.kt @@ -4,6 +4,7 @@ import dev.typetype.server.models.ErrorResponse import dev.typetype.server.services.AdminSettingsService import dev.typetype.server.services.AuthCookieHelpers import dev.typetype.server.services.AuthService +import dev.typetype.server.services.AuthSessionConfig import dev.typetype.server.services.HomeRecommendationWarmup import io.ktor.http.HttpStatusCode import io.ktor.server.application.call @@ -17,6 +18,7 @@ fun Route.registerRoutes( authService: AuthService, adminSettingsService: AdminSettingsService, warmupService: HomeRecommendationWarmup, + sessionConfig: AuthSessionConfig, ): Unit { get("/auth/register/status") { val bootstrapAvailable = !authService.hasAdmin() @@ -50,7 +52,7 @@ fun Route.registerRoutes( try { val token = authService.register(req.email, req.password, req.name) authService.verify(token.accessToken)?.let(warmupService::markActive) - AuthCookieHelpers.setRefreshCookie(call.response, token.refreshToken) + AuthCookieHelpers.setRefreshCookie(call.response, token.refreshToken, sessionConfig) call.respond(SessionResponse(token.accessToken)) } catch (e: Exception) { call.respond(HttpStatusCode.BadRequest, ErrorResponse("Registration failed")) diff --git a/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackRecovery.kt b/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackRecovery.kt index 8f7f312f..1880d371 100644 --- a/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackRecovery.kt +++ b/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackRecovery.kt @@ -17,7 +17,10 @@ internal class SabrPlaybackRecovery(private val sessionStore: SabrSessionStore) sessionStore.invalidatePlaybackInfo(holder.key.videoId) return RETRY_FRESH_SESSION } - if (failure.contains("protected no-media")) { + if ( + failure.contains("protected no-media") || + failure.contains("attestation required", ignoreCase = true) + ) { sessionStore.recoverProtectedPlaybackInfo(holder) return RETRY_FRESH_SESSION } diff --git a/src/main/kotlin/dev/typetype/server/routes/SearchRoutes.kt b/src/main/kotlin/dev/typetype/server/routes/SearchRoutes.kt index 5cc053dd..e5313e24 100644 --- a/src/main/kotlin/dev/typetype/server/routes/SearchRoutes.kt +++ b/src/main/kotlin/dev/typetype/server/routes/SearchRoutes.kt @@ -29,7 +29,8 @@ fun Route.searchRoutes( ?: return@get call.respond(HttpStatusCode.BadRequest, ErrorResponse("Missing or invalid 'service' parameter")) if (serviceId !in VALID_SERVICE_IDS) return@get call.respond(HttpStatusCode.BadRequest, ErrorResponse("Invalid 'service' parameter")) - when (val result = searchService.filters(serviceId = serviceId)) { + val contentFilter = call.request.queryParameters["contentFilter"] + when (val result = searchService.filters(serviceId = serviceId, contentFilter = contentFilter)) { is ExtractionResult.Success -> call.respond(result.data) is ExtractionResult.BadRequest -> call.respond(HttpStatusCode.BadRequest, ErrorResponse(result.message)) is ExtractionResult.Failure -> call.respond(HttpStatusCode.UnprocessableEntity, ErrorResponse(result.message)) @@ -44,11 +45,14 @@ fun Route.searchRoutes( return@get call.respond(HttpStatusCode.BadRequest, ErrorResponse("Invalid 'service' parameter")) val nextpage = call.request.queryParameters["nextpage"] val contentFilter = call.request.queryParameters["contentFilter"] - val sortFilter = call.request.queryParameters["sortFilter"] + val filters = buildList { + addAll(call.request.queryParameters.getAll("filter").orEmpty().filter(String::isNotBlank)) + call.request.queryParameters["sortFilter"]?.takeIf(String::isNotBlank)?.let(::add) + }.distinct() val access = call.accessProfileOrRespond(authService, accessControlService, adminSettingsService) ?: return@get val blocked = access.userId?.let { blockedService?.profileFor(it) } ?: BlockedContentProfile.empty - when (val result = searchService.search(query = query, serviceId = serviceId, nextpage = nextpage, contentFilter = contentFilter, sortFilter = sortFilter)) { + when (val result = searchService.search(query, serviceId, nextpage, contentFilter, filters)) { is ExtractionResult.Success -> call.respond(result.data.filterAllowed(access.profile).filterBlocked(blocked)) is ExtractionResult.BadRequest -> call.respond(HttpStatusCode.BadRequest, ErrorResponse(result.message)) is ExtractionResult.Failure -> call.respond(HttpStatusCode.UnprocessableEntity, ErrorResponse(result.message)) diff --git a/src/main/kotlin/dev/typetype/server/routes/StreamRouteDependencies.kt b/src/main/kotlin/dev/typetype/server/routes/StreamRouteDependencies.kt index 04bcb785..6598572a 100644 --- a/src/main/kotlin/dev/typetype/server/routes/StreamRouteDependencies.kt +++ b/src/main/kotlin/dev/typetype/server/routes/StreamRouteDependencies.kt @@ -4,12 +4,14 @@ import dev.typetype.server.models.StreamResponse import dev.typetype.server.services.AccessControlService import dev.typetype.server.services.AdminSettingsService import dev.typetype.server.services.AuthService +import dev.typetype.server.services.BlockedService import dev.typetype.server.services.PublicHlsManifestTokenService internal data class StreamRouteDependencies( val authService: AuthService?, val accessControlService: AccessControlService?, val adminSettingsService: AdminSettingsService?, + val blockedService: BlockedService?, val publicHlsManifestTokenService: PublicHlsManifestTokenService?, val sabrStreamContractFilter: (suspend (String, StreamResponse) -> StreamResponse)?, ) diff --git a/src/main/kotlin/dev/typetype/server/routes/StreamRoutes.kt b/src/main/kotlin/dev/typetype/server/routes/StreamRoutes.kt index 872aa9f2..81f60626 100644 --- a/src/main/kotlin/dev/typetype/server/routes/StreamRoutes.kt +++ b/src/main/kotlin/dev/typetype/server/routes/StreamRoutes.kt @@ -6,8 +6,11 @@ import dev.typetype.server.models.StreamResponse import dev.typetype.server.services.AccessControlService import dev.typetype.server.services.AdminSettingsService import dev.typetype.server.services.AuthService +import dev.typetype.server.services.BlockedContentProfile +import dev.typetype.server.services.BlockedService import dev.typetype.server.services.PublicHlsManifestTokenService import dev.typetype.server.services.StreamService +import dev.typetype.server.services.filterBlocked import dev.typetype.server.services.filterAllowed import dev.typetype.server.services.withSabrManifestUrls import io.ktor.http.HttpHeaders @@ -24,6 +27,7 @@ fun Route.streamRoutes( authService: AuthService? = null, accessControlService: AccessControlService? = null, adminSettingsService: AdminSettingsService? = null, + blockedService: BlockedService? = null, publicHlsManifestTokenService: PublicHlsManifestTokenService? = null, nicoNicoStreamService: StreamService = streamService, bilibiliStreamService: StreamService = streamService, @@ -34,6 +38,7 @@ fun Route.streamRoutes( authService = authService, accessControlService = accessControlService, adminSettingsService = adminSettingsService, + blockedService = blockedService, publicHlsManifestTokenService = publicHlsManifestTokenService, sabrStreamContractFilter = sabrStreamContractFilter, ) @@ -69,11 +74,26 @@ private fun Route.streamRoute( dependencies.adminSettingsService, ) ?: return@get val accessProfile = access.profile + val blockedProfile = access.userId + ?.let { dependencies.blockedService?.profileFor(it) } + ?: BlockedContentProfile.empty + if (blockedProfile.blocksVideo(url)) { + return@get call.respond( + HttpStatusCode.Forbidden, + ErrorResponse("Video is blocked", "content_blocked"), + ) + } when (val result = streamService.getStreamInfo(url)) { is ExtractionResult.Success -> { if (!accessProfile.allowsUploader(result.data.uploaderUrl, result.data.uploaderName)) { return@get call.respond(HttpStatusCode.Forbidden, ErrorResponse("Channel is not allowed")) } + if (!blockedProfile.allowsRequestedVideo(url, result.data.uploaderUrl, result.data.uploaderName)) { + return@get call.respond( + HttpStatusCode.Forbidden, + ErrorResponse("Channel is blocked", "content_blocked"), + ) + } val selected = if (deliveryMode.isSabr()) { result.data.withSabrManifestUrls().onlySabrStreams() } else { @@ -81,6 +101,7 @@ private fun Route.streamRoute( } val filtered = selected .filterAllowed(accessProfile) + .filterBlocked(blockedProfile) .withSignedPublicHlsUrl( deliveryMode.isSabr() && selected.isLive || access.userId != null && !access.allowGuest, dependencies.publicHlsManifestTokenService, @@ -98,7 +119,7 @@ private fun Route.streamRoute( } call.response.headers.append( HttpHeaders.CacheControl, - if (accessProfile.enabled) AUTHENTICATED_STREAMS_CACHE_CONTROL else STREAMS_CACHE_CONTROL, + if (access.userId != null) AUTHENTICATED_STREAMS_CACHE_CONTROL else STREAMS_CACHE_CONTROL, ) call.respond(data) } diff --git a/src/main/kotlin/dev/typetype/server/routes/UserDataRoutes.kt b/src/main/kotlin/dev/typetype/server/routes/UserDataRoutes.kt index 0c169e0c..eab6f335 100644 --- a/src/main/kotlin/dev/typetype/server/routes/UserDataRoutes.kt +++ b/src/main/kotlin/dev/typetype/server/routes/UserDataRoutes.kt @@ -38,6 +38,6 @@ internal fun Route.userDataRoutes( bugReportRoutes(bugReportService, authService) restoreRoutes(restoreService, authService) typeTypeBackupRoutes(svc.typeTypeBackupService, authService) - homeRecommendationRoutes(svc.homeRecommendationService, authService, svc.accessControlService) - homeRecommendationShortsRoutes(svc.homeRecommendationService, authService, svc.accessControlService) + homeRecommendationRoutes(svc.homeRecommendationService, authService, svc.blockedService, svc.accessControlService) + homeRecommendationShortsRoutes(svc.homeRecommendationService, authService, svc.blockedService, svc.accessControlService) } diff --git a/src/main/kotlin/dev/typetype/server/routes/YouTubeSubtitleResponse.kt b/src/main/kotlin/dev/typetype/server/routes/YouTubeSubtitleResponse.kt new file mode 100644 index 00000000..ddd76ea1 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/routes/YouTubeSubtitleResponse.kt @@ -0,0 +1,72 @@ +package dev.typetype.server.routes + +import dev.typetype.server.models.ErrorResponse +import dev.typetype.server.preserveTooManyRequestsBody +import dev.typetype.server.services.YouTubeSubtitleContentResult +import io.ktor.http.ContentType +import io.ktor.http.HttpHeaders +import io.ktor.http.HttpStatusCode +import io.ktor.server.application.ApplicationCall +import io.ktor.server.response.respond +import io.ktor.server.response.respondBytes + +internal suspend fun ApplicationCall.respondYouTubeSubtitle(result: YouTubeSubtitleContentResult) { + when (result) { + is YouTubeSubtitleContentResult.Ready -> { + response.headers.append( + HttpHeaders.CacheControl, + if (result.isLive) LIVE_CACHE_CONTROL else VOD_CACHE_CONTROL, + safeOnly = false, + ) + respondBytes(result.content, ContentType.parse(result.format.contentType)) + } + YouTubeSubtitleContentResult.InvalidRequest -> respondSubtitleError( + HttpStatusCode.BadRequest, + "Invalid YouTube subtitle URL", + "subtitle_request_invalid", + ) + YouTubeSubtitleContentResult.NotFound -> respondSubtitleError( + HttpStatusCode.NotFound, + "Subtitle track not found", + "subtitle_track_not_found", + ) + YouTubeSubtitleContentResult.Throttled -> respondSubtitleError( + HttpStatusCode.TooManyRequests, + "YouTube temporarily throttled subtitle retrieval", + "subtitle_upstream_throttled", + ) + YouTubeSubtitleContentResult.Expired -> respondSubtitleError( + HttpStatusCode.BadGateway, + "YouTube subtitle URL expired after refresh", + "subtitle_url_expired", + ) + YouTubeSubtitleContentResult.InvalidPayload -> respondSubtitleError( + HttpStatusCode.BadGateway, + "YouTube returned invalid subtitle content", + "subtitle_payload_invalid", + ) + YouTubeSubtitleContentResult.Unavailable -> respondSubtitleError( + HttpStatusCode.BadGateway, + "YouTube subtitle retrieval failed", + "subtitle_upstream_unavailable", + ) + } +} + +internal suspend fun ApplicationCall.respondYouTubeSubtitleInvalidRequest() = respondSubtitleError( + HttpStatusCode.BadRequest, + "Invalid YouTube subtitle request", + "subtitle_request_invalid", +) + +private suspend fun ApplicationCall.respondSubtitleError( + status: HttpStatusCode, + message: String, + code: String, +) { + if (status == HttpStatusCode.TooManyRequests) preserveTooManyRequestsBody() + respond(status, ErrorResponse(message, code)) +} + +private const val VOD_CACHE_CONTROL = "public, max-age=21600, stale-while-revalidate=3600" +private const val LIVE_CACHE_CONTROL = "no-store" diff --git a/src/main/kotlin/dev/typetype/server/routes/YouTubeSubtitleRoutes.kt b/src/main/kotlin/dev/typetype/server/routes/YouTubeSubtitleRoutes.kt new file mode 100644 index 00000000..f568222d --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/routes/YouTubeSubtitleRoutes.kt @@ -0,0 +1,44 @@ +package dev.typetype.server.routes + +import dev.typetype.server.services.YouTubeSubtitleDeliveryService +import dev.typetype.server.services.YouTubeSubtitleFormat +import dev.typetype.server.services.YouTubeSubtitleSelection +import dev.typetype.server.services.YouTubeSubtitleVariant +import dev.typetype.server.services.isValidSubtitleTag +import dev.typetype.server.services.isValidYouTubeVideoId +import io.ktor.server.routing.Route +import io.ktor.server.routing.get + +internal fun Route.youtubeSubtitleRoutes(service: YouTubeSubtitleDeliveryService) { + get("/subtitles/youtube/{videoId}") { + val selection = call.subtitleSelection() + ?: return@get call.respondYouTubeSubtitleInvalidRequest() + call.respondYouTubeSubtitle(service.fetch(selection)) + } +} + +private fun io.ktor.server.application.ApplicationCall.subtitleSelection(): YouTubeSubtitleSelection? { + val videoId = parameters["videoId"]?.trim().orEmpty() + val language = request.queryParameters["language"]?.trim().orEmpty() + val variant = YouTubeSubtitleVariant.from(request.queryParameters["variant"]) + val format = YouTubeSubtitleFormat.from(request.queryParameters["format"] ?: "vtt") + val sourceLanguage = request.queryParameters["sourceLanguage"]?.trim()?.takeIf(String::isNotEmpty) + val translation = request.queryParameters["translation"]?.trim()?.takeIf(String::isNotEmpty) + val trackName = request.queryParameters["name"]?.trim()?.takeIf(String::isNotEmpty) + if (!isValidYouTubeVideoId(videoId) || !isValidSubtitleTag(language)) return null + if (variant == null || format == null) return null + if (sourceLanguage != null && !isValidSubtitleTag(sourceLanguage)) return null + if (translation != null && !isValidSubtitleTag(translation)) return null + if (trackName != null && trackName.length > MAX_TRACK_NAME_LENGTH) return null + return YouTubeSubtitleSelection( + videoId = videoId, + language = language, + variant = variant, + format = format, + sourceLanguage = sourceLanguage, + translationLanguage = translation, + trackName = trackName, + ) +} + +private const val MAX_TRACK_NAME_LENGTH = 128 diff --git a/src/main/kotlin/dev/typetype/server/services/AllowedChannelsService.kt b/src/main/kotlin/dev/typetype/server/services/AllowedChannelsService.kt index 31bbfc4d..fba4aea1 100644 --- a/src/main/kotlin/dev/typetype/server/services/AllowedChannelsService.kt +++ b/src/main/kotlin/dev/typetype/server/services/AllowedChannelsService.kt @@ -86,3 +86,7 @@ internal fun normalizeChannelKey(value: String): String = value.trim() .substringBefore('?') .removeSuffix("/") .replace("http://", "https://") + .replace( + Regex("^https://(?:www\\.|m\\.|music\\.)youtube\\.com", RegexOption.IGNORE_CASE), + "https://youtube.com", + ) diff --git a/src/main/kotlin/dev/typetype/server/services/AuthCookieHelpers.kt b/src/main/kotlin/dev/typetype/server/services/AuthCookieHelpers.kt index 708329f2..40fbc3dd 100644 --- a/src/main/kotlin/dev/typetype/server/services/AuthCookieHelpers.kt +++ b/src/main/kotlin/dev/typetype/server/services/AuthCookieHelpers.kt @@ -12,37 +12,37 @@ object AuthCookieHelpers { fun extractRefreshToken(call: ApplicationCall): String? = call.request.cookies[REFRESH_COOKIE_NAME] - fun setRefreshCookie(response: ApplicationResponse, token: String) { + fun setRefreshCookie(response: ApplicationResponse, token: String, config: AuthSessionConfig) { response.cookies.append( Cookie( name = REFRESH_COOKIE_NAME, value = token, httpOnly = true, - secure = true, + secure = !config.allowInsecureCookies, path = REFRESH_COOKIE_PATH, - maxAge = REFRESH_TTL_SECONDS.toInt(), + maxAge = config.refreshTtlSeconds.toInt(), encoding = CookieEncoding.RAW, - extensions = mapOf("SameSite" to "None"), + extensions = mapOf("SameSite" to config.sameSite), ) ) response.headers.append(HttpHeaders.AccessControlAllowCredentials, "true") } - fun clearRefreshCookie(response: ApplicationResponse) { + fun clearRefreshCookie(response: ApplicationResponse, config: AuthSessionConfig) { response.cookies.append( Cookie( name = REFRESH_COOKIE_NAME, value = "", httpOnly = true, - secure = true, + secure = !config.allowInsecureCookies, path = REFRESH_COOKIE_PATH, maxAge = 0, encoding = CookieEncoding.RAW, - extensions = mapOf("SameSite" to "None"), + extensions = mapOf("SameSite" to config.sameSite), ) ) response.headers.append(HttpHeaders.AccessControlAllowCredentials, "true") } - - const val REFRESH_TTL_SECONDS = 30L * 24L * 60L * 60L + private val AuthSessionConfig.sameSite: String + get() = if (allowInsecureCookies) "Lax" else "None" } diff --git a/src/main/kotlin/dev/typetype/server/services/AuthService.kt b/src/main/kotlin/dev/typetype/server/services/AuthService.kt index 4ed320f3..21a5eb59 100644 --- a/src/main/kotlin/dev/typetype/server/services/AuthService.kt +++ b/src/main/kotlin/dev/typetype/server/services/AuthService.kt @@ -12,10 +12,14 @@ import org.jetbrains.exposed.v1.jdbc.transactions.transaction import java.util.UUID import java.util.Date -open class AuthService(private val jwtSecret: String, private val hasUsersProbe: (() -> Boolean)? = null) { +open class AuthService( + private val jwtSecret: String, + private val hasUsersProbe: (() -> Boolean)? = null, + sessionConfig: AuthSessionConfig = AuthSessionConfig(), +) { private val accessCodec = AuthAccessTokenCodec(jwtSecret) private val sessionStore = AuthSessionStore() - private val tokenIssuer = AuthTokenIssuer(accessCodec, sessionStore) + private val tokenIssuer = AuthTokenIssuer(accessCodec, sessionStore, sessionConfig) private val sessionRefresher = AuthSessionRefresher(sessionStore, tokenIssuer) private val sessionVerifier = AuthSessionVerifier(accessCodec, sessionStore) private val sessionRevoker = AuthSessionRevoker(sessionStore) diff --git a/src/main/kotlin/dev/typetype/server/services/AuthSessionConfig.kt b/src/main/kotlin/dev/typetype/server/services/AuthSessionConfig.kt new file mode 100644 index 00000000..b51322b7 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/services/AuthSessionConfig.kt @@ -0,0 +1,32 @@ +package dev.typetype.server.services + +data class AuthSessionConfig( + val refreshTtlDays: Long = DEFAULT_REFRESH_TTL_DAYS, + val allowInsecureCookies: Boolean = false, +) { + val refreshTtlMs: Long = refreshTtlDays * MILLIS_PER_DAY + val refreshTtlSeconds: Long = refreshTtlDays * SECONDS_PER_DAY + + companion object { + const val DEFAULT_REFRESH_TTL_DAYS = 30L + const val MIN_REFRESH_TTL_DAYS = 1L + const val MAX_REFRESH_TTL_DAYS = 365L + private const val MILLIS_PER_DAY = 24L * 60L * 60L * 1000L + private const val SECONDS_PER_DAY = 24L * 60L * 60L + + fun fromEnvironment(read: (String) -> String? = System::getenv): AuthSessionConfig = + AuthSessionConfig( + refreshTtlDays = read("AUTH_SESSION_TTL_DAYS") + ?.trim() + ?.toLongOrNull() + ?.coerceIn(MIN_REFRESH_TTL_DAYS, MAX_REFRESH_TTL_DAYS) + ?: DEFAULT_REFRESH_TTL_DAYS, + allowInsecureCookies = read("AUTH_ALLOW_INSECURE_COOKIES").isEnabled(), + ) + + private fun String?.isEnabled(): Boolean = when (this?.trim()?.lowercase()) { + "1", "true", "yes", "on" -> true + else -> false + } + } +} diff --git a/src/main/kotlin/dev/typetype/server/services/AuthTokenIssuer.kt b/src/main/kotlin/dev/typetype/server/services/AuthTokenIssuer.kt index 1c9ce5e1..82eee7c3 100644 --- a/src/main/kotlin/dev/typetype/server/services/AuthTokenIssuer.kt +++ b/src/main/kotlin/dev/typetype/server/services/AuthTokenIssuer.kt @@ -5,13 +5,14 @@ import java.util.UUID class AuthTokenIssuer( private val accessCodec: AuthAccessTokenCodec, private val sessionStore: AuthSessionStore, + private val sessionConfig: AuthSessionConfig, ) { fun issue(userId: String, sessionId: String = UUID.randomUUID().toString()): AuthSessionTokens? { val now = System.currentTimeMillis() val refreshToken = UUID.randomUUID().toString() + UUID.randomUUID().toString() val refreshHash = AuthRefreshTokenHasher.hash(refreshToken) val accessToken = accessCodec.issue(userId = userId, sessionId = sessionId) - val expiresAt = now + REFRESH_TTL_MS + val expiresAt = now + sessionConfig.refreshTtlMs val ok = sessionStore.upsert( sessionId = sessionId, userId = userId, @@ -22,8 +23,4 @@ class AuthTokenIssuer( ) return if (ok) AuthSessionTokens(accessToken = accessToken, refreshToken = refreshToken) else null } - - companion object { - private const val REFRESH_TTL_MS = 30L * 24L * 60L * 60L * 1000L - } } diff --git a/src/main/kotlin/dev/typetype/server/services/BlockedContentFilters.kt b/src/main/kotlin/dev/typetype/server/services/BlockedContentFilters.kt index fcba974b..0048911d 100644 --- a/src/main/kotlin/dev/typetype/server/services/BlockedContentFilters.kt +++ b/src/main/kotlin/dev/typetype/server/services/BlockedContentFilters.kt @@ -1,9 +1,23 @@ package dev.typetype.server.services +import dev.typetype.server.models.HomeRecommendationsResponse import dev.typetype.server.models.SearchPageResponse +import dev.typetype.server.models.StreamResponse + +internal fun HomeRecommendationsResponse.filterBlocked(profile: BlockedContentProfile): HomeRecommendationsResponse = copy( + items = items.filter { + profile.allowsVideo(it.url, it.title, it.uploaderUrl, it.uploaderName) + }, +) internal fun SearchPageResponse.filterBlocked(profile: BlockedContentProfile): SearchPageResponse = copy( items = items.filter { profile.allowsVideo(it.url, it.title, it.uploaderUrl, it.uploaderName) }, channels = channels.filter { profile.allowsChannel(url = it.url, name = it.name) }, playlists = playlists.filter { profile.allowsChannel(url = "", name = it.uploaderName) }, ) + +internal fun StreamResponse.filterBlocked(profile: BlockedContentProfile): StreamResponse = copy( + relatedStreams = relatedStreams.filter { + profile.allowsVideo(it.url, it.title, it.uploaderUrl, it.uploaderName) + }, +) diff --git a/src/main/kotlin/dev/typetype/server/services/BlockedContentProfile.kt b/src/main/kotlin/dev/typetype/server/services/BlockedContentProfile.kt index 07581898..9595097f 100644 --- a/src/main/kotlin/dev/typetype/server/services/BlockedContentProfile.kt +++ b/src/main/kotlin/dev/typetype/server/services/BlockedContentProfile.kt @@ -9,20 +9,37 @@ data class BlockedContentProfile( val keywords: List, ) { fun allowsVideo(url: String, title: String, uploaderUrl: String, uploaderName: String): Boolean = - videos.none { normalizeUrl(it.url) == normalizeUrl(url) } && + !blocksVideo(url) && keywords.none { containsBlockedKeyword(title, it.keyword) } && - allowsChannel(uploaderUrl, uploaderName) + !blocksChannel(uploaderUrl, uploaderName) - fun allowsChannel(url: String, name: String): Boolean = channels.none { item -> + fun allowsRequestedVideo(url: String, uploaderUrl: String, uploaderName: String): Boolean = + !blocksVideo(url) && !blocksChannel(uploaderUrl, uploaderName) + + fun blocksVideo(url: String): Boolean { + val normalized = normalizeBlockedVideoKey(url) + return normalized.isNotBlank() && videos.any { normalizeBlockedVideoKey(it.url) == normalized } + } + + fun blocksChannel(url: String, name: String): Boolean = channels.any { item -> val blockedUrl = normalizeChannelKey(item.url) - val blockedName = item.name?.trim().orEmpty() + val blockedName = normalizeBlockedKeyword(item.name.orEmpty()) blockedUrl.isNotBlank() && blockedUrl == normalizeChannelKey(url) || - blockedName.isNotBlank() && blockedName.equals(name.trim(), ignoreCase = true) + blockedName.isNotBlank() && blockedName == normalizeBlockedKeyword(name) } + fun allowsChannel(url: String, name: String): Boolean = !blocksChannel(url, name) + companion object { val empty = BlockedContentProfile(videos = emptyList(), channels = emptyList(), keywords = emptyList()) } } -private fun normalizeUrl(value: String): String = value.trim().trimEnd('/') +private val YOUTUBE_VIDEO_ID = + Regex("(?:[?&]v=|/(?:shorts|embed|live)/|youtu\\.be/)([A-Za-z0-9_-]{6,})", RegexOption.IGNORE_CASE) + +internal fun normalizeBlockedVideoKey(value: String): String { + val trimmed = value.trim() + val youtubeId = YOUTUBE_VIDEO_ID.find(trimmed)?.groupValues?.get(1) + return youtubeId?.let { "youtube:video:$it" } ?: normalizeChannelKey(trimmed) +} diff --git a/src/main/kotlin/dev/typetype/server/services/CachedSearchService.kt b/src/main/kotlin/dev/typetype/server/services/CachedSearchService.kt index 26249ae8..82cf5ee4 100644 --- a/src/main/kotlin/dev/typetype/server/services/CachedSearchService.kt +++ b/src/main/kotlin/dev/typetype/server/services/CachedSearchService.kt @@ -15,14 +15,24 @@ class CachedSearchService( serviceId: Int, nextpage: String?, contentFilter: String?, - sortFilter: String?, + filters: List, ): ExtractionResult = PublicExtractionCache.getOrLoad( cache = cache, area = "search", - key = PublicCacheKey.of("search-v2", serviceId.toString(), query, nextpage, contentFilter, sortFilter), + key = PublicCacheKey.of( + "search-v3", + serviceId.toString(), + query, + nextpage, + contentFilter, + *filters.sorted().toTypedArray(), + ), serializer = SearchPageResponse.serializer(), ttlSeconds = { PublicCachePolicy.searchTtl(serviceId, nextpage) }, - ) { delegate.search(query, serviceId, nextpage, contentFilter, sortFilter) } + ) { delegate.search(query, serviceId, nextpage, contentFilter, filters) } - override suspend fun filters(serviceId: Int): ExtractionResult = delegate.filters(serviceId) + override suspend fun filters( + serviceId: Int, + contentFilter: String?, + ): ExtractionResult = delegate.filters(serviceId, contentFilter) } diff --git a/src/main/kotlin/dev/typetype/server/services/PipePipeSearchService.kt b/src/main/kotlin/dev/typetype/server/services/PipePipeSearchService.kt index a78eee6d..a94cfb61 100644 --- a/src/main/kotlin/dev/typetype/server/services/PipePipeSearchService.kt +++ b/src/main/kotlin/dev/typetype/server/services/PipePipeSearchService.kt @@ -7,7 +7,10 @@ import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.withContext import kotlinx.coroutines.withTimeout import org.schabi.newpipe.extractor.NewPipe +import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandlerFactory import org.schabi.newpipe.extractor.search.SearchInfo +import org.schabi.newpipe.extractor.search.filter.Filter +import org.schabi.newpipe.extractor.search.filter.FilterItem class PipePipeSearchService : SearchService { @@ -16,35 +19,46 @@ class PipePipeSearchService : SearchService { serviceId: Int, nextpage: String?, contentFilter: String?, - sortFilter: String?, + filters: List, ): ExtractionResult = withContext(Dispatchers.IO) { - if (serviceId == YOUTUBE_SERVICE_ID && sortFilter != null) { - return@withContext ExtractionResult.BadRequest("Sort filters are unavailable for YouTube") - } - val page = if (nextpage != null) { runCatching { nextpage.toPage() } .getOrElse { return@withContext ExtractionResult.BadRequest("Invalid nextpage cursor") } } else null + val service = runCatching { NewPipe.getService(serviceId) } + .getOrElse { return@withContext ExtractionResult.Failure(it.message ?: "Search failed") } + val factory = service.searchQHFactory + val selectedContentFilter = when (val resolution = factory.resolveContentFilter(contentFilter)) { + is SearchFilterResolution.Valid -> resolution.items + is SearchFilterResolution.Invalid -> + return@withContext ExtractionResult.BadRequest(resolution.message) + } + val availableFilters = factory.filtersFor(selectedContentFilter) + val selectedFilters = if (filters.isEmpty()) { + availableFilters.defaultSearchFilters() + } else { + when (val resolution = availableFilters.resolveSearchFilters(filters)) { + is SearchFilterResolution.Valid -> resolution.items + is SearchFilterResolution.Invalid -> + return@withContext ExtractionResult.BadRequest(resolution.message) + } + } + val queryHandler = runCatching { + factory.fromQuery( + query, + selectedContentFilter.ifEmpty { null }, + selectedFilters.ifEmpty { null }, + ) + }.getOrElse { + return@withContext ExtractionResult.Failure(it.message ?: "Search failed") + } + val contentKind = contentFilter.toSearchContentKind(selectedContentFilter) + runCatching { withExtractionRetry { withTimeout(30_000L) { - val service = NewPipe.getService(serviceId) - val factory = service.searchQHFactory - val selectedContentFilter = if (contentFilter == null) { - factory.availableContentFilter.defaultSearchFilter() - } else { - factory.availableContentFilter.findSearchFilter(contentFilter) - } - val selectedSortFilter = factory.availableSortFilter.findSearchFilter(sortFilter) - val queryHandler = factory.fromQuery( - query, - selectedContentFilter.ifEmpty { null }, - selectedSortFilter.ifEmpty { null }, - ) - val contentKind = contentFilter.toSearchContentKind(selectedContentFilter) if (page == null) { SearchInfo.getInfo(service, queryHandler).toSearchPageResponse().filteredBy(contentKind) } else { @@ -54,29 +68,46 @@ class PipePipeSearchService : SearchService { } }.fold( onSuccess = { ExtractionResult.Success(it) }, - onFailure = { ExtractionResult.Failure(it.message ?: "Search failed") } + onFailure = { ExtractionResult.Failure(it.message ?: "Search failed") }, ) } - override suspend fun filters(serviceId: Int): ExtractionResult = + override suspend fun filters( + serviceId: Int, + contentFilter: String?, + ): ExtractionResult = withContext(Dispatchers.IO) { + val factory = runCatching { NewPipe.getService(serviceId).searchQHFactory } + .getOrElse { return@withContext ExtractionResult.Failure(it.message ?: "Search filters failed") } + val selectedContentFilter = when (val resolution = factory.resolveContentFilter(contentFilter)) { + is SearchFilterResolution.Valid -> resolution.items + is SearchFilterResolution.Invalid -> + return@withContext ExtractionResult.BadRequest(resolution.message) + } + val availableFilters = factory.filtersFor(selectedContentFilter) runCatching { - val factory = NewPipe.getService(serviceId).searchQHFactory SearchFiltersResponse( - contentFilters = factory.availableContentFilter.toSearchFilterOptions(), - sortFilters = if (serviceId == YOUTUBE_SERVICE_ID) { - emptyList() - } else { - factory.availableSortFilter.toSearchFilterOptions() - }, + contentFilters = factory.availableContentFilter.toSearchContentFilterOptions(), + sortFilters = availableFilters.toSearchFilterOptions(), + filterGroups = availableFilters.toSearchFilterGroups(), ) }.fold( onSuccess = { ExtractionResult.Success(it) }, onFailure = { ExtractionResult.Failure(it.message ?: "Search filters failed") }, ) } +} - private companion object { - const val YOUTUBE_SERVICE_ID = 0 +private fun SearchQueryHandlerFactory.resolveContentFilter(value: String?): SearchFilterResolution { + if (value == null) return SearchFilterResolution.Valid(availableContentFilter.defaultSearchFilter()) + val selected = availableContentFilter.findSearchFilter(value) + return if (selected.isEmpty()) { + SearchFilterResolution.Invalid("Unknown content filter") + } else { + SearchFilterResolution.Valid(selected) } } + +private fun SearchQueryHandlerFactory.filtersFor(contentFilter: List): Filter? = + contentFilter.firstOrNull()?.let { getContentFilterSortFilterVariant(it.identifier) } + ?: availableSortFilter diff --git a/src/main/kotlin/dev/typetype/server/services/SabrAdaptiveInitialization.kt b/src/main/kotlin/dev/typetype/server/services/SabrAdaptiveInitialization.kt new file mode 100644 index 00000000..de684c89 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/services/SabrAdaptiveInitialization.kt @@ -0,0 +1,43 @@ +package dev.typetype.server.services + +import dev.typetype.server.cache.CacheService +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.runInterruptible +import org.schabi.newpipe.extractor.localization.Localization +import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat + +internal object SabrAdaptiveInitialization { + private val localization = Localization("en", "US") + + suspend fun fetch( + holder: SabrSessionHolder, + format: YoutubeSabrFormat, + cache: CacheService?, + timeoutMs: Long = 2_000L, + ): ByteArray? { + SabrInitializationData.fetch(holder.key.videoId, format, cache)?.let { + holder.session.streamState.ingestInitializationData(format, it) + return it + } + val data = fetchRange(holder, format, timeoutMs) ?: return null + SabrInitializationData.remember(holder.key.videoId, format, data, cache) + return data + } + + suspend fun fetchRange( + holder: SabrSessionHolder, + format: YoutubeSabrFormat, + timeoutMs: Long, + ): ByteArray? { + val poToken = holder.session.streamState.poToken?.takeIf { it.isNotEmpty() } + ?: holder.playerContextToken?.streamingPoTokenBytesFor(holder.info) + ?: return null + return runCatchingNonCancellation { + runInterruptible(Dispatchers.IO) { + holder.withPlayerContext { + fetchInitializationData(format, localization, timeoutMs, poToken) + } + } + }.getOrNull() + } +} diff --git a/src/main/kotlin/dev/typetype/server/services/SabrDownloadInitialization.kt b/src/main/kotlin/dev/typetype/server/services/SabrDownloadInitialization.kt index 8941e0f3..62f296e5 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrDownloadInitialization.kt +++ b/src/main/kotlin/dev/typetype/server/services/SabrDownloadInitialization.kt @@ -1,33 +1,15 @@ package dev.typetype.server.services -import kotlinx.coroutines.Dispatchers -import kotlinx.coroutines.runInterruptible -import org.schabi.newpipe.extractor.localization.Localization import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat internal object SabrDownloadInitialization { - private val localization = Localization("en", "US") - suspend fun fetch( store: SabrSessionStore, holder: SabrSessionHolder, format: YoutubeSabrFormat, ): ByteArray? { - SabrInitializationData.fetch(holder.key.videoId, format, store.initCache)?.let { - holder.session.streamState.ingestInitializationData(format, it) - return it - } - val poToken = holder.session.streamState.poToken?.takeIf { it.isNotEmpty() } - ?: holder.playerContextToken?.streamingPoTokenBytesFor(holder.info) + val direct = SabrAdaptiveInitialization.fetch(holder, format, store.initCache, DIRECT_TIMEOUT_MS) ?: return store.fetchInitializationData(holder, format) - val direct = runCatchingNonCancellation { - runInterruptible(Dispatchers.IO) { - holder.withPlayerContext { - fetchInitializationData(format, localization, DIRECT_TIMEOUT_MS, poToken) - } - } - }.getOrNull() ?: return store.fetchInitializationData(holder, format) - SabrInitializationData.remember(holder.key.videoId, format, direct, store.initCache) return direct } diff --git a/src/main/kotlin/dev/typetype/server/services/SabrSessionPumpLoop.kt b/src/main/kotlin/dev/typetype/server/services/SabrSessionPumpLoop.kt index 44ef828e..1f3953ed 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrSessionPumpLoop.kt +++ b/src/main/kotlin/dev/typetype/server/services/SabrSessionPumpLoop.kt @@ -69,6 +69,12 @@ internal class SabrSessionPumpLoop( runtime: SabrPumpRuntime, ): Boolean { preparePumpEviction(holder) + if (holder.prepareStartupBootstrapPump()) { + holder.setPlaybackState(SabrPlaybackState.REQUESTING) + pumpOnce(holder, localization, runtime) + holder.setPlaybackState(SabrPlaybackState.IDLE) + return true + } holder.consumeRefetch()?.let { request -> if (holder.session.isBeyondEnd(request) && !holder.isFutureLiveRequest(request)) { holder.clearSegmentDemand(request) diff --git a/src/main/kotlin/dev/typetype/server/services/SabrSessionStore.kt b/src/main/kotlin/dev/typetype/server/services/SabrSessionStore.kt index 231a7675..a76ea88a 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrSessionStore.kt +++ b/src/main/kotlin/dev/typetype/server/services/SabrSessionStore.kt @@ -184,10 +184,7 @@ internal class SabrSessionStore( holder.liveInitialization(format)?.let { return it } val request = SabrSegmentRequest.initialization(format) holder.session.getCachedSegment(request)?.let { segmentCache.put(holder, it); return it.data } - SabrInitializationData.fetch(holder.key.videoId, format, initCache)?.let { - holder.session.streamState.ingestInitializationData(format, it) - return it - } + SabrAdaptiveInitialization.fetch(holder, format, initCache)?.let { return it } SabrInitializationData.bootstrap(holder, format, initCache)?.let { return it } val segment = pump.fetchSegment(holder, request) ?: return null segmentCache.put(holder, segment) diff --git a/src/main/kotlin/dev/typetype/server/services/SabrStartupBootstrap.kt b/src/main/kotlin/dev/typetype/server/services/SabrStartupBootstrap.kt index 32fd1f51..47797f98 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrStartupBootstrap.kt +++ b/src/main/kotlin/dev/typetype/server/services/SabrStartupBootstrap.kt @@ -1,6 +1,7 @@ package dev.typetype.server.services internal fun SabrSessionHolder.prepareStartupBootstrapPump(): Boolean { + if (expectsLive() || !hasPendingSeek()) return false if (session.requestNumber == 0) { session.streamState.setPlayerTimeMs(0L) return true diff --git a/src/main/kotlin/dev/typetype/server/services/SearchFilterMappers.kt b/src/main/kotlin/dev/typetype/server/services/SearchFilterMappers.kt index 8050188a..80223e8e 100644 --- a/src/main/kotlin/dev/typetype/server/services/SearchFilterMappers.kt +++ b/src/main/kotlin/dev/typetype/server/services/SearchFilterMappers.kt @@ -1,20 +1,62 @@ package dev.typetype.server.services +import dev.typetype.server.models.SearchFilterGroup import dev.typetype.server.models.SearchFilterOption import org.schabi.newpipe.extractor.search.filter.Filter +import org.schabi.newpipe.extractor.search.filter.FilterGroup import org.schabi.newpipe.extractor.search.filter.FilterItem +internal sealed interface SearchFilterResolution { + data class Valid(val items: List) : SearchFilterResolution + data class Invalid(val message: String) : SearchFilterResolution +} + +internal fun Filter?.toSearchContentFilterOptions(): List = entries() + .mapIndexed { index, entry -> entry.toOption(isDefault = index == 0, includeGroup = true) } + internal fun Filter?.toSearchFilterOptions(): List = this?.filterGroups - ?.flatMap { group -> group.filterItems.map { item -> item.toSearchFilterOption(group.groupName.orEmpty()) } } - ?: emptyList() + ?.flatMap { group -> + group.filterItems.mapIndexed { index, item -> + FilterEntry(group, item).toOption(group.onlyOneCheckable && index == 0, includeGroup = true) + } + }.orEmpty() + +internal fun Filter?.toSearchFilterGroups(): List = this?.filterGroups + ?.map { group -> + SearchFilterGroup( + key = "${group.groupName.orEmpty()}|${group.identifier}", + label = group.groupName.orEmpty(), + multiSelect = !group.onlyOneCheckable, + options = group.filterItems.mapIndexed { index, item -> + FilterEntry(group, item).toOption(group.onlyOneCheckable && index == 0) + }, + ) + }.orEmpty() internal fun Filter?.findSearchFilter(value: String?): List = value?.let { raw -> - this?.filterGroups - ?.flatMap { group -> group.filterItems.map { group.groupName.orEmpty() to it } } - ?.firstOrNull { (groupName, item) -> item.searchFilterValue(groupName) == raw } - ?.let { listOf(it.second) } + entries().firstOrNull { it.value == raw }?.let { listOf(it.item) } } ?: emptyList() +internal fun Filter?.resolveSearchFilters(values: List): SearchFilterResolution { + if (values.isEmpty()) return SearchFilterResolution.Valid(emptyList()) + val requested = values.toSet() + val selected = entries().filter { it.value in requested } + if (selected.size != requested.size) return SearchFilterResolution.Invalid("Unknown search filter") + val conflict = selected.groupBy { it.group.identifier } + .values + .firstOrNull { entries -> entries.first().group.onlyOneCheckable && entries.size > 1 } + if (conflict != null) { + val groupName = conflict.first().group.groupName.orEmpty().ifBlank { "filter" } + return SearchFilterResolution.Invalid("Only one '$groupName' filter can be selected") + } + return SearchFilterResolution.Valid(selected.map(FilterEntry::item)) +} + +internal fun Filter?.defaultSearchFilters(): List = this?.filterGroups + ?.filter(FilterGroup::onlyOneCheckable) + ?.mapNotNull { it.filterItems.firstOrNull() } + .orEmpty() + internal fun Filter?.defaultSearchFilter(): List = this?.filterGroups ?.firstOrNull() ?.filterItems @@ -22,10 +64,19 @@ internal fun Filter?.defaultSearchFilter(): List = this?.filterGroup ?.let { listOf(it) } ?: emptyList() -private fun FilterItem.toSearchFilterOption(groupName: String): SearchFilterOption = SearchFilterOption( - value = searchFilterValue(groupName), - label = if (groupName.isBlank()) name else "$groupName: $name", -) +private data class FilterEntry(val group: FilterGroup, val item: FilterItem) { + val value: String = item.searchFilterValue(group.groupName.orEmpty()) + + fun toOption(isDefault: Boolean, includeGroup: Boolean = false): SearchFilterOption { + val groupName = group.groupName.orEmpty() + val label = if (includeGroup && groupName.isNotBlank()) "$groupName: ${item.name}" else item.name + return SearchFilterOption(value = value, label = label, isDefault = isDefault) + } +} + +private fun Filter?.entries(): List = this?.filterGroups + ?.flatMap { group -> group.filterItems.map { item -> FilterEntry(group, item) } } + .orEmpty() private fun FilterItem.searchFilterValue(groupName: String): String = listOf(groupName, identifier.toString(), name).joinToString("|") diff --git a/src/main/kotlin/dev/typetype/server/services/SearchService.kt b/src/main/kotlin/dev/typetype/server/services/SearchService.kt index 0cc413c2..16353179 100644 --- a/src/main/kotlin/dev/typetype/server/services/SearchService.kt +++ b/src/main/kotlin/dev/typetype/server/services/SearchService.kt @@ -5,6 +5,16 @@ import dev.typetype.server.models.SearchFiltersResponse import dev.typetype.server.models.SearchPageResponse interface SearchService { - suspend fun search(query: String, serviceId: Int, nextpage: String? = null, contentFilter: String? = null, sortFilter: String? = null): ExtractionResult - suspend fun filters(serviceId: Int): ExtractionResult + suspend fun search( + query: String, + serviceId: Int, + nextpage: String? = null, + contentFilter: String? = null, + filters: List = emptyList(), + ): ExtractionResult + + suspend fun filters( + serviceId: Int, + contentFilter: String? = null, + ): ExtractionResult } diff --git a/src/main/kotlin/dev/typetype/server/services/StreamYouTubeSubtitleResolver.kt b/src/main/kotlin/dev/typetype/server/services/StreamYouTubeSubtitleResolver.kt new file mode 100644 index 00000000..85643b1f --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/services/StreamYouTubeSubtitleResolver.kt @@ -0,0 +1,83 @@ +package dev.typetype.server.services + +import dev.typetype.server.models.ExtractionResult +import dev.typetype.server.models.StreamResponse +import dev.typetype.server.models.SubtitleItem +import kotlinx.coroutines.CancellationException +import kotlinx.coroutines.TimeoutCancellationException +import kotlinx.coroutines.withTimeout +import okhttp3.HttpUrl.Companion.toHttpUrlOrNull + +internal fun interface YouTubeSubtitleTrackResolver { + suspend fun resolve(selection: YouTubeSubtitleSelection): YouTubeSubtitleResolution +} + +internal class StreamYouTubeSubtitleResolver( + private val streamService: StreamService, + private val fetchInventory: suspend (String) -> YouTubeSubtitleInventoryResult, +) : YouTubeSubtitleTrackResolver { + override suspend fun resolve(selection: YouTubeSubtitleSelection): YouTubeSubtitleResolution = try { + withTimeout(RESOLUTION_TIMEOUT_MS) { + val stream = streamService.streamForSubtitle(selection.videoId) + val inventory = fetchInventory(selection.videoId) + val tracks = (inventory as? YouTubeSubtitleInventoryResult.Ready)?.tracks + ?.takeIf(List::isNotEmpty) + ?: stream?.subtitles + ?: return@withTimeout YouTubeSubtitleResolution.Unavailable + val track = tracks.firstOrNull { it.matchesYouTubeSubtitle(selection) } + ?: return@withTimeout YouTubeSubtitleResolution.NotFound + val content = track.contentForYouTubeSubtitle(selection) + ?: return@withTimeout YouTubeSubtitleResolution.NotFound + YouTubeSubtitleResolution.Ready( + ResolvedYouTubeSubtitle( + content = content, + isUrl = true, + isLive = stream?.isLiveSubtitleContent() == true, + ), + ) + } + } catch (_: TimeoutCancellationException) { + YouTubeSubtitleResolution.Unavailable + } catch (error: CancellationException) { + throw error + } catch (_: Throwable) { + YouTubeSubtitleResolution.Unavailable + } + + private companion object { + const val RESOLUTION_TIMEOUT_MS = 30_000L + } +} + +private suspend fun StreamService.streamForSubtitle(videoId: String): StreamResponse? = + when (val result = getStreamInfo("https://www.youtube.com/watch?v=$videoId")) { + is ExtractionResult.Success -> result.data + is ExtractionResult.BadRequest, is ExtractionResult.Failure -> null + } + +private fun StreamResponse.isLiveSubtitleContent(): Boolean = isLive || isLiveContent + +internal fun SubtitleItem.matchesYouTubeSubtitle(selection: YouTubeSubtitleSelection): Boolean { + val parsedUrl = url.toHttpUrlOrNull() + val sourceLanguage = parsedUrl?.queryParameter("lang") ?: languageTag + val requestedSource = selection.sourceLanguage ?: selection.language + if (!sourceLanguage.equals(requestedSource, ignoreCase = true) && + !languageTag.equals(selection.language, ignoreCase = true) + ) return false + val auto = isAutoGenerated || parsedUrl?.queryParameter("kind") == "asr" || + parsedUrl?.queryParameter("vssId")?.startsWith("a.") == true + if (auto != (selection.variant == YouTubeSubtitleVariant.Auto)) return false + return selection.trackName == null || parsedUrl?.queryParameter("name") == selection.trackName +} + +internal fun SubtitleItem.contentForYouTubeSubtitle(selection: YouTubeSubtitleSelection): String? { + val parsedUrl = url.toHttpUrlOrNull()?.takeIf { isYouTubeTimedTextUrl(it.toString()) } ?: return null + return parsedUrl.newBuilder() + .setQueryParameter("fmt", selection.format.value) + .apply { + if (selection.translationLanguage == null) removeAllQueryParameters("tlang") + else setQueryParameter("tlang", selection.translationLanguage) + } + .build() + .toString() +} diff --git a/src/main/kotlin/dev/typetype/server/services/TypetypeTokenSabrPoTokenProvider.kt b/src/main/kotlin/dev/typetype/server/services/TypetypeTokenSabrPoTokenProvider.kt index 78527ecc..9495bcc1 100644 --- a/src/main/kotlin/dev/typetype/server/services/TypetypeTokenSabrPoTokenProvider.kt +++ b/src/main/kotlin/dev/typetype/server/services/TypetypeTokenSabrPoTokenProvider.kt @@ -11,19 +11,8 @@ internal class TypetypeTokenSabrPoTokenProvider( ) : SabrPoTokenProvider { constructor(tokenServiceUrl: String) : this(TypetypeTokenSabrTokenClient(tokenServiceUrl)) - override fun getPoToken(info: YoutubeSabrInfo, streamState: YoutubeSabrStreamState): ByteArray? = - fetch(info, forceRefresh = false) - - override fun getPoToken( - info: YoutubeSabrInfo, - streamState: YoutubeSabrStreamState, - forceRefresh: Boolean, - ): ByteArray? = fetch(info, forceRefresh) - - private fun fetch(info: YoutubeSabrInfo, forceRefresh: Boolean): ByteArray? { - val token = if (!forceRefresh && initialToken?.videoId == info.videoId) initialToken else { - tokenClient.fetch(info.videoId, refreshVideo = forceRefresh) - } + override fun getPoToken(info: YoutubeSabrInfo, streamState: YoutubeSabrStreamState): ByteArray? { + val token = if (initialToken?.videoId == info.videoId) initialToken else tokenClient.fetch(info.videoId) return token?.streamingPoTokenBytesFor(info) ?: token?.let { throw SabrRecoverableException(SABR_TOKEN_BINDING_FAILURE) } } diff --git a/src/main/kotlin/dev/typetype/server/services/TypetypeTokenYoutubeSessionPoTokenProvider.kt b/src/main/kotlin/dev/typetype/server/services/TypetypeTokenYoutubeSessionPoTokenProvider.kt index f0fbad22..abb8ca67 100644 --- a/src/main/kotlin/dev/typetype/server/services/TypetypeTokenYoutubeSessionPoTokenProvider.kt +++ b/src/main/kotlin/dev/typetype/server/services/TypetypeTokenYoutubeSessionPoTokenProvider.kt @@ -28,6 +28,8 @@ internal class TypetypeTokenYoutubeSessionPoTokenProvider( override fun getSessionPoToken( clientName: String, + clientVersion: String, + userAgent: String?, localization: Localization, contentCountry: ContentCountry, loggedIn: Boolean, diff --git a/src/main/kotlin/dev/typetype/server/services/TypetypeYoutubeSessionPoTokenProvider.kt b/src/main/kotlin/dev/typetype/server/services/TypetypeYoutubeSessionPoTokenProvider.kt index f4f28006..72bff27b 100644 --- a/src/main/kotlin/dev/typetype/server/services/TypetypeYoutubeSessionPoTokenProvider.kt +++ b/src/main/kotlin/dev/typetype/server/services/TypetypeYoutubeSessionPoTokenProvider.kt @@ -21,6 +21,8 @@ internal object TypetypeYoutubeSessionPoTokenProvider : YoutubeSessionPoTokenPro override fun getSessionPoToken( clientName: String, + clientVersion: String, + userAgent: String?, localization: Localization, contentCountry: ContentCountry, loggedIn: Boolean, diff --git a/src/main/kotlin/dev/typetype/server/services/YouTubeSubtitleCache.kt b/src/main/kotlin/dev/typetype/server/services/YouTubeSubtitleCache.kt new file mode 100644 index 00000000..03297d24 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/services/YouTubeSubtitleCache.kt @@ -0,0 +1,75 @@ +package dev.typetype.server.services + +import dev.typetype.server.cache.CacheJson +import dev.typetype.server.cache.CacheService +import kotlinx.serialization.Serializable +import java.security.MessageDigest +import java.time.Duration +import java.util.HexFormat + +internal class YouTubeSubtitleCache(private val sharedCache: CacheService?) { + private val vod = BoundedExpiringCache( + maxEntries = 128, + maxWeight = MAX_MEMORY_BYTES, + ttl = Duration.ofSeconds(VOD_TTL_SECONDS), + weigher = { it.content.size.toLong() }, + ) + private val live = BoundedExpiringCache( + maxEntries = 32, + maxWeight = MAX_LIVE_MEMORY_BYTES, + ttl = Duration.ofSeconds(LIVE_TTL_SECONDS), + weigher = { it.content.size.toLong() }, + ) + + suspend fun get(selection: YouTubeSubtitleSelection): YouTubeSubtitleContentResult.Ready? { + val key = selection.key() + vod.get(key)?.let { return it } + live.get(key)?.let { return it } + val encoded = runCatching { sharedCache?.get(key) }.getOrNull() ?: return null + val cached = runCatching { CacheJson.decodeFromString(encoded) }.getOrNull() + ?: return null + val ready = cached.toReady(selection.format) ?: return null + memoryCache(cached.isLive).put(key, ready) + return ready + } + + suspend fun put(selection: YouTubeSubtitleSelection, value: YouTubeSubtitleContentResult.Ready) { + val key = selection.key() + memoryCache(value.isLive).put(key, value) + val cached = CachedYouTubeSubtitle(value.content.toString(Charsets.UTF_8), value.isLive) + runCatching { + sharedCache?.set( + key, + CacheJson.encodeToString(CachedYouTubeSubtitle.serializer(), cached), + if (value.isLive) LIVE_TTL_SECONDS else VOD_TTL_SECONDS, + ) + } + } + + private fun memoryCache(isLive: Boolean) = if (isLive) live else vod + + private fun YouTubeSubtitleSelection.key(): String { + val digest = MessageDigest.getInstance("SHA-256").digest(cacheKey.encodeToByteArray()) + return "$CACHE_PREFIX:${HexFormat.of().formatHex(digest)}" + } + + @Serializable + private data class CachedYouTubeSubtitle(val content: String, val isLive: Boolean) { + fun toReady(format: YouTubeSubtitleFormat): YouTubeSubtitleContentResult.Ready? { + val bytes = content.encodeToByteArray() + return if (isValidSubtitlePayload(bytes, format)) { + YouTubeSubtitleContentResult.Ready(bytes, format, isLive) + } else { + null + } + } + } + + private companion object { + const val CACHE_PREFIX = "youtube-subtitle:v1" + const val VOD_TTL_SECONDS = 21_600L + const val LIVE_TTL_SECONDS = 5L + const val MAX_MEMORY_BYTES = 64L * 1024 * 1024 + const val MAX_LIVE_MEMORY_BYTES = 8L * 1024 * 1024 + } +} diff --git a/src/main/kotlin/dev/typetype/server/services/YouTubeSubtitleContentFetcher.kt b/src/main/kotlin/dev/typetype/server/services/YouTubeSubtitleContentFetcher.kt new file mode 100644 index 00000000..acb9e662 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/services/YouTubeSubtitleContentFetcher.kt @@ -0,0 +1,104 @@ +package dev.typetype.server.services + +import dev.typetype.server.REQUEST_ID_HEADER +import dev.typetype.server.currentRequestId +import kotlinx.coroutines.suspendCancellableCoroutine +import okhttp3.Call +import okhttp3.Callback +import okhttp3.HttpUrl.Companion.toHttpUrl +import okhttp3.OkHttpClient +import okhttp3.Request +import okhttp3.Response +import java.io.IOException +import kotlin.coroutines.resume + +internal fun interface YouTubeSubtitleContentFetcher { + suspend fun fetch(url: String, format: YouTubeSubtitleFormat): YouTubeSubtitleFetchResult +} + +internal class OkHttpYouTubeSubtitleContentFetcher( + private val client: OkHttpClient, +) : YouTubeSubtitleContentFetcher { + override suspend fun fetch(url: String, format: YouTubeSubtitleFormat): YouTubeSubtitleFetchResult { + if (!isYouTubeTimedTextUrl(url)) return YouTubeSubtitleFetchResult.Unavailable + val request = Request.Builder() + .url(url) + .header("Accept", "${format.contentType},*/*;q=0.8") + .header("Origin", YOUTUBE_ORIGIN) + .header("Referer", "$YOUTUBE_ORIGIN/") + .header("User-Agent", OkHttpProxyService.BROWSER_USER_AGENT) + .apply { currentRequestId()?.let { header(REQUEST_ID_HEADER, it) } } + .build() + return client.executeSubtitleRequest(request, format) + } +} + +internal class TokenYouTubeSubtitleContentFetcher( + private val client: OkHttpClient, + baseUrl: String, + private val directFetcher: YouTubeSubtitleContentFetcher, +) : YouTubeSubtitleContentFetcher { + private val endpoint = baseUrl.toHttpUrl().newBuilder() + .addPathSegments("subtitles/content") + .build() + + override suspend fun fetch(url: String, format: YouTubeSubtitleFormat): YouTubeSubtitleFetchResult { + if (format == YouTubeSubtitleFormat.Ttml) return directFetcher.fetch(url, format) + if (!isYouTubeTimedTextUrl(url)) return YouTubeSubtitleFetchResult.Unavailable + val request = Request.Builder() + .url(endpoint.newBuilder().addQueryParameter("url", url).build()) + .header("Accept", format.contentType) + .apply { currentRequestId()?.let { header(REQUEST_ID_HEADER, it) } } + .build() + return client.executeSubtitleRequest(request, format) + } +} + +private suspend fun OkHttpClient.executeSubtitleRequest( + request: Request, + format: YouTubeSubtitleFormat, +): YouTubeSubtitleFetchResult = suspendCancellableCoroutine { continuation -> + val call = newCall(request) + continuation.invokeOnCancellation { call.cancel() } + call.enqueue(object : Callback { + override fun onFailure(call: Call, e: IOException) { + if (continuation.isActive) continuation.resume(YouTubeSubtitleFetchResult.Unavailable) + } + + override fun onResponse(call: Call, response: Response) { + val result = runCatching { response.use { readSubtitleResponse(it, format) } } + .getOrDefault(YouTubeSubtitleFetchResult.Unavailable) + if (continuation.isActive) continuation.resume(result) + } + }) +} + +private fun readSubtitleResponse(response: Response, format: YouTubeSubtitleFormat): YouTubeSubtitleFetchResult { + if (response.code == 429) return YouTubeSubtitleFetchResult.Throttled + if (response.code == 403 || response.code == 404 || response.code == 410) { + return YouTubeSubtitleFetchResult.Expired + } + if (!response.isSuccessful) return YouTubeSubtitleFetchResult.Unavailable + val body = response.body + if (body.contentLength() > MAX_SUBTITLE_BYTES) return YouTubeSubtitleFetchResult.InvalidPayload + return runCatching { body.byteStream().readNBytes(MAX_SUBTITLE_BYTES + 1) } + .fold( + onSuccess = { bytes -> + if (isValidSubtitlePayload(bytes, format)) YouTubeSubtitleFetchResult.Ready(bytes) + else YouTubeSubtitleFetchResult.InvalidPayload + }, + onFailure = { YouTubeSubtitleFetchResult.Unavailable }, + ) +} + +private const val YOUTUBE_ORIGIN = "https://m.youtube.com" +internal const val MAX_SUBTITLE_BYTES = 5 * 1024 * 1024 + +internal fun isValidSubtitlePayload(content: ByteArray, format: YouTubeSubtitleFormat): Boolean { + if (content.isEmpty() || content.size > MAX_SUBTITLE_BYTES) return false + val text = content.toString(Charsets.UTF_8).trimStart() + return when (format) { + YouTubeSubtitleFormat.Vtt -> text.startsWith("WEBVTT") + YouTubeSubtitleFormat.Ttml -> text.startsWith(">() + private val upstreamPermits = Semaphore(MAX_CONCURRENT_UPSTREAM_REQUESTS) + + suspend fun fetch(selection: YouTubeSubtitleSelection): YouTubeSubtitleContentResult { + cache.get(selection)?.let { return it } + val pending = CompletableDeferred() + val existing = inFlight.putIfAbsent(selection.cacheKey, pending) + if (existing != null) return existing.await() + if (!upstreamPermits.tryAcquire()) { + pending.complete(YouTubeSubtitleContentResult.Unavailable) + inFlight.remove(selection.cacheKey, pending) + return YouTubeSubtitleContentResult.Unavailable + } + return try { + val result = cache.get(selection) ?: load(selection) + if (result is YouTubeSubtitleContentResult.Ready) cache.put(selection, result) + pending.complete(result) + result + } catch (error: Throwable) { + pending.completeExceptionally(error) + throw error + } finally { + inFlight.remove(selection.cacheKey, pending) + upstreamPermits.release() + } + } + + private suspend fun load(selection: YouTubeSubtitleSelection): YouTubeSubtitleContentResult { + repeat(MAX_RESOLUTION_ATTEMPTS) { attempt -> + when (val resolution = resolver.resolve(selection)) { + YouTubeSubtitleResolution.NotFound -> return YouTubeSubtitleContentResult.NotFound + YouTubeSubtitleResolution.Throttled -> return YouTubeSubtitleContentResult.Throttled + YouTubeSubtitleResolution.Unavailable -> return YouTubeSubtitleContentResult.Unavailable + is YouTubeSubtitleResolution.Ready -> { + val result = fetchResolved(resolution.track, selection.format) + if (result == YouTubeSubtitleFetchResult.Expired && attempt < MAX_RESOLUTION_ATTEMPTS - 1) { + return@repeat + } + return result.toContentResult(selection.format, resolution.track.isLive) + } + } + } + return YouTubeSubtitleContentResult.Expired + } + + private suspend fun fetchResolved( + track: ResolvedYouTubeSubtitle, + format: YouTubeSubtitleFormat, + ): YouTubeSubtitleFetchResult { + if (track.isUrl) return fetcher.fetch(track.content, format) + val bytes = track.content.encodeToByteArray() + return if (isValidSubtitlePayload(bytes, format)) YouTubeSubtitleFetchResult.Ready(bytes) + else YouTubeSubtitleFetchResult.InvalidPayload + } + + private fun YouTubeSubtitleFetchResult.toContentResult( + format: YouTubeSubtitleFormat, + isLive: Boolean, + ): YouTubeSubtitleContentResult = when (this) { + is YouTubeSubtitleFetchResult.Ready -> YouTubeSubtitleContentResult.Ready(content, format, isLive) + YouTubeSubtitleFetchResult.Expired -> YouTubeSubtitleContentResult.Expired + YouTubeSubtitleFetchResult.Throttled -> YouTubeSubtitleContentResult.Throttled + YouTubeSubtitleFetchResult.InvalidPayload -> YouTubeSubtitleContentResult.InvalidPayload + YouTubeSubtitleFetchResult.Unavailable -> YouTubeSubtitleContentResult.Unavailable + } + + private companion object { + const val MAX_RESOLUTION_ATTEMPTS = 2 + const val MAX_CONCURRENT_UPSTREAM_REQUESTS = 64 + } +} diff --git a/src/main/kotlin/dev/typetype/server/services/YouTubeSubtitleService.kt b/src/main/kotlin/dev/typetype/server/services/YouTubeSubtitleService.kt index 6cb1998b..0d1f5367 100644 --- a/src/main/kotlin/dev/typetype/server/services/YouTubeSubtitleService.kt +++ b/src/main/kotlin/dev/typetype/server/services/YouTubeSubtitleService.kt @@ -52,6 +52,7 @@ internal class YouTubeSubtitleService(private val httpClient: OkHttpClient, priv }) } } + } internal sealed interface YouTubeSubtitleInventoryResult { diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubePlayerClient.kt b/src/main/kotlin/dev/typetype/server/services/YoutubePlayerClient.kt index 62d276cf..90fda099 100644 --- a/src/main/kotlin/dev/typetype/server/services/YoutubePlayerClient.kt +++ b/src/main/kotlin/dev/typetype/server/services/YoutubePlayerClient.kt @@ -2,6 +2,6 @@ package dev.typetype.server.services internal enum class YoutubePlayerClient(val value: String) { MWEB("mweb"), - WEB_SAFARI("web_safari"), + VISIONOS("visionos"), TV_DOWNGRADED("tv_downgraded"), } diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeScopedSearchService.kt b/src/main/kotlin/dev/typetype/server/services/YoutubeScopedSearchService.kt index 92f001a1..cf0af470 100644 --- a/src/main/kotlin/dev/typetype/server/services/YoutubeScopedSearchService.kt +++ b/src/main/kotlin/dev/typetype/server/services/YoutubeScopedSearchService.kt @@ -10,16 +10,18 @@ class YoutubeScopedSearchService(private val delegate: SearchService) : SearchSe serviceId: Int, nextpage: String?, contentFilter: String?, - sortFilter: String?, + filters: List, ): ExtractionResult = if (serviceId == YOUTUBE_SERVICE_ID) { YoutubeSessionTokenScope.withoutCredentials { - delegate.search(query, serviceId, nextpage, contentFilter, sortFilter) + delegate.search(query, serviceId, nextpage, contentFilter, filters) } } else { - delegate.search(query, serviceId, nextpage, contentFilter, sortFilter) + delegate.search(query, serviceId, nextpage, contentFilter, filters) } - override suspend fun filters(serviceId: Int): ExtractionResult = - delegate.filters(serviceId) + override suspend fun filters( + serviceId: Int, + contentFilter: String?, + ): ExtractionResult = delegate.filters(serviceId, contentFilter) } diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutActivitySignalService.kt b/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutActivitySignalService.kt index 4c3fdabe..c14db660 100644 --- a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutActivitySignalService.kt +++ b/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutActivitySignalService.kt @@ -38,7 +38,7 @@ object YoutubeTakeoutActivitySignalService { private fun parseFavorites(html: String): List { return likedRegex.findAll(html).mapNotNull { match -> val title = decode(match.groupValues[2]) - if (isUnavailable(title)) return@mapNotNull null + if (YoutubeTakeoutUnavailableItem.matches(title)) return@mapNotNull null val source = decode(match.groupValues[1]) + " " + title val videoUrl = watchUrlRegex.find(source)?.value?.replace("http://", "https://") ?: return@mapNotNull null @@ -52,9 +52,6 @@ object YoutubeTakeoutActivitySignalService { }.toList() } - private fun isUnavailable(title: String): Boolean = - YoutubeTakeoutTextNormalizer.normalize(title) in unavailableTitles - private fun decode(value: String): String { return value .replace(" ", " ") @@ -66,14 +63,4 @@ object YoutubeTakeoutActivitySignalService { .replace(spacesRegex, " ") .trim() } - - private val unavailableTitles = setOf( - "deleted video", - "private video", - "video unavailable", - "video deleted", - "video indisponible", - "video privee", - "video supprimee", - ) } diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutHistoryParser.kt b/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutHistoryParser.kt index 472a3b34..1cee2b5a 100644 --- a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutHistoryParser.kt +++ b/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutHistoryParser.kt @@ -13,6 +13,7 @@ object YoutubeTakeoutHistoryParser { return rowRegex.findAll(resolvedHtml).mapNotNull { match -> val url = extractUrl(match.groupValues[1]) ?: return@mapNotNull null val title = decode(match.groupValues[2]) + if (YoutubeTakeoutUnavailableItem.matches(title)) return@mapNotNull null val channelUrl = match.groupValues[3].takeIf { it.isNotBlank() }.orEmpty() val channelName = decode(match.groupValues[4]).ifBlank { "Unknown channel" } val watchedAt = parseDate(decode(match.groupValues[5])) diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutParserService.kt b/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutParserService.kt index 562e56c9..1196917a 100644 --- a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutParserService.kt +++ b/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutParserService.kt @@ -32,6 +32,7 @@ class YoutubeTakeoutParserService { }.toMap() val rawPlaylistItems = mutableMapOf>() scan.playlistItemsRows.forEach { row -> + if (YoutubeTakeoutRowParser.isUnavailablePlaylistItem(scan.playlistItemsHeader, row)) return@forEach val parsed = runCatching { YoutubeTakeoutRowParser.parsePlaylistItem(scan.playlistItemsHeader, row) }.getOrNull() if (parsed == null) { errors += "Invalid playlist item row" diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutRowParser.kt b/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutRowParser.kt index 0fa8fd81..d9cf8d5f 100644 --- a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutRowParser.kt +++ b/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutRowParser.kt @@ -33,6 +33,7 @@ object YoutubeTakeoutRowParser { fun parsePlaylistItem(header: List, row: List): Pair? { val values = header.zip(row).toMap() + if (isUnavailablePlaylistItem(values)) return null val playlistKey = values.pickExact("playlist source key") ?: values.pickHeader(YoutubeTakeoutSchemaHints::isPlaylistIdHeader) ?: values.pickHeader(YoutubeTakeoutSchemaHints::isPlaylistTitleHeader) @@ -63,6 +64,14 @@ object YoutubeTakeoutRowParser { ) } + fun isUnavailablePlaylistItem(header: List, row: List): Boolean = + isUnavailablePlaylistItem(header.zip(row).toMap()) + + private fun isUnavailablePlaylistItem(values: Map): Boolean = + values.pickHeader(YoutubeTakeoutSchemaHints::isVideoTitleHeader) + ?.let(YoutubeTakeoutUnavailableItem::matches) + ?: false + private fun Map.pickExact(vararg keys: String): String? { val normalized = entries.associate { YoutubeTakeoutSchemaHints.normalize(it.key) to it.value.trim() } return keys.asSequence().mapNotNull { normalized[it] }.firstOrNull { it.isNotBlank() } diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutUnavailableItem.kt b/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutUnavailableItem.kt new file mode 100644 index 00000000..808d72b7 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutUnavailableItem.kt @@ -0,0 +1,18 @@ +package dev.typetype.server.services + +internal object YoutubeTakeoutUnavailableItem { + fun matches(title: String): Boolean = + YoutubeTakeoutTextNormalizer.normalize(title) in TITLES + + private val TITLES = setOf( + "deleted video", + "private video", + "video deleted", + "video indisponible", + "video no disponible", + "video privee", + "video privado", + "video supprimee", + "video unavailable", + ) +} diff --git a/src/test/kotlin/dev/typetype/server/AuthCookieHelpersTest.kt b/src/test/kotlin/dev/typetype/server/AuthCookieHelpersTest.kt index 307ee201..574f3bda 100644 --- a/src/test/kotlin/dev/typetype/server/AuthCookieHelpersTest.kt +++ b/src/test/kotlin/dev/typetype/server/AuthCookieHelpersTest.kt @@ -1,6 +1,7 @@ package dev.typetype.server import dev.typetype.server.services.AuthCookieHelpers +import dev.typetype.server.services.AuthSessionConfig import io.ktor.client.request.get import io.ktor.client.request.header import io.ktor.client.statement.bodyAsText @@ -10,6 +11,8 @@ import io.ktor.server.routing.get import io.ktor.server.routing.routing import io.ktor.server.testing.testApplication import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertFalse +import org.junit.jupiter.api.Assertions.assertTrue import org.junit.jupiter.api.Test class AuthCookieHelpersTest { @@ -27,4 +30,39 @@ class AuthCookieHelpersTest { } assertEquals("abc123", response.bodyAsText()) } + + @Test + fun `secure refresh cookie remains the default`() = testApplication { + application { + routing { + get("/probe") { + AuthCookieHelpers.setRefreshCookie(call.response, "abc123", AuthSessionConfig(refreshTtlDays = 45)) + call.respondText("ok") + } + } + } + + val cookie = client.get("/probe").headers.getAll(HttpHeaders.SetCookie).orEmpty().joinToString("; ") + assertTrue(cookie.contains("Max-Age=3888000")) + assertTrue(cookie.contains("Secure")) + assertTrue(cookie.contains("SameSite=None")) + } + + @Test + fun `explicit http compatibility cookie uses lax same site`() = testApplication { + application { + routing { + get("/probe") { + val config = AuthSessionConfig(refreshTtlDays = 7, allowInsecureCookies = true) + AuthCookieHelpers.setRefreshCookie(call.response, "abc123", config) + call.respondText("ok") + } + } + } + + val cookie = client.get("/probe").headers.getAll(HttpHeaders.SetCookie).orEmpty().joinToString("; ") + assertTrue(cookie.contains("Max-Age=604800")) + assertFalse(cookie.contains("Secure")) + assertTrue(cookie.contains("SameSite=Lax")) + } } diff --git a/src/test/kotlin/dev/typetype/server/AuthServiceCoreTest.kt b/src/test/kotlin/dev/typetype/server/AuthServiceCoreTest.kt index 158ac57e..487f90fb 100644 --- a/src/test/kotlin/dev/typetype/server/AuthServiceCoreTest.kt +++ b/src/test/kotlin/dev/typetype/server/AuthServiceCoreTest.kt @@ -1,7 +1,9 @@ package dev.typetype.server import dev.typetype.server.db.tables.UsersTable +import dev.typetype.server.db.tables.SessionsTable import dev.typetype.server.services.AuthService +import dev.typetype.server.services.AuthSessionConfig import org.jetbrains.exposed.v1.core.eq import org.jetbrains.exposed.v1.jdbc.selectAll import org.jetbrains.exposed.v1.jdbc.transactions.transaction @@ -76,6 +78,23 @@ class AuthServiceCoreTest { assertEquals(expectedUser, refreshed?.let { service.verify(it.accessToken) }) } + @Test + fun `configured refresh lifetime is stored for new sessions`() { + val before = System.currentTimeMillis() + val service = AuthService( + "test-secret", + sessionConfig = AuthSessionConfig(refreshTtlDays = 3), + ) + + service.register("ttl@test.local", "secret-1", "TTL") + val expiresAt = transaction { + SessionsTable.selectAll().single()[SessionsTable.expiresAt] + } + + val expected = before + 3L * 24L * 60L * 60L * 1000L + assertTrue(expiresAt in expected..(expected + 5_000L)) + } + @Test fun `login supports public username identifier`() { val service = AuthService("test-secret") diff --git a/src/test/kotlin/dev/typetype/server/BlockedStreamRoutesTest.kt b/src/test/kotlin/dev/typetype/server/BlockedStreamRoutesTest.kt new file mode 100644 index 00000000..15cb41be --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/BlockedStreamRoutesTest.kt @@ -0,0 +1,120 @@ +package dev.typetype.server + +import dev.typetype.server.models.ExtractionResult +import dev.typetype.server.routes.streamRoutes +import dev.typetype.server.services.AuthService +import dev.typetype.server.services.BlockedService +import dev.typetype.server.services.StreamService +import io.ktor.client.request.get +import io.ktor.client.request.headers +import io.ktor.client.statement.bodyAsText +import io.ktor.http.HttpHeaders +import io.ktor.http.HttpStatusCode +import io.ktor.serialization.kotlinx.json.json +import io.ktor.server.application.install +import io.ktor.server.plugins.contentnegotiation.ContentNegotiation +import io.ktor.server.routing.routing +import io.ktor.server.testing.testApplication +import io.mockk.coEvery +import io.mockk.mockk +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertFalse +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.BeforeAll +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test + +class BlockedStreamRoutesTest { + private val auth = AuthService.fixed(TEST_USER_ID) + private val blocked = BlockedService() + private val streams: StreamService = mockk() + + companion object { + @BeforeAll + @JvmStatic + fun initDb() { + TestDatabase.setup() + } + } + + @BeforeEach + fun clean() { + TestDatabase.truncateAll() + } + + @Test + fun `blocked video cannot be extracted through an equivalent url`() = testApplication { + blocked.addVideo(TEST_USER_ID, "https://www.youtube.com/watch?v=blocked-video") + application { installRoutes() } + + val response = get("https://youtu.be/blocked-video") + + assertEquals(HttpStatusCode.Forbidden, response.status) + assertTrue(response.bodyAsText().contains("\"code\":\"content_blocked\"")) + } + + @Test + fun `blocked channel cannot be extracted`() = testApplication { + blocked.addChannel(TEST_USER_ID, "https://www.youtube.com/@blocked", "Blocked") + coEvery { streams.getStreamInfo(any()) } returns ExtractionResult.Success( + sabrResponse().copy( + uploaderName = "Blocked", + uploaderUrl = "https://m.youtube.com/@blocked/", + ), + ) + application { installRoutes() } + + val response = get("https://youtube.com/watch?v=channel-video") + + assertEquals(HttpStatusCode.Forbidden, response.status) + assertTrue(response.bodyAsText().contains("\"code\":\"content_blocked\"")) + } + + @Test + fun `authenticated stream filters blocked related videos and disables shared caching`() = + testApplication { + blocked.addChannel(TEST_USER_ID, "https://youtube.com/@blocked", "Blocked") + coEvery { streams.getStreamInfo(any()) } returns ExtractionResult.Success( + sabrResponse().copy( + relatedStreams = listOf( + testVideoItem().copy( + title = "Hidden", + uploaderName = "Blocked", + uploaderUrl = "https://www.youtube.com/@blocked", + ), + testVideoItem().copy(title = "Visible", url = "https://youtube.com/watch?v=visible-video"), + ), + ), + ) + application { installRoutes() } + + val response = get("https://youtube.com/watch?v=source-video") + val body = response.bodyAsText() + + assertEquals(HttpStatusCode.OK, response.status) + assertEquals("no-store", response.headers[HttpHeaders.CacheControl]) + assertFalse(body.contains("\"title\":\"Hidden\"")) + assertTrue(body.contains("\"title\":\"Visible\"")) + } + + private fun io.ktor.server.application.Application.installRoutes() { + install(ContentNegotiation) { json() } + routing { + streamRoutes( + streamService = streams, + authService = auth, + blockedService = blocked, + ) + } + } + + private suspend fun io.ktor.server.testing.ApplicationTestBuilder.get(url: String) = + client.get("/streams/youtube/sabr?url=${java.net.URLEncoder.encode(url, Charsets.UTF_8)}") { + headers.append(HttpHeaders.Authorization, "Bearer test-jwt") + } + + private fun sabrResponse() = testStreamResponse( + videoOnlyStreams = listOf(testVideoStream().copy(deliveryMethod = "sabr")), + audioStreams = listOf(testAudioStream(deliveryMethod = "sabr")), + ) +} diff --git a/src/test/kotlin/dev/typetype/server/ExtractionTest.kt b/src/test/kotlin/dev/typetype/server/ExtractionTest.kt index 52483ea5..858fe6a2 100644 --- a/src/test/kotlin/dev/typetype/server/ExtractionTest.kt +++ b/src/test/kotlin/dev/typetype/server/ExtractionTest.kt @@ -27,7 +27,7 @@ private object NoOpCache : CacheService { class ExtractionTest { private val service = PipePipeStreamService(NoOpCache, YouTubeSubtitleService(OkHttpClient(), "http://localhost:8081"), BilibiliRelatedService()) - private val safariClassicService = YoutubePlayerClientStreamService(service, YoutubePlayerClient.WEB_SAFARI) + private val visionOsClassicService = YoutubePlayerClientStreamService(service, YoutubePlayerClient.VISIONOS) @BeforeAll fun setup() { @@ -35,8 +35,8 @@ class ExtractionTest { } @Test - fun `YouTube classic Safari path has playable streams without SABR`() = kotlinx.coroutines.runBlocking { - val result = safariClassicService.getStreamInfo("https://www.youtube.com/watch?v=dQw4w9WgXcQ") + fun `YouTube classic visionOS path has playable streams without SABR`() = kotlinx.coroutines.runBlocking { + val result = visionOsClassicService.getStreamInfo("https://www.youtube.com/watch?v=dQw4w9WgXcQ") assertTrue(result is ExtractionResult.Success) val data = (result as ExtractionResult.Success).data val streams = data.videoStreams + data.videoOnlyStreams diff --git a/src/test/kotlin/dev/typetype/server/FakeCacheService.kt b/src/test/kotlin/dev/typetype/server/FakeCacheService.kt index 89b43033..ac2319ec 100644 --- a/src/test/kotlin/dev/typetype/server/FakeCacheService.kt +++ b/src/test/kotlin/dev/typetype/server/FakeCacheService.kt @@ -1,9 +1,10 @@ package dev.typetype.server import dev.typetype.server.cache.CacheService +import java.util.concurrent.ConcurrentHashMap class FakeCacheService : CacheService { - private val values = mutableMapOf() + private val values = ConcurrentHashMap() override suspend fun get(key: String): String? = values[key] diff --git a/src/test/kotlin/dev/typetype/server/HomeRecommendationRoutesTest.kt b/src/test/kotlin/dev/typetype/server/HomeRecommendationRoutesTest.kt index 57ceb9ea..027baf2c 100644 --- a/src/test/kotlin/dev/typetype/server/HomeRecommendationRoutesTest.kt +++ b/src/test/kotlin/dev/typetype/server/HomeRecommendationRoutesTest.kt @@ -60,7 +60,7 @@ class HomeRecommendationRoutesTest { private fun withApp(block: suspend ApplicationTestBuilder.() -> Unit) = testApplication { application { install(ContentNegotiation) { json() } - routing { homeRecommendationRoutes(service, auth) } + routing { homeRecommendationRoutes(service, auth, resolverDeps.blockedService) } } block() } diff --git a/src/test/kotlin/dev/typetype/server/HomeRecommendationRoutesValidationTest.kt b/src/test/kotlin/dev/typetype/server/HomeRecommendationRoutesValidationTest.kt index 29efdf8d..502d9a57 100644 --- a/src/test/kotlin/dev/typetype/server/HomeRecommendationRoutesValidationTest.kt +++ b/src/test/kotlin/dev/typetype/server/HomeRecommendationRoutesValidationTest.kt @@ -57,7 +57,7 @@ class HomeRecommendationRoutesValidationTest { private fun withApp(block: suspend ApplicationTestBuilder.() -> Unit) = testApplication { application { install(ContentNegotiation) { json() } - routing { homeRecommendationRoutes(service, auth) } + routing { homeRecommendationRoutes(service, auth, resolverDeps.blockedService) } } block() } diff --git a/src/test/kotlin/dev/typetype/server/HomeRecommendationShortsDebugRoutesTest.kt b/src/test/kotlin/dev/typetype/server/HomeRecommendationShortsDebugRoutesTest.kt index bbe768d2..0a36b92e 100644 --- a/src/test/kotlin/dev/typetype/server/HomeRecommendationShortsDebugRoutesTest.kt +++ b/src/test/kotlin/dev/typetype/server/HomeRecommendationShortsDebugRoutesTest.kt @@ -60,7 +60,7 @@ class HomeRecommendationShortsDebugRoutesTest { fun `shorts endpoint returns source debug payload when enabled`() = testApplication { application { install(ContentNegotiation) { json() } - routing { homeRecommendationShortsRoutes(service, auth) } + routing { homeRecommendationShortsRoutes(service, auth, resolverDeps.blockedService) } } val response = client.get("/recommendations/shorts?limit=5&debug=true") { headers.append(HttpHeaders.Authorization, "Bearer test-jwt") diff --git a/src/test/kotlin/dev/typetype/server/HomeRecommendationShortsRoutesTest.kt b/src/test/kotlin/dev/typetype/server/HomeRecommendationShortsRoutesTest.kt index 4cdc03a5..fafcf6d9 100644 --- a/src/test/kotlin/dev/typetype/server/HomeRecommendationShortsRoutesTest.kt +++ b/src/test/kotlin/dev/typetype/server/HomeRecommendationShortsRoutesTest.kt @@ -62,7 +62,7 @@ class HomeRecommendationShortsRoutesTest { private fun withApp(block: suspend ApplicationTestBuilder.() -> Unit) = testApplication { application { install(ContentNegotiation) { json() } - routing { homeRecommendationShortsRoutes(service, auth) } + routing { homeRecommendationShortsRoutes(service, auth, resolverDeps.blockedService) } } block() } diff --git a/src/test/kotlin/dev/typetype/server/SearchFilterMappersTest.kt b/src/test/kotlin/dev/typetype/server/SearchFilterMappersTest.kt new file mode 100644 index 00000000..1356f083 --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/SearchFilterMappersTest.kt @@ -0,0 +1,114 @@ +package dev.typetype.server + +import dev.typetype.server.models.ExtractionResult +import dev.typetype.server.services.PipePipeSearchService +import dev.typetype.server.services.SearchFilterResolution +import dev.typetype.server.services.VALID_SERVICE_IDS +import dev.typetype.server.services.defaultSearchFilter +import dev.typetype.server.services.defaultSearchFilters +import dev.typetype.server.services.findSearchFilter +import dev.typetype.server.services.resolveSearchFilters +import dev.typetype.server.services.toSearchContentFilterOptions +import dev.typetype.server.services.toSearchFilterGroups +import kotlinx.coroutines.test.runTest +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertFalse +import org.junit.jupiter.api.Assertions.assertInstanceOf +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test +import org.schabi.newpipe.extractor.NewPipe +import org.schabi.newpipe.extractor.services.youtube.search.filter.YoutubeSearchSortFilter +import org.schabi.newpipe.extractor.services.youtube.search.filter.protobuf.DateFilter +import org.schabi.newpipe.extractor.services.youtube.search.filter.protobuf.LenFilter +import org.schabi.newpipe.extractor.services.youtube.search.filter.protobuf.SortOrder +import org.schabi.newpipe.extractor.services.youtube.search.filter.protobuf.TypeFilter + +class SearchFilterMappersTest { + private val factory = NewPipe.getService(0).searchQHFactory + private val content = factory.availableContentFilter.defaultSearchFilter() + private val filters = factory.getContentFilterSortFilterVariant(content.first().identifier) + + @Test + fun `YouTube filters expose exclusive and multi-select groups`() { + val groups = filters.toSearchFilterGroups() + + assertEquals(listOf("sortby", "upload_date", "duration", "features"), groups.map { it.label }) + assertFalse(groups.first { it.label == "duration" }.multiSelect) + assertTrue(groups.first { it.label == "features" }.multiSelect) + assertTrue(groups.first { it.label == "sortby" }.options.first().isDefault) + assertFalse(groups.first { it.label == "features" }.options.first().isDefault) + } + + @Test + fun `YouTube filters resolve selections across groups`() { + val groups = filters.toSearchFilterGroups() + val values = listOf( + groups.option("sortby", "sort_view"), + groups.option("upload_date", "past_week"), + groups.option("duration", "short_video"), + groups.option("features", "HD"), + groups.option("features", "Subtitles"), + ) + + val resolution = filters.resolveSearchFilters(values) + + assertInstanceOf(SearchFilterResolution.Valid::class.java, resolution) + assertEquals(5, (resolution as SearchFilterResolution.Valid).items.size) + val url = factory.fromQuery("kotlin", content, resolution.items).url + val request = YoutubeSearchSortFilter().decodeSp(url.substringAfter("&sp=")) + + assertEquals(SortOrder.views.value.toLong(), request.sorted) + assertEquals(DateFilter.week.value.toLong(), request.filter.date) + assertEquals(LenFilter.duration_short.value.toLong(), request.filter.length) + assertTrue(request.filter.is_hd) + assertTrue(request.filter.subtitles) + } + + @Test + fun `YouTube content filters retain their type with default selections`() { + val contentFilters = factory.availableContentFilter + val videoValue = contentFilters.toSearchContentFilterOptions() + .first { it.label.endsWith("videos") } + .value + val videoContent = contentFilters.findSearchFilter(videoValue) + val videoFilters = factory.getContentFilterSortFilterVariant(videoContent.first().identifier) + + val url = factory.fromQuery("kotlin", videoContent, videoFilters.defaultSearchFilters()).url + val request = YoutubeSearchSortFilter().decodeSp(url.substringAfter("&sp=")) + + assertEquals(TypeFilter.video.value.toLong(), request.filter.type) + } + + @Test + fun `YouTube filters reject conflicting exclusive selections`() { + val sortOptions = filters.toSearchFilterGroups().first { it.label == "sortby" }.options + + val resolution = filters.resolveSearchFilters(sortOptions.take(2).map { it.value }) + + assertInstanceOf(SearchFilterResolution.Invalid::class.java, resolution) + assertTrue((resolution as SearchFilterResolution.Invalid).message.contains("sortby")) + } + + @Test + fun `YouTube filters reject unknown selections`() { + val resolution = filters.resolveSearchFilters(listOf("unknown")) + + assertInstanceOf(SearchFilterResolution.Invalid::class.java, resolution) + } + + @Test + fun `all supported services expose filter capabilities`() = runTest { + val service = PipePipeSearchService() + + VALID_SERVICE_IDS.forEach { serviceId -> + assertInstanceOf( + ExtractionResult.Success::class.java, + service.filters(serviceId, null), + "service $serviceId", + ) + } + } +} + +private fun List.option(group: String, label: String): String = + first { it.label == group }.options.first { it.label == label }.value diff --git a/src/test/kotlin/dev/typetype/server/SearchFiltersRoutesTest.kt b/src/test/kotlin/dev/typetype/server/SearchFiltersRoutesTest.kt index a9f6e521..2d020801 100644 --- a/src/test/kotlin/dev/typetype/server/SearchFiltersRoutesTest.kt +++ b/src/test/kotlin/dev/typetype/server/SearchFiltersRoutesTest.kt @@ -1,6 +1,7 @@ package dev.typetype.server import dev.typetype.server.models.ExtractionResult +import dev.typetype.server.models.SearchFilterGroup import dev.typetype.server.models.SearchFilterOption import dev.typetype.server.models.SearchFiltersResponse import dev.typetype.server.models.SearchPageResponse @@ -26,10 +27,18 @@ class SearchFiltersRoutesTest { @Test fun `GET search filters returns supported filters`() = withApp { - coEvery { searchService.filters(0) } returns ExtractionResult.Success( + coEvery { searchService.filters(0, null) } returns ExtractionResult.Success( SearchFiltersResponse( contentFilters = listOf(SearchFilterOption(value = "type|1|Videos", label = "Type: Videos")), sortFilters = listOf(SearchFilterOption(value = "sort|2|Upload date", label = "Sort: Upload date")), + filterGroups = listOf( + SearchFilterGroup( + key = "sort|2", + label = "Sort", + multiSelect = false, + options = listOf(SearchFilterOption(value = "sort|2|Upload date", label = "Upload date")), + ) + ), ) ) val response = client.get("/search/filters?service=0") @@ -37,6 +46,23 @@ class SearchFiltersRoutesTest { assertEquals(HttpStatusCode.OK, response.status) assertTrue(body.contains("contentFilters")) assertTrue(body.contains("Upload date")) + assertTrue(body.contains("filterGroups")) + } + + @Test + fun `GET search filters passes selected content filter`() = withApp { + coEvery { searchService.filters(0, "content") } returns ExtractionResult.Success( + SearchFiltersResponse( + contentFilters = emptyList(), + sortFilters = emptyList(), + filterGroups = emptyList(), + ) + ) + + val response = client.get("/search/filters?service=0&contentFilter=content") + + assertEquals(HttpStatusCode.OK, response.status) + coVerify { searchService.filters(0, "content") } } @Test @@ -44,9 +70,11 @@ class SearchFiltersRoutesTest { coEvery { searchService.search(any(), any(), any(), any(), any()) } returns ExtractionResult.Success( SearchPageResponse(items = emptyList(), nextpage = null, searchSuggestion = null, isCorrectedSearch = false) ) - val response = client.get("/search?q=test&service=0&contentFilter=content&sortFilter=sort") + val response = client.get( + "/search?q=test&service=0&contentFilter=content&filter=views&filter=short&sortFilter=legacy" + ) assertEquals(HttpStatusCode.OK, response.status) - coVerify { searchService.search("test", 0, null, "content", "sort") } + coVerify { searchService.search("test", 0, null, "content", listOf("views", "short", "legacy")) } } private fun withApp(block: suspend io.ktor.server.testing.ApplicationTestBuilder.() -> Unit) = testApplication { diff --git a/src/test/kotlin/dev/typetype/server/TypetypeTokenYoutubeSessionPoTokenProviderTest.kt b/src/test/kotlin/dev/typetype/server/TypetypeTokenYoutubeSessionPoTokenProviderTest.kt index b97ce165..1d9638f9 100644 --- a/src/test/kotlin/dev/typetype/server/TypetypeTokenYoutubeSessionPoTokenProviderTest.kt +++ b/src/test/kotlin/dev/typetype/server/TypetypeTokenYoutubeSessionPoTokenProviderTest.kt @@ -24,7 +24,7 @@ class TypetypeTokenYoutubeSessionPoTokenProviderTest { visitorDataFetcher = { _, _ -> calls += 1; "unused" }, ) - val result = provider.getSessionPoToken("WEB", localization, country, false) + val result = provider.getSessionPoToken("WEB", "1.0", "test-user-agent", localization, country, false) assertNull(result) assertEquals(0, calls) @@ -40,8 +40,8 @@ class TypetypeTokenYoutubeSessionPoTokenProviderTest { visitorDataFetcher = { _, _ -> visitorCalls += 1; "visitor-one" }, ) - val first = provider.getSessionPoToken("TV", localization, country, true) - val second = provider.getSessionPoToken("WEB", localization, country, true) + val first = provider.getSessionPoToken("TV", "1.0", "test-user-agent", localization, country, true) + val second = provider.getSessionPoToken("WEB", "2.0", "test-user-agent", localization, country, true) assertEquals("visitor-one", first?.visitorData) assertEquals("token-for-visitor-one", first?.poToken) @@ -58,10 +58,10 @@ class TypetypeTokenYoutubeSessionPoTokenProviderTest { visitorDataFetcher = { _, _ -> "visitor-${++index}" }, ) ServiceList.YouTube.setTokens("SID=one; SAPISID=one") - val first = provider.getSessionPoToken("WEB", localization, country, true) + val first = provider.getSessionPoToken("WEB", "1.0", "test-user-agent", localization, country, true) ServiceList.YouTube.setTokens("SID=two; SAPISID=two") - val second = provider.getSessionPoToken("WEB", localization, country, true) + val second = provider.getSessionPoToken("WEB", "1.0", "test-user-agent", localization, country, true) assertEquals("visitor-1", first?.visitorData) assertEquals("visitor-2", second?.visitorData) diff --git a/src/test/kotlin/dev/typetype/server/YouTubeSubtitleProxyRoutesTest.kt b/src/test/kotlin/dev/typetype/server/YouTubeSubtitleProxyRoutesTest.kt new file mode 100644 index 00000000..2bca9ee5 --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/YouTubeSubtitleProxyRoutesTest.kt @@ -0,0 +1,144 @@ +package dev.typetype.server + +import dev.typetype.server.routes.proxyRoutes +import dev.typetype.server.routes.youtubeSubtitleRoutes +import dev.typetype.server.services.ProxyService +import dev.typetype.server.services.ResolvedYouTubeSubtitle +import dev.typetype.server.services.YouTubeSubtitleCache +import dev.typetype.server.services.YouTubeSubtitleDeliveryService +import dev.typetype.server.services.YouTubeSubtitleFetchResult +import dev.typetype.server.services.YouTubeSubtitleResolution +import io.ktor.client.request.get +import io.ktor.client.request.header +import io.ktor.client.request.parameter +import io.ktor.client.statement.bodyAsText +import io.ktor.http.HttpHeaders +import io.ktor.http.HttpStatusCode +import io.ktor.serialization.kotlinx.json.json +import io.ktor.server.application.install +import io.ktor.server.plugins.contentnegotiation.ContentNegotiation +import io.ktor.server.routing.routing +import io.ktor.server.testing.testApplication +import io.mockk.coVerify +import io.mockk.mockk +import kotlinx.serialization.json.Json +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test + +class YouTubeSubtitleProxyRoutesTest { + private val proxyService: ProxyService = mockk(relaxed = true) + + @Test + fun `dedicated YouTube subtitle route returns cacheable WebVTT`() = testApplication { + val service = subtitleService(YouTubeSubtitleFetchResult.Ready(VTT)) + application { + install(ContentNegotiation) { json() } + routing { youtubeSubtitleRoutes(service) } + } + + val response = client.get("/subtitles/youtube/abcdefghijk") { + parameter("language", "en") + parameter("variant", "manual") + parameter("format", "vtt") + } + + assertEquals(HttpStatusCode.OK, response.status) + assertTrue(response.headers[HttpHeaders.ContentType]?.startsWith("text/vtt") == true) + assertEquals("public, max-age=21600, stale-while-revalidate=3600", response.headers[HttpHeaders.CacheControl]) + assertTrue(response.bodyAsText().startsWith("WEBVTT")) + } + + @Test + fun `live YouTube subtitle route disables response caching`() = testApplication { + val service = subtitleService(YouTubeSubtitleFetchResult.Ready(VTT), isLive = true) + application { + install(ContentNegotiation) { json() } + routing { youtubeSubtitleRoutes(service) } + } + + val response = client.get("/subtitles/youtube/abcdefghijk") { + parameter("language", "en") + parameter("variant", "auto") + } + + assertEquals(HttpStatusCode.OK, response.status) + assertEquals("no-store", response.headers[HttpHeaders.CacheControl]) + } + + @Test + fun `legacy timed text proxy uses dedicated subtitle delivery`() = testApplication { + val service = subtitleService(YouTubeSubtitleFetchResult.Ready(VTT)) + application { + install(ContentNegotiation) { json() } + routing { proxyRoutes(proxyService, service) } + } + + val response = client.get("/proxy") { + parameter("url", "https://www.youtube.com/api/timedtext?v=abcdefghijk&lang=en") + } + + assertEquals(HttpStatusCode.OK, response.status) + assertTrue(response.bodyAsText().startsWith("WEBVTT")) + coVerify(exactly = 0) { proxyService.pipe(any(), any(), any()) } + } + + @Test + fun `YouTube subtitle throttle returns a typed request id error`() = testApplication { + val service = subtitleService(YouTubeSubtitleFetchResult.Throttled) + application { + installRequestObservability() + install(ContentNegotiation) { json(Json { encodeDefaults = true }) } + configureCompression() + configureStatusPages() + routing { youtubeSubtitleRoutes(service) } + } + + val response = client.get("/subtitles/youtube/abcdefghijk") { + header(REQUEST_ID_HEADER, "subtitle-request-123") + header(HttpHeaders.AcceptEncoding, "gzip") + parameter("language", "en") + parameter("variant", "manual") + } + + assertEquals(HttpStatusCode.TooManyRequests, response.status) + assertEquals("subtitle-request-123", response.headers[REQUEST_ID_HEADER]) + assertEquals(null, response.headers[HttpHeaders.ContentEncoding]) + val body = response.bodyAsText() + assertTrue(body.contains("\"code\":\"subtitle_upstream_throttled\"")) + assertTrue(body.contains("\"requestId\":\"subtitle-request-123\"")) + } + + @Test + fun `invalid dedicated subtitle selection returns typed bad request`() = testApplication { + val service = subtitleService(YouTubeSubtitleFetchResult.Ready(VTT)) + application { + installRequestObservability() + install(ContentNegotiation) { json(Json { encodeDefaults = true }) } + configureStatusPages() + routing { youtubeSubtitleRoutes(service) } + } + + val response = client.get("/subtitles/youtube/not-valid") { + parameter("language", "en") + parameter("variant", "manual") + } + + assertEquals(HttpStatusCode.BadRequest, response.status) + assertTrue(response.bodyAsText().contains("\"code\":\"subtitle_request_invalid\"")) + } + + private fun subtitleService( + fetchResult: YouTubeSubtitleFetchResult, + isLive: Boolean = false, + ) = YouTubeSubtitleDeliveryService( + resolver = { YouTubeSubtitleResolution.Ready(ResolvedYouTubeSubtitle(TIMED_TEXT_URL, true, isLive)) }, + fetcher = { _, _ -> fetchResult }, + cache = YouTubeSubtitleCache(null), + ) + + private companion object { + val VTT = "WEBVTT\n\n00:00.000 --> 00:01.000\nHello".encodeToByteArray() + const val TIMED_TEXT_URL = "https://www.youtube.com/api/timedtext?v=abcdefghijk&lang=en&fmt=vtt" + } +} diff --git a/src/test/kotlin/dev/typetype/server/YoutubeAuthenticatedExtractionProbeTest.kt b/src/test/kotlin/dev/typetype/server/YoutubeAuthenticatedExtractionProbeTest.kt index ffe357b0..0f01f486 100644 --- a/src/test/kotlin/dev/typetype/server/YoutubeAuthenticatedExtractionProbeTest.kt +++ b/src/test/kotlin/dev/typetype/server/YoutubeAuthenticatedExtractionProbeTest.kt @@ -53,12 +53,14 @@ class YoutubeAuthenticatedExtractionProbeTest { ) val service = YoutubePlayerClientFallbackStreamService( pipePipe, - listOf(YoutubePlayerClient.TV_DOWNGRADED, YoutubePlayerClient.WEB_SAFARI), + listOf(YoutubePlayerClient.TV_DOWNGRADED, YoutubePlayerClient.VISIONOS), ) val result = YoutubeSessionTokenScope.withCredentials(credentials) { val token = NewPipe.getYoutubeSessionPoTokenProvider()?.getSessionPoToken( "TV", + "1.0", + "test-user-agent", localization, contentCountry, true, diff --git a/src/test/kotlin/dev/typetype/server/YoutubeProxySelectorTest.kt b/src/test/kotlin/dev/typetype/server/YoutubeProxySelectorTest.kt index e41bd942..fb0f5be5 100644 --- a/src/test/kotlin/dev/typetype/server/YoutubeProxySelectorTest.kt +++ b/src/test/kotlin/dev/typetype/server/YoutubeProxySelectorTest.kt @@ -22,11 +22,9 @@ class YoutubeProxySelectorTest { "https://i.ytimg.com/vi/id/hqdefault.jpg", "https://yt3.googleusercontent.com/avatar", ).forEach { url -> - val proxies = selector.select(URI(url)) - val proxy = proxies.first() + val proxy = selector.select(URI(url)).single() assertEquals(Proxy.Type.HTTP, proxy.type()) assertEquals(InetSocketAddress.createUnresolved("proxy.internal", 8080), proxy.address()) - assertEquals(Proxy.NO_PROXY, proxies.last()) } } diff --git a/src/test/kotlin/dev/typetype/server/YoutubeTakeoutParserServiceTest.kt b/src/test/kotlin/dev/typetype/server/YoutubeTakeoutParserServiceTest.kt index 55970acc..5b960a53 100644 --- a/src/test/kotlin/dev/typetype/server/YoutubeTakeoutParserServiceTest.kt +++ b/src/test/kotlin/dev/typetype/server/YoutubeTakeoutParserServiceTest.kt @@ -137,6 +137,47 @@ class YoutubeTakeoutParserServiceTest { Files.deleteIfExists(zip) } + @Test + fun `parse omits unavailable videos from takeout collections and activity`() { + val zip = Files.createTempFile("yt-takeout-unavailable-", ".zip") + ZipOutputStream(Files.newOutputStream(zip)).use { out -> + out.writeEntry( + "Takeout/YouTube/playlists/playlists.csv", + "Playlist ID,Playlist Title\nPL123456,Imported\n", + ) + out.writeEntry( + "Takeout/YouTube/playlists/Imported.csv", + "Video ID,Video Title\nkeep000001,Available title\ngone000001,Deleted video\n", + ) + out.writeEntry( + "Takeout/YouTube/playlists/Watch later.csv", + "Video ID,Video Title\nkeep000002,Another title\ngone000002,Vidéo privée\n", + ) + out.writeEntry( + "Takeout/YouTube/playlists/Liked videos.csv", + "Video ID,Video Title\nkeep000003,Liked title\ngone000003,Video no disponible\n", + ) + out.writeEntry( + "Takeout/My Activity/YouTube/watch-history.html", + """ + You watched Watched title
+ 1 Jan 2026, 12:00:00 CET
+ You watched Video unavailable
+ 1 Jan 2026, 13:00:00 CET
+ """.trimIndent(), + ) + } + + val parsed = YoutubeTakeoutParserService().parse(zip) + + assertEquals(listOf("keep000001"), parsed.playlistItems["Imported"]?.map { it.url.substringAfter("v=") }) + assertEquals(listOf("keep000002"), parsed.watchLater.map { it.url.substringAfter("v=") }) + assertEquals(listOf("keep000003"), parsed.favorites.map { it.videoUrl.substringAfter("v=") }) + assertEquals(listOf("keep000004"), parsed.history.map { it.url.substringAfter("v=") }) + assertTrue(parsed.errors.isEmpty()) + Files.deleteIfExists(zip) + } + private fun createZip(): Path { val zip = Files.createTempFile("yt-takeout-parser-", ".zip") ZipOutputStream(Files.newOutputStream(zip)).use { out -> @@ -158,4 +199,10 @@ class YoutubeTakeoutParserServiceTest { } return zip } + + private fun ZipOutputStream.writeEntry(path: String, content: String) { + putNextEntry(ZipEntry(path)) + write(content.toByteArray()) + closeEntry() + } } diff --git a/src/test/kotlin/dev/typetype/server/routes/SabrPlaybackRecoveryTest.kt b/src/test/kotlin/dev/typetype/server/routes/SabrPlaybackRecoveryTest.kt index 181e1aa8..0cc7b3d7 100644 --- a/src/test/kotlin/dev/typetype/server/routes/SabrPlaybackRecoveryTest.kt +++ b/src/test/kotlin/dev/typetype/server/routes/SabrPlaybackRecoveryTest.kt @@ -102,4 +102,17 @@ class SabrPlaybackRecoveryTest { assertEquals(emptyList(), recovery.retryVideoItags()) coVerify(exactly = 1) { store.recoverProtectedPlaybackInfo(holder) } } + + @Test + fun `attestation rejection refreshes context and requests fresh session`() = runTest { + val holder = mockk() + val store = mockk() + every { holder.terminalFailure() } returns + "SABR error: SABR attestation required: status=3, policy=true" + every { holder.key } returns SabrSessionKey("video", "user", 140, null, 137, 900_000L) + coEvery { store.recoverProtectedPlaybackInfo(holder) } returns Unit + + assertEquals("retry_fresh_session", SabrPlaybackRecovery(store).action(holder)) + coVerify(exactly = 1) { store.recoverProtectedPlaybackInfo(holder) } + } } diff --git a/src/test/kotlin/dev/typetype/server/services/AuthSessionConfigTest.kt b/src/test/kotlin/dev/typetype/server/services/AuthSessionConfigTest.kt new file mode 100644 index 00000000..859a731e --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/services/AuthSessionConfigTest.kt @@ -0,0 +1,40 @@ +package dev.typetype.server.services + +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertFalse +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test + +class AuthSessionConfigTest { + @Test + fun `defaults keep secure thirty day sessions`() { + val config = AuthSessionConfig.fromEnvironment { null } + + assertEquals(30L, config.refreshTtlDays) + assertFalse(config.allowInsecureCookies) + } + + @Test + fun `session duration is bounded`() { + val belowMinimum = AuthSessionConfig.fromEnvironment { name -> + if (name == "AUTH_SESSION_TTL_DAYS") "0" else null + } + val aboveMaximum = AuthSessionConfig.fromEnvironment { name -> + if (name == "AUTH_SESSION_TTL_DAYS") "900" else null + } + + assertEquals(1L, belowMinimum.refreshTtlDays) + assertEquals(365L, aboveMaximum.refreshTtlDays) + } + + @Test + fun `insecure cookies require an explicit enabled value`() { + val disabled = AuthSessionConfig.fromEnvironment { "unexpected" } + val enabled = AuthSessionConfig.fromEnvironment { name -> + if (name == "AUTH_ALLOW_INSECURE_COOKIES") "yes" else null + } + + assertFalse(disabled.allowInsecureCookies) + assertTrue(enabled.allowInsecureCookies) + } +} diff --git a/src/test/kotlin/dev/typetype/server/services/BlockedContentProfileTest.kt b/src/test/kotlin/dev/typetype/server/services/BlockedContentProfileTest.kt new file mode 100644 index 00000000..198aa1d3 --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/services/BlockedContentProfileTest.kt @@ -0,0 +1,64 @@ +package dev.typetype.server.services + +import dev.typetype.server.models.BlockedItem +import dev.typetype.server.models.BlockedKeywordItem +import org.junit.jupiter.api.Assertions.assertFalse +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test + +class BlockedContentProfileTest { + @Test + fun `matches equivalent youtube video urls`() { + val profile = profile( + videos = listOf( + BlockedItem(url = "https://www.youtube.com/watch?v=AbC_123-xyZ", blockedAt = 1), + ), + ) + + assertTrue(profile.blocksVideo("https://youtu.be/AbC_123-xyZ?t=12")) + assertTrue(profile.blocksVideo("https://m.youtube.com/shorts/AbC_123-xyZ")) + } + + @Test + fun `matches youtube channel hosts and normalized names`() { + val profile = profile( + channels = listOf( + BlockedItem( + url = "http://www.youtube.com/@Example/?view=0", + name = "Ｔｅｓｔ Channel", + blockedAt = 1, + ), + ), + ) + + assertTrue(profile.blocksChannel("https://m.youtube.com/@Example/", "Other")) + assertTrue(profile.blocksChannel("", "test channel")) + } + + @Test + fun `filters video url channel and keyword independently`() { + val profile = profile( + videos = listOf(BlockedItem(url = "https://youtube.com/watch?v=blocked-video", blockedAt = 1)), + channels = listOf(BlockedItem("https://youtube.com/@blocked", "Blocked", null, 1)), + keywords = listOf(BlockedKeywordItem("spoiler", 1)), + ) + + assertFalse(profile.allowsVideo("https://youtu.be/blocked-video", "One", "", "")) + assertFalse( + profile.allowsVideo( + "https://youtube.com/watch?v=other-video", + "Two", + "https://www.youtube.com/@blocked", + "Blocked", + ), + ) + assertFalse(profile.allowsVideo("https://youtube.com/watch?v=third-video", "A spoiler", "", "")) + assertTrue(profile.allowsVideo("https://youtube.com/watch?v=visible-video", "Visible", "", "")) + } + + private fun profile( + videos: List = emptyList(), + channels: List = emptyList(), + keywords: List = emptyList(), + ) = BlockedContentProfile(videos, channels, keywords) +} diff --git a/src/test/kotlin/dev/typetype/server/services/BlockedHomeRecommendationsFilterTest.kt b/src/test/kotlin/dev/typetype/server/services/BlockedHomeRecommendationsFilterTest.kt new file mode 100644 index 00000000..d2ec5ab7 --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/services/BlockedHomeRecommendationsFilterTest.kt @@ -0,0 +1,49 @@ +package dev.typetype.server.services + +import dev.typetype.server.models.BlockedItem +import dev.typetype.server.models.BlockedKeywordItem +import dev.typetype.server.models.HomeRecommendationsResponse +import dev.typetype.server.models.VideoItem +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Test + +class BlockedHomeRecommendationsFilterTest { + @Test + fun `filters cached recommendations with current blocked profile`() { + val response = HomeRecommendationsResponse( + items = listOf( + video("blocked-video", "Keep", "Allowed", "channel:allowed"), + video("allowed", "Keep", "Blocked", "channel:blocked"), + video("keyword", "Hide this topic", "Allowed", "channel:allowed"), + video("visible", "Keep", "Allowed", "channel:allowed"), + ), + nextCursor = null, + hasMore = false, + ) + val profile = BlockedContentProfile( + videos = listOf(BlockedItem(url = "https://youtube.com/watch?v=blocked-video")), + channels = listOf(BlockedItem(url = "channel:blocked", name = "Blocked")), + keywords = listOf(BlockedKeywordItem(keyword = "hide this")), + ) + + assertEquals(listOf("visible"), response.filterBlocked(profile).items.map { it.id }) + } + + private fun video(id: String, title: String, uploaderName: String, uploaderUrl: String) = VideoItem( + id = id, + title = title, + url = "https://youtube.com/watch?v=$id", + thumbnailUrl = "", + uploaderName = uploaderName, + uploaderUrl = uploaderUrl, + uploaderAvatarUrl = "", + duration = 60, + viewCount = 0, + uploadDate = "", + uploaded = 0, + streamType = "video_stream", + isShortFormContent = false, + uploaderVerified = false, + shortDescription = null, + ) +} diff --git a/src/test/kotlin/dev/typetype/server/services/SabrAdaptiveInitializationTest.kt b/src/test/kotlin/dev/typetype/server/services/SabrAdaptiveInitializationTest.kt new file mode 100644 index 00000000..6c2f8a78 --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/services/SabrAdaptiveInitializationTest.kt @@ -0,0 +1,79 @@ +package dev.typetype.server.services + +import io.mockk.every +import io.mockk.mockk +import io.mockk.verify +import kotlinx.coroutines.runBlocking +import org.junit.jupiter.api.Assertions.assertArrayEquals +import org.junit.jupiter.api.Assertions.assertNull +import org.junit.jupiter.api.Test +import org.schabi.newpipe.extractor.localization.Localization +import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat +import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrInfo +import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrSession +import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrStreamState + +class SabrAdaptiveInitializationTest { + @Test + fun `fetches initialization range with the active streaming token`() = runBlocking { + val fixture = fixture(byteArrayOf(1, 2, 3)) + val expected = byteArrayOf(4, 5, 6) + every { + fixture.session.fetchInitializationData( + fixture.format, + any(), + 2_000L, + match { it.contentEquals(fixture.poToken) }, + ) + } returns expected + + val actual = SabrAdaptiveInitialization.fetchRange(fixture.holder, fixture.format, 2_000L) + + assertArrayEquals(expected, actual) + } + + @Test + fun `returns null when the adaptive range request fails`() = runBlocking { + val fixture = fixture(byteArrayOf(1)) + every { + fixture.session.fetchInitializationData( + fixture.format, + any(), + 2_000L, + any(), + ) + } throws java.io.IOException("range unavailable") + + assertNull(SabrAdaptiveInitialization.fetchRange(fixture.holder, fixture.format, 2_000L)) + } + + @Test + fun `does not request a range without a streaming token`() = runBlocking { + val fixture = fixture(null) + + assertNull(SabrAdaptiveInitialization.fetchRange(fixture.holder, fixture.format, 2_000L)) + verify(exactly = 0) { + fixture.session.fetchInitializationData(any(), any(), any(), any()) + } + } + + private fun fixture(poToken: ByteArray?): Fixture { + val holder = mockk() + val session = mockk() + val state = mockk() + val format = mockk() + every { holder.session } returns session + every { holder.playerContextToken } returns null + every { holder.info } returns mockk() + every { session.streamState } returns state + every { state.poToken } returns poToken + return Fixture(holder, session, format, poToken ?: byteArrayOf()) + } + + private data class Fixture( + val holder: SabrSessionHolder, + val session: YoutubeSabrSession, + val format: YoutubeSabrFormat, + val poToken: ByteArray, + ) +} diff --git a/src/test/kotlin/dev/typetype/server/services/SabrSeekRepositionPumpTest.kt b/src/test/kotlin/dev/typetype/server/services/SabrSeekRepositionPumpTest.kt index 4edb2ff2..ca8ae740 100644 --- a/src/test/kotlin/dev/typetype/server/services/SabrSeekRepositionPumpTest.kt +++ b/src/test/kotlin/dev/typetype/server/services/SabrSeekRepositionPumpTest.kt @@ -3,6 +3,7 @@ package dev.typetype.server.services import io.mockk.every import io.mockk.mockk import io.mockk.verify +import io.mockk.verifyOrder import kotlinx.coroutines.test.runTest import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Assertions.assertFalse @@ -59,6 +60,7 @@ class SabrSeekRepositionPumpTest { val request = SabrSegmentRequest.media(video, 24) val session = mockk(relaxed = true) every { session.streamState } returns mockk(relaxed = true) + every { session.requestNumber } returns 2 every { session.getCachedSegment(any()) } returns null every { session.pumpOnceStreamingForDemand(any(), request) } returns mockk(relaxed = true) val holder = holder(session, audio, video) @@ -78,6 +80,45 @@ class SabrSeekRepositionPumpTest { } } + @Test + fun `cold playback bootstraps before applying saved position`() = runTest { + SabrSegmentDemandTracker.clearAll() + try { + val audio = format(140, true) + val video = format(137, false) + val request = SabrSegmentRequest.media(video, 180) + val session = mockk(relaxed = true) + val state = mockk(relaxed = true) + var requestNumber = 0 + every { session.streamState } returns state + every { session.requestNumber } answers { requestNumber } + every { session.getCachedSegment(any()) } returns null + every { session.pumpOnceStreaming(any()) } answers { + requestNumber = 1 + 2 + } + every { state.getMaxSegment(audio) } returns 1 + every { state.getMaxSegment(video) } returns 1 + every { session.pumpOnceStreamingForDemand(any(), request) } returns mockk(relaxed = true) + val holder = holder(session, audio, video) + holder.setRequestedSeekTimeMs(900_000L) + holder.requestSegmentDemand(request) + holder.requestForwardSeek(request) + var rounds = 0 + + SabrSessionPumpLoop().run({ rounds++ < 2 }, holder, intervalMs = 0L) + + verifyOrder { + state.setPlayerTimeMs(0L) + session.pumpOnceStreaming(any()) + session.prepareForForwardJump(request, 900_000L) + session.pumpOnceStreamingForDemand(any(), request) + } + } finally { + SabrSegmentDemandTracker.clearAll() + } + } + @Test fun `forward seek discovered beyond end is discarded`() = runTest { SabrSegmentDemandTracker.clearAll() diff --git a/src/test/kotlin/dev/typetype/server/services/SabrSessionPlayerContextTest.kt b/src/test/kotlin/dev/typetype/server/services/SabrSessionPlayerContextTest.kt index 3f7d2f86..b95c4f07 100644 --- a/src/test/kotlin/dev/typetype/server/services/SabrSessionPlayerContextTest.kt +++ b/src/test/kotlin/dev/typetype/server/services/SabrSessionPlayerContextTest.kt @@ -57,6 +57,8 @@ class SabrSessionPlayerContextTest { private fun currentToken() = TypetypeYoutubeSessionPoTokenProvider.getSessionPoToken( "MWEB", + "2.20260801.00.00", + "test-user-agent", Localization("en", "US"), ContentCountry("US"), false, diff --git a/src/test/kotlin/dev/typetype/server/services/StreamYouTubeSubtitleResolverTest.kt b/src/test/kotlin/dev/typetype/server/services/StreamYouTubeSubtitleResolverTest.kt new file mode 100644 index 00000000..53713cf4 --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/services/StreamYouTubeSubtitleResolverTest.kt @@ -0,0 +1,97 @@ +package dev.typetype.server.services + +import dev.typetype.server.models.ExtractionResult +import dev.typetype.server.models.SubtitleItem +import dev.typetype.server.testStreamResponse +import kotlinx.coroutines.runBlocking +import okhttp3.HttpUrl.Companion.toHttpUrl +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertFalse +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test + +class StreamYouTubeSubtitleResolverTest { + @Test + fun `resolution uses fresh inventory and preserves live state`() = runBlocking { + val resolver = StreamYouTubeSubtitleResolver( + streamService = streamService( + listOf(track("https://www.youtube.com/api/timedtext?v=abcdefghijk&lang=de", false)), + isLive = true, + ), + fetchInventory = { + YouTubeSubtitleInventoryResult.Ready( + listOf(track("https://www.youtube.com/api/timedtext?v=abcdefghijk&lang=en", false)), + ) + }, + ) + + val result = resolver.resolve(selection(YouTubeSubtitleVariant.Manual)) + + val ready = result as YouTubeSubtitleResolution.Ready + assertTrue(ready.track.isLive) + assertEquals("en", ready.track.content.toHttpUrl().queryParameter("lang")) + } + + @Test + fun `stream inventory is used when fresh inventory is unavailable`() = runBlocking { + val resolver = StreamYouTubeSubtitleResolver( + streamService = streamService( + listOf(track("https://www.youtube.com/api/timedtext?v=abcdefghijk&lang=en", false)), + ), + fetchInventory = { YouTubeSubtitleInventoryResult.Unavailable }, + ) + + assertTrue(resolver.resolve(selection(YouTubeSubtitleVariant.Manual)) is YouTubeSubtitleResolution.Ready) + } + + @Test + fun `manual and automatic tracks remain distinct`() { + val manual = track("https://www.youtube.com/api/timedtext?v=abcdefghijk&lang=en", false) + val automatic = track("https://www.youtube.com/api/timedtext?v=abcdefghijk&lang=en&kind=asr", true) + + assertTrue(manual.matchesYouTubeSubtitle(selection(YouTubeSubtitleVariant.Manual))) + assertFalse(manual.matchesYouTubeSubtitle(selection(YouTubeSubtitleVariant.Auto))) + assertTrue(automatic.matchesYouTubeSubtitle(selection(YouTubeSubtitleVariant.Auto))) + assertFalse(automatic.matchesYouTubeSubtitle(selection(YouTubeSubtitleVariant.Manual))) + } + + @Test + fun `translated selection preserves source language and requested translation`() { + val track = track( + "https://www.youtube.com/api/timedtext?v=abcdefghijk&lang=en&kind=asr&tlang=de", + true, + ) + val selection = selection(YouTubeSubtitleVariant.Auto).copy( + language = "fr", + sourceLanguage = "en", + translationLanguage = "fr", + ) + + assertTrue(track.matchesYouTubeSubtitle(selection)) + val resolved = requireNotNull(track.contentForYouTubeSubtitle(selection)).toHttpUrl() + assertEquals("en", resolved.queryParameter("lang")) + assertEquals("fr", resolved.queryParameter("tlang")) + assertEquals("vtt", resolved.queryParameter("fmt")) + } + + private fun streamService(subtitles: List, isLive: Boolean = false) = object : StreamService { + override suspend fun getStreamInfo(url: String) = ExtractionResult.Success( + testStreamResponse().copy(subtitles = subtitles, isLive = isLive, isLiveContent = isLive), + ) + } + + private fun track(url: String, automatic: Boolean) = SubtitleItem( + url = url, + mimeType = "application/ttml+xml", + languageTag = "en", + displayLanguageName = "English", + isAutoGenerated = automatic, + ) + + private fun selection(variant: YouTubeSubtitleVariant) = YouTubeSubtitleSelection( + videoId = "abcdefghijk", + language = "en", + variant = variant, + format = YouTubeSubtitleFormat.Vtt, + ) +} diff --git a/src/test/kotlin/dev/typetype/server/services/TypetypeTokenSabrTokenClientTest.kt b/src/test/kotlin/dev/typetype/server/services/TypetypeTokenSabrTokenClientTest.kt index a713f17d..54b3a704 100644 --- a/src/test/kotlin/dev/typetype/server/services/TypetypeTokenSabrTokenClientTest.kt +++ b/src/test/kotlin/dev/typetype/server/services/TypetypeTokenSabrTokenClientTest.kt @@ -48,18 +48,18 @@ class TypetypeTokenSabrTokenClientTest { } @Test - fun providerForceRefreshUsesVideoRefreshOnly(): Unit { + fun providerFetchesVideoTokenWithoutRefresh(): Unit { val recorder = PotokenRequestRecorder() val client = TypetypeTokenSabrTokenClient("https://token.example", recorder.client) val provider = TypetypeTokenSabrPoTokenProvider(client) - val token = provider.getPoToken(info("visitor"), mockk(), true) + val token = provider.getPoToken(info("visitor"), mockk()) assertNotNull(token) assertArrayEquals(byteArrayOf(2), token) val url = recorder.urls.single() assertEquals("video", url.queryParameter("videoId")) assertNull(url.queryParameter("refresh")) - assertEquals("true", url.queryParameter("refreshVideo")) + assertNull(url.queryParameter("refreshVideo")) } @Test @@ -70,13 +70,13 @@ class TypetypeTokenSabrTokenClientTest { ) val error = assertThrows(SabrRecoverableException::class.java) { - provider.getPoToken(info("visitor"), mockk(), true) + provider.getPoToken(info("visitor"), mockk()) } assertEquals(SABR_TOKEN_BINDING_FAILURE, error.message) val url = recorder.urls.single() assertNull(url.queryParameter("refresh")) - assertEquals("true", url.queryParameter("refreshVideo")) + assertNull(url.queryParameter("refreshVideo")) } private fun info(expectedVisitorData: String): YoutubeSabrInfo = mockk { diff --git a/src/test/kotlin/dev/typetype/server/services/TypetypeYoutubeSessionPoTokenProviderTest.kt b/src/test/kotlin/dev/typetype/server/services/TypetypeYoutubeSessionPoTokenProviderTest.kt index 01a8bf23..7d80673c 100644 --- a/src/test/kotlin/dev/typetype/server/services/TypetypeYoutubeSessionPoTokenProviderTest.kt +++ b/src/test/kotlin/dev/typetype/server/services/TypetypeYoutubeSessionPoTokenProviderTest.kt @@ -42,6 +42,8 @@ class TypetypeYoutubeSessionPoTokenProviderTest { private fun currentToken() = TypetypeYoutubeSessionPoTokenProvider.getSessionPoToken( "MWEB", + "2.20260801.00.00", + "test-user-agent", Localization("en", "US"), ContentCountry("US"), false, diff --git a/src/test/kotlin/dev/typetype/server/services/YouTubeSubtitleContentFetcherTest.kt b/src/test/kotlin/dev/typetype/server/services/YouTubeSubtitleContentFetcherTest.kt new file mode 100644 index 00000000..478e17e0 --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/services/YouTubeSubtitleContentFetcherTest.kt @@ -0,0 +1,138 @@ +package dev.typetype.server.services + +import kotlinx.coroutines.test.runTest +import okhttp3.MediaType.Companion.toMediaType +import okhttp3.OkHttpClient +import okhttp3.Protocol +import okhttp3.Response +import okhttp3.ResponseBody.Companion.toResponseBody +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test + +class YouTubeSubtitleContentFetcherTest { + @Test + fun `fetcher sends browser context and accepts bounded WebVTT`() = runTest { + var accept: String? = null + var origin: String? = null + val fetcher = fetcher(VTT, "text/vtt") { request -> + accept = request.header("Accept") + origin = request.header("Origin") + } + + val result = fetcher.fetch(TIMED_TEXT_URL, YouTubeSubtitleFormat.Vtt) + + assertTrue(result is YouTubeSubtitleFetchResult.Ready) + assertTrue(accept?.startsWith("text/vtt") == true) + assertEquals("https://m.youtube.com", origin) + } + + @Test + fun `fetcher classifies throttle and expired URLs`() = runTest { + assertEquals( + YouTubeSubtitleFetchResult.Throttled, + fetcher("throttled", code = 429).fetch(TIMED_TEXT_URL, YouTubeSubtitleFormat.Vtt), + ) + assertEquals( + YouTubeSubtitleFetchResult.Expired, + fetcher("expired", code = 403).fetch(TIMED_TEXT_URL, YouTubeSubtitleFormat.Vtt), + ) + } + + @Test + fun `fetcher rejects an unexpected success payload`() = runTest { + assertEquals( + YouTubeSubtitleFetchResult.InvalidPayload, + fetcher("challenge", "text/html") + .fetch(TIMED_TEXT_URL, YouTubeSubtitleFormat.Vtt), + ) + } + + @Test + fun `token fetcher keeps WebVTT retrieval on the token service`() = runTest { + var request: okhttp3.Request? = null + val fetcher = tokenFetcher(VTT, observe = { request = it }) + + val result = fetcher.fetch(TIMED_TEXT_URL, YouTubeSubtitleFormat.Vtt) + + assertTrue(result is YouTubeSubtitleFetchResult.Ready) + assertEquals("token", request?.url?.host) + assertEquals("/subtitles/content", request?.url?.encodedPath) + assertEquals(TIMED_TEXT_URL, request?.url?.queryParameter("url")) + } + + @Test + fun `token fetcher preserves typed throttling`() = runTest { + val result = tokenFetcher("throttled", code = 429) + .fetch(TIMED_TEXT_URL, YouTubeSubtitleFormat.Vtt) + + assertEquals(YouTubeSubtitleFetchResult.Throttled, result) + } + + @Test + fun `token fetcher leaves TTML on the direct path`() = runTest { + var directFormat: YouTubeSubtitleFormat? = null + val fetcher = tokenFetcher( + body = VTT, + observe = { error("Token must not receive TTML requests") }, + direct = { _, format -> + directFormat = format + YouTubeSubtitleFetchResult.Ready(TTML.encodeToByteArray()) + }, + ) + + val result = fetcher.fetch(TIMED_TEXT_URL, YouTubeSubtitleFormat.Ttml) + + assertTrue(result is YouTubeSubtitleFetchResult.Ready) + assertEquals(YouTubeSubtitleFormat.Ttml, directFormat) + } + + private fun fetcher( + body: String, + contentType: String = "application/json", + code: Int = 200, + observe: (okhttp3.Request) -> Unit = {}, + ): OkHttpYouTubeSubtitleContentFetcher { + val client = OkHttpClient.Builder() + .addInterceptor { chain -> + observe(chain.request()) + Response.Builder() + .request(chain.request()) + .protocol(Protocol.HTTP_1_1) + .code(code) + .message("test") + .body(body.toResponseBody(contentType.toMediaType())) + .build() + } + .build() + return OkHttpYouTubeSubtitleContentFetcher(client) + } + + private fun tokenFetcher( + body: String, + code: Int = 200, + observe: (okhttp3.Request) -> Unit = {}, + direct: suspend (String, YouTubeSubtitleFormat) -> YouTubeSubtitleFetchResult = + { _, _ -> YouTubeSubtitleFetchResult.Unavailable }, + ): TokenYouTubeSubtitleContentFetcher { + val client = OkHttpClient.Builder() + .addInterceptor { chain -> + observe(chain.request()) + Response.Builder() + .request(chain.request()) + .protocol(Protocol.HTTP_1_1) + .code(code) + .message("test") + .body(body.toResponseBody("text/vtt".toMediaType())) + .build() + } + .build() + return TokenYouTubeSubtitleContentFetcher(client, "http://token/", YouTubeSubtitleContentFetcher(direct)) + } + + private companion object { + const val VTT = "WEBVTT\n\n00:00.000 --> 00:01.000\nHello" + const val TTML = "" + const val TIMED_TEXT_URL = "https://www.youtube.com/api/timedtext?v=abcdefghijk&lang=en&fmt=vtt" + } +} diff --git a/src/test/kotlin/dev/typetype/server/services/YouTubeSubtitleContractTest.kt b/src/test/kotlin/dev/typetype/server/services/YouTubeSubtitleContractTest.kt new file mode 100644 index 00000000..b6994c18 --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/services/YouTubeSubtitleContractTest.kt @@ -0,0 +1,38 @@ +package dev.typetype.server.services + +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertNull +import org.junit.jupiter.api.Test + +class YouTubeSubtitleContractTest { + @Test + fun `legacy timed text selection preserves track and translation`() { + val selection = subtitleSelectionFromTimedTextUrl( + "https://www.youtube.com/api/timedtext?v=abcdefghijk&lang=en&kind=asr" + + "&name=English&tlang=fr&expire=123&sig=secret", + ) + + requireNotNull(selection) + assertEquals("abcdefghijk", selection.videoId) + assertEquals("fr", selection.language) + assertEquals("en", selection.sourceLanguage) + assertEquals("fr", selection.translationLanguage) + assertEquals("English", selection.trackName) + assertEquals(YouTubeSubtitleVariant.Auto, selection.variant) + assertEquals(YouTubeSubtitleFormat.Vtt, selection.format) + } + + @Test + fun `legacy selection rejects non YouTube and invalid video IDs`() { + assertNull( + subtitleSelectionFromTimedTextUrl( + "https://example.com/api/timedtext?v=abcdefghijk&lang=en", + ), + ) + assertNull( + subtitleSelectionFromTimedTextUrl( + "https://www.youtube.com/api/timedtext?v=short&lang=en", + ), + ) + } +} diff --git a/src/test/kotlin/dev/typetype/server/services/YouTubeSubtitleDeliveryServiceTest.kt b/src/test/kotlin/dev/typetype/server/services/YouTubeSubtitleDeliveryServiceTest.kt new file mode 100644 index 00000000..b09e96ca --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/services/YouTubeSubtitleDeliveryServiceTest.kt @@ -0,0 +1,149 @@ +package dev.typetype.server.services + +import kotlinx.coroutines.CompletableDeferred +import kotlinx.coroutines.async +import kotlinx.coroutines.test.runTest +import kotlinx.coroutines.yield +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertInstanceOf +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test +import java.util.concurrent.atomic.AtomicInteger + +class YouTubeSubtitleDeliveryServiceTest { + @Test + fun `VOD subtitle content is cached by stable selection`() = runTest { + val resolutions = AtomicInteger() + val fetches = AtomicInteger() + val service = service( + resolver = { + resolutions.incrementAndGet() + readyTrack() + }, + fetcher = { _, _ -> + fetches.incrementAndGet() + YouTubeSubtitleFetchResult.Ready(VTT) + }, + ) + + val first = service.fetch(SELECTION) + val second = service.fetch(SELECTION) + + assertTrue(first is YouTubeSubtitleContentResult.Ready) + assertEquals(first, second) + assertEquals(1, resolutions.get()) + assertEquals(1, fetches.get()) + } + + @Test + fun `simultaneous subtitle requests share one upstream fetch`() = runTest { + val resolutions = AtomicInteger() + val started = CompletableDeferred() + val release = CompletableDeferred() + val service = service( + resolver = { + resolutions.incrementAndGet() + started.complete(Unit) + release.await() + readyTrack() + }, + fetcher = { _, _ -> YouTubeSubtitleFetchResult.Ready(VTT) }, + ) + + val first = async { service.fetch(SELECTION) } + started.await() + val second = async { service.fetch(SELECTION) } + yield() + release.complete(Unit) + + assertEquals(first.await(), second.await()) + assertEquals(1, resolutions.get()) + } + + @Test + fun `expired subtitle URL is refreshed once`() = runTest { + val resolutions = AtomicInteger() + val fetches = AtomicInteger() + val service = service( + resolver = { + resolutions.incrementAndGet() + readyTrack() + }, + fetcher = { _, _ -> + if (fetches.incrementAndGet() == 1) YouTubeSubtitleFetchResult.Expired + else YouTubeSubtitleFetchResult.Ready(VTT) + }, + ) + + val result = service.fetch(SELECTION) + + assertTrue(result is YouTubeSubtitleContentResult.Ready) + assertEquals(2, resolutions.get()) + assertEquals(2, fetches.get()) + } + + @Test + fun `repeated expired subtitle URL returns stable typed result`() = runTest { + val resolutions = AtomicInteger() + val service = service( + resolver = { + resolutions.incrementAndGet() + readyTrack() + }, + fetcher = { _, _ -> YouTubeSubtitleFetchResult.Expired }, + ) + + assertEquals(YouTubeSubtitleContentResult.Expired, service.fetch(SELECTION)) + assertEquals(2, resolutions.get()) + } + + @Test + fun `inline TTML is returned without an upstream content request`() = runTest { + val fetches = AtomicInteger() + val selection = SELECTION.copy(format = YouTubeSubtitleFormat.Ttml) + val service = service( + resolver = { + YouTubeSubtitleResolution.Ready( + ResolvedYouTubeSubtitle(TTML.decodeToString(), isUrl = false, isLive = false), + ) + }, + fetcher = { _, _ -> + fetches.incrementAndGet() + YouTubeSubtitleFetchResult.Unavailable + }, + ) + + val result = service.fetch(selection) + + val ready = assertInstanceOf(YouTubeSubtitleContentResult.Ready::class.java, result) + assertTrue(ready.content.contentEquals(TTML)) + assertEquals(YouTubeSubtitleFormat.Ttml, ready.format) + assertEquals(false, ready.isLive) + assertEquals(0, fetches.get()) + } + + private fun service( + resolver: suspend (YouTubeSubtitleSelection) -> YouTubeSubtitleResolution, + fetcher: suspend (String, YouTubeSubtitleFormat) -> YouTubeSubtitleFetchResult, + ) = YouTubeSubtitleDeliveryService( + resolver = YouTubeSubtitleTrackResolver(resolver), + fetcher = YouTubeSubtitleContentFetcher(fetcher), + cache = YouTubeSubtitleCache(null), + ) + + private fun readyTrack(isLive: Boolean = false) = YouTubeSubtitleResolution.Ready( + ResolvedYouTubeSubtitle(TIMED_TEXT_URL, isUrl = true, isLive = isLive), + ) + + private companion object { + val VTT = "WEBVTT\n\n00:00.000 --> 00:01.000\nHello".encodeToByteArray() + val TTML = "".encodeToByteArray() + const val TIMED_TEXT_URL = "https://www.youtube.com/api/timedtext?v=abcdefghijk&lang=en&fmt=vtt" + val SELECTION = YouTubeSubtitleSelection( + videoId = "abcdefghijk", + language = "en", + variant = YouTubeSubtitleVariant.Manual, + format = YouTubeSubtitleFormat.Vtt, + ) + } +} diff --git a/src/test/kotlin/dev/typetype/server/services/YouTubeSubtitleServiceTest.kt b/src/test/kotlin/dev/typetype/server/services/YouTubeSubtitleServiceTest.kt index 66a31a45..b581b964 100644 --- a/src/test/kotlin/dev/typetype/server/services/YouTubeSubtitleServiceTest.kt +++ b/src/test/kotlin/dev/typetype/server/services/YouTubeSubtitleServiceTest.kt @@ -4,9 +4,11 @@ import kotlinx.coroutines.test.runTest import okhttp3.MediaType.Companion.toMediaType import okhttp3.OkHttpClient import okhttp3.Protocol +import okhttp3.Request import okhttp3.Response import okhttp3.ResponseBody.Companion.toResponseBody import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertTrue import org.junit.jupiter.api.Test class YouTubeSubtitleServiceTest { @@ -40,15 +42,21 @@ class YouTubeSubtitleServiceTest { ) } - private fun service(body: String): YouTubeSubtitleService { + private fun service( + body: String, + contentType: String = "application/json", + code: Int = 200, + observeRequest: (Request) -> Unit = {}, + ): YouTubeSubtitleService { val client = OkHttpClient.Builder() .addInterceptor { chain -> + observeRequest(chain.request()) Response.Builder() .request(chain.request()) .protocol(Protocol.HTTP_1_1) - .code(200) + .code(code) .message("test") - .body(body.toResponseBody("application/json".toMediaType())) + .body(body.toResponseBody(contentType.toMediaType())) .build() } .build() diff --git a/src/test/kotlin/dev/typetype/server/services/YoutubePlayerClientFallbackStreamServiceTest.kt b/src/test/kotlin/dev/typetype/server/services/YoutubePlayerClientFallbackStreamServiceTest.kt index be2e9474..c3545fc5 100644 --- a/src/test/kotlin/dev/typetype/server/services/YoutubePlayerClientFallbackStreamServiceTest.kt +++ b/src/test/kotlin/dev/typetype/server/services/YoutubePlayerClientFallbackStreamServiceTest.kt @@ -24,14 +24,14 @@ class YoutubePlayerClientFallbackStreamServiceTest { } val service = YoutubePlayerClientFallbackStreamService( delegate, - listOf(YoutubePlayerClient.WEB_SAFARI, YoutubePlayerClient.TV_DOWNGRADED), + listOf(YoutubePlayerClient.VISIONOS, YoutubePlayerClient.TV_DOWNGRADED), ) val result = service.getStreamInfo(YOUTUBE_URL) assertSame(success, result) assertEquals( - listOf(YoutubePlayerClient.WEB_SAFARI.value, YoutubePlayerClient.TV_DOWNGRADED.value), + listOf(YoutubePlayerClient.VISIONOS.value, YoutubePlayerClient.TV_DOWNGRADED.value), observed, ) assertEquals(YoutubePlayerClient.MWEB.value, NewPipe.getYoutubePlayerClient()) @@ -49,13 +49,13 @@ class YoutubePlayerClientFallbackStreamServiceTest { } val service = YoutubePlayerClientFallbackStreamService( delegate, - listOf(YoutubePlayerClient.WEB_SAFARI, YoutubePlayerClient.TV_DOWNGRADED), + listOf(YoutubePlayerClient.VISIONOS, YoutubePlayerClient.TV_DOWNGRADED), ) val result = service.getStreamInfo(YOUTUBE_URL) assertSame(success, result) - assertEquals(listOf(YoutubePlayerClient.WEB_SAFARI.value), observed) + assertEquals(listOf(YoutubePlayerClient.VISIONOS.value), observed) } private companion object { diff --git a/src/test/kotlin/dev/typetype/server/services/YoutubePlayerClientStreamServiceTest.kt b/src/test/kotlin/dev/typetype/server/services/YoutubePlayerClientStreamServiceTest.kt index edc77c56..75b7584a 100644 --- a/src/test/kotlin/dev/typetype/server/services/YoutubePlayerClientStreamServiceTest.kt +++ b/src/test/kotlin/dev/typetype/server/services/YoutubePlayerClientStreamServiceTest.kt @@ -18,15 +18,15 @@ import org.schabi.newpipe.extractor.NewPipe class YoutubePlayerClientStreamServiceTest { @Test - fun `classic extraction selects web safari and restores mweb`() = runBlocking { + fun `classic extraction selects visionOS and restores mweb`() = runBlocking { NewPipe.setYoutubePlayerClient(YoutubePlayerClient.MWEB.value) val observed = mutableListOf() val delegate = recordingService(observed) - val service = YoutubePlayerClientStreamService(delegate, YoutubePlayerClient.WEB_SAFARI) + val service = YoutubePlayerClientStreamService(delegate, YoutubePlayerClient.VISIONOS) service.getStreamInfo(YOUTUBE_URL) - assertEquals(listOf(YoutubePlayerClient.WEB_SAFARI.value), observed) + assertEquals(listOf(YoutubePlayerClient.VISIONOS.value), observed) assertEquals(YoutubePlayerClient.MWEB.value, NewPipe.getYoutubePlayerClient()) } @@ -52,7 +52,7 @@ class YoutubePlayerClientStreamServiceTest { } } val sabr = YoutubePlayerClientStreamService(delegate, YoutubePlayerClient.MWEB) - val classic = YoutubePlayerClientStreamService(delegate, YoutubePlayerClient.WEB_SAFARI) + val classic = YoutubePlayerClientStreamService(delegate, YoutubePlayerClient.VISIONOS) val sabrJobs = List(2) { launch { sabr.getStreamInfo(YOUTUBE_URL) } } repeat(2) { sabrEntered.receive() } @@ -65,7 +65,7 @@ class YoutubePlayerClientStreamServiceTest { assertTrue(classicEntered.get()) assertEquals(4, observations.count { it == YoutubePlayerClient.MWEB.value }) - assertEquals(2, observations.count { it == YoutubePlayerClient.WEB_SAFARI.value }) + assertEquals(2, observations.count { it == YoutubePlayerClient.VISIONOS.value }) assertEquals(YoutubePlayerClient.MWEB.value, NewPipe.getYoutubePlayerClient()) }