From 4c1f82aa228239b041c7d3657f1117abd7516991 Mon Sep 17 00:00:00 2001 From: oSumAtrIX Date: Tue, 21 May 2024 02:41:34 +0200 Subject: [PATCH] fix(YouTube - Client spoof): Spoof client to fix playback (#637) Co-authored-by: LisoUseInAIKyrios <118716522+LisoUseInAIKyrios@users.noreply.github.com> --- .../youtube/patches/VideoInformation.java | 2 +- .../patches/spoof/SpoofClientPatch.java | 264 ++++++++++++++++++ .../patches/spoof/SpoofSignaturePatch.java | 17 +- .../patches/spoof/StoryboardRenderer.java | 34 +-- .../patches/spoof/requests/PlayerRoutes.java | 3 +- .../requests/StoryboardRendererRequester.java | 27 +- .../youtube/settings/Settings.java | 4 +- .../InnertubeContext$ClientInfo.java | 5 + 8 files changed, 304 insertions(+), 52 deletions(-) create mode 100644 app/src/main/java/app/revanced/integrations/youtube/patches/spoof/SpoofClientPatch.java create mode 100644 stub/src/main/java/com/google/protos/youtube/api/innertube/InnertubeContext$ClientInfo.java diff --git a/app/src/main/java/app/revanced/integrations/youtube/patches/VideoInformation.java b/app/src/main/java/app/revanced/integrations/youtube/patches/VideoInformation.java index 57751ff02a..8cf525f3c7 100644 --- a/app/src/main/java/app/revanced/integrations/youtube/patches/VideoInformation.java +++ b/app/src/main/java/app/revanced/integrations/youtube/patches/VideoInformation.java @@ -81,7 +81,7 @@ public static boolean playerParametersAreShort(@NonNull String parameters) { /** * Injection point. */ - public static String newPlayerResponseSignature(@NonNull String signature, boolean isShortAndOpeningOrPlaying) { + public static String newPlayerResponseSignature(@NonNull String signature, String videoId, boolean isShortAndOpeningOrPlaying) { final boolean isShort = playerParametersAreShort(signature); playerResponseVideoIdIsShort = isShort; if (!isShort || isShortAndOpeningOrPlaying) { diff --git a/app/src/main/java/app/revanced/integrations/youtube/patches/spoof/SpoofClientPatch.java b/app/src/main/java/app/revanced/integrations/youtube/patches/spoof/SpoofClientPatch.java new file mode 100644 index 0000000000..93c904667b --- /dev/null +++ b/app/src/main/java/app/revanced/integrations/youtube/patches/spoof/SpoofClientPatch.java @@ -0,0 +1,264 @@ +package app.revanced.integrations.youtube.patches.spoof; + +import static app.revanced.integrations.youtube.patches.spoof.requests.StoryboardRendererRequester.getStoryboardRenderer; + +import android.net.Uri; + +import androidx.annotation.Nullable; + +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; + +import app.revanced.integrations.shared.Logger; +import app.revanced.integrations.shared.Utils; +import app.revanced.integrations.youtube.patches.VideoInformation; +import app.revanced.integrations.youtube.settings.Settings; + +@SuppressWarnings("unused") +public class SpoofClientPatch { + private static final boolean SPOOF_CLIENT_ENABLED = Settings.SPOOF_CLIENT.get(); + private static final boolean SPOOF_CLIENT_USE_IOS = Settings.SPOOF_CLIENT_USE_IOS.get(); + private static final boolean SPOOF_CLIENT_STORYBOARD = SPOOF_CLIENT_ENABLED && !SPOOF_CLIENT_USE_IOS; + + /** + * Any unreachable ip address. Used to intentionally fail requests. + */ + private static final String UNREACHABLE_HOST_URI_STRING = "https://127.0.0.0"; + private static final Uri UNREACHABLE_HOST_URI = Uri.parse(UNREACHABLE_HOST_URI_STRING); + + @Nullable + private static volatile Future lastStoryboardFetched; + + private static final Map> storyboardCache = + Collections.synchronizedMap(new LinkedHashMap<>(100) { + private static final int CACHE_LIMIT = 100; + + @Override + protected boolean removeEldestEntry(Entry eldest) { + return size() > CACHE_LIMIT; // Evict the oldest entry if over the cache limit. + } + }); + + /** + * Injection point. + * Blocks /get_watch requests by returning a localhost URI. + * + * @param playerRequestUri The URI of the player request. + * @return Localhost URI if the request is a /get_watch request, otherwise the original URI. + */ + public static Uri blockGetWatchRequest(Uri playerRequestUri) { + if (SPOOF_CLIENT_ENABLED) { + try { + String path = playerRequestUri.getPath(); + + if (path != null && path.contains("get_watch")) { + Logger.printDebug(() -> "Blocking: " + playerRequestUri + " by returning: " + UNREACHABLE_HOST_URI_STRING); + + return UNREACHABLE_HOST_URI; + } + } catch (Exception ex) { + Logger.printException(() -> "blockGetWatchRequest failure", ex); + } + } + + return playerRequestUri; + } + + /** + * Injection point. + * Blocks /initplayback requests. + * For iOS, an unreachable host URL can be used, but for Android Testsuite, this is not possible. + */ + public static String blockInitPlaybackRequest(String originalUrlString) { + if (SPOOF_CLIENT_ENABLED) { + try { + var originalUri = Uri.parse(originalUrlString); + String path = originalUri.getPath(); + + if (path != null && path.contains("initplayback")) { + String replacementUriString = (getSpoofClientType() == ClientType.IOS) + ? UNREACHABLE_HOST_URI_STRING + // TODO: Ideally, a local proxy could be setup and block + // the request the same way as Burp Suite is capable of + // because that way the request is never sent to YouTube unnecessarily. + // Just using localhost unfortunately does not work. + : originalUri.buildUpon().clearQuery().build().toString(); + + Logger.printDebug(() -> "Blocking: " + originalUrlString + " by returning: " + replacementUriString); + + return replacementUriString; + } + } catch (Exception ex) { + Logger.printException(() -> "blockInitPlaybackRequest failure", ex); + } + } + + return originalUrlString; + } + + private static ClientType getSpoofClientType() { + if (SPOOF_CLIENT_USE_IOS) { + return ClientType.IOS; + } + + StoryboardRenderer renderer = getRenderer(false); + if (renderer == null) { + // Video is private or otherwise not available. + // Test client still works for video playback, but seekbar thumbnails are not available. + // Use iOS client instead. + Logger.printDebug(() -> "Using iOS client for paid or otherwise restricted video"); + return ClientType.IOS; + } + + if (renderer.isLiveStream) { + // Test client does not support live streams. + // Use the storyboard renderer information to fallback to iOS if a live stream is opened. + Logger.printDebug(() -> "Using iOS client for livestream: " + renderer.videoId); + return ClientType.IOS; + } + + return ClientType.ANDROID_TESTSUITE; + } + + /** + * Injection point. + */ + public static int getClientTypeId(int originalClientTypeId) { + if (SPOOF_CLIENT_ENABLED) { + return getSpoofClientType().id; + } + + return originalClientTypeId; + } + + /** + * Injection point. + */ + public static String getClientVersion(String originalClientVersion) { + if (SPOOF_CLIENT_ENABLED) { + return getSpoofClientType().version; + } + + return originalClientVersion; + } + + /** + * Injection point. + */ + public static boolean isClientSpoofingEnabled() { + return SPOOF_CLIENT_ENABLED; + } + + // + // Storyboard. + // + + /** + * Injection point. + */ + public static String setPlayerResponseVideoId(String parameters, String videoId, boolean isShortAndOpeningOrPlaying) { + if (SPOOF_CLIENT_STORYBOARD) { + try { + // VideoInformation is not a dependent patch, and only this single helper method is used. + // Hook can be called when scrolling thru the feed and a Shorts shelf is present. + // Ignore these videos. + if (!isShortAndOpeningOrPlaying && VideoInformation.playerParametersAreShort(parameters)) { + Logger.printDebug(() -> "Ignoring Short: " + videoId); + return parameters; + } + + Future storyboard = storyboardCache.get(videoId); + if (storyboard == null) { + storyboard = Utils.submitOnBackgroundThread(() -> getStoryboardRenderer(videoId)); + storyboardCache.put(videoId, storyboard); + lastStoryboardFetched = storyboard; + + // Block until the renderer fetch completes. + // This is desired because if this returns without finishing the fetch + // then video will start playback but the storyboard is not ready yet. + getRenderer(true); + } else { + lastStoryboardFetched = storyboard; + // No need to block on the fetch since it previously loaded. + } + + } catch (Exception ex) { + Logger.printException(() -> "setPlayerResponseVideoId failure", ex); + } + } + + return parameters; // Return the original value since we are observing and not modifying. + } + + @Nullable + private static StoryboardRenderer getRenderer(boolean waitForCompletion) { + var future = lastStoryboardFetched; + if (future != null) { + try { + if (waitForCompletion || future.isDone()) { + return future.get(20000, TimeUnit.MILLISECONDS); // Any arbitrarily large timeout. + } // else, return null. + } catch (TimeoutException ex) { + Logger.printDebug(() -> "Could not get renderer (get timed out)"); + } catch (ExecutionException | InterruptedException ex) { + // Should never happen. + Logger.printException(() -> "Could not get renderer", ex); + } + } + return null; + } + + /** + * Injection point. + * Called from background threads and from the main thread. + */ + @Nullable + public static String getStoryboardRendererSpec(String originalStoryboardRendererSpec) { + if (SPOOF_CLIENT_STORYBOARD) { + StoryboardRenderer renderer = getRenderer(false); + + if (renderer != null) { + if (!renderer.isLiveStream && renderer.spec != null) { + return renderer.spec; + } + } + } + + return originalStoryboardRendererSpec; + } + + /** + * Injection point. + */ + public static int getRecommendedLevel(int originalLevel) { + if (SPOOF_CLIENT_STORYBOARD) { + StoryboardRenderer renderer = getRenderer(false); + + if (renderer != null) { + if (!renderer.isLiveStream && renderer.recommendedLevel != null) { + return renderer.recommendedLevel; + } + } + } + + return originalLevel; + } + + private enum ClientType { + ANDROID_TESTSUITE(30, "1.9"), + IOS(5, Utils.getAppVersionName()); + + final int id; + final String version; + + ClientType(int id, String version) { + this.id = id; + this.version = version; + } + } +} diff --git a/app/src/main/java/app/revanced/integrations/youtube/patches/spoof/SpoofSignaturePatch.java b/app/src/main/java/app/revanced/integrations/youtube/patches/spoof/SpoofSignaturePatch.java index 6573467a37..41f03ed781 100644 --- a/app/src/main/java/app/revanced/integrations/youtube/patches/spoof/SpoofSignaturePatch.java +++ b/app/src/main/java/app/revanced/integrations/youtube/patches/spoof/SpoofSignaturePatch.java @@ -85,7 +85,7 @@ private static StoryboardRenderer getRenderer(boolean waitForCompletion) { * * @param parameters Original protobuf parameter value. */ - public static String spoofParameter(String parameters, boolean isShortAndOpeningOrPlaying) { + public static String spoofParameter(String parameters, String videoId, boolean isShortAndOpeningOrPlaying) { try { Logger.printDebug(() -> "Original protobuf parameter value: " + parameters); @@ -152,12 +152,12 @@ private static String getStoryboardRendererSpec(String originalStoryboardRendere if (Settings.SPOOF_SIGNATURE.get() && !useOriginalStoryboardRenderer) { StoryboardRenderer renderer = getRenderer(false); if (renderer != null) { - if (returnNullIfLiveStream && renderer.isLiveStream()) { + if (returnNullIfLiveStream && renderer.isLiveStream) { return null; } - String spec = renderer.getSpec(); - if (spec != null) { - return spec; + + if (renderer.spec != null) { + return renderer.spec; } } } @@ -191,8 +191,9 @@ public static int getRecommendedLevel(int originalLevel) { if (Settings.SPOOF_SIGNATURE.get() && !useOriginalStoryboardRenderer) { StoryboardRenderer renderer = getRenderer(false); if (renderer != null) { - Integer recommendedLevel = renderer.getRecommendedLevel(); - if (recommendedLevel != null) return recommendedLevel; + if (renderer.recommendedLevel != null) { + return renderer.recommendedLevel; + } } } @@ -214,7 +215,7 @@ public static boolean getSeekbarThumbnailOverrideValue() { // Show empty thumbnails so the seek time and chapters still show up. return true; } - return renderer.getSpec() != null; + return renderer.spec != null; } /** diff --git a/app/src/main/java/app/revanced/integrations/youtube/patches/spoof/StoryboardRenderer.java b/app/src/main/java/app/revanced/integrations/youtube/patches/spoof/StoryboardRenderer.java index 8e33729724..45bb86ebcc 100644 --- a/app/src/main/java/app/revanced/integrations/youtube/patches/spoof/StoryboardRenderer.java +++ b/app/src/main/java/app/revanced/integrations/youtube/patches/spoof/StoryboardRenderer.java @@ -4,42 +4,30 @@ import org.jetbrains.annotations.NotNull; -@Deprecated public final class StoryboardRenderer { + public final String videoId; @Nullable - private final String spec; - private final boolean isLiveStream; + public final String spec; + public final boolean isLiveStream; + /** + * Recommended image quality level, or NULL if no recommendation exists. + */ @Nullable - private final Integer recommendedLevel; + public final Integer recommendedLevel; - public StoryboardRenderer(@Nullable String spec, boolean isLiveStream, @Nullable Integer recommendedLevel) { + public StoryboardRenderer(String videoId, @Nullable String spec, boolean isLiveStream, @Nullable Integer recommendedLevel) { + this.videoId = videoId; this.spec = spec; this.isLiveStream = isLiveStream; this.recommendedLevel = recommendedLevel; } - @Nullable - public String getSpec() { - return spec; - } - - public boolean isLiveStream() { - return isLiveStream; - } - - /** - * @return Recommended image quality level, or NULL if no recommendation exists. - */ - @Nullable - public Integer getRecommendedLevel() { - return recommendedLevel; - } - @NotNull @Override public String toString() { return "StoryboardRenderer{" + - "isLiveStream=" + isLiveStream + + "videoId=" + videoId + + ", isLiveStream=" + isLiveStream + ", spec='" + spec + '\'' + ", recommendedLevel=" + recommendedLevel + '}'; diff --git a/app/src/main/java/app/revanced/integrations/youtube/patches/spoof/requests/PlayerRoutes.java b/app/src/main/java/app/revanced/integrations/youtube/patches/spoof/requests/PlayerRoutes.java index 7909387f22..1927b1d68a 100644 --- a/app/src/main/java/app/revanced/integrations/youtube/patches/spoof/requests/PlayerRoutes.java +++ b/app/src/main/java/app/revanced/integrations/youtube/patches/spoof/requests/PlayerRoutes.java @@ -10,7 +10,6 @@ import java.io.IOException; import java.net.HttpURLConnection; -@Deprecated final class PlayerRoutes { private static final String YT_API_URL = "https://www.youtube.com/youtubei/v1/"; static final Route.CompiledRoute GET_STORYBOARD_SPEC_RENDERER = new Route( @@ -27,7 +26,7 @@ final class PlayerRoutes { /** * TCP connection and HTTP read timeout */ - private static final int CONNECTION_TIMEOUT_MILLISECONDS = 4 * 1000; // 4 Seconds. + private static final int CONNECTION_TIMEOUT_MILLISECONDS = 10 * 1000; // 10 Seconds. static { JSONObject innerTubeBody = new JSONObject(); diff --git a/app/src/main/java/app/revanced/integrations/youtube/patches/spoof/requests/StoryboardRendererRequester.java b/app/src/main/java/app/revanced/integrations/youtube/patches/spoof/requests/StoryboardRendererRequester.java index 31d8c3ef5d..0cbec19400 100644 --- a/app/src/main/java/app/revanced/integrations/youtube/patches/spoof/requests/StoryboardRendererRequester.java +++ b/app/src/main/java/app/revanced/integrations/youtube/patches/spoof/requests/StoryboardRendererRequester.java @@ -19,17 +19,8 @@ import static app.revanced.integrations.shared.StringRef.str; import static app.revanced.integrations.youtube.patches.spoof.requests.PlayerRoutes.*; -@Deprecated public class StoryboardRendererRequester { - /** - * For videos that have no storyboard. - * Usually for low resolution videos as old as YouTube itself. - * Does not include paid videos where the renderer fetch fails. - */ - private static final StoryboardRenderer emptyStoryboard - = new StoryboardRenderer(null, false, null); - private StoryboardRendererRequester() { } @@ -69,9 +60,9 @@ private static JSONObject fetchPlayerResponse(@NonNull String requestBody, boole null, showToastOnIOException || BaseSettings.DEBUG_TOAST_ON_ERROR.get()); connection.disconnect(); } catch (SocketTimeoutException ex) { - handleConnectionError(str("revanced_spoof_storyboard_timeout"), ex, showToastOnIOException); + handleConnectionError(str("revanced_spoof_client_storyboard_timeout"), ex, showToastOnIOException); } catch (IOException ex) { - handleConnectionError(str("revanced_spoof_storyboard_io_exception", ex.getMessage()), + handleConnectionError(str("revanced_spoof_client_storyboard_io_exception", ex.getMessage()), ex, showToastOnIOException); } catch (Exception ex) { Logger.printException(() -> "Spoof storyboard fetch failed", ex); // Should never happen. @@ -98,22 +89,23 @@ private static boolean isPlayabilityStatusOk(@NonNull JSONObject playerResponse) * @return StoryboardRenderer or null if playabilityStatus is not OK. */ @Nullable - private static StoryboardRenderer getStoryboardRendererUsingBody(@NonNull String innerTubeBody, + private static StoryboardRenderer getStoryboardRendererUsingBody(String videoId, + @NonNull String innerTubeBody, boolean showToastOnIOException) { final JSONObject playerResponse = fetchPlayerResponse(innerTubeBody, showToastOnIOException); if (playerResponse != null && isPlayabilityStatusOk(playerResponse)) - return getStoryboardRendererUsingResponse(playerResponse); + return getStoryboardRendererUsingResponse(videoId, playerResponse); return null; } @Nullable - private static StoryboardRenderer getStoryboardRendererUsingResponse(@NonNull JSONObject playerResponse) { + private static StoryboardRenderer getStoryboardRendererUsingResponse(@NonNull String videoId, @NonNull JSONObject playerResponse) { try { Logger.printDebug(() -> "Parsing response: " + playerResponse); if (!playerResponse.has("storyboards")) { Logger.printDebug(() -> "Using empty storyboard"); - return emptyStoryboard; + return new StoryboardRenderer(videoId, null, false, null); } final JSONObject storyboards = playerResponse.getJSONObject("storyboards"); final boolean isLiveStream = storyboards.has("playerLiveStoryboardSpecRenderer"); @@ -123,6 +115,7 @@ private static StoryboardRenderer getStoryboardRendererUsingResponse(@NonNull JS final var rendererElement = storyboards.getJSONObject(storyboardsRendererTag); StoryboardRenderer renderer = new StoryboardRenderer( + videoId, rendererElement.getString("spec"), isLiveStream, rendererElement.has("recommendedLevel") @@ -144,11 +137,11 @@ private static StoryboardRenderer getStoryboardRendererUsingResponse(@NonNull JS public static StoryboardRenderer getStoryboardRenderer(@NonNull String videoId) { Objects.requireNonNull(videoId); - var renderer = getStoryboardRendererUsingBody( + var renderer = getStoryboardRendererUsingBody(videoId, String.format(ANDROID_INNER_TUBE_BODY, videoId), false); if (renderer == null) { Logger.printDebug(() -> videoId + " not available using Android client"); - renderer = getStoryboardRendererUsingBody( + renderer = getStoryboardRendererUsingBody(videoId, String.format(TV_EMBED_INNER_TUBE_BODY, videoId, videoId), true); if (renderer == null) { Logger.printDebug(() -> videoId + " not available using TV embedded client"); diff --git a/app/src/main/java/app/revanced/integrations/youtube/settings/Settings.java b/app/src/main/java/app/revanced/integrations/youtube/settings/Settings.java index b71e1809fa..04a08d410c 100644 --- a/app/src/main/java/app/revanced/integrations/youtube/settings/Settings.java +++ b/app/src/main/java/app/revanced/integrations/youtube/settings/Settings.java @@ -236,6 +236,8 @@ public class Settings extends BaseSettings { "revanced_spoof_device_dimensions_user_dialog_message"); public static final BooleanSetting BYPASS_URL_REDIRECTS = new BooleanSetting("revanced_bypass_url_redirects", TRUE); public static final BooleanSetting ANNOUNCEMENTS = new BooleanSetting("revanced_announcements", TRUE); + public static final BooleanSetting SPOOF_CLIENT = new BooleanSetting("revanced_spoof_client", TRUE, true, "revanced_spoof_client_user_dialog_message"); + public static final BooleanSetting SPOOF_CLIENT_USE_IOS = new BooleanSetting("revanced_spoof_client_use_ios", FALSE, true, parent(SPOOF_CLIENT)); @Deprecated public static final StringSetting DEPRECATED_ANNOUNCEMENT_LAST_HASH = new StringSetting("revanced_announcement_last_hash", ""); public static final IntegerSetting ANNOUNCEMENT_LAST_ID = new IntegerSetting("revanced_announcement_last_id", -1); @@ -244,7 +246,7 @@ public class Settings extends BaseSettings { // Debugging /** * When enabled, share the debug logs with care. - * The buffer contains select user data, including the client ip address and information that could identify the YT account. + * The buffer contains select user data, including the client ip address and information that could identify the end user. */ public static final BooleanSetting DEBUG_PROTOBUFFER = new BooleanSetting("revanced_debug_protobuffer", FALSE, parent(BaseSettings.DEBUG)); diff --git a/stub/src/main/java/com/google/protos/youtube/api/innertube/InnertubeContext$ClientInfo.java b/stub/src/main/java/com/google/protos/youtube/api/innertube/InnertubeContext$ClientInfo.java new file mode 100644 index 0000000000..f517608f20 --- /dev/null +++ b/stub/src/main/java/com/google/protos/youtube/api/innertube/InnertubeContext$ClientInfo.java @@ -0,0 +1,5 @@ +package com.google.protos.youtube.api.innertube; + +public class InnertubeContext$ClientInfo { + public int r; +}