Add geo-bypass for video info

Author: omarroth

src/invidious/helpers/proxy.cr

@@ -0,0 +1,143 @@
+# See https://github.com/crystal-lang/crystal/issues/2963
+class HTTPProxy
+  getter proxy_host : String
+  getter proxy_port : Int32
+  getter options : Hash(Symbol, String)
+  getter tls : OpenSSL::SSL::Context::Client?
+
+  def initialize(@proxy_host, @proxy_port = 80, @options = {} of Symbol => String)
+  end
+
+  def open(host, port, tls = nil, connection_options = {} of Symbol => Float64 | Nil)
+    dns_timeout = connection_options.fetch(:dns_timeout, nil)
+    connect_timeout = connection_options.fetch(:connect_timeout, nil)
+    read_timeout = connection_options.fetch(:read_timeout, nil)
+
+    socket = TCPSocket.new @proxy_host, @proxy_port, dns_timeout, connect_timeout
+    socket.read_timeout = read_timeout if read_timeout
+    socket.sync = true
+
+    socket << "CONNECT #{host}:#{port} HTTP/1.1\r\n"
+
+    if options[:user]?
+      credentials = Base64.strict_encode("#{options[:user]}:#{options[:password]}")
+      credentials = "#{credentials}\n".gsub(/\s/, "")
+      socket << "Proxy-Authorization: Basic #{credentials}\r\n"
+    end
+
+    socket << "\r\n"
+
+    resp = parse_response(socket)
+
+    if resp[:code]? == 200
+      {% if !flag?(:without_openssl) %}
+        if tls
+          tls_socket = OpenSSL::SSL::Socket::Client.new(socket, context: tls, sync_close: true, hostname: host)
+          socket = tls_socket
+        end
+      {% end %}
+
+      return socket
+    else
+      socket.close
+      raise IO::Error.new(resp.inspect)
+    end
+  end
+
+  private def parse_response(socket)
+    resp = {} of Symbol => Int32 | String | Hash(String, String)
+
+    begin
+      version, code, reason = socket.gets.as(String).chomp.split(/ /, 3)
+
+      headers = {} of String => String
+
+      while (line = socket.gets.as(String)) && (line.chomp != "")
+        name, value = line.split(/:/, 2)
+        headers[name.strip] = value.strip
+      end
+
+      resp[:version] = version
+      resp[:code] = code.to_i
+      resp[:reason] = reason
+      resp[:headers] = headers
+    rescue
+    end
+
+    return resp
+  end
+end
+
+class HTTPClient < HTTP::Client
+  def set_proxy(proxy : HTTPProxy)
+    begin
+      @socket = proxy.open(host: @host, port: @port, tls: @tls, connection_options: proxy_connection_options)
+    rescue IO::Error
+      @socket = nil
+    end
+  end
+
+  def proxy_connection_options
+    opts = {} of Symbol => Float64 | Nil
+
+    opts[:dns_timeout] = @dns_timeout
+    opts[:connect_timeout] = @connect_timeout
+    opts[:read_timeout] = @read_timeout
+
+    return opts
+  end
+end
+
+def get_proxies(country_code = "US")
+  client = HTTP::Client.new(URI.parse("http://spys.one"))
+  client.read_timeout = 10.seconds
+  client.connect_timeout = 10.seconds
+
+  headers = HTTP::Headers.new
+  headers["User-Agent"] = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36"
+  headers["Accept"] = "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8"
+  headers["Accept-Language"] = "Accept-Language: en-US,en;q=0.9"
+  headers["Host"] = "spys.one"
+  headers["Content-Type"] = "application/x-www-form-urlencoded"
+  body = {
+    "xpp" => "2",
+    "xf1" => "0",
+    "xf2" => "2",
+    "xf4" => "1",
+    "xf5" => "1",
+  }
+  response = client.post("/free-proxy-list/#{country_code}/", headers, form: body)
+  response = XML.parse_html(response.body)
+
+  proxies = [] of {ip: String, port: Int32, score: Float64}
+  response = response.xpath_nodes(%q(//table))[1]
+  response.xpath_nodes(%q(.//tr)).each do |node|
+    if !node["onmouseover"]?
+      next
+    end
+
+    ip = node.xpath_node(%q(.//td[1]/font[2])).to_s.match(/<font class="spy14">(?<address>[^<]+)</).not_nil!["address"]
+    port = 3128
+
+    latency = node.xpath_node(%q(.//td[6])).not_nil!.content.to_f
+    speed = node.xpath_node(%q(.//td[7]/font/table)).not_nil!["width"].to_f
+    uptime = node.xpath_node(%q(.//td[8]/font/acronym)).not_nil!
+
+    # Skip proxies that are down
+    if uptime["title"].ends_with? "?"
+      next
+    end
+
+    if md = uptime.content.match(/^\d+/)
+      uptime = md[0].to_f
+    else
+      next
+    end
+
+    score = (uptime*4 + speed*2 + latency)/7
+
+    proxies << {ip: ip, port: port, score: score}
+  end
+
+  return proxies
+end

src/invidious/videos.cr

@@ -108,6 +108,9 @@ CAPTION_LANGUAGES = {
   "Zulu",
 }
 
+REGIONS        = {"AD", "AE", "AF", "AG", "AI", "AL", "AM", "AO", "AQ", "AR", "AS", "AT", "AU", "AW", "AX", "AZ", "BA", "BB", "BD", "BE", "BF", "BG", "BH", "BI", "BJ", "BL", "BM", "BN", "BO", "BQ", "BR", "BS", "BT", "BV", "BW", "BY", "BZ", "CA", "CC", "CD", "CF", "CG", "CH", "CI", "CK", "CL", "CM", "CN", "CO", "CR", "CU", "CV", "CW", "CX", "CY", "CZ", "DE", "DJ", "DK", "DM", "DO", "DZ", "EC", "EE", "EG", "EH", "ER", "ES", "ET", "FI", "FJ", "FK", "FM", "FO", "FR", "GA", "GB", "GD", "GE", "GF", "GG", "GH", "GI", "GL", "GM", "GN", "GP", "GQ", "GR", "GS", "GT", "GU", "GW", "GY", "HK", "HM", "HN", "HR", "HT", "HU", "ID", "IE", "IL", "IM", "IN", "IO", "IQ", "IR", "IS", "IT", "JE", "JM", "JO", "JP", "KE", "KG", "KH", "KI", "KM", "KN", "KP", "KR", "KW", "KY", "KZ", "LA", "LB", "LC", "LI", "LK", "LR", "LS", "LT", "LU", "LV", "LY", "MA", "MC", "MD", "ME", "MF", "MG", "MH", "MK", "ML", "MM", "MN", "MO", "MP", "MQ", "MR", "MS", "MT", "MU", "MV", "MW", "MX", "MY", "MZ", "NA", "NC", "NE", "NF", "NG", "NI", "NL", "NO", "NP", "NR", "NU", "NZ", "OM", "PA", "PE", "PF", "PG", "PH", "PK", "PL", "PM", "PN", "PR", "PS", "PT", "PW", "PY", "QA", "RE", "RO", "RS", "RU", "RW", "SA", "SB", "SC", "SD", "SE", "SG", "SH", "SI", "SJ", "SK", "SL", "SM", "SN", "SO", "SR", "SS", "ST", "SV", "SX", "SY", "SZ", "TC", "TD", "TF", "TG", "TH", "TJ", "TK", "TL", "TM", "TN", "TO", "TR", "TT", "TV", "TW", "TZ", "UA", "UG", "UM", "US", "UY", "UZ", "VA", "VC", "VE", "VG", "VI", "VN", "VU", "WF", "WS", "YE", "YT", "ZA", "ZM", "ZW"}
+BYPASS_REGIONS = {"CA", "DE", "FR", "JP", "RU", "UK"}
+
 VIDEO_THUMBNAILS = {
   {name: "default", url: "default", height: 90, width: 120},
   {name: "medium", url: "mqdefault", height: 180, width: 320},
@@ -409,6 +412,49 @@ def fetch_video(id)
   html = html_channel.receive
   info = info_channel.receive
 
+  if info["reason"]? && info["reason"].includes? "your country"
+    bypass_channel = Channel({HTTP::Params | Nil, XML::Node | Nil}).new
+
+    BYPASS_REGIONS.each do |country_code|
+      spawn do
+        begin
+          proxies = get_proxies(country_code)
+
+          # Try not to overload single proxy
+          proxy = proxies[0, 5].sample(1)[0]
+          proxy = HTTPProxy.new(proxy_host: proxy[:ip], proxy_port: proxy[:port])
+
+          client = HTTPClient.new(URI.parse("https://www.youtube.com"))
+          client.read_timeout = 10.seconds
+          client.connect_timeout = 10.seconds
+          client.set_proxy(proxy)
+
+          proxy_info = client.get("/get_video_info?video_id=#{id}&el=detailpage&ps=default&eurl=&gl=US&hl=en&disable_polymer=1")
+          proxy_info = HTTP::Params.parse(proxy_info.body)
+
+          if !proxy_info["reason"]?
+            proxy_html = client.get("/watch?v=#{id}&bpctr=#{Time.new.epoch + 2000}&gl=US&hl=en&disable_polymer=1")
+            proxy_html = XML.parse_html(proxy_html.body)
+
+            bypass_channel.send({proxy_info, proxy_html})
+          else
+            bypass_channel.send({nil, nil})
+          end
+        rescue ex
+          bypass_channel.send({nil, nil})
+        end
+      end
+    end
+
+    BYPASS_REGIONS.size.times do
+      response = bypass_channel.receive
+      if response[0] || response[1]
+        info = response[0].not_nil!
+        html = response[1].not_nil!
+      end
+    end
+  end
+
   if info["reason"]?
     raise info["reason"]
   end

src/invidious/views/watch.ecr

@@ -199,6 +199,15 @@ get_youtube_comments();
             <p id="Wilson">Wilson Score : <%= video.wilson_score.round(4) %></p>
             <p id="Rating">Rating : <%= rating.round(4) %> / 5</p>
             <p id="Engagement">Engagement : <%= engagement.round(2) %>%</p>
+            <% if video.allowed_regions.size != REGIONS.size %>
+                <p id="AllowedRegions">
+                <% if video.allowed_regions.size < REGIONS.size / 2 %>
+                Whitelisted regions: <%= video.allowed_regions.join(", ") %>
+                <% else %>
+                Blacklisted regions: <%= (REGIONS.to_a - video.allowed_regions).join(", ") %>
+                <% end %>
+                </p>
+            <% end %>
             <% if ad_slots %>
             <p id="Slots">Ad Slots : <%= ad_slots %></p>
             <% end %>