Added cipher guessing

Rakefile

@@ -2,12 +2,13 @@ require 'rubygems'
 require 'bundler/setup'
 require 'rake/testtask'
 
-ALL_INTEGRATION = FileList["spec/integration/*.rb", "spec/integration/*/*.rb"]
-ALL_UNIT        = FileList["spec/unit/*/*.rb"]
+SKIPPED_INTEGRATION = FileList["spec/integration/youtube/cipher_guesser_spec.rb"]
+ALL_INTEGRATION     = FileList["spec/integration/*.rb", "spec/integration/*/*.rb"] - SKIPPED_INTEGRATION
+ALL_UNIT            = FileList["spec/unit/*.rb", "spec/unit/*/*.rb"]
 
-task :default => [:all]
+task :default => [:test_all]
 
-Rake::TestTask.new(:all) do |t|
+Rake::TestTask.new(:test_all) do |t|
   t.test_files =  ALL_INTEGRATION + ALL_UNIT
 end
 
@@ -34,3 +35,11 @@ end
 Rake::TestTask.new(:test_cipher_loader) do |t|
   t.test_files = FileList["spec/integration/youtube/cipher_loader_spec.rb"]
 end
+
+Rake::TestTask.new(:test_cipher_guesser) do |t|
+  t.test_files = FileList["spec/unit/youtube/cipher_guesser_spec.rb"]
+end
+
+Rake::TestTask.new(:test_decipherer) do |t|
+  t.test_files = FileList["spec/unit/youtube/decipherer_spec.rb"]
+end

bin/helper/downloader.rb

@@ -15,15 +15,17 @@ def download(download_queue, params)
                                                  name,
                                                  :save_dir => params[:save_dir],
                                                  :tool => params[:tool] && params[:tool].to_sym
-      unless result
+      if result
+        puts "Download for #{name} successful."
+        url_name[:on_downloaded].call(true) if url_name[:on_downloaded]
+        ViddlRb::AudioHelper.extract(name, params[:save_dir]) if params[:extract_audio]
+      else
+        url_name[:on_downloaded].call(false) if url_name[:on_downloaded]
         if params[:abort_on_failure]
           raise DownloadFailedError, "Download for #{name} failed."
         else
           puts "Download for #{name} failed. Moving onto next file."
         end
-      else
-        puts "Download for #{name} successful."
-        ViddlRb::AudioHelper.extract(name, params[:save_dir]) if params[:extract_audio]
       end
     end
   end

plugins/youtube.rb

@@ -1,17 +1,16 @@
 class Youtube < PluginBase
 
+  #TODO: TEST THIS: https://www.youtube.com/watch?v=Qapou-3-fM8&list=PL_Z529zmzNGcOVBJA0MgjjQoKiBcmMQWh
+
   # this will be called by the main app to check whether this plugin is responsible for the url passed
   def self.matches_provider?(url)
     url.include?("youtube.com") || url.include?("youtu.be")
   end
 
   def self.get_urls_and_filenames(url, options = {})
+    initialize_components(options)
 
-    @url_resolver   = UrlResolver.new
-    @video_resolver = VideoResolver.new(Decipherer.new(CipherLoader.new))
-    @format_picker  = FormatPicker.new(options)
-
-    urls = @url_resolver.get_all_urls(url, options[:filter])
+    urls   = @url_resolver.get_all_urls(url, options[:filter])
     videos = get_videos(urls)
 
     return_value = videos.map do |video|
@@ -22,6 +21,14 @@ def self.get_urls_and_filenames(url, options = {})
     return_value.empty? ? download_error("No videos could be downloaded.") : return_value
   end
 
+  def self.initialize_components(options)
+    @cipher_io      = CipherIO.new
+    coordinator     = DecipherCoordinator.new(Decipherer.new(@cipher_io), CipherGuesser.new)
+    @video_resolver = VideoResolver.new(coordinator)
+    @url_resolver   = UrlResolver.new
+    @format_picker  = FormatPicker.new(options)
+  end
+
   def self.notify(message)
     puts "[YOUTUBE] #{message}"
   end
@@ -36,17 +43,26 @@ def self.get_videos(urls)
         @video_resolver.get_video(url)
       rescue VideoResolver::VideoRemovedError
         notify "The video #{url} has been removed."
+        nil
       rescue => e
         notify "Error getting the video: #{e.message}"
+        nil
       end
     end
-
     videos.reject(&:nil?)
   end
 
   def self.make_url_filname_hash(video, format)
     url = video.get_download_url(format.itag)
     name = PluginBase.make_filename_safe(video.title) + ".#{format.extension}"
-    {url: url, name: name}
+    {url: url, name: name, on_downloaded: make_downloaded_callback(video)}
+  end
+
+  def self.make_downloaded_callback(video)
+    return nil unless video.signature_guess?
+
+    lambda do |success|
+      @cipher_io.add_cipher(video.cipher_version, video.cipher_operations) if success
+    end
   end
 end

plugins/youtube/cipher_guesser.rb

@@ -0,0 +1,114 @@
+require 'open-uri'
+
+class CipherGuesser
+  class CipherGuessError < StandardError; end
+
+  JS_URL = "http://s.ytimg.com/yts/jsbin/html5player-%s.js"
+
+  def guess(cipher_version)
+    js   = download_player_javascript(cipher_version)
+    body = extract_decipher_function_body(js)
+
+    parse_function_body(body)
+  end
+
+  private
+
+  def download_player_javascript(cipher_version)
+    open(JS_URL % cipher_version).read
+  end
+
+  def extract_decipher_function_body(js)
+    function_name  = js[decipher_function_name_regex, 1]
+    function_regex = get_function_regex(function_name)
+    match          = function_regex.match(js)
+
+    raise(CipherGuessError, "Could not extract the decipher function") unless match
+    match[:brace]
+  end
+
+  def parse_function_body(body)
+    lines = body.split(";")
+
+    remove_non_decipher_lines!(lines)
+    do_pre_transformations!(lines)
+
+    lines.map do |line|
+      if /\(\w+,(?<index>\d+)\)/     =~ line  # calling a two argument function (swap)
+        "w#{index}"
+      elsif /slice\((?<index>\d+)\)/ =~ line  # calling slice
+        "s#{index}"
+      elsif /reverse\(\)/            =~ line  # calling reverse
+        "r"
+      else
+        raise "Cannot parse line: #{line}"
+      end
+    end
+  end
+
+  def remove_non_decipher_lines!(lines)
+    # The first line splits the string into an array and the last joins and returns
+    lines.delete_at(0)
+    lines.delete_at(-1)
+  end
+
+  def do_pre_transformations!(lines)
+    change_inline_swap_to_function_call(lines) if inline_swap?(lines)
+  end
+
+  def inline_swap?(lines)
+    # Defining a variable = inline swap function
+    lines.any? { |line| line.include?("var ") }
+  end
+
+  def change_inline_swap_to_function_call(lines)
+    start_index = lines.find_index { |line| line.include?("var ") }
+    swap_lines  = lines.slice!(start_index, 3)  # inline swap is 3 lines long
+    i1, i2      = get_swap_indices(swap_lines)
+
+    lines.insert(start_index, "swap(#{i1},#{i2})")
+    lines
+  end
+
+  def get_swap_indices(lines)
+    i1 = lines.first[/(\d+)/, 1]
+    i2 = lines.last[/(\d+)/, 1]
+    [i1, i2]
+  end
+
+  def decipher_function_name_regex
+    # Find "C" in this: var A = B.sig || C (B.s)
+    /
+    \.sig
+    \s*
+    \|\|
+    (\w+)
+    \(
+    /x
+  end
+
+  def get_function_regex(function_name)
+  # Match the function function_name (that has one argument)
+    /
+    #{function_name}
+    \(
+    \w+
+    \)
+    #{function_body_regex}
+    /x
+  end
+
+  def function_body_regex
+  # Match nested braces
+    /
+    (?<brace>
+    {
+    (
+    [^{}]
+    | \g<brace>
+    )*
+    }
+    )
+    /x
+  end
+end

plugins/youtube/cipher_loader.rb → plugins/youtube/cipher_io.rb

@@ -4,7 +4,7 @@
 require 'openssl'
 require 'yaml'
 
-class CipherLoader
+class CipherIO
 
   CIPHER_YAML_URL = "https://raw.github.com/rb2k/viddl-rb/master/plugins/youtube/ciphers.yml"
   CIPHER_YAML_PATH = File.join(ViddlRb::UtilityHelper.base_path, "plugins/youtube/ciphers.yml")
@@ -23,24 +23,29 @@ def load_ciphers
     @ciphers.dup
   end
 
+  def add_cipher(version, operations)
+    File.open(CIPHER_YAML_PATH, "a") do |file|
+      file.puts("#{version}: #{operations}")
+    end
+  end
+
   private
 
   def update_ciphers
-    return if get_server_file_size == get_local_file_size
+    server_etag = get_server_etag
+    return if server_etag == @ciphers["ETag"]
 
     @ciphers.merge!(download_server_ciphers)
+    @ciphers["ETag"] = server_etag
     save_local_ciphers(@ciphers)
   end
 
-  def get_local_file_size
-    File.size(CIPHER_YAML_PATH)
-  end
-
-  def get_server_file_size 
+  def get_server_etag
     uri = URI.parse(CIPHER_YAML_URL)
     http = make_http(uri)
     head = Net::HTTP::Head.new(uri.request_uri)
-    http.request(head)["Content-Length"].to_i
+    etag = http.request(head)["ETag"]
+    etag.gsub('"', '')  # remove leading and trailing quotes
   end
 
   def make_http(uri)

plugins/youtube/decipher_coordinator.rb

@@ -0,0 +1,28 @@
+
+class DecipherCoordinator
+
+  def initialize(decipherer, cipher_guesser)
+    @decipherer = decipherer
+    @cipher_guesser = cipher_guesser
+  end
+
+  def get_decipher_data(cipher_version)
+    ops = @decipherer.get_operations(cipher_version)
+    Youtube.notify "Cipher guess: no"
+    {version: cipher_version, operations: ops.join(" "), guess?: false}
+
+  rescue Decipherer::UnknownCipherVersionError => e
+    ops = @cipher_guesser.guess(cipher_version)
+    Youtube.notify "Cipher guess: yes"
+    {version: cipher_version, operations: ops.join(" "), guess?: true}
+
+  rescue Decipherer::UnknownCipherOperationError => e
+    Youtube.notify "Failed to parse the cipher from the Youtube player version #{cipher_version}\n" +
+                   "Please submit a bug report at https://github.com/rb2k/viddl-rb"
+    raise e
+  end
+
+  def decipher_with_operations(cipher, operations)
+    @decipherer.decipher_with_operations(cipher, operations)
+  end
+end

plugins/youtube/decipherer.rb

@@ -1,18 +1,31 @@
 
 class Decipherer
 
-  class UnknownCipherVersionError < StandardError; end 
   class UnknownCipherOperationError < StandardError; end
 
+  class UnknownCipherVersionError < StandardError
+
+    attr_reader :cipher_version
+
+    def initialize(cipher_version)
+      super("Unknown cipher version: #{cipher_version}")
+      @cipher_version = cipher_version
+    end
+  end
+
   def initialize(loader)
     @ciphers = loader.load_ciphers
   end
 
   def decipher_with_version(cipher, cipher_version)
-    operations = @ciphers[cipher_version] 
-    raise UnknownCipherVersionError.new("Unknown cipher version: #{cipher_version}") unless operations
+    ops = get_operations(cipher_version)
+    decipher_with_operations(cipher, ops)
+  end
 
-    decipher_with_operations(cipher, operations.split)
+  def get_operations(cipher_version)
+    operations = @ciphers[cipher_version] 
+    raise UnknownCipherVersionError.new(cipher_version) unless operations
+    operations.split
   end
 
   def decipher_with_operations(cipher, operations)