diff --git a/.document b/.document new file mode 100644 index 0000000..7f80cfc --- /dev/null +++ b/.document @@ -0,0 +1,3 @@ +- +ChangeLog.md +LICENSE.txt diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..ac5b454 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +doc/ +pkg/ diff --git a/.rspec b/.rspec new file mode 100644 index 0000000..660778b --- /dev/null +++ b/.rspec @@ -0,0 +1 @@ +--colour --format documentation diff --git a/.yardopts b/.yardopts new file mode 100644 index 0000000..6cf835d --- /dev/null +++ b/.yardopts @@ -0,0 +1 @@ +--markup markdown --title "flv-dl Documentation" --protected diff --git a/ChangeLog.md b/ChangeLog.md new file mode 100644 index 0000000..5b7cba0 --- /dev/null +++ b/ChangeLog.md @@ -0,0 +1,4 @@ +### 0.1.0 / 2012-07-14 + +* Initial release: + diff --git a/LICENSE.txt b/LICENSE.txt new file mode 100644 index 0000000..286dc96 --- /dev/null +++ b/LICENSE.txt @@ -0,0 +1,20 @@ +Copyright (c) 2012 Postmodern + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/README.md b/README.md new file mode 100644 index 0000000..45d6d16 --- /dev/null +++ b/README.md @@ -0,0 +1,53 @@ +# flv-dl + +* [Homepage](https://github.com/sophsec/flv-dl#readme) +* [Issues](https://github.com/sophsec/flv-dl/issues) +* [Documentation](http://rubydoc.info/gems/flv-dl/frames) +* [Email](mailto:postmodern.mod3 at gmail.com) + +## Description + +Downloads or plays Flash Video (flv) file directly from their web-page. + +## Why? + +Because **fuck flash**. + +## Features + +* Extracts `flashvars` from. + * `param` tags. + * `embed` / `object` tags. + * JavaScript + +## Synopsis + +Downloads a video: + + flv-dl "URL" + +Plays a video: + + flv-dl -p totem "URL" + +List available formats / URLs: + + flv-dl -U "URL" + +Dumps the collected `flashvars`: + + flv-dl -D "URL" + +## Requirements + +* [nokogiri](https://github.com/tenderlove/nokogiri) ~> 1.4 + +## Install + + $ gem install flv-dl + +## Copyright + +Copyright (c) 2012 Postmodern + +See {file:LICENSE.txt} for details. diff --git a/Rakefile b/Rakefile new file mode 100644 index 0000000..3f3a260 --- /dev/null +++ b/Rakefile @@ -0,0 +1,40 @@ +# encoding: utf-8 + +require 'rubygems' +require 'rake' + +begin + gem 'rubygems-tasks', '~> 0.2' + require 'rubygems/tasks' + + Gem::Tasks.new +rescue LoadError => e + warn e.message + warn "Run `gem install rubygems-tasks` to install 'rubygems/tasks'." +end + +begin + gem 'rspec', '~> 2.4' + require 'rspec/core/rake_task' + + RSpec::Core::RakeTask.new +rescue LoadError => e + task :spec do + abort "Please run `gem install rspec` to install RSpec." + end +end + +task :test => :spec +task :default => :spec + +begin + gem 'yard', '~> 0.7' + require 'yard' + + YARD::Rake::YardocTask.new +rescue LoadError => e + task :yard do + abort "Please run `gem install yard` to install YARD." + end +end +task :doc => :yard diff --git a/bin/flv-dl b/bin/flv-dl new file mode 100755 index 0000000..720985d --- /dev/null +++ b/bin/flv-dl @@ -0,0 +1,124 @@ +#!/usr/bin/env ruby + +require 'net/http' +require 'optparse' +require 'pp' + +require 'flv/video' + +options = { + :mode => :play, + :player => ENV['VIDEO_PLAYER'], + :format => :flv +} + +optparser = OptionParser.new do |opts| + opts.banner = "Usage: #{File.basename($0)} [options] URL" + + opts.on('-o','--output PATH','Path to download the video to') do |output| + options[:mode] = :download + options[:output] = output + end + + opts.on('-p','--play [COMMAND]','Play the video URL') do |player| + options[:mode] = :play + options[:player] = player if player + end + + opts.on('-f',"--format [#{FLV::Page::FORMATS.keys.join(', ')}]",'The video format') do |format| + options[:format] = format.to_sym + end + + opts.on('-F', '--formats','Lists the available video formats') do + options[:mode] = :list + options[:list] = :formats + end + + opts.on('-U', '--urls','Lists the available video URLs') do + options[:mode] = :list + options[:list] = :urls + end + + opts.on('-D', '--dump','Dumps the flashvars') do + options[:mode] = :list + options[:list] = :flashvars + end +end + +optparser.parse! + +video = FLV::Page.new(ARGV[0]) + +if options[:list] + case options[:list] + when :formats + puts(*video.formats) + when :urls + video.video_urls.each do |format,url| + puts "#{format}: #{url}" + end + when :flashvars + pp video.flashvars + end +else + unless video.flashvars + $stderr.puts "Could not extract flashvars from #{video.url}" + exit -1 + end + + unless video.formats.include?(options[:format]) + $stderr.puts "Unknown format: #{options[:format]}" + exit -1 + end + + video_url = video.video_urls[options[:format]] + + case options[:mode] + when :play + unless options[:player] + $stderr.puts "Must specify the video player via --play or $VIDEO_PLAYER" + exit -1 + end + + system(options[:player],video_url) + when :download + unless options[:output] + $stderr.puts "Must specify --output PATH" + exit -1 + end + + puts ">>> Requesting #{video_url} ..." + + http = Net::HTTP.new(video_url.host,video_url.port) + + if video_url.scheme == 'https' + require 'net/https' + + http.use_ssl = true + http.verify_mode = OpenSSL::SSL::VERIFY_NONE + end + + request = Net::HTTP::Get.new(video_url.request_uri) + request['Referer'] = video_url + request['User-Agent'] = 'Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; WOW64; Trident/6.0)' + + http.request(request) do |response| + size, total = 0, response.header['Content-Length'].to_i + + video_path = options.fetch(:output) + + puts ">>> Downloading to #{video_path.dump} ..." + + File.open(video_path,"wb") do |file| + response.read_body do |chunk| + file.write(chunk) + + size += chunk.size + printf "\r>>> [%d / %d] %d%% ...", size, total, ((size * 100) / total) + end + end + + puts "\n>>> Download complete!" + end + end +end diff --git a/flv-dl.gemspec b/flv-dl.gemspec new file mode 100644 index 0000000..9e696ae --- /dev/null +++ b/flv-dl.gemspec @@ -0,0 +1,60 @@ +# encoding: utf-8 + +require 'yaml' + +Gem::Specification.new do |gem| + gemspec = YAML.load_file('gemspec.yml') + + gem.name = gemspec.fetch('name') + gem.version = gemspec.fetch('version') do + lib_dir = File.join(File.dirname(__FILE__),'lib') + $LOAD_PATH << lib_dir unless $LOAD_PATH.include?(lib_dir) + + require 'flv/dl/version' + Flv::Dl::VERSION + end + + gem.summary = gemspec['summary'] + gem.description = gemspec['description'] + gem.licenses = Array(gemspec['license']) + gem.authors = Array(gemspec['authors']) + gem.email = gemspec['email'] + gem.homepage = gemspec['homepage'] + + glob = lambda { |patterns| gem.files & Dir[*patterns] } + + gem.files = `git ls-files`.split($/) + gem.files = glob[gemspec['files']] if gemspec['files'] + + gem.executables = gemspec.fetch('executables') do + glob['bin/*'].map { |path| File.basename(path) } + end + gem.default_executable = gem.executables.first if Gem::VERSION < '1.7.' + + gem.extensions = glob[gemspec['extensions'] || 'ext/**/extconf.rb'] + gem.test_files = glob[gemspec['test_files'] || '{test/{**/}*_test.rb'] + gem.extra_rdoc_files = glob[gemspec['extra_doc_files'] || '*.{txt,md}'] + + gem.require_paths = Array(gemspec.fetch('require_paths') { + %w[ext lib].select { |dir| File.directory?(dir) } + }) + + gem.requirements = gemspec['requirements'] + gem.required_ruby_version = gemspec['required_ruby_version'] + gem.required_rubygems_version = gemspec['required_rubygems_version'] + gem.post_install_message = gemspec['post_install_message'] + + split = lambda { |string| string.split(/,\s*/) } + + if gemspec['dependencies'] + gemspec['dependencies'].each do |name,versions| + gem.add_dependency(name,split[versions]) + end + end + + if gemspec['development_dependencies'] + gemspec['development_dependencies'].each do |name,versions| + gem.add_development_dependency(name,split[versions]) + end + end +end diff --git a/gemspec.yml b/gemspec.yml new file mode 100644 index 0000000..93dfc2b --- /dev/null +++ b/gemspec.yml @@ -0,0 +1,18 @@ +name: flv-dl +version: 0.1.0 +summary: Downloads Flash Video (flv) files from web-pages. +description: + Downloads or plays Flash Video (flv) files directly from web-pages. + +license: MIT +authors: Postmodern +email: postmodern.mod3@gmail.com +homepage: https://github.com/sophsec/flv-dl#readme + +dependencies: + nokogiri: ~> 1.4 + +development_dependencies: + rubygems-tasks: ~> 0.2 + rspec: ~> 2.4 + yard: ~> 0.7 diff --git a/lib/flv/video.rb b/lib/flv/video.rb new file mode 100644 index 0000000..2e1f0ca --- /dev/null +++ b/lib/flv/video.rb @@ -0,0 +1,128 @@ +require 'uri' +require 'open-uri' +require 'nokogiri' +require 'json' +require 'uri/query_params' + +module FLV + class Video + + FORMATS = { + :mp4 => ['mp4_url'], + :flv => ['flv_url', 'flv', 'video_url', 'file'], + :flv_h264 => ['flv_h264'] + } + + attr_reader :url + + def initialize(url) + @url = URI(url) + @doc = Nokogiri::HTML(open(@url)) + end + + def title + @title ||= @doc.at('//title').inner_text + end + + def flashvars + @flashvars ||= ( + extract_flashvars_from_html || + extract_flashvars_from_embedded_html || + extract_flashvars_from_javascript || + {} + ) + end + + def formats + @formats ||= FORMATS.keys.select do |format| + FORMATS[format].any? { |var| flashvars.has_key?(var) } + end + end + + def video_urls + @video_urls ||= begin + urls = {} + + FORMATS.each do |format,vars| + vars.each do |var| + if flashvars.has_key?(var) + urls[format] = @url.merge(flashvars[var]) + next + end + end + end + + urls + end + end + + protected + + FLASHVARS_XPATHS = [ + '//@flashvars', + '//param[@name="flashvars"]/@value', + '//param[@name="FlashVars"]/@value' + ] + + def extract_flashvars_from_html(doc=@doc) + if (flashvars = doc.at(FLASHVARS_XPATHS.join('|'))) + URI::QueryParams.parse(flashvars.value) + end + end + + EMBEDDED_FLASHVARS_XPATHS = [ + '//*[contains(.,"flashvars")]', + '//*[contains(.,"FlashVars")]' + ] + + def extract_flashvars_from_embedded_html + if (attr = @doc.at(EMBEDDED_FLASHVARS_XPATHS.join('|'))) + html = Nokogiri::HTML(attr.inner_text) + + extract_flashvars_from_html(html) + end + end + + FLASHVARS_JAVASCRIPT_XPATHS = [ + '//script[contains(.,"flashvars")]', + '//script[contains(.,"jwplayer")]', + ] + + def extract_flashvars_from_javascript + if (script = @doc.at(FLASHVARS_JAVASCRIPT_XPATHS.join('|'))) + code = script.inner_text + + return extract_flashvars_from_javascript_function(code) || + extract_flashvars_from_javascript_hash(code) + end + end + + FLASHVARS_FUNCTION_REGEXP = /['"]flashvars['"]\s*,\s*['"]([^'"]+)['"]/ + + def extract_flashvars_from_javascript_function(code) + if (match = code.match(FLASHVARS_FUNCTION_REGEXP)) + URI::QueryParams.parse(match[1]) + end + end + + FLASHVARS_HASH_REGEXP = /(?:flashvars\s*=\s*|jwplayer\(['"][a-z]+['"]\).setup\()(\{[^\}]+\})/m + + def extract_flashvars_from_javascript_hash(code) + regexp = lambda { |vars| + + } + + flashvars = {} + + vars = FORMATS.values.flatten + regexp = /['"](#{Regexp.union(vars)})['"]:\s*['"]([^'"]+)['"]/ + + if (match = code.match(regexp)) + flashvars[match[1]] = URI.decode(match[2]) + end + + return flashvars + end + + end +end diff --git a/spec/dl_spec.rb b/spec/dl_spec.rb new file mode 100644 index 0000000..bc2812e --- /dev/null +++ b/spec/dl_spec.rb @@ -0,0 +1,8 @@ +require 'spec_helper' +require 'flv/dl' + +describe Flv::Dl do + it "should have a VERSION constant" do + subject.const_get('VERSION').should_not be_empty + end +end diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb new file mode 100644 index 0000000..974875c --- /dev/null +++ b/spec/spec_helper.rb @@ -0,0 +1,5 @@ +gem 'rspec', '~> 2.4' +require 'rspec' +require 'flv/dl/version' + +include Flv::Dl