Permalink
Browse files

Initial commit.

  • Loading branch information...
0 parents commit c733384cd9d077fa6e801e733014516e0aef1813 @postmodern postmodern committed Jul 14, 2012
Showing with 467 additions and 0 deletions.
  1. +3 −0 .document
  2. +2 −0 .gitignore
  3. +1 −0 .rspec
  4. +1 −0 .yardopts
  5. +4 −0 ChangeLog.md
  6. +20 −0 LICENSE.txt
  7. +53 −0 README.md
  8. +40 −0 Rakefile
  9. +124 −0 bin/flv-dl
  10. +60 −0 flv-dl.gemspec
  11. +18 −0 gemspec.yml
  12. +128 −0 lib/flv/video.rb
  13. +8 −0 spec/dl_spec.rb
  14. +5 −0 spec/spec_helper.rb
3 .document
@@ -0,0 +1,3 @@
+-
+ChangeLog.md
+LICENSE.txt
2 .gitignore
@@ -0,0 +1,2 @@
+doc/
+pkg/
1 .rspec
@@ -0,0 +1 @@
+--colour --format documentation
1 .yardopts
@@ -0,0 +1 @@
+--markup markdown --title "flv-dl Documentation" --protected
4 ChangeLog.md
@@ -0,0 +1,4 @@
+### 0.1.0 / 2012-07-14
+
+* Initial release:
+
20 LICENSE.txt
@@ -0,0 +1,20 @@
+Copyright (c) 2012 Postmodern
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
53 README.md
@@ -0,0 +1,53 @@
+# flv-dl
+
+* [Homepage](https://github.com/sophsec/flv-dl#readme)
+* [Issues](https://github.com/sophsec/flv-dl/issues)
+* [Documentation](http://rubydoc.info/gems/flv-dl/frames)
+* [Email](mailto:postmodern.mod3 at gmail.com)
+
+## Description
+
+Downloads or plays Flash Video (flv) file directly from their web-page.
+
+## Why?
+
+Because **fuck flash**.
+
+## Features
+
+* Extracts `flashvars` from.
+ * `param` tags.
+ * `embed` / `object` tags.
+ * JavaScript
+
+## Synopsis
+
+Downloads a video:
+
+ flv-dl "URL"
+
+Plays a video:
+
+ flv-dl -p totem "URL"
+
+List available formats / URLs:
+
+ flv-dl -U "URL"
+
+Dumps the collected `flashvars`:
+
+ flv-dl -D "URL"
+
+## Requirements
+
+* [nokogiri](https://github.com/tenderlove/nokogiri) ~> 1.4
+
+## Install
+
+ $ gem install flv-dl
+
+## Copyright
+
+Copyright (c) 2012 Postmodern
+
+See {file:LICENSE.txt} for details.
40 Rakefile
@@ -0,0 +1,40 @@
+# encoding: utf-8
+
+require 'rubygems'
+require 'rake'
+
+begin
+ gem 'rubygems-tasks', '~> 0.2'
+ require 'rubygems/tasks'
+
+ Gem::Tasks.new
+rescue LoadError => e
+ warn e.message
+ warn "Run `gem install rubygems-tasks` to install 'rubygems/tasks'."
+end
+
+begin
+ gem 'rspec', '~> 2.4'
+ require 'rspec/core/rake_task'
+
+ RSpec::Core::RakeTask.new
+rescue LoadError => e
+ task :spec do
+ abort "Please run `gem install rspec` to install RSpec."
+ end
+end
+
+task :test => :spec
+task :default => :spec
+
+begin
+ gem 'yard', '~> 0.7'
+ require 'yard'
+
+ YARD::Rake::YardocTask.new
+rescue LoadError => e
+ task :yard do
+ abort "Please run `gem install yard` to install YARD."
+ end
+end
+task :doc => :yard
124 bin/flv-dl
@@ -0,0 +1,124 @@
+#!/usr/bin/env ruby
+
+require 'net/http'
+require 'optparse'
+require 'pp'
+
+require 'flv/video'
+
+options = {
+ :mode => :play,
+ :player => ENV['VIDEO_PLAYER'],
+ :format => :flv
+}
+
+optparser = OptionParser.new do |opts|
+ opts.banner = "Usage: #{File.basename($0)} [options] URL"
+
+ opts.on('-o','--output PATH','Path to download the video to') do |output|
+ options[:mode] = :download
+ options[:output] = output
+ end
+
+ opts.on('-p','--play [COMMAND]','Play the video URL') do |player|
+ options[:mode] = :play
+ options[:player] = player if player
+ end
+
+ opts.on('-f',"--format [#{FLV::Page::FORMATS.keys.join(', ')}]",'The video format') do |format|
+ options[:format] = format.to_sym
+ end
+
+ opts.on('-F', '--formats','Lists the available video formats') do
+ options[:mode] = :list
+ options[:list] = :formats
+ end
+
+ opts.on('-U', '--urls','Lists the available video URLs') do
+ options[:mode] = :list
+ options[:list] = :urls
+ end
+
+ opts.on('-D', '--dump','Dumps the flashvars') do
+ options[:mode] = :list
+ options[:list] = :flashvars
+ end
+end
+
+optparser.parse!
+
+video = FLV::Page.new(ARGV[0])
+
+if options[:list]
+ case options[:list]
+ when :formats
+ puts(*video.formats)
+ when :urls
+ video.video_urls.each do |format,url|
+ puts "#{format}: #{url}"
+ end
+ when :flashvars
+ pp video.flashvars
+ end
+else
+ unless video.flashvars
+ $stderr.puts "Could not extract flashvars from #{video.url}"
+ exit -1
+ end
+
+ unless video.formats.include?(options[:format])
+ $stderr.puts "Unknown format: #{options[:format]}"
+ exit -1
+ end
+
+ video_url = video.video_urls[options[:format]]
+
+ case options[:mode]
+ when :play
+ unless options[:player]
+ $stderr.puts "Must specify the video player via --play or $VIDEO_PLAYER"
+ exit -1
+ end
+
+ system(options[:player],video_url)
+ when :download
+ unless options[:output]
+ $stderr.puts "Must specify --output PATH"
+ exit -1
+ end
+
+ puts ">>> Requesting #{video_url} ..."
+
+ http = Net::HTTP.new(video_url.host,video_url.port)
+
+ if video_url.scheme == 'https'
+ require 'net/https'
+
+ http.use_ssl = true
+ http.verify_mode = OpenSSL::SSL::VERIFY_NONE
+ end
+
+ request = Net::HTTP::Get.new(video_url.request_uri)
+ request['Referer'] = video_url
+ request['User-Agent'] = 'Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; WOW64; Trident/6.0)'
+
+ http.request(request) do |response|
+ size, total = 0, response.header['Content-Length'].to_i
+
+ video_path = options.fetch(:output)
+
+ puts ">>> Downloading to #{video_path.dump} ..."
+
+ File.open(video_path,"wb") do |file|
+ response.read_body do |chunk|
+ file.write(chunk)
+
+ size += chunk.size
+ printf "\r>>> [%d / %d] %d%% ...", size, total, ((size * 100) / total)
+ end
+ end
+
+ puts "\n>>> Download complete!"
+ end
+ end
+end
60 flv-dl.gemspec
@@ -0,0 +1,60 @@
+# encoding: utf-8
+
+require 'yaml'
+
+Gem::Specification.new do |gem|
+ gemspec = YAML.load_file('gemspec.yml')
+
+ gem.name = gemspec.fetch('name')
+ gem.version = gemspec.fetch('version') do
+ lib_dir = File.join(File.dirname(__FILE__),'lib')
+ $LOAD_PATH << lib_dir unless $LOAD_PATH.include?(lib_dir)
+
+ require 'flv/dl/version'
+ Flv::Dl::VERSION
+ end
+
+ gem.summary = gemspec['summary']
+ gem.description = gemspec['description']
+ gem.licenses = Array(gemspec['license'])
+ gem.authors = Array(gemspec['authors'])
+ gem.email = gemspec['email']
+ gem.homepage = gemspec['homepage']
+
+ glob = lambda { |patterns| gem.files & Dir[*patterns] }
+
+ gem.files = `git ls-files`.split($/)
+ gem.files = glob[gemspec['files']] if gemspec['files']
+
+ gem.executables = gemspec.fetch('executables') do
+ glob['bin/*'].map { |path| File.basename(path) }
+ end
+ gem.default_executable = gem.executables.first if Gem::VERSION < '1.7.'
+
+ gem.extensions = glob[gemspec['extensions'] || 'ext/**/extconf.rb']
+ gem.test_files = glob[gemspec['test_files'] || '{test/{**/}*_test.rb']
+ gem.extra_rdoc_files = glob[gemspec['extra_doc_files'] || '*.{txt,md}']
+
+ gem.require_paths = Array(gemspec.fetch('require_paths') {
+ %w[ext lib].select { |dir| File.directory?(dir) }
+ })
+
+ gem.requirements = gemspec['requirements']
+ gem.required_ruby_version = gemspec['required_ruby_version']
+ gem.required_rubygems_version = gemspec['required_rubygems_version']
+ gem.post_install_message = gemspec['post_install_message']
+
+ split = lambda { |string| string.split(/,\s*/) }
+
+ if gemspec['dependencies']
+ gemspec['dependencies'].each do |name,versions|
+ gem.add_dependency(name,split[versions])
+ end
+ end
+
+ if gemspec['development_dependencies']
+ gemspec['development_dependencies'].each do |name,versions|
+ gem.add_development_dependency(name,split[versions])
+ end
+ end
+end
18 gemspec.yml
@@ -0,0 +1,18 @@
+name: flv-dl
+version: 0.1.0
+summary: Downloads Flash Video (flv) files from web-pages.
+description:
+ Downloads or plays Flash Video (flv) files directly from web-pages.
+
+license: MIT
+authors: Postmodern
+email: postmodern.mod3@gmail.com
+homepage: https://github.com/sophsec/flv-dl#readme
+
+dependencies:
+ nokogiri: ~> 1.4
+
+development_dependencies:
+ rubygems-tasks: ~> 0.2
+ rspec: ~> 2.4
+ yard: ~> 0.7
128 lib/flv/video.rb
@@ -0,0 +1,128 @@
+require 'uri'
+require 'open-uri'
+require 'nokogiri'
+require 'json'
+require 'uri/query_params'
+
+module FLV
+ class Video
+
+ FORMATS = {
+ :mp4 => ['mp4_url'],
+ :flv => ['flv_url', 'flv', 'video_url', 'file'],
+ :flv_h264 => ['flv_h264']
+ }
+
+ attr_reader :url
+
+ def initialize(url)
+ @url = URI(url)
+ @doc = Nokogiri::HTML(open(@url))
+ end
+
+ def title
+ @title ||= @doc.at('//title').inner_text
+ end
+
+ def flashvars
+ @flashvars ||= (
+ extract_flashvars_from_html ||
+ extract_flashvars_from_embedded_html ||
+ extract_flashvars_from_javascript ||
+ {}
+ )
+ end
+
+ def formats
+ @formats ||= FORMATS.keys.select do |format|
+ FORMATS[format].any? { |var| flashvars.has_key?(var) }
+ end
+ end
+
+ def video_urls
+ @video_urls ||= begin
+ urls = {}
+
+ FORMATS.each do |format,vars|
+ vars.each do |var|
+ if flashvars.has_key?(var)
+ urls[format] = @url.merge(flashvars[var])
+ next
+ end
+ end
+ end
+
+ urls
+ end
+ end
+
+ protected
+
+ FLASHVARS_XPATHS = [
+ '//@flashvars',
+ '//param[@name="flashvars"]/@value',
+ '//param[@name="FlashVars"]/@value'
+ ]
+
+ def extract_flashvars_from_html(doc=@doc)
+ if (flashvars = doc.at(FLASHVARS_XPATHS.join('|')))
+ URI::QueryParams.parse(flashvars.value)
+ end
+ end
+
+ EMBEDDED_FLASHVARS_XPATHS = [
+ '//*[contains(.,"flashvars")]',
+ '//*[contains(.,"FlashVars")]'
+ ]
+
+ def extract_flashvars_from_embedded_html
+ if (attr = @doc.at(EMBEDDED_FLASHVARS_XPATHS.join('|')))
+ html = Nokogiri::HTML(attr.inner_text)
+
+ extract_flashvars_from_html(html)
+ end
+ end
+
+ FLASHVARS_JAVASCRIPT_XPATHS = [
+ '//script[contains(.,"flashvars")]',
+ '//script[contains(.,"jwplayer")]',
+ ]
+
+ def extract_flashvars_from_javascript
+ if (script = @doc.at(FLASHVARS_JAVASCRIPT_XPATHS.join('|')))
+ code = script.inner_text
+
+ return extract_flashvars_from_javascript_function(code) ||
+ extract_flashvars_from_javascript_hash(code)
+ end
+ end
+
+ FLASHVARS_FUNCTION_REGEXP = /['"]flashvars['"]\s*,\s*['"]([^'"]+)['"]/
+
+ def extract_flashvars_from_javascript_function(code)
+ if (match = code.match(FLASHVARS_FUNCTION_REGEXP))
+ URI::QueryParams.parse(match[1])
+ end
+ end
+
+ FLASHVARS_HASH_REGEXP = /(?:flashvars\s*=\s*|jwplayer\(['"][a-z]+['"]\).setup\()(\{[^\}]+\})/m
+
+ def extract_flashvars_from_javascript_hash(code)
+ regexp = lambda { |vars|
+
+ }
+
+ flashvars = {}
+
+ vars = FORMATS.values.flatten
+ regexp = /['"](#{Regexp.union(vars)})['"]:\s*['"]([^'"]+)['"]/
+
+ if (match = code.match(regexp))
+ flashvars[match[1]] = URI.decode(match[2])
+ end
+
+ return flashvars
+ end
+
+ end
+end
8 spec/dl_spec.rb
@@ -0,0 +1,8 @@
+require 'spec_helper'
+require 'flv/dl'
+
+describe Flv::Dl do
+ it "should have a VERSION constant" do
+ subject.const_get('VERSION').should_not be_empty
+ end
+end
5 spec/spec_helper.rb
@@ -0,0 +1,5 @@
+gem 'rspec', '~> 2.4'
+require 'rspec'
+require 'flv/dl/version'
+
+include Flv::Dl

0 comments on commit c733384

Please sign in to comment.