Browse files

download scriptfiles

  • Loading branch information...
1 parent 61d9d04 commit c3ced2a81a2e4d214b72f5a0c779d38a6850d712 @bronson bronson committed Aug 24, 2010
Showing with 54 additions and 5 deletions.
  1. +1 −0 .gitignore
  2. +51 −4 dumper
  3. +2 −1 scraper
View
1 .gitignore
@@ -1,2 +1,3 @@
token
repos
+scripts
View
55 dumper
@@ -1,20 +1,67 @@
#!/bin/env ruby
require 'rubygems'
+require 'open-uri'
require 'json'
require 'grit'
+require 'zip/zipfilesystem' # rubyzip gem
+
repos_dir = 'repos'
+scripts_dir = 'scripts'
+token = File.read('token').chomp
-script = JSON.parse($stdin.read)
Dir.mkdir repos_dir unless test ?d, repos_dir
+Dir.mkdir scripts_dir unless test ?d, scripts_dir
+
+script = JSON.parse($stdin.read)
repo_path = File.join(repos_dir, script['name'] + '.git')
+$repo_path = repo_path
repo = Grit::Repo.init_bare_or_open(repo_path)
+def download(url, dest)
+ puts "Downloading #{url} to #{dest}"
+ open(url, 'rb') { |u|
+ File.open(dest, 'wb') { |f|
+ f.write(u.read)
+ }
+ }
+end
+
+def copy_file(filename, contents)
+ puts "Copying #{filename} into #{$repo_path}"
+end
+
+def copy_zip(zip_path)
+ Zip::ZipFile.open(zip_path) do |zipfile|
+ zipfile.each do |file|
+ raise "add symlinks!" if file.symlink?
+ copy_file(file.name, file.get_input_stream.read)
+ end
+ end
+end
+
+# github = Octopussy::Client.new(:login => 'vim-scripts', :token => token)
+
script['versions'].reverse.each do |version|
- puts "#{version['date']} #{version['script_version']}"
+ scriptname = "#{version['date']} #{version['script_version']} #{version['filename']}"
+ scriptdir = File.join(scripts_dir, script['name'])
+ Dir.mkdir scriptdir unless test ?d, scriptdir
+ scriptfile = File.join(scriptdir, scriptname)
+ download(version['url'], scriptfile) unless test ?f, scriptfile
+ # todo: might want to do a HEAD request and match file sizes
+
+ case scriptfile
+ when /\.vim$/ then copy_file(scriptfile, File.read(scriptfile))
+ when /\.zip$/ then copy_zip(scriptfile)
+ when /\.tar\.gz$|\.tgz$/ then puts "can't handle .tar.gz"
+ when /\.tar\.bz2?$|\.tbz2?$/ then puts "can't handle .tar.bz"
+ when /\.gz$/ then puts "can't handle .gz"
+ when /\.bz2?$/ then puts "can't handle .bz"
+ when /\.vba$/ then puts "can't handle .vba"
+ end
+
+ exit
end
-# token = File.read('token').chomp
-# client = Octopussy::Client.new(:login => 'vim-scripts', :token => token)
View
3 scraper
@@ -31,7 +31,8 @@ def scrape_script(script_id)
s[:description] = doc.at('td[text()="description"]').parent.next_sibling.children.first.inner_text.gsub("\r", "\n")
s[:install_details] = doc.at('td[text()="install details"]').parent.next_sibling.children.first.inner_text.gsub("\r", "\n")
s[:versions] = doc.search('a[@href*="download_script.php?"]').to_a.map do |a|
- v = {:url => 'http://www.vim.org/scripts/' + a.attributes['href']}
+ v = {:url => 'http://www.vim.org/scripts/' + a.attributes['href'],
+ :filename => a.inner_text}
row = a.parent
v[:script_version] = row.siblings_at(1).inner_text
v[:date] = row.siblings_at(2).inner_text

0 comments on commit c3ced2a

Please sign in to comment.