Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
branch: master
Fetching contributors…

Cannot retrieve contributors at this time

149 lines (143 sloc) 4.669 kb
dep 'mirror has assets' do
define_var :mirror_prefix, :default => '/srv/http' #L{ "http://#{var(:mirror_path).p.basename}" }
define_var :local_path, :default => :mirror_domain
def scanned_urls
(var(:mirror_prefix) / var(:local_path)).glob("**/*").select {|f|
f[/\.(html?|css)$/i]
}.map {|f|
f.p.read.scan(/url\(['"]?([^)'"]+)['"]?\)/).flatten
}.flatten.uniq
end
def asset_map
scanned_urls.group_by {|url|
url[/^(http\:)?\/\//] ? url.scan(/^[http\:]*\/\/([^\/]+)/).flatten.first : var(:mirror_domain)
}.map_values {|domain,urls|
urls.map {|url| url.sub(/^(http\:)?\/\/[^\/]+\//, '') }
}
end
def nonexistent_asset_map
asset_map.map_values {|domain,assets|
assets.reject {|asset|
path = var(:mirror_prefix) / domain / asset
path.exists? && !path.empty?
}
}
end
met? { nonexistent_asset_map.values.all? &:empty? }
meet {
nonexistent_asset_map.each_pair {|domain,assets|
assets.each {|asset|
shell "mkdir -p '#{var(:mirror_prefix) / domain / File.dirname(asset)}'"
log_shell "Downloading http://#{domain}/#{asset}", "wget -O '#{var(:mirror_prefix) / domain / asset}' '#{File.join "http://#{domain}", asset}'"
}
}
}
end
meta :twitter do
def users
"~/Desktop/rc7/campers.txt".p.read.split(/\n+/).uniq.map {|name| name.sub(/^@/, '') }
end
def avatars
users.map {|user|
path = "~/Desktop/rc7/avatars/".p.glob("#{user}.*").first
path.p unless path.nil?
}.compact
end
def missing_avatars
avatars.reject {|avatar|
avatar.exists? && !avatar.empty?
}
end
end
dep 'avatars mirrored.twitter' do
define_var :twitter_pass, :default => L{ 'secret' }
met? { missing_avatars.empty? }
meet {
require 'rubygems'
require 'twitter'
client = Twitter::Base.new(Twitter::HTTPAuth.new(var(:twitter_username), var(:twitter_pass)))
cd "~/Desktop/rc7/avatars", :create => true do
missing_avatars.each {|name|
begin
url = client.user(name)['profile_image_url'].sub(/_normal(\.[a-zA-Z]+)$/) { $1 }
Babushka::Archive.download url, name
rescue Twitter::NotFound
log_error "#{name}: 404."
rescue Twitter::InformTwitter
log_error "#{name}: Fail whale!"
rescue JSON::ParserError
log_error "#{name}: Bad JSON."
end
}
end
}
end
dep 'avatars renamed.twitter' do
# requires 'twitter avatars mirrored'
met? { (avatars - missing_avatars).all? {|avatar| avatar.to_s[/\.[jpengif]{3,4}$/] } }
meet {
(avatars - missing_avatars).each {|avatar|
type = shell("file '#{avatar}'").scan(/([A-Z]+) image/).flatten.first
unless type.nil?
ext = type.downcase
shell "mv '#{avatar}' '#{avatar}.#{ext}'"
end
}
}
after {
log "These ones are broken:"
log avatars.reject {|avatar| avatar.to_s[/\.[jpengif]{3,4}$/] }.join("\n")
}
end
dep 'gravatars mirrored' do
def users
"~/Desktop/rc7/emails.txt".p.read.split(/\n+/).uniq
end
def missing_avatars
users.reject {|user| "~/Desktop/rc7/gravatars/#{user}.jpg".p.exists? }
end
met? { missing_avatars.empty? }
meet {
require 'digest/md5'
cd "~/Desktop/rc7/gravatars", :create => true do
missing_avatars.each {|email|
Babushka::Archive.download "http://gravatar.com/avatar/#{Digest::MD5.hexdigest(email)}.jpg?s=512&d=404", "#{email}.jpg"
}
end
}
end
dep 'google ajax libs mirrored' do
define_var :mirror_root, :default => '/srv/http/ajax.googleapis.com'
def search_libstate doc, key
doc.search("dd.al-libstate[text()*='#{key}:']").text.gsub("#{key}:", '').strip
end
def urls
require 'rubygems'
require 'hpricot'
require 'net/http'
Hpricot(Net::HTTP.get(URI.parse('http://code.google.com/apis/ajaxlibs/documentation/'))).search('.al-liblist').map {|lib|
lib_name = search_libstate(lib, 'name')
versions = search_libstate(lib, 'versions').split(/[, ]+/)
[search_libstate(lib, 'path'), search_libstate(lib, 'path(u)')].squash.map {|path_template|
versions.map {|version|
URI.parse path_template.gsub(versions.last, version)
}
}
}.flatten
end
def missing_urls
urls.tap {|urls|
log "#{urls.length} items to consider."
}.reject {|url| (var(:mirror_root) / url.path).exists? }.tap {|present|
log "Of those, we have #{present.length}."
}
end
met? { missing_urls.empty? }
meet {
missing_urls.each {|url|
cd var(:mirror_root) / url.path.p.dirname, :create => true do
Babushka::Archive.download url
end
}
}
end
Jump to Line
Something went wrong with that request. Please try again.