Skip to content

Commit

Permalink
do not try to download files again
Browse files Browse the repository at this point in the history
Signed-off-by: David A. Cuadrado <krawek@gmail.com>
  • Loading branch information
dcu committed May 30, 2009
1 parent 553509b commit 0793861
Show file tree
Hide file tree
Showing 2 changed files with 18 additions and 4 deletions.
18 changes: 15 additions & 3 deletions lib/rrs_fetcher.rb
Original file line number Diff line number Diff line change
Expand Up @@ -21,8 +21,10 @@ def add_download(*links)
end

def start_dowload
while !download_list.empty?
download @download_list.pop
while !@download_list.empty?
url = @download_list.pop.strip
next if url.empty? || downloaded?(url)
download url
end
end

Expand All @@ -33,7 +35,7 @@ def download(to_download)
agent = "Mozilla Firefox 1.5"

cmd = "wget -U '#{agent}' '#{parser.download_link}' --post-data='mirror=on&x=67&y=50'"
file = parser.download_link.split("/").last # FIXME: check if download_link is valid
file = parser.filename # FIXME: check if download_link is valid

case @app
when :curl
Expand All @@ -47,6 +49,16 @@ def download(to_download)
end

private
def downloaded?(url)
filename = url.to_s.split("/").last

if File.exist?(filename)
$stderr.puts "Already downloaded: #{filename}"
return true
end
false
end

def wait_until_ready(parser)
while parser.metadata.include?(:minutes)
show_delay(parser.metadata[:minutes] * 60)
Expand Down
4 changes: 3 additions & 1 deletion lib/rrs_parser.rb
Original file line number Diff line number Diff line change
@@ -1,9 +1,10 @@
class RSSParser
attr_reader :page, :metadata, :url, :download_link, :form_action
attr_reader :page, :metadata, :url, :download_link, :filename, :form_action
def initialize(url)
@page = Hpricot(Net::HTTP.get(URI.parse(url)))
@url = url
@download_link = ""
@filename = ""

find_form_action
end
Expand All @@ -18,6 +19,7 @@ def parse_next_page
if line =~ /.*name="dlf"\saction="(\S*)"/
$stderr.puts "-> Downloading #{$1}..."
@download_link = $1
@filename = $1.to_s.split("/").last
end

if line =~ /var\s*c\s*=\s*(\d+)/
Expand Down

0 comments on commit 0793861

Please sign in to comment.