Set read timeout to 300

katkad · Oct 11, 2015 · 085b850 · 085b850
1 parent 2471ab1
commit 085b850
Showing 1 changed file with 7 additions and 7 deletions.
diff --git a/scraper.rb b/scraper.rb
@@ -91,7 +91,7 @@ def parse_elektr_aukcie(doc, institution)
 
 SITE = 'http://www.genpro.gov.sk'
 url = "http://www.genpro.gov.sk/objednavky-faktury-a-zmluvy-28dd.html"
-doc = Nokogiri::HTML(open(url))
+doc = Nokogiri::HTML(open(url, :read_timeout => 300))
 
 navig = doc.xpath('//ul[@class="navigacia"]/li/a')
 navig.each do |nav|
@@ -101,10 +101,10 @@ def parse_elektr_aukcie(doc, institution)
     next
   end
 
-  html = open(SITE + nav['href'])
+  html = open(SITE + nav['href'], :read_timeout => 300)
   doc = Nokogiri::HTML(html)
   doc.xpath('//li[@class="activ"]/ul/li/a').each do |lnk_node|
-    htm = open(SITE + lnk_node['href'])
+    htm = open(SITE + lnk_node['href'], :read_timeout => 300)
     doc  = Nokogiri::HTML(htm)
 
     case lnk_node
@@ -113,7 +113,7 @@ def parse_elektr_aukcie(doc, institution)
       divs.each do |div|
         months = div.xpath('.//a')
         months.each do |month|
-          doc = Nokogiri::HTML(open(SITE + lnk_node['href'] + month['href']))
+          doc = Nokogiri::HTML(open(SITE + lnk_node['href'] + month['href'], :read_timeout => 300))
           parse_it(doc, "objednavky", institution)
         end
       end
@@ -122,7 +122,7 @@ def parse_elektr_aukcie(doc, institution)
       divs.each do |div|
         months = div.xpath('.//a')
         months.each do |month|
-          doc = Nokogiri::HTML(open(SITE + lnk_node['href'] + month['href']))
+          doc = Nokogiri::HTML(open(SITE + lnk_node['href'] + month['href'], :read_timeout => 300))
           parse_it(doc, "faktury", institution)
         end
       end
@@ -131,15 +131,15 @@ def parse_elektr_aukcie(doc, institution)
       divs.each do |div|
         months = div.xpath('.//a')
         months.each do |month|
-          doc = Nokogiri::HTML(open(SITE + lnk_node['href'] + month['href']))
+          doc = Nokogiri::HTML(open(SITE + lnk_node['href'] + month['href'], :read_timeout => 300))
           parse_it(doc, "zmluvy", institution)
         end
       end
     when /z\303\241kazky s n\303\255zkou hodnotou/u then
       parse_zsnh(doc, institution)
     when /elektronick\303\251 aukcie/u then
       doc.xpath('//div[@class="content"]/div/span/a').each do |a|
-        doc = Nokogiri::HTML(open(SITE + lnk_node['href'] + a['href']))
+        doc = Nokogiri::HTML(open(SITE + lnk_node['href'] + a['href'], :read_timeout => 300))
         parse_elektr_aukcie(doc, institution)
       end
     end