Added skip-download for debugging

bin/oddb2xml

@@ -13,6 +13,7 @@ def help
 #$0 ver.#{Oddb2xml::VERSION}
 Usage:
   oddb2xml [option]
+    produced files are found under data
     -a T, --append=T     Additional target. T, only 'nonpharma' is available.
     -c F, --compress=F   Compress format F. {tar.gz|zip}
     -f F, --format=F     File format F, default is xml. {xml|dat}
@@ -24,6 +25,10 @@ Usage:
     -t S, --tag-suffix=S XML tag suffix S. Default is none. [A-z0-9]
                          If S is given, it is also used as prefix of filename.
     -x N, --context=N    context N {product|address}. product is default.
+                         
+                         For debugging purposes
+    --skip-download      skips downloading files it the file is already under data/downloads.
+                         Downloaded files are saved under data/downloads
     -h,   --help         Show this help message.
 EOS
 end
@@ -38,6 +43,7 @@ opts = {
   :tag_suffix   => nil,
   :debug        => false,
   :ean14        => false,
+  :skip_download=> false,
 }
 
 parser.on('-a v', '--append v',     /^nonpharma$/)  {|v| opts[:nonpharma] = true }
@@ -48,7 +54,7 @@ parser.on('-i v', '--include v',    /^ean14$/)      {|v| opts[:ean14] = true }
 parser.on('-t v', '--tag-suffix v', /^[A-z0-9]*$/i) {|v| opts[:tag_suffix] = v.upcase }
 parser.on('-x v', '--context v',    /^addr(ess)*$/i){|v| opts[:address] = true }
 parser.on('-p v', '--price v',      /^zurrose$/)    {|v| opts[:price] = v.intern }
-#parser.on('--debug') {|v| opts[:debug] = true }
+parser.on('--skip-download') {|v| opts[:skip_download] = true }
 parser.on_tail('-h', '--help') { puts help; exit }
 
 args = ARGV.dup
@@ -81,6 +87,6 @@ then you will not see anymore warnings
   end
   ui.run
 rescue Interrupt
-  puts
   exit
 end
+puts "#{Time.now}: #{__LINE__} done"; $stdout.sync

lib/oddb2xml/builder.rb

@@ -22,7 +22,8 @@ class Builder
                   :infos, :packs, :prices,
                   :ean14, :tag_suffix,
                   :companies, :people
-    def initialize
+    def initialize(args = {})
+      @options    = args
       @subject    = nil
       @index      = {}
       @items      = {}

lib/oddb2xml/cli.rb

@@ -5,6 +5,7 @@
 require 'oddb2xml/downloader'
 require 'oddb2xml/extractor'
 require 'oddb2xml/compressor'
+require 'oddb2xml/util'
 
 module Oddb2xml
   class Cli
@@ -14,6 +15,7 @@ class Cli
     LANGUAGES = %w[DE FR] # EN does not exist
     def initialize(args)
       @options = args
+      Oddb2xml.save_options(@options)
       @mutex = Mutex.new
       # product
       @items = {} # Items from Preparations.xml in BAG
@@ -88,7 +90,7 @@ def run
     def build
       begin
         files.each_pair do |sbj, file|
-          builder = Builder.new do |builder|
+          builder = Builder.new(@options) do |builder|
             if @options[:address]
               builder.subject   = sbj
               builder.companies = @companies
@@ -143,7 +145,7 @@ def build
       rescue Interrupt
         files.values.each do |file|
           if File.exist? file
-            File.unlink file
+            File.unlink(file) # we don't save it as it might be only partly downloaded
           end
         end
         raise Interrupt

lib/oddb2xml/compressor.rb

@@ -36,13 +36,11 @@ def finalize!
         end
         if File.exists? @compress_file
           @contents.each do |file|
-            File.unlink file
+            Oddb2xml.download_finished(file, )
           end
         end
-      rescue Errno::ENOENT, StandardError
-        if File.exists? @compress_file
-          File.unlink @compress_file
-        end
+      rescue Errno::ENOENT, StandardError  => e
+        Oddb2xml.download_finished(@compress_file)
         return false
       end
       return true

lib/oddb2xml/downloader.rb

@@ -9,18 +9,22 @@ module DownloadMethod
     private
     def download_as(file, option='r')
       data = nil
-      begin
-        response = @agent.get(@url)
-        response.save_as(file)
-        response = nil # win
+      # require 'pry'; binding.pry
+      if Oddb2xml.skip_download(file)
         io = File.open(file, option)
         data = io.read
-      rescue Timeout::Error, Errno::ETIMEDOUT
-        retrievable? ? retry : raise
-      ensure
-        io.close if io and !io.closed? # win
-        if File.exists?(file)
-          File.unlink(file)
+      else
+        begin
+          response = @agent.get(@url)
+          response.save_as(file)
+          response = nil # win
+          io = File.open(file, option)
+          data = io.read
+        rescue Timeout::Error, Errno::ETIMEDOUT
+          retrievable? ? retry : raise
+        ensure
+          io.close if io and !io.closed? # win
+          Oddb2xml.download_finished(file)
         end
       end
       return data
@@ -153,17 +157,17 @@ def download
         if @options[:debug]
           FileUtils.copy(File.expand_path("../../../spec/data/#{file}", __FILE__), '.')
         else
-          response = @agent.get(@url)
-          response.save_as(file)
-          response = nil # win
+          unless Oddb2xml.skip_download(file)
+            response = @agent.get(@url)
+            response.save_as(file)
+            response = nil # win
+          end
         end
-        return read_xml_form_zip(/^Preparation/iu, file)
+        read_xml_form_zip(/^Preparation/iu, file)
       rescue Timeout::Error, Errno::ETIMEDOUT
         retrievable? ? retry : raise
       ensure
-        if File.exists?(file)
-          File.unlink(file)
-        end
+        Oddb2xml.download_finished(file)
       end
     end
   end
@@ -239,19 +243,19 @@ def download
         page = @agent.get(@url)
         if link_node = page.search(@xpath).first
           link = Mechanize::Page::Link.new(link_node, @agent, page)
-          response = link.click
-          response.save_as(file)
-          response = nil # win
+          unless Oddb2xml.skip_download(file)
+            response = link.click
+            response.save_as(file)
+            response = nil # win
+          end
         end
         io = File.open(file, 'rb')
         return io.read
       rescue Timeout::Error, Errno::ETIMEDOUT
         retrievable? ? retry : raise
       ensure
         io.close if io and !io.closed?
-        if File.exists?(file)
-          File.unlink(file)
-        end
+        Oddb2xml.download_finished(file)
       end
     end
   end
@@ -264,37 +268,29 @@ def init
     def download
       file = "swissmedic_info.zip"
       begin
-        response = nil
-        if home = @agent.get(@url)
-          form = home.form_with(:id => 'Form1')
-          bttn = form.button_with(:name => 'ctl00$MainContent$btnOK')
-          if page = form.submit(bttn)
-            form = page.form_with(:id => 'Form1')
-            bttn = form.button_with(:name => 'ctl00$MainContent$BtnYes')
-            response = form.submit(bttn)
+        unless Oddb2xml.skip_download(file)
+          response = nil
+          if home = @agent.get(@url)
+            form = home.form_with(:id => 'Form1')
+            bttn = form.button_with(:name => 'ctl00$MainContent$btnOK')
+            if page = form.submit(bttn)
+              form = page.form_with(:id => 'Form1')
+              bttn = form.button_with(:name => 'ctl00$MainContent$BtnYes')
+              response = form.submit(bttn)
+            end
+          end
+          if response          
+            response.save_as(file)
+            response = nil # win
           end
         end
-        if response
-          response.save_as(file)
-          response = nil # win
-        end
-        return read_xml_form_zip(/^AipsDownload_/iu, file)
+        read_xml_form_zip(/^AipsDownload_/iu, file)
       rescue Timeout::Error, Errno::ETIMEDOUT
         retrievable? ? retry : raise
       rescue NoMethodError
         # pass
       ensure
-        # win
-        deleted = false
-        if File.exists?(file)
-          until deleted
-            begin
-              File.unlink(file)
-              deleted = true
-            rescue Errno::EACCES # Permission Denied on Windows
-            end
-          end
-        end
+        Oddb2xml.download_finished(file)
       end
     end
   end

lib/oddb2xml/util.rb

@@ -0,0 +1,28 @@
+module Oddb2xml
+  Backup = "#{Dir.pwd}/data/download"
+  @options = {}
+
+  def Oddb2xml.save_options(options)
+    @options = options
+  end
+
+  def Oddb2xml.skip_download(file)
+    dest = "#{Backup}/#{File.basename(file)}"
+    return false unless @options[:skip_download]
+    if File.exists?(dest)
+      FileUtils.cp(dest, file, :verbose => false, :preserve => true)
+      return true
+    end
+    false
+  end
+
+  def Oddb2xml.download_finished(file)
+    dest = "#{Backup}/#{File.basename(file)}"
+    FileUtils.makedirs(Backup)
+    FileUtils.cp(file, dest, :verbose => false)
+    begin
+      File.unlink(file) if File.exists?(file)
+    rescue Errno::EACCES # Permission Denied on Windows      
+    end
+  end                            
+end

spec/extractor_spec.rb

@@ -1,6 +1,7 @@
 # encoding: utf-8
 
 require 'spec_helper'
+require "#{Dir.pwd}/lib/oddb2xml/downloader"
 
 describe Oddb2xml::Extractor do
   it "pending"
@@ -41,27 +42,42 @@
   it "pending"
 end
 
-describe Oddb2xml::SwissmedicExtractor do
+describe Oddb2xml::BMUpdateExtractor do
   it "pending"
 end
 
-describe Oddb2xml::MigelExtractor do
+describe Oddb2xml::LppvExtractor do
   it "pending"
 end
 
-describe Oddb2xml::SwissmedicInfoExtractor do
+describe Oddb2xml::SwissIndexExtractor do
   it "pending"
 end
 
-describe Oddb2xml::EphaExtractor do
+describe Oddb2xml::MigelExtractor do
   it "pending"
 end
 
-describe Oddb2xml::MedregbmExtractor do
+describe Oddb2xml::SwissmedicInfoExtractor do
+  context 'when transfer.dat is empty' do
+    subject { Oddb2xml::SwissmedicInfoExtractor.new("") }
+    it { expect(subject.to_hash).to be_empty }
+  end
+  context 'can parse swissmedic_packages.xls' do
+    it {
+        filename = File.join(File.dirname(__FILE__), 'data/swissmedic_packages.xls')
+        bin = IO.read(filename)
+        @packs = Oddb2xml::SwissmedicExtractor.new(bin, :package).to_hash
+        expect(@packs.size).to eq(8) 
+       }
+  end
+end
+
+describe Oddb2xml::EphaExtractor do
   it "pending"
 end
 
-describe Oddb2xml::ZurroseExtractor do
+describe Oddb2xml::MedregbmExtractor do
   it "pending"
 end