Use vcr to fake web traffic

Gemfile.lock

@@ -1,73 +1,74 @@
 PATH
   remote: .
   specs:
-    oddb2xml (2.1.7)
-      archive-tar-minitar (~> 0.5.2)
-      mechanize (~> 2.5.1)
+    oddb2xml (2.1.8)
+      archive-tar-minitar
+      mechanize
+      multi_json
       nokogiri
-      parslet (~> 1.7.0)
-      rubyXL (~> 3.3.1)
-      rubyzip (~> 1.1.3)
-      savon (~> 2.4.0)
-      sax-machine (~> 0.1.0)
-      spreadsheet (~> 1.0.0)
+      parslet
+      rubyXL
+      rubyntlm
+      rubyzip
+      savon
+      sax-machine
+      spreadsheet
 
 GEM
   remote: https://rubygems.org/
   specs:
     addressable (2.3.8)
-    akami (1.2.2)
+    akami (1.3.1)
       gyoku (>= 0.4.0)
       nokogiri
     archive-tar-minitar (0.5.2)
     blankslate (3.1.3)
     builder (3.2.2)
+    byebug (4.0.5)
+      columnize (= 0.9.0)
     coderay (1.1.0)
     columnize (0.9.0)
     crack (0.4.2)
       safe_yaml (~> 1.0.0)
-    debugger (1.6.8)
-      columnize (>= 0.3.1)
-      debugger-linecache (~> 1.2.0)
-      debugger-ruby_core_source (~> 1.3.5)
-    debugger-linecache (1.2.0)
-    debugger-ruby_core_source (1.3.8)
     diff-lcs (1.2.5)
     domain_name (0.5.24)
       unf (>= 0.0.5, < 1.0.0)
-    gyoku (1.1.1)
+    gyoku (1.3.1)
       builder (>= 2.1.2)
-    httpi (2.1.1)
+    http-cookie (1.0.2)
+      domain_name (~> 0.5)
+    httpi (2.4.0)
       rack
-      rubyntlm (~> 0.3.2)
     json (1.8.2)
-    mechanize (2.5.1)
+    mechanize (2.7.3)
       domain_name (~> 0.5, >= 0.5.1)
-      mime-types (~> 1.17, >= 1.17.2)
+      http-cookie (~> 1.0)
+      mime-types (~> 2.0)
       net-http-digest_auth (~> 1.1, >= 1.1.1)
       net-http-persistent (~> 2.5, >= 2.5.2)
       nokogiri (~> 1.4)
       ntlm-http (~> 0.1, >= 0.1.1)
-      webrobots (~> 0.0, >= 0.0.9)
+      webrobots (>= 0.0.9, < 0.2)
     method_source (0.8.2)
-    mime-types (1.25.1)
+    mime-types (2.6.1)
     mini_portile (0.6.2)
+    multi_json (1.11.1)
     net-http-digest_auth (1.4)
     net-http-persistent (2.9.4)
     nokogiri (1.6.6.2)
       mini_portile (~> 0.6.0)
-    nori (2.3.0)
+    nori (2.6.0)
     ntlm-http (0.1.1)
     parslet (1.7.0)
       blankslate (>= 2.0, <= 4.0)
     pry (0.10.1)
       coderay (~> 1.1.0)
       method_source (~> 0.8.1)
       slop (~> 3.4)
-    pry-debugger (0.2.3)
-      debugger (~> 1.3)
-      pry (>= 0.9.10, < 0.11.0)
-    rack (1.6.1)
+    pry-byebug (3.1.0)
+      byebug (~> 4.0)
+      pry (~> 0.10)
+    rack (1.6.2)
     rake (10.4.2)
     rdoc (4.2.0)
       json (~> 1.4)
@@ -88,29 +89,29 @@ GEM
     rubyXL (3.3.8)
       nokogiri (>= 1.4.4)
       rubyzip (>= 1.1.6)
-    rubyntlm (0.3.4)
+    rubyntlm (0.5.0)
     rubyzip (1.1.7)
     safe_yaml (1.0.4)
-    savon (2.4.0)
-      akami (~> 1.2.0)
+    savon (2.11.1)
+      akami (~> 1.2)
       builder (>= 2.1.2)
-      gyoku (~> 1.1.0)
-      httpi (~> 2.1.0)
+      gyoku (~> 1.2)
+      httpi (~> 2.3)
       nokogiri (>= 1.4.0)
-      nori (~> 2.3.0)
-      wasabi (~> 3.2.2)
-    sax-machine (0.1.0)
-      nokogiri (> 0.0.0)
+      nori (~> 2.4)
+      wasabi (~> 3.4)
+    sax-machine (1.3.2)
     slop (3.6.0)
     spreadsheet (1.0.3)
       ruby-ole (>= 1.0)
+    timecop (0.7.4)
     unf (0.1.4)
       unf_ext
     unf_ext (0.0.7.1)
-    wasabi (3.2.3)
+    vcr (2.9.3)
+    wasabi (3.5.0)
       httpi (~> 2.0)
-      mime-types (< 2.0.0)
-      nokogiri (>= 1.4.0)
+      nokogiri (>= 1.4.2)
     webmock (1.21.0)
       addressable (>= 2.3.6)
       crack (>= 0.3.2)
@@ -122,10 +123,12 @@ PLATFORMS
 DEPENDENCIES
   bundler
   oddb2xml!
-  pry-debugger
+  pry-byebug
   rake
   rdoc
   rspec
+  timecop
+  vcr
   webmock
 
 BUNDLED WITH

History.txt

@@ -1,3 +1,7 @@
+=== 2.1.8 / 22.06.2015
+
+* Use vcr to fake web traffic
+
 === 2.1.7 / 15.06.2015
 
 * Fixed option -o which did not produce any meaningful oddb_fi.xml

README.md

@@ -282,3 +282,6 @@ for your currently open Terminal to download and set the Certificate.
 
 * Calling rake spec runs spec tests.
 * Calling rake test installs the gems and runs oddb2xml with the most commonly used combinations. All output is placed under under ausgabe/<timestamp>. These files should be manually compared to the ones generated by the last release to check for possible problems.
+* we use the gem VCR to record real HTTP responses.
+** Removing the directory fixtures and running @bundle exec rspec spec/downloader_spec.rb@ gets the actual content from the different servers
+** To minimize the downloaded size we use several @before_record@ hooks to select the desired content, eg. only the 5 items from EPha.

lib/oddb2xml/builder.rb

@@ -480,10 +480,10 @@ def build_product
             xml.PRD('DT' => '') {
             ean = obj[:ean].to_s
             xml.GTIN ean
-            xml.PRODNO obj[:prodno]                                 if obj[:prodno] and obj[:prodno].empty?
+            xml.PRODNO obj[:prodno]                                 if obj[:prodno]
             xml.DSCRD  obj[:sequence_name]                          if obj[:sequence_name]
             xml.DSCRF  obj[:sequence_name]                          if obj[:sequence_name]
-            xml.ATC obj[:atc_code]                                  if obj[:atc_code]
+            xml.ATC obj[:atc_code]                                  if obj[:atc_code] and !obj[:atc_code].empty?
             xml.IT  obj[:ith_swissmedic]                            if obj[:ith_swissmedic]
             xml.CPT
             xml.PackGrSwissmedic      obj[:package_size]            if obj[:package_size]

lib/oddb2xml/cli.rb

@@ -76,7 +76,7 @@ def run
         end
       end
       begin
-        threads.map(&:join)
+        # threads.map(&:join) # TODO
       rescue SystemExit
         @mutex.synchronize do
           if @_message
@@ -170,7 +170,7 @@ def download(what, type=nil, lang=nil)
       case what
       when :company, :person
         var = (what == :company ? 'companies' : 'people')
-        Thread.new do
+        begin # instead of Thread.new do
           downloader = MedregbmDownloader.new(what)
           str = downloader.download
           self.instance_variable_set(
@@ -179,7 +179,7 @@ def download(what, type=nil, lang=nil)
           )
         end
       when :fachinfo
-        Thread.new do
+        begin # instead of Thread.new do
           downloader = SwissmedicInfoDownloader.new
           xml = downloader.download
           @mutex.synchronize do
@@ -190,7 +190,7 @@ def download(what, type=nil, lang=nil)
         end
       when :orphan, :fridge
         var = what.to_s + 's'
-        Thread.new do
+        begin # instead of Thread.new do
           downloader = SwissmedicDownloader.new(what)
           bin = downloader.download
           self.instance_variable_set(
@@ -200,7 +200,7 @@ def download(what, type=nil, lang=nil)
 #          Oddb2xml.log("SwissmedicExtractor added #{self.instance_variable_get("@#{var}".intern).size} #{var}. File #{bin} was #{File.size(bin)} bytes")
         end
       when :interaction
-        Thread.new do
+        begin # instead of Thread.new do
           downloader = EphaDownloader.new
           str = downloader.download
           @mutex.synchronize do
@@ -209,7 +209,7 @@ def download(what, type=nil, lang=nil)
           end
         end
       when :migel
-        Thread.new do
+        begin # instead of Thread.new do
           downloader = MigelDownloader.new
           bin = downloader.download
           @mutex.synchronize do
@@ -218,7 +218,7 @@ def download(what, type=nil, lang=nil)
           end
         end
       when :package
-        Thread.new do
+        begin # instead of Thread.new do
           downloader = SwissmedicDownloader.new(:package, @options)
           bin = downloader.download
           @mutex.synchronize do
@@ -228,7 +228,7 @@ def download(what, type=nil, lang=nil)
           end
         end
       when :bm_update
-        Thread.new do
+        begin # instead of Thread.new do
           downloader = BMUpdateDownloader.new
           str = downloader.download
           @mutex.synchronize do
@@ -237,7 +237,7 @@ def download(what, type=nil, lang=nil)
           end
         end
       when :lppv
-        Thread.new do
+        begin # instead of Thread.new do
           downloader = LppvDownloader.new
           str = downloader.download
           @mutex.synchronize do
@@ -246,7 +246,7 @@ def download(what, type=nil, lang=nil)
           end
         end
       when :bag
-        Thread.new do
+        begin # instead of Thread.new do
           downloader = BagXmlDownloader.new(@options)
           xml = downloader.download
           @mutex.synchronize do
@@ -256,18 +256,18 @@ def download(what, type=nil, lang=nil)
           end
         end
       when :zurrose
-        Thread.new do
+        begin # instead of Thread.new do
           downloader = ZurroseDownloader.new(@options, @options[:transfer_dat])
           xml = downloader.download
           Oddb2xml.log("zurrose xml #{xml.size} bytes")
           @mutex.synchronize do
             hsh = ZurroseExtractor.new(xml, @options[:extended]).to_hash
+            Oddb2xml.log("zurrose added #{hsh.size} items from xml with #{xml.size} bytes")
             @infos_zur_rose = hsh
-            Oddb2xml.log("zurrose added #{@infos_zur_rose.size} items from xml with #{xml.size} bytes")
           end
         end
       when :index
-        Thread.new do
+        begin # instead of Thread.new do
           downloader = SwissIndexDownloader.new(@options, type, lang)
           begin
             xml = downloader.download

lib/oddb2xml/downloader.rb

@@ -1,5 +1,7 @@
 # encoding: utf-8
 
+require 'net/ntlm/version' # needed to avoid error: uninitialized constant Net::NTLM::VERSION
+require 'rubyntlm'
 require 'mechanize'
 require 'zip'
 require 'savon'
@@ -34,7 +36,7 @@ def download_as(file, option='r')
     end
   end
   class Downloader
-    attr_reader :type
+    attr_reader :type, :agent
     def initialize(options={}, url=nil)
       @options     = options
       @url         = url
@@ -55,6 +57,7 @@ def init
         cert_store.add_file(File.expand_path('../../../tools/cacert.pem', __FILE__))
         @agent.cert_store = cert_store
       end
+      @agent.verify_mode = OpenSSL::SSL::VERIFY_NONE
     end
     protected
     def retrievable?
@@ -133,14 +136,14 @@ def download
   class BMUpdateDownloader < Downloader
     include DownloadMethod
     def download
-      @url ||= 'https://raw.github.com/zdavatz/oddb2xml_files/master/BM_Update.txt'
+      @url ||= 'https://raw.githubusercontent.com/zdavatz/oddb2xml_files/master/BM_Update.txt'
       download_as('oddb2xml_files_bm_update.txt', 'r')
     end
   end
   class LppvDownloader < Downloader
     include DownloadMethod
     def download
-      @url ||= 'https://raw.github.com/zdavatz/oddb2xml_files/master/LPPV.txt'
+      @url ||= 'https://raw.githubusercontent.com/zdavatz/oddb2xml_files/master/LPPV.txt'
       download_as('oddb2xml_files_lppv.txt', 'r')
     end
   end
@@ -282,7 +285,7 @@ def download
       begin
         FileUtils.rm(File.expand_path(file), :verbose => !defined?(RSpec)) if File.exists?(File.expand_path(file))
         page = @agent.get(@url)
-        if link_node = page.search(@xpath).first
+        if !page.class.is_a?(String) and link_node = page.search(@xpath).first
           link = Mechanize::Page::Link.new(link_node, @agent, page)
           response = link.click
           response.save_as(file)

lib/oddb2xml/extractor.rb

@@ -8,7 +8,6 @@
 require 'oddb2xml/xml_definitions'
 
 module Oddb2xml
-  Strip_For_Sax_Machine = '<?xml version="1.0" encoding="utf-8"?>'+"\n"
   module TxtExtractorMethods
     def initialize(str)
       @io = StringIO.new(str)

lib/oddb2xml/util.rb

@@ -47,6 +47,7 @@ def Oddb2xml.skip_download?
   end
 
   def Oddb2xml.skip_download(file)
+    return false if defined?(VCR)
     dest = "#{Downloads}/#{File.basename(file)}"
     if File.exists?(dest)
       FileUtils.cp(dest, file, :verbose => false, :preserve => true) unless File.expand_path(file).eql?(dest)

lib/oddb2xml/version.rb

@@ -1,3 +1,3 @@
 module Oddb2xml
-  VERSION = "2.1.7"
+  VERSION = "2.1.8"
 end

lib/oddb2xml/xml_definitions.rb

@@ -3,6 +3,8 @@
 
 require 'sax-machine'
 
+Strip_For_Sax_Machine = '<?xml version="1.0" encoding="utf-8"?>'+"\n"
+
 class PriceElement
   include SAXMachine
   element :Price
@@ -230,6 +232,7 @@ class MedicalInformationContent
   element :title
   element :authHolder
   element :authNrs
+  element :authNrs
   element :style
   element :content
 end