Swissmedic has different header lines

ngiger · Jun 8, 2015 · 6af5a87 · 6af5a87
1 parent 6830908
commit 6af5a87
Show file tree

Hide file tree

Showing 7 changed files with 47 additions and 54 deletions.
diff --git a/Gemfile b/Gemfile
@@ -7,5 +7,10 @@ group :development do
 end
 
 group :debugger do
-  gem 'pry-debugger'
+  if RUBY_VERSION.match(/^1/)
+    gem 'pry-debugger'
+  else
+    gem 'pry-byebug'
+  end
 end unless RUBY_VERSION =~ /^1\.8/
+
diff --git a/Gemfile.lock b/Gemfile.lock
@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    swissmedic-diff (0.1.8)
+    swissmedic-diff (0.1.9)
       nokogiri
       rubyXL
       rubyzip
@@ -10,44 +10,44 @@ PATH
 GEM
   remote: https://rubygems.org/
   specs:
-    ansi (1.4.3)
+    ansi (1.5.0)
     builder (3.2.2)
     coderay (1.1.0)
-    columnize (0.8.9)
+    columnize (0.9.0)
     debugger (1.6.8)
       columnize (>= 0.3.1)
       debugger-linecache (~> 1.2.0)
       debugger-ruby_core_source (~> 1.3.5)
     debugger-linecache (1.2.0)
-    debugger-ruby_core_source (1.3.5)
-    hoe (3.12.0)
+    debugger-ruby_core_source (1.3.8)
+    hoe (3.13.1)
       rake (>= 0.8, < 11.0)
     method_source (0.8.2)
-    mini_portile (0.6.0)
-    minitest (5.3.5)
-    minitest-reporters (1.0.5)
+    mini_portile (0.6.2)
+    minitest (5.7.0)
+    minitest-reporters (1.0.16)
       ansi
       builder
       minitest (>= 5.0)
       ruby-progressbar
-    nokogiri (1.6.2.1)
-      mini_portile (= 0.6.0)
-    pry (0.10.0)
+    nokogiri (1.6.6.2)
+      mini_portile (~> 0.6.0)
+    pry (0.10.1)
       coderay (~> 1.1.0)
       method_source (~> 0.8.1)
       slop (~> 3.4)
     pry-debugger (0.2.3)
       debugger (~> 1.3)
       pry (>= 0.9.10, < 0.11.0)
-    rake (10.3.2)
-    ruby-ole (1.2.11.7)
-    ruby-progressbar (1.5.1)
-    rubyXL (3.1.2)
+    rake (10.4.2)
+    ruby-ole (1.2.11.8)
+    ruby-progressbar (1.7.5)
+    rubyXL (3.3.8)
       nokogiri (>= 1.4.4)
       rubyzip (>= 1.1.6)
-    rubyzip (1.1.6)
-    slop (3.5.0)
-    spreadsheet (0.9.7)
+    rubyzip (1.1.7)
+    slop (3.6.0)
+    spreadsheet (1.0.3)
       ruby-ole (>= 1.0)
 
 PLATFORMS

diff --git a/History.txt b/History.txt
@@ -1,3 +1,7 @@
+=== 0.1.9 / 08.06.2015
+
+* Swissmedic has different header lines
+
 === 0.1.8 / 07.07.2014
 
 * Support parsing xlsx of Swissmedic July

diff --git a/lib/swissmedic-diff.rb b/lib/swissmedic-diff.rb
@@ -19,11 +19,11 @@
 #Authors::   Hannes Wyss (hwyss@ywesee.com), Masaomi Hatakeyama (mhatakeyama@ywesee.com)
 #Version::   0.1.4 2013-10-16 commit c30af5c15f6b8101f8f84cb482dfd09ab20729d6
 #Copyright:: Copyright (C) ywesee GmbH, 2010. All rights reserved.
-#License::   GPLv2.0 Compliance 
+#License::   GPLv2.0 Compliance
 #Source::    http://scm.ywesee.com/?p=swissmedic-diff/.git;a=summary
 class SwissmedicDiff
   module Diff
-    COLUMNS = [ :iksnr, :seqnr, :name_base, :company, 
+    COLUMNS = [ :iksnr, :seqnr, :name_base, :company,
                 :index_therapeuticus, :atc_class, :production_science,
                 :registration_date, :sequence_date, :expiry_date, :ikscd,
                 :size, :unit, :ikscat, :substances, :composition,
@@ -102,9 +102,6 @@ def diff(target, latest, ignore = [])
       Spreadsheet.client_encoding = 'UTF-8'
       tbook = Spreadsheet.open(target)
       sheet = tbook.worksheet(0)
-      if new_column = cell(sheet.row(2), COLUMNS.size)
-        raise "New column #{COLUMNS.size} (#{new_column})"
-      end
       idx, prr, prp = nil
       multiples = {}
       each_valid_row(tbook) { |row|
@@ -137,7 +134,7 @@ def diff(target, latest, ignore = [])
           (changes[iksnr].concat flags).uniq!
           updates.push row unless flags.empty?
         else
-          replacements.store [ iksnr, seqnr, cell(row, column(:size)), 
+          replacements.store [ iksnr, seqnr, cell(row, column(:size)),
                                 cell(row, column(:unit)) ], row
           flags = changes[iksnr]
           flags.push(:sequence).uniq! unless(flags.include? :new)
@@ -146,7 +143,7 @@ def diff(target, latest, ignore = [])
       }
       @diff.replacements = reps = {}
       known_pacs.each { |(iksnr, pacnr), row|
-        key = [iksnr, '%02i' % cell(row, column(:seqnr)).to_i, 
+        key = [iksnr, '%02i' % cell(row, column(:seqnr)).to_i,
                       cell(row, column(:size)), cell(row, column(:unit))]
         if(rep = replacements[key])
           changes[iksnr].push :replaced_package
@@ -157,7 +154,7 @@ def diff(target, latest, ignore = [])
       changes.delete_if { |iksnr, flags| flags.empty? }
       @diff.package_deletions = known_pacs.collect { |key, row|
         ## the keys in known_pacs don't include the sequence number (which
-        #  would prevent us from properly recognizing multi-sequence-Packages), 
+        #  would prevent us from properly recognizing multi-sequence-Packages),
         #  so we need complete the path to the package now
         key[1,0] = '%02i' % cell(row, column(:seqnr)).to_i
         key
@@ -205,7 +202,7 @@ def _known_data(latest, known_regs, known_seqs, known_pacs, newest_rows)
         known_regs.store [iksnr], row
         known_seqs.store [iksnr, seqnr], row
         known_pacs.store [iksnr, pacnr, idx], row
-        (newest_rows[iksnr] ||= {})[pacnr] = row                            
+        (newest_rows[iksnr] ||= {})[pacnr] = row
       }
     end
     def name(diff, iksnr)
@@ -236,7 +233,7 @@ def rows_diff(row, other, ignore = [])
     def to_s(sort=:group)
       @diff ||= nil
       return '' unless @diff
-      @diff.changes.sort_by { |iksnr, flags| 
+      @diff.changes.sort_by { |iksnr, flags|
         _sort_by(sort, iksnr, flags)
       }.collect { |iksnr, flags|
         if(flags.include? :new)
@@ -245,7 +242,7 @@ def to_s(sort=:group)
           "- " << describe(@diff, iksnr)
         else
           "> " << describe(@diff, iksnr) << "; " \
-            << flags.collect { |flag| describe_flag(@diff, iksnr, flag) 
+            << flags.collect { |flag| describe_flag(@diff, iksnr, flag)
           }.compact.join(", ")
         end
       }.join("\n")
@@ -279,7 +276,7 @@ def _comparable(key, row, idx)
         end
       end
     end
-  
+
     #=== iterate over all valid rows of a swissmedic Packungen.xls
     #
     # Iterates over all rows, ignoring Tierarzneimittel and
@@ -312,7 +309,7 @@ def each_valid_row(spreadsheet)
         yield row
       }
     end
-    
+
     def rows_to_skip(spreadsheet)
       # Packungen.xls of swissmedic before October 2013 had  3 leading rows
       # Packungen.xls of swissmedic after  October 2013 have 4 leading rows
@@ -325,7 +322,7 @@ def rows_to_skip(spreadsheet)
       end
       j
     end
-    
+
   end
   include Diff
 end
diff --git a/lib/version.rb b/lib/version.rb
@@ -1,3 +1,3 @@
 class SwissmedicDiff
-  VERSION = '0.1.8'
+  VERSION = '0.1.9'
 end
diff --git a/test/data/Packungen_2014_small.xlsx b/test/data/Packungen_2014_small.xlsx
diff --git a/test/test_swissmedic-diff.rb b/test/test_swissmedic-diff.rb
@@ -35,8 +35,8 @@ def test_diff_xls_and_xlsx
       assert(result.changes.flatten.index('Zulassungs-Nummer') == nil, "Should not find Zulassungs-Nummer in changes")
       # puts "Got #{result.news.size} news, #{result.changes.size} changes, #{result.updates.size} updates."
       assert_equal(8, result.news.size)
-      assert_equal(3, result.changes.size)
-      assert_equal(1, result.updates.size)
+      assert_equal(4, result.changes.size)
+      assert_equal(2, result.updates.size)
       assert(result.news.first.index('00280'), "Should find 00280 in news")
       assert(result.news.flatten.index('65034'), "Should find 65034 in news")
       assert(result.news.flatten.index('00277') == nil, "Should not find 00277 in news")
@@ -79,26 +79,14 @@ def test_diff_xlsx_and_xls
     end
     def test_diff_xlsx_and_xlsx
       @diff = SwissmedicDiff.new
-      last_month = File.expand_path 'data/Packungen_2014_small.xlsx',  File.dirname(__FILE__)
+      last_month = File.expand_path 'data/Packungen_2014_01.xlsx',  File.dirname(__FILE__)
       this_month = File.expand_path 'data/Packungen_2014_small.xlsx',  File.dirname(__FILE__)
       result = @diff.diff last_month, this_month, [:atc_class, :sequence_date]
-      assert_equal({}, result.changes)
+      assert_equal({"00277"=>[:expiry_date, :indication_sequence]}, result.changes)
+      assert_equal(1, result.updates.size)
       assert_equal([], result.news)
-      assert_equal([], result.updates)
     end
 
-    # This is not a unit test as it takes way too long (> 1 minute)
-    # Instead it might just tell you how to test with real data
-    def test_real_diff
-      @diff = SwissmedicDiff.new
-      last_month = File.expand_path 'data/Packungen-2013.08.16.xls',  File.dirname(__FILE__)
-      this_month = File.expand_path 'data/Packungen-2013.11.04.xls',  File.dirname(__FILE__)
-      result = @diff.diff last_month, this_month, [:atc_class, :sequence_date]
-      assert(result.changes.flatten.index('Zulassungs-Nummer') == nil, "Should not find Zulassungs-Nummer in changes")
-      assert(result.news.flatten.index('Zulassungs-Nummer') == nil, "Should not find Zulassungs-Nummer in changes")
-      assert(result.news.flatten.index('00277') == nil, "Should not find 00277 in news")
-    end if false
-
     def test_iterate
       diff = SwissmedicDiff.new
       strings = []
@@ -187,9 +175,8 @@ def test_diff
       assert_equal '005', result.replacements.values.first
     end
     def test_diff_error_column
-      assert_raises(RuntimeError) { 
-        @diff.diff(@data_error_column, @older)
-      }
+      res = @diff.diff(@data_error_column, @older)
+      assert_equal(OpenStruct, res.class)
     end
 
     # if row.size < COLUMNS.size/2