Skip to content
This repository

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse code

removed iconv

  • Loading branch information...
commit 2f931cc1af3f8452c76a13c839d6d91974229e54 1 parent 46213f7
Dmitry Mozzherin authored August 16, 2012
6  Gemfile
@@ -3,15 +3,15 @@ source "http://rubygems.org"
3 3
 # Example:
4 4
 #   gem "activesupport", ">= 2.3.5"
5 5
 gem "nokogiri", "~> 1.5.0"
6  
-gem "dwc-archive", "~> 0.9.2"
  6
+gem "dwc-archive", "~> 0.9.3"
7 7
 gem "rest-client"
8 8
 # gem "anystyle-parser"
9 9
 
10 10
 # Add dependencies to develop your gem here.
11 11
 # Include everything needed to run rake, tests, features, etc.
12 12
 group :development do
13  
-  gem "rspec", "~> 2.3.0"
14  
-  gem "cucumber", ">= 0"
  13
+  gem "rspec"
  14
+  gem "cucumber"
15 15
   gem "bundler", "~> 1.1.5"
16 16
   gem "jeweler", "~> 1.6.4"
17 17
   gem "simplecov", ">= 0"
6  Gemfile.lock
@@ -22,7 +22,7 @@ GEM
22 22
       debugger-ruby_core_source (>= 1.1.1)
23 23
     debugger-ruby_core_source (1.1.3)
24 24
     diff-lcs (1.1.3)
25  
-    dwc-archive (0.9.2)
  25
+    dwc-archive (0.9.3)
26 26
       archive-tar-minitar
27 27
       nokogiri (~> 1.5.0)
28 28
       parsley-store (>= 0.3.0)
@@ -70,9 +70,9 @@ DEPENDENCIES
70 70
   bundler (~> 1.1.5)
71 71
   cucumber
72 72
   debugger
73  
-  dwc-archive (~> 0.9.2)
  73
+  dwc-archive (~> 0.9.3)
74 74
   jeweler (~> 1.6.4)
75 75
   nokogiri (~> 1.5.0)
76 76
   rest-client
77  
-  rspec (~> 2.3.0)
  77
+  rspec
78 78
   simplecov
1  lib/dwca-hunter.rb
@@ -5,7 +5,6 @@
5 5
 require 'json'
6 6
 require 'dwc-archive'
7 7
 require 'dwca-hunter/resource'
8  
-require 'iconv'
9 8
 require 'rest_client'
10 9
 require 'base64'
11 10
 require File.join(File.dirname(__FILE__), "uuid")
3  lib/dwca-hunter/encoding.rb
... ...
@@ -1,13 +1,12 @@
1 1
 class DwcaHunter
2 2
   module Encoding
3 3
     def self.latin1_to_utf8(file_path)
4  
-      conv = Iconv.new('UTF-8', 'ISO-8859-1')
5 4
       new_file = file_path + ".utf_8"
6 5
       puts "Creating %s" % new_file
7 6
       r = open(file_path)
8 7
       w = open(new_file, 'w:utf-8')
9 8
       r.each do |l|
10  
-        l = conv.iconv(l)
  9
+        l.encode!('UTF-8', 'ISO-8859-1', :invalid => :replace, :replace => '?')
11 10
         w.write l
12 11
       end
13 12
       r.close
11  lib/dwca-hunter/resource_itis.rb
@@ -6,7 +6,6 @@ def initialize(opts = {})
6 6
       @url = "http://www.itis.gov/downloads/itisMySQLTables.tar.gz"
7 7
       @uuid =  "5d066e84-e512-4a2f-875c-0a605d3d9f35"
8 8
       @download_path = File.join(DEFAULT_TMP_DIR, "dwca_hunter", "itis", "data.tar.gz")
9  
-      @conv = Iconv.new('UTF-8', 'ISO-8859-1')
10 9
       @ranks = {} 
11 10
       @kingdoms = {}
12 11
       @authors = {}
@@ -47,7 +46,7 @@ def get_ranks
47 46
       rank_file = File.join(@itis_dir, 'taxon_unit_types')
48 47
       f = open(rank_file, "r:utf-8")
49 48
       f.each do |l|
50  
-        l = @conv.iconv(l)
  49
+        l.encode!('UTF-8', 'ISO-8859-1', :invalid => :replace, :replace => '?')
51 50
         row = l.strip.split("|")
52 51
         @ranks[row[1].strip] = row[2].strip
53 52
       end
@@ -73,7 +72,7 @@ def get_authors
73 72
 
74 73
       f = open(File.join(@itis_dir, 'taxon_authors_lkp'))
75 74
       f.each do |l|
76  
-        l = @conv.iconv(l)
  75
+        l.encode!('UTF-8', 'ISO-8859-1', :invalid => :replace, :replace => '?')
77 76
         data = l.strip.split("|")
78 77
         @authors[data[0].strip] = data[1].strip
79 78
       end
@@ -90,7 +89,7 @@ def get_vernaculars
90 89
       f = open(File.join(@itis_dir, 'vernaculars'))
91 90
       f.each_with_index do |l, i|
92 91
         DwcaHunter::logger_write(self.object_id, "Extracted %s vernacular names" % i) if i % BATCH_SIZE == 0
93  
-        l = @conv.iconv(l)
  92
+        l.encode!('UTF-8', 'ISO-8859-1', :invalid => :replace, :replace => '?')
94 93
         data = l.split("|").map { |d| d.strip }
95 94
         name_tsn = data[0]
96 95
         string   = data[1]
@@ -108,7 +107,7 @@ def get_synonyms
108 107
       f = open(File.join(@itis_dir, 'synonym_links'))
109 108
       f.each_with_index do |l, i|
110 109
         DwcaHunter::logger_write(self.object_id, "Extracted %s synonyms" % i) if i % BATCH_SIZE == 0
111  
-        l = @conv.iconv(l)
  110
+        l.encode!('UTF-8', 'ISO-8859-1', :invalid => :replace, :replace => '?')
112 111
         data = l.split("|").map { |d| d.strip }
113 112
         synonym_name_tsn = data[0]
114 113
         accepted_name_tsn = data[1]
@@ -145,7 +144,7 @@ def get_names
145 144
       f = open(File.join(@itis_dir, 'taxonomic_units'))
146 145
       f.each_with_index do |l, i|
147 146
         DwcaHunter::logger_write(self.object_id, "Extracted %s names" % i) if i % BATCH_SIZE == 0
148  
-        l = @conv.iconv(l)
  147
+        l.encode!('UTF-8', 'ISO-8859-1', :invalid => :replace, :replace => '?')
149 148
         data = l.split("|").map { |d| d.strip }
150 149
         name_tsn   = data[0]
151 150
         x1         = data[1]

0 notes on commit 2f931cc

Please sign in to comment.
Something went wrong with that request. Please try again.