Fixed field extraction in open and index

mikisvaz · Sep 8, 2010 · 23f6a8b · 23f6a8b
1 parent 9c5e81f
commit 23f6a8b
Show file tree

Hide file tree

Showing 2 changed files with 7 additions and 4 deletions.
diff --git a/lib/rbbt/util/index.rb b/lib/rbbt/util/index.rb
@@ -8,7 +8,7 @@ module Index
   # where each element points to the first element in the row. +lexicon+
   # is the file containing the data.
   def self.index(lexicon, options = {}) 
-    options = {:sep => "\t|\\|", :case_sensitive => true}.merge(options)
+    options = {:sep => "\t", :sep2 => '\|', :case_sensitive => true}.merge(options)
 
 
     data = Open.to_hash(lexicon, options)

diff --git a/lib/rbbt/util/open.rb b/lib/rbbt/util/open.rb
@@ -17,10 +17,13 @@ def self.func_match_field(entities, options = {})
   end
 
   def self.fields(line, sep = "\t")
-    chunks = line.chomp.split(/(#{sep})/).select{|c| c !~ /^#{sep}$/ }
+    line << sep
+    line << "PLACEHOLDER"
+    chunks = line.split(/(#{sep})/).select{|c| c !~ /^#{sep}$/ }
     if line =~ /#{sep}$/
       chunks << ""
     end
+    chunks.pop
     chunks
   end
 
@@ -197,7 +200,7 @@ def self.to_hash(input, options = {})
     sep2    = options[:sep2]    || "|"
     single  = options[:single]  
     single  = false if single.nil?
-    flatten = options[:flatten] || single
+    flatten = options[:flatten]
     flatten = single if flatten.nil?
 
     extra = [extra] if extra && ! extra.is_a?( Array)
@@ -214,7 +217,7 @@ def self.to_hash(input, options = {})
       next if exclude and exclude.call(l)
       next if select  and ! select.call(l)
 
-      row_fields = self.fields(l, sep)
+      row_fields = self.fields(l.chomp, sep)
       id = row_fields[native]
       next if id.nil? || id == ""