Fixing tokenization bug causing single quotes to not be removed

twitter · Feb 4, 2013 · 371869c · 371869c
1 parent 475ffcf
commit 371869c
Show file tree

Hide file tree

Showing 2 changed files with 2 additions and 2 deletions.
diff --git a/lib/twitter_cldr/tokenizers/base.rb b/lib/twitter_cldr/tokenizers/base.rb
@@ -34,7 +34,7 @@ def tokenize_format(text)
               content = token.match(regexes[token_type][:content])[1]
               ret << CompositeToken.new(tokenize_format(content))
             else
-              ret << Token.new(:value => token, :type => token_type)  # .gsub(/\A\'/, "").chomp("'")
+              ret << Token.new(:value => token, :type => token_type)
             end
           end
           ret

diff --git a/lib/twitter_cldr/tokenizers/calendars/datetime_tokenizer.rb b/lib/twitter_cldr/tokenizers/calendars/datetime_tokenizer.rb
@@ -20,7 +20,7 @@ def initialize(options = {})
             DateTokenizer::TOKEN_SPLITTER_REGEX,
             TimeTokenizer::TOKEN_SPLITTER_REGEX
           ),
-          :else => //
+          :else => /([^\s]+)/  # groups of non-space chars
         }
 
         @token_type_regexes = {