Fix tests

lib/twitter_cldr/resources/timezone_tests_importer.rb

@@ -10,7 +10,6 @@
 module TwitterCldr
   module Resources
 
-    # This class should be used with JRuby in 1.9 mode
     class TimezoneTestsImporter < Importer
       TEST_TIME = Time.utc(2019, 11, 17, 0, 0, 0)
 

lib/twitter_cldr/tokenizers/calendars/date_time_tokenizer.rb

@@ -16,20 +16,6 @@ def tokenizer
             TokenRecognizer.new(:plaintext, //)
           ])
         end
-
-        def full_tokenizer
-          @full_tokenizer ||= begin
-            new_tok = Tokenizer.union(
-              data_reader.date_reader.tokenizer.tokenizer,
-              data_reader.time_reader.tokenizer.tokenizer
-            ) do |recognizer|
-              recognizer.token_type != :plaintext
-            end
-
-            new_tok.recognizers << TokenRecognizer.new(:plaintext, //)
-            new_tok
-          end
-        end
       end
 
       attr_reader :data_reader
@@ -39,25 +25,19 @@ def initialize(data_reader)
       end
 
       def tokenize(pattern)
-        expand_tokens(tokenizer.tokenize(pattern))
+        expand_tokens(
+          PatternTokenizer.new(data_reader, tokenizer).tokenize(pattern)
+        )
       end
 
       # Tokenizes mixed date and time pattern strings,
       # used to tokenize the additional date format patterns.
       def full_tokenize(pattern)
-        full_tokenizer.tokenize(pattern)
+        PatternTokenizer.new(data_reader, full_tokenizer).tokenize(pattern)
       end
 
       protected
 
-      def tokenizer
-        @tokenizer ||= PatternTokenizer.new(data_reader, self.class.tokenizer)
-      end
-
-      def full_tokenizer
-        @full_tokenizer ||= PatternTokenizer.new(data_reader, self.class.full_tokenizer)
-      end
-
       def expand_tokens(tokens)
         tokens.inject([]) do |ret, token|
           ret + case token.type
@@ -81,6 +61,24 @@ def expand_time(token)
         time_reader.tokenizer.tokenize(time_reader.pattern)
       end
 
+      def full_tokenizer
+        @@full_tokenizer ||= begin
+          new_tok = Tokenizer.union(
+            data_reader.date_reader.tokenizer.tokenizer,
+            data_reader.time_reader.tokenizer.tokenizer
+          ) do |recognizer|
+            recognizer.token_type != :plaintext
+          end
+
+          new_tok.recognizers << TokenRecognizer.new(:plaintext, //)
+          new_tok
+        end
+      end
+
+      def tokenizer
+        self.class.tokenizer
+      end
+
     end
   end
 end

lib/twitter_cldr/tokenizers/calendars/date_tokenizer.rb

@@ -24,13 +24,11 @@ def initialize(data_reader)
       end
 
       def tokenize(pattern)
-        tokenizer.tokenize(pattern)
+        PatternTokenizer.new(data_reader, tokenizer).tokenize(pattern)
       end
 
-      private
-
       def tokenizer
-        @tokenizer ||= PatternTokenizer.new(data_reader, self.class.tokenizer)
+        self.class.tokenizer
       end
 
     end

lib/twitter_cldr/tokenizers/calendars/time_tokenizer.rb

@@ -23,13 +23,11 @@ def initialize(data_reader)
       end
 
       def tokenize(pattern)
-        tokenizer.tokenize(pattern)
+        PatternTokenizer.new(data_reader, tokenizer).tokenize(pattern)
       end
 
-      private
-
       def tokenizer
-        @tokenizer ||= PatternTokenizer.new(data_reader, self.class.tokenizer)
+        self.class.tokenizer
       end
 
     end