Separate data/logic for the Tokenizer module

lib/twitter_cldr/js/compiler.rb

@@ -113,14 +113,14 @@ def implementation_renderers
           :parser                          => TwitterCldr::Js::Renderers::ImplementationRenderers::Parsers::ParserRenderer,
           :segmentation_parser             => TwitterCldr::Js::Renderers::ImplementationRenderers::Parsers::SegmentationParserRenderer,
           :unicode_regex_parser            => TwitterCldr::Js::Renderers::ImplementationRenderers::Parsers::UnicodeRegexParserRenderer,
-          :token                           => TwitterCldr::Js::Renderers::DataRenderers::Tokenizers::TokenRenderer,
-          :composite_token                 => TwitterCldr::Js::Renderers::DataRenderers::Tokenizers::CompositeTokenRenderer,
-          :tokenizer                       => TwitterCldr::Js::Renderers::DataRenderers::Tokenizers::TokenizerRenderer,
-          :segmentation_tokenizer          => TwitterCldr::Js::Renderers::DataRenderers::Tokenizers::SegmentationTokenizerRenderer,
-          :unicode_regex_tokenizer         => TwitterCldr::Js::Renderers::DataRenderers::Tokenizers::UnicodeRegexTokenizerRenderer,
-          :rbnf_tokenizer                  => TwitterCldr::Js::Renderers::DataRenderers::Tokenizers::RBNFTokenizerRenderer,
-          :number_tokenizer                => TwitterCldr::Js::Renderers::DataRenderers::Tokenizers::NumberTokenizerRenderer,
-          :pattern_tokenizer               => TwitterCldr::Js::Renderers::DataRenderers::Tokenizers::PatternTokenizerRenderer,
+          :token                           => TwitterCldr::Js::Renderers::ImplementationRenderers::Tokenizers::TokenRenderer,
+          :composite_token                 => TwitterCldr::Js::Renderers::ImplementationRenderers::Tokenizers::CompositeTokenRenderer,
+          :tokenizer                       => TwitterCldr::Js::Renderers::ImplementationRenderers::Tokenizers::TokenizerRenderer,
+          :segmentation_tokenizer          => TwitterCldr::Js::Renderers::ImplementationRenderers::Tokenizers::SegmentationTokenizerRenderer,
+          :unicode_regex_tokenizer         => TwitterCldr::Js::Renderers::ImplementationRenderers::Tokenizers::UnicodeRegexTokenizerRenderer,
+          :rbnf_tokenizer                  => TwitterCldr::Js::Renderers::ImplementationRenderers::Tokenizers::RBNFTokenizerRenderer,
+          :number_tokenizer                => TwitterCldr::Js::Renderers::ImplementationRenderers::Tokenizers::NumberTokenizerRenderer,
+          :pattern_tokenizer               => TwitterCldr::Js::Renderers::ImplementationRenderers::Tokenizers::PatternTokenizerRenderer,
           :numbers                         => TwitterCldr::Js::Renderers::ImplementationRenderers::Numbers::NumbersRenderer,
           :rbnf                            => TwitterCldr::Js::Renderers::ImplementationRenderers::Numbers::RBNF::RBNFRenderer,
           :number_data_reader              => TwitterCldr::Js::Renderers::ImplementationRenderers::Numbers::RBNF::NumberDataReaderRenderer,

lib/twitter_cldr/js/renderers.rb

@@ -17,6 +17,7 @@ module ImplementationRenderers
         autoload :Utils,                                  'twitter_cldr/js/renderers/implementation/utils'
         autoload :Numbers,                                'twitter_cldr/js/renderers/implementation/numbers'
         autoload :Parsers,                                'twitter_cldr/js/renderers/implementation/parsers'
+        autoload :Tokenizers,                             'twitter_cldr/js/renderers/implementation/tokenizers'
 
       module Shared
           autoload :BidiRenderer,                         'twitter_cldr/js/renderers/implementation/shared/bidi_renderer'
@@ -34,16 +35,6 @@ module Shared
           autoload :TerritoriesContainmentRenderer,       'twitter_cldr/js/renderers/implementation/shared/territories_containment_renderer'
         end
 
-        module Tokenizers
-          autoload :TokenRenderer,                        'twitter_cldr/js/renderers/data/tokenizers/token'
-          autoload :CompositeTokenRenderer,               'twitter_cldr/js/renderers/data/tokenizers/composite_token'
-          autoload :TokenizerRenderer,                    'twitter_cldr/js/renderers/data/tokenizers/tokenizer'
-          autoload :SegmentationTokenizerRenderer,        'twitter_cldr/js/renderers/data/tokenizers/segmentation_tokenizer'
-          autoload :PatternTokenizerRenderer,             'twitter_cldr/js/renderers/data/tokenizers/pattern_tokenizer_renderer'
-          autoload :UnicodeRegexTokenizerRenderer,        'twitter_cldr/js/renderers/data/tokenizers/unicode_regex/unicode_regex_tokenizer'
-          autoload :RBNFTokenizerRenderer,                'twitter_cldr/js/renderers/data/tokenizers/numbers/rbnf_renderer'
-          autoload :NumberTokenizerRenderer,              'twitter_cldr/js/renderers/data/tokenizers/numbers/number_tokenizer_renderer'
-        end
       end
 
       module DataRenderers

lib/twitter_cldr/js/renderers/implementation/tokenizers.rb

@@ -0,0 +1,48 @@
+# encoding: UTF-8
+
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
+module TwitterCldr
+  module Js
+    module Renderers
+      module ImplementationRenderers
+        module Tokenizers
+
+        	class NumberTokenizerRenderer < TwitterCldr::Js::Renderers::Base
+            set_template "mustache/data/tokenizers/numbers/number_tokenizer.coffee"
+          end
+
+          class RBNFTokenizerRenderer < TwitterCldr::Js::Renderers::Base
+            set_template "mustache/data/tokenizers/numbers/rbnf_tokenizer.coffee"
+          end
+
+          class UnicodeRegexTokenizerRenderer < TwitterCldr::Js::Renderers::Base
+            set_template "mustache/data/tokenizers/unicode_regex/unicode_regex_tokenizer.coffee"
+          end
+
+          class CompositeTokenRenderer < TwitterCldr::Js::Renderers::Base
+            set_template "mustache/data/tokenizers/composite_token.coffee"
+          end
+
+          class PatternTokenizerRenderer < TwitterCldr::Js::Renderers::Base
+            set_template "mustache/data/tokenizers/pattern_tokenizer.coffee"
+          end
+
+          class SegmentationTokenizerRenderer < TwitterCldr::Js::Renderers::Base
+            set_template "mustache/data/tokenizers/segmentation_tokenizer.coffee"
+          end
+
+          class TokenRenderer < TwitterCldr::Js::Renderers::Base
+            set_template "mustache/data/tokenizers/token.coffee"
+          end
+
+          class TokenizerRenderer < TwitterCldr::Js::Renderers::Base
+            set_template "mustache/data/tokenizers/tokenizer.coffee"
+          end
+
+        end
+      end
+    end
+  end
+end