Adding a bunch of new languages - also, spanish and french calendars …

…changed -- need to investigate why {{date}} has turned into {1}

lib/ext/date.rb

@@ -20,5 +20,9 @@ def method_missing(method, *args, &block)
         raise "Method not supported"
       end
     end
+
+    def to_s
+      self.to_default_s
+    end
   end
 end

lib/ext/datetime.rb

@@ -20,5 +20,9 @@ def method_missing(method, *args, &block)
         raise "Method not supported"
       end
     end
+
+    def to_s
+      self.to_default_s
+    end
   end
 end

lib/tokenizers/base.rb

@@ -6,16 +6,15 @@ class Base
 
       def initialize(options = {})
         @locale = options[:locale] || TwitterCldr::DEFAULT_LOCALE
-        @type = options[:type] || :default
         self.init_resources
         self.init_placeholders
       end
 
       protected
 
-      def tokens_for(key)
+      def tokens_for(key, type)
         final = []
-        tokens = self.expand_pattern(self.pattern_for(self.traverse(key)))
+        tokens = self.expand_pattern(self.pattern_for(self.traverse(key)), type)
 
         tokens.each do |token|
           if token.is_a?(Token)
@@ -39,19 +38,19 @@ def traverse(needle, haystack = @resource)
         final
       end
 
-      def expand_pattern(format_str)
+      def expand_pattern(format_str, type)
         if format_str.is_a?(Symbol)
           # symbols mean another path was given
-          self.expand_pattern(self.pattern_for(self.traverse(format_str)))
+          self.expand_pattern(self.pattern_for(self.traverse(format_str)), type)
         else
           parts = tokenize_pattern(format_str)
           final = []
 
           parts.each do |part|
             case part[:type]
               when :placeholder then
-                placeholder = @placeholders[part[:value].to_sym]
-                final += placeholder ? placeholder.tokens : []
+                placeholder = self.choose_placeholder(part[:value], @placeholders)
+                final += placeholder ? placeholder.tokens(:type => type) : []
               else
                 final << part
             end
@@ -63,18 +62,29 @@ def expand_pattern(format_str)
 
       def tokenize_pattern(pattern_str)
         results = []
-        pattern_str.split(/(\{\{\w*\}\}|\'\w+\')/).each do |piece|
+        pattern_str.split(/(\{\{?\w*\}?\}|\'\w+\')/).each do |piece|
           unless piece.empty?
             case piece[0].chr
               when "{"
-                results << { :value => piece[2..-3], :type => :placeholder }
+                results << { :value => piece, :type => :placeholder }
               else
                 results << { :value => piece, :type => :plaintext }
             end
           end
         end
         results
       end
+
+      def choose_placeholder(token, placeholders)
+        if token[0..1] == "{{"
+          token_value = token[2..-3]
+          found = placeholders.find { |placeholder| placeholder[:name].to_s == token_value }
+        else
+          found = placeholders[token[1..-2].to_i]
+        end
+
+        found ? found[:object] : nil
+      end
     end
   end
 end

lib/tokenizers/date_tokenizer.rb

@@ -2,7 +2,7 @@ module TwitterCldr
   module Tokenizers
     class DateTokenizer < TwitterCldr::Tokenizers::DateTimeTokenizer
       TOKEN_SPLITTER_REGEX = /(\'[\w\s-]+\'|G{1,5}|y+|Y+|Q{1,4}|q{1,5}|M{1,5}|L{1,5}|d{1,2}|F{1}|E{1,5}|e{1,5}|c{1,5})/
-      TOKEN_TYPE_REGEXES = [{ :type => :pattern, :regex => /^[G{1,5}|y+|Y+|Q{1,4}|q{1,5}|M{1,5}|L{1,5}|d{1,2}|F{1}|E{1,5}|e{1,5}|c{1,5}]/ },
+      TOKEN_TYPE_REGEXES = [{ :type => :pattern, :regex => /^(?:G{1,5}|y+|Y+|Q{1,4}|q{1,5}|M{1,5}|L{1,5}|d{1,2}|F{1}|E{1,5}|e{1,5}|c{1,5})/ },
                             { :type => :plaintext, :regex => // }]
       PATHS = { :default => "calendars.gregorian.formats.date.default",
                 :full => "calendars.gregorian.formats.date.full",
@@ -12,7 +12,7 @@ class DateTokenizer < TwitterCldr::Tokenizers::DateTimeTokenizer
 
       def tokens(options = {})
         type = options[:type] || :default
-        self.tokens_for(PATHS[type])
+        self.tokens_for(PATHS[type], type)
       end
 
       protected

lib/tokenizers/datetime_tokenizer.rb

@@ -17,8 +17,7 @@ def initialize(options = {})
 
       def tokens(options = {})
         type = options[:type] || :default
-        @placeholders.each_pair { |name, placeholder| placeholder.type = type }
-        self.tokens_for(PATHS[type])
+        self.tokens_for(PATHS[type], type)
       end
 
       def calendar
@@ -36,8 +35,8 @@ def init_resources
       end
 
       def init_placeholders
-        @placeholders = { :date => TwitterCldr::Tokenizers::DateTokenizer.new(:locale => @locale),
-                          :time => TwitterCldr::Tokenizers::TimeTokenizer.new(:locale => @locale) }
+        @placeholders = [{ :name => :date, :object => TwitterCldr::Tokenizers::DateTokenizer.new(:locale => @locale) },
+                         { :name => :time, :object => TwitterCldr::Tokenizers::TimeTokenizer.new(:locale => @locale) }]
       end
 
       def pattern_for(resource)

lib/tokenizers/time_tokenizer.rb

@@ -12,7 +12,7 @@ class TimeTokenizer < TwitterCldr::Tokenizers::DateTimeTokenizer
 
       def tokens(options = {})
         type = options[:type] || :default
-        self.tokens_for(PATHS[type])
+        self.tokens_for(PATHS[type], type)
       end
 
       protected

lib/twitter_cldr.rb

@@ -4,8 +4,9 @@
 require 'active_support'
 
 # require patches for extending Ruby functionality
-require 'ext/date'
 require 'ext/datetime'
+require 'ext/date'
+require 'ext/time'
 require 'ext/fixnum'
 require 'ext/float'
 require 'ext/hash'

resources/es/calendars.yml

@@ -57,13 +57,13 @@ es:
         datetime: 
           default: :"calendars.gregorian.formats.datetime.medium"
           full: 
-            pattern: "{{date}} {{time}}"
+            pattern: "{1} {0}"
           long: 
-            pattern: "{{date}} {{time}}"
+            pattern: "{1} {0}"
           medium: 
-            pattern: "{{date}} {{time}}"
+            pattern: "{1} {0}"
           short: 
-            pattern: "{{date}} {{time}}"
+            pattern: "{1} {0}"
         time: 
           default: :"calendars.gregorian.formats.time.medium"
           full: 

resources/fr/calendars.yml

@@ -57,13 +57,13 @@ fr:
         datetime: 
           default: :"calendars.gregorian.formats.datetime.medium"
           full: 
-            pattern: "{{date}} {{time}}"
+            pattern: "{1} {0}"
           long: 
-            pattern: "{{date}} {{time}}"
+            pattern: "{1} {0}"
           medium: 
-            pattern: "{{date}} {{time}}"
+            pattern: "{1} {0}"
           short: 
-            pattern: "{{date}} {{time}}"
+            pattern: "{1} {0}"
         time: 
           default: :"calendars.gregorian.formats.time.medium"
           full: