Liquid UTF-8 support

lib/liquid.rb

@@ -20,14 +20,15 @@
 # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
 module Liquid
+  WordRegex                   = RUBY_VERSION < "1.9" ? '\w' : '[[:word:]]'
   FilterSeparator             = /\|/
   ArgumentSeparator           = ','
   FilterArgumentSeparator     = ':'
   VariableAttributeSeparator  = '.'
   TagStart                    = /\{\%/
   TagEnd                      = /\%\}/
-  VariableSignature           = /\(?[\w\-\.\[\]]\)?/
-  VariableSegment             = /[\w\-]/
+  VariableSignature           = /\(?[#{WordRegex}\-\.\[\]]\)?/o
+  VariableSegment             = /[#{WordRegex}\-]/o
   VariableStart               = /\{\{/
   VariableEnd                 = /\}\}/
   VariableIncompleteEnd       = /\}\}?/
@@ -38,7 +39,7 @@ module Liquid
   OtherFilterArgument         = /#{ArgumentSeparator}(?:#{StrictQuotedFragment})/o
   SpacelessFilter             = /^(?:'[^']+'|"[^"]+"|[^'"])*#{FilterSeparator}(?:#{StrictQuotedFragment})(?:#{FirstFilterArgument}(?:#{OtherFilterArgument})*)?/o
   Expression                  = /(?:#{QuotedFragment}(?:#{SpacelessFilter})*)/o
-  TagAttributes               = /(\w+)\s*\:\s*(#{QuotedFragment})/o
+  TagAttributes               = /(#{WordRegex}+)\s*\:\s*(#{QuotedFragment})/o
   AnyStartingTag              = /\{\{|\{\%/
   PartialTemplateParser       = /#{TagStart}.*?#{TagEnd}|#{VariableStart}.*?#{VariableIncompleteEnd}/o
   TemplateParser              = /(#{PartialTemplateParser}|#{AnyStartingTag})/o

lib/liquid/block.rb

@@ -3,7 +3,7 @@ module Liquid
   class Block < Tag
     IsTag             = /^#{TagStart}/o
     IsVariable        = /^#{VariableStart}/o
-    FullToken         = /^#{TagStart}\s*(\w+)\s*(.*)?#{TagEnd}$/o
+    FullToken         = /^#{TagStart}\s*(#{WordRegex}+)\s*(.*)?#{TagEnd}$/o
     ContentOfVariable = /^#{VariableStart}(.*)#{VariableEnd}$/o
 
     def parse(tokens)

lib/liquid/htmltags.rb

@@ -1,6 +1,6 @@
 module Liquid
   class TableRow < Block
-    Syntax = /(\w+)\s+in\s+(#{QuotedFragment}+)/o
+    Syntax = /(#{WordRegex}+)\s+in\s+(#{QuotedFragment}+)/o
 
     def initialize(tag_name, markup, tokens)
       if markup =~ Syntax

lib/liquid/tags/capture.rb

@@ -12,7 +12,7 @@ module Liquid
   # in a sidebar or footer.
   #
   class Capture < Block
-    Syntax = /(\w+)/
+    Syntax = /(#{WordRegex}+)/o
 
     def initialize(tag_name, markup, tokens)
       if markup =~ Syntax

lib/liquid/tags/for.rb

@@ -44,7 +44,7 @@ module Liquid
   # forloop.last:: Returns true if the item is the last item.
   #
   class For < Block                                             
-    Syntax = /(\w+)\s+in\s+(#{QuotedFragment}+)\s*(reversed)?/o
+    Syntax = /(#{WordRegex}+)\s+in\s+(#{QuotedFragment}+)\s*(reversed)?/ou
 
     def initialize(tag_name, markup, tokens)
       if markup =~ Syntax

lib/liquid/variable.rb

@@ -23,7 +23,7 @@ def initialize(markup)
         if match[2].match(/#{FilterSeparator}\s*(.*)/o)
           filters = Regexp.last_match(1).scan(FilterParser)
           filters.each do |f|
-            if matches = f.match(/\s*(\w+)(?:\s*#{FilterArgumentSeparator}(.*))?/)
+            if matches = f.match(/\s*(#{WordRegex}+)(?:\s*#{FilterArgumentSeparator}(.*))?/)
               filtername = matches[1]
               filterargs = matches[2].to_s.scan(/(?:\A|#{ArgumentSeparator})\s*((?:\w+\s*\:\s*)?#{QuotedFragment})/o).flatten
               @filters << [filtername, filterargs]

test/liquid/assign_test.rb

@@ -12,7 +12,13 @@ def test_assigned_variable
                            '{% assign foo = values %}.{{ foo[1] }}.',
                            'values' => %w{foo bar baz})
   end
-
+
+  def test_assigned_utf8_variable
+    assert_template_result('.bar.',
+                           "{% assign foo\u6000 = values %}.{{ foo\u6000[1] }}.",
+                           'values' => %w{foo bar baz})
+  end
+
   def test_assign_with_filter
     assert_template_result('.bar.',
                            '{% assign foo = values | split: "," %}.{{ foo[1] }}.',

test/liquid/block_test.rb

@@ -51,6 +51,14 @@ def test_with_custom_tag
     end
   end
 
+  def test_with_custom_utf8_tag
+    Liquid::Template.register_tag("testtag\u6000", Block)
+
+    assert_nothing_thrown do
+      template = Liquid::Template.parse( "{% testtag\u6000 something\u6000 %} {% endtesttag\u6000 %}")
+    end
+  end
+
   private
     def block_types(nodelist)
       nodelist.collect { |node| node.class }

test/liquid/capture_test.rb

@@ -7,6 +7,10 @@ def test_captures_block_content_in_variable
     assert_template_result("test string", "{% capture 'var' %}test string{% endcapture %}{{var}}", {})
   end
 
+  def test_captures_block_content_in_utf8_variable
+    assert_template_result("test string", "{% capture var\u6000 %}test string{% endcapture %}{{var\u6000}}", {})
+  end
+
   def test_capture_to_variable_from_outer_scope_if_existing
     template_source = <<-END_TEMPLATE
     {% assign var = '' %}

test/liquid/context_test.rb

@@ -98,6 +98,11 @@ def test_variables
     assert_equal nil, @context['nil']
   end
 
+  def test_utf8_variables
+    @context["chinese\u6000variable"] = 'chinese'
+    assert_equal 'chinese', @context["chinese\u6000variable"]
+  end
+
   def test_variables_not_existing
     assert_equal nil, @context['does_not_exist']
   end

test/liquid/tags/for_tag_test.rb

@@ -25,6 +25,11 @@ def test_for
     assert_template_result(expected,template,'array' => [1,2,3])
   end
 
+  def test_utf8_for
+    assigns = {"array\u6000chinese" => [1,2,3]}
+    assert_template_result('123', "{% for item\u6000chinese in array\u6000chinese %}{{ item\u6000chinese }}{% endfor %}", assigns)
+  end
+
   def test_for_reversed
     assigns = {'array' => [ 1, 2, 3] }
     assert_template_result('321','{%for item in array reversed %}{{item}}{%endfor%}',assigns)

test/liquid/tags/html_tag_test.rb

@@ -26,6 +26,12 @@ def test_html_table
                            'numbers' => [])
   end
 
+  def test_utf8_html_table
+    assert_template_result("<tr class=\"row1\">\n<td class=\"col1\"> 1 </td></tr>\n",
+                           "{% tablerow n\u6000 in numbers\u6000 %} {{n\u6000}} {% endtablerow %}",
+                           "numbers\u6000" => [1])
+  end
+
   def test_html_table_with_different_cols
     assert_template_result("<tr class=\"row1\">\n<td class=\"col1\"> 1 </td><td class=\"col2\"> 2 </td><td class=\"col3\"> 3 </td><td class=\"col4\"> 4 </td><td class=\"col5\"> 5 </td></tr>\n<tr class=\"row2\"><td class=\"col1\"> 6 </td></tr>\n",
                            '{% tablerow n in numbers cols:5%} {{n}} {% endtablerow %}',

test/liquid/variable_test.rb

@@ -50,6 +50,12 @@ def test_filters
     assert_equal [["things",["\"%Y, okay?\"","'the other one'"]]], var.filters
   end
 
+  def test_utf8_filters
+    var = Variable.new("foo | chinese\u6000filter: value\u6000")
+    assert_equal 'foo', var.name
+    assert_equal [["chinese\u6000filter",["value\u6000"]]], var.filters
+  end
+
   def test_filter_with_date_parameter
 
     var = Variable.new(%! '2006-06-06' | date: "%m/%d/%Y"!)