diff --git a/lib/liquid.rb b/lib/liquid.rb index 82676d0e8..0f550f571 100644 --- a/lib/liquid.rb +++ b/lib/liquid.rb @@ -20,14 +20,15 @@ # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. module Liquid + WordRegex = RUBY_VERSION < "1.9" ? '\w' : '[[:word:]]' FilterSeparator = /\|/ ArgumentSeparator = ',' FilterArgumentSeparator = ':' VariableAttributeSeparator = '.' TagStart = /\{\%/ TagEnd = /\%\}/ - VariableSignature = /\(?[\w\-\.\[\]]\)?/ - VariableSegment = /[\w\-]/ + VariableSignature = /\(?[#{WordRegex}\-\.\[\]]\)?/o + VariableSegment = /[#{WordRegex}\-]/o VariableStart = /\{\{/ VariableEnd = /\}\}/ VariableIncompleteEnd = /\}\}?/ @@ -38,7 +39,7 @@ module Liquid OtherFilterArgument = /#{ArgumentSeparator}(?:#{StrictQuotedFragment})/o SpacelessFilter = /^(?:'[^']+'|"[^"]+"|[^'"])*#{FilterSeparator}(?:#{StrictQuotedFragment})(?:#{FirstFilterArgument}(?:#{OtherFilterArgument})*)?/o Expression = /(?:#{QuotedFragment}(?:#{SpacelessFilter})*)/o - TagAttributes = /(\w+)\s*\:\s*(#{QuotedFragment})/o + TagAttributes = /(#{WordRegex}+)\s*\:\s*(#{QuotedFragment})/o AnyStartingTag = /\{\{|\{\%/ PartialTemplateParser = /#{TagStart}.*?#{TagEnd}|#{VariableStart}.*?#{VariableIncompleteEnd}/o TemplateParser = /(#{PartialTemplateParser}|#{AnyStartingTag})/o diff --git a/lib/liquid/block.rb b/lib/liquid/block.rb index 8ea87b08e..602f2c01e 100644 --- a/lib/liquid/block.rb +++ b/lib/liquid/block.rb @@ -3,7 +3,7 @@ module Liquid class Block < Tag IsTag = /^#{TagStart}/o IsVariable = /^#{VariableStart}/o - FullToken = /^#{TagStart}\s*(\w+)\s*(.*)?#{TagEnd}$/o + FullToken = /^#{TagStart}\s*(#{WordRegex}+)\s*(.*)?#{TagEnd}$/o ContentOfVariable = /^#{VariableStart}(.*)#{VariableEnd}$/o def parse(tokens) diff --git a/lib/liquid/htmltags.rb b/lib/liquid/htmltags.rb index 78424e655..60c9a898f 100644 --- a/lib/liquid/htmltags.rb +++ b/lib/liquid/htmltags.rb @@ -1,6 +1,6 @@ module Liquid class TableRow < Block - Syntax = /(\w+)\s+in\s+(#{QuotedFragment}+)/o + Syntax = /(#{WordRegex}+)\s+in\s+(#{QuotedFragment}+)/o def initialize(tag_name, markup, tokens) if markup =~ Syntax diff --git a/lib/liquid/tags/capture.rb b/lib/liquid/tags/capture.rb index 2f67a0b29..67552e771 100644 --- a/lib/liquid/tags/capture.rb +++ b/lib/liquid/tags/capture.rb @@ -12,7 +12,7 @@ module Liquid # in a sidebar or footer. # class Capture < Block - Syntax = /(\w+)/ + Syntax = /(#{WordRegex}+)/o def initialize(tag_name, markup, tokens) if markup =~ Syntax diff --git a/lib/liquid/tags/for.rb b/lib/liquid/tags/for.rb index 8d2b27bf7..a09eb5d33 100644 --- a/lib/liquid/tags/for.rb +++ b/lib/liquid/tags/for.rb @@ -44,7 +44,7 @@ module Liquid # forloop.last:: Returns true if the item is the last item. # class For < Block - Syntax = /(\w+)\s+in\s+(#{QuotedFragment}+)\s*(reversed)?/o + Syntax = /(#{WordRegex}+)\s+in\s+(#{QuotedFragment}+)\s*(reversed)?/ou def initialize(tag_name, markup, tokens) if markup =~ Syntax diff --git a/lib/liquid/variable.rb b/lib/liquid/variable.rb index 5758fb0c0..330d5f8fc 100644 --- a/lib/liquid/variable.rb +++ b/lib/liquid/variable.rb @@ -23,7 +23,7 @@ def initialize(markup) if match[2].match(/#{FilterSeparator}\s*(.*)/o) filters = Regexp.last_match(1).scan(FilterParser) filters.each do |f| - if matches = f.match(/\s*(\w+)(?:\s*#{FilterArgumentSeparator}(.*))?/) + if matches = f.match(/\s*(#{WordRegex}+)(?:\s*#{FilterArgumentSeparator}(.*))?/) filtername = matches[1] filterargs = matches[2].to_s.scan(/(?:\A|#{ArgumentSeparator})\s*((?:\w+\s*\:\s*)?#{QuotedFragment})/o).flatten @filters << [filtername, filterargs] diff --git a/test/liquid/assign_test.rb b/test/liquid/assign_test.rb index 883378f0c..1145b33ae 100644 --- a/test/liquid/assign_test.rb +++ b/test/liquid/assign_test.rb @@ -12,7 +12,13 @@ def test_assigned_variable '{% assign foo = values %}.{{ foo[1] }}.', 'values' => %w{foo bar baz}) end - + + def test_assigned_utf8_variable + assert_template_result('.bar.', + "{% assign foo\u6000 = values %}.{{ foo\u6000[1] }}.", + 'values' => %w{foo bar baz}) + end + def test_assign_with_filter assert_template_result('.bar.', '{% assign foo = values | split: "," %}.{{ foo[1] }}.', diff --git a/test/liquid/block_test.rb b/test/liquid/block_test.rb index f4b44941d..4e2c1906f 100644 --- a/test/liquid/block_test.rb +++ b/test/liquid/block_test.rb @@ -51,6 +51,14 @@ def test_with_custom_tag end end + def test_with_custom_utf8_tag + Liquid::Template.register_tag("testtag\u6000", Block) + + assert_nothing_thrown do + template = Liquid::Template.parse( "{% testtag\u6000 something\u6000 %} {% endtesttag\u6000 %}") + end + end + private def block_types(nodelist) nodelist.collect { |node| node.class } diff --git a/test/liquid/capture_test.rb b/test/liquid/capture_test.rb index cf27ff1d4..00c63e85d 100644 --- a/test/liquid/capture_test.rb +++ b/test/liquid/capture_test.rb @@ -7,6 +7,10 @@ def test_captures_block_content_in_variable assert_template_result("test string", "{% capture 'var' %}test string{% endcapture %}{{var}}", {}) end + def test_captures_block_content_in_utf8_variable + assert_template_result("test string", "{% capture var\u6000 %}test string{% endcapture %}{{var\u6000}}", {}) + end + def test_capture_to_variable_from_outer_scope_if_existing template_source = <<-END_TEMPLATE {% assign var = '' %} diff --git a/test/liquid/context_test.rb b/test/liquid/context_test.rb index e50b23730..c156379ef 100644 --- a/test/liquid/context_test.rb +++ b/test/liquid/context_test.rb @@ -98,6 +98,11 @@ def test_variables assert_equal nil, @context['nil'] end + def test_utf8_variables + @context["chinese\u6000variable"] = 'chinese' + assert_equal 'chinese', @context["chinese\u6000variable"] + end + def test_variables_not_existing assert_equal nil, @context['does_not_exist'] end diff --git a/test/liquid/tags/for_tag_test.rb b/test/liquid/tags/for_tag_test.rb index edfdc88b8..13e883ff3 100644 --- a/test/liquid/tags/for_tag_test.rb +++ b/test/liquid/tags/for_tag_test.rb @@ -25,6 +25,11 @@ def test_for assert_template_result(expected,template,'array' => [1,2,3]) end + def test_utf8_for + assigns = {"array\u6000chinese" => [1,2,3]} + assert_template_result('123', "{% for item\u6000chinese in array\u6000chinese %}{{ item\u6000chinese }}{% endfor %}", assigns) + end + def test_for_reversed assigns = {'array' => [ 1, 2, 3] } assert_template_result('321','{%for item in array reversed %}{{item}}{%endfor%}',assigns) diff --git a/test/liquid/tags/html_tag_test.rb b/test/liquid/tags/html_tag_test.rb index 0815ba129..d4ffa34e6 100644 --- a/test/liquid/tags/html_tag_test.rb +++ b/test/liquid/tags/html_tag_test.rb @@ -26,6 +26,12 @@ def test_html_table 'numbers' => []) end + def test_utf8_html_table + assert_template_result("\n 1 \n", + "{% tablerow n\u6000 in numbers\u6000 %} {{n\u6000}} {% endtablerow %}", + "numbers\u6000" => [1]) + end + def test_html_table_with_different_cols assert_template_result("\n 1 2 3 4 5 \n 6 \n", '{% tablerow n in numbers cols:5%} {{n}} {% endtablerow %}', diff --git a/test/liquid/variable_test.rb b/test/liquid/variable_test.rb index 217a885cf..9c1c0e56d 100644 --- a/test/liquid/variable_test.rb +++ b/test/liquid/variable_test.rb @@ -50,6 +50,12 @@ def test_filters assert_equal [["things",["\"%Y, okay?\"","'the other one'"]]], var.filters end + def test_utf8_filters + var = Variable.new("foo | chinese\u6000filter: value\u6000") + assert_equal 'foo', var.name + assert_equal [["chinese\u6000filter",["value\u6000"]]], var.filters + end + def test_filter_with_date_parameter var = Variable.new(%! '2006-06-06' | date: "%m/%d/%Y"!)