Merge pull request #1905 from Shopify/catlee/invalid_utf8

Raise SyntaxError on invalid UTF8 strings in lexer/tokenizer

Author: catlee

History.md

@@ -2,6 +2,8 @@
 
 ## 5.7.3 (unreleased)
 
+* Raise Liquid::SyntaxError when parsing invalidly encoded strings
+
 ## 5.7.2 2025-01-31
 
 * Fix array filters to not support nested properties

lib/liquid/lexer.rb

@@ -161,6 +161,12 @@ def tokenize(ss)
         end
         # rubocop:enable Metrics/BlockNesting
         output << EOS
+      rescue ::ArgumentError => e
+        if e.message == "invalid byte sequence in #{ss.string.encoding}"
+          raise SyntaxError, "Invalid byte sequence in #{ss.string.encoding}"
+        else
+          raise
+        end
       end
 
       def raise_syntax_error(start_pos, ss)

lib/liquid/tokenizer.rb

@@ -103,6 +103,12 @@ def next_text_token
 
       pos = @ss.pos -= 2
       @source.byteslice(start, pos - start)
+    rescue ::ArgumentError => e
+      if e.message == "invalid byte sequence in #{@ss.string.encoding}"
+        raise SyntaxError, "Invalid byte sequence in #{@ss.string.encoding}"
+      else
+        raise
+      end
     end
 
     def next_variable_token

test/unit/lexer_unit_test.rb

@@ -131,6 +131,16 @@ def test_tokenize_incomplete_expression
     assert_equal([[:id, "false"], [:number, "1"], [:end_of_string]], tokenize("false 1"))
   end
 
+  def test_error_with_invalid_utf8
+    error = assert_raises(SyntaxError) do
+      tokenize("\x00\xff")
+    end
+    assert_equal(
+      'Liquid syntax error: Invalid byte sequence in UTF-8',
+      error.message,
+    )
+  end
+
   private
 
   def tokenize(input)

test/unit/template_unit_test.rb

@@ -35,4 +35,15 @@ class TemplateSubclass < Liquid::Template
   def test_template_inheritance
     assert_equal("foo", TemplateSubclass.parse("foo").render)
   end
+
+  def test_invalid_utf8
+    input = "\xff\x00"
+    error = assert_raises(SyntaxError) do
+      Liquid::Tokenizer.new(source: input, string_scanner: StringScanner.new(input))
+    end
+    assert_equal(
+      'Liquid syntax error: Invalid byte sequence in UTF-8',
+      error.message,
+    )
+  end
 end