Strict Liquid Syntax (Part 2) + Render Performance (#441)

* Updated Capitalize filter to be inline with Reference Library. * Idea discussed and originally developed by @robin-parker * Render performance rewrite * Whitespace control handling changes, resolves #411 * Remove non-standard short-hand notation for comments and literal tags. See Shopify/liquid@f85bea2 * PR code review changes to syntax - no functional changes * Address PR review + additional code coverage * Resolves #413 * Implements SortNatural Filter and changes Sort to be case-sensitive in line with spec * Code formatting changes per PR review
dotliquid · Jul 28, 2021 · e18c34c · e18c34c
1 parent 32950d5
commit e18c34c
Show file tree

Hide file tree

Showing 21 changed files with 899 additions and 330 deletions.
diff --git a/appveyor.yml b/appveyor.yml
@@ -1,5 +1,5 @@
-# Build script for dotliquid, see https://www.appveyor.com/docs/appveyor-yml for reference
-version: 2.1.{build}
+# Build script for dotliquid, see https://www.appveyor.com/docs/appveyor-yml for reference
+version: 2.2.{build}
 image: Visual Studio 2017
 
 cache:

diff --git a/src/DotLiquid.Tests/ContextTests.cs b/src/DotLiquid.Tests/ContextTests.cs
@@ -171,11 +171,16 @@ private class ExpandoModel
         #endregion
 
         private Context _context;
+        private Context _contextV22;
 
         [OneTimeSetUp]
         public void SetUp()
         {
             _context = new Context(CultureInfo.InvariantCulture);
+            _contextV22 = new Context(CultureInfo.InvariantCulture)
+            {
+                SyntaxCompatibilityLevel = SyntaxCompatibility.DotLiquid22
+            };
         }
 
         [Test]
@@ -344,25 +349,31 @@ public void TestAddFilter()
             Context context = new Context(CultureInfo.InvariantCulture);
             context.AddFilters(new[] { typeof(TestFilters) });
             Assert.AreEqual("hi? hi!", context.Invoke("hi", new List<object> { "hi?" }));
+            context.SyntaxCompatibilityLevel = SyntaxCompatibility.DotLiquid22;
+            Assert.AreEqual("hi? hi!", context.Invoke("hi", new List<object> { "hi?" }));
 
             context = new Context(CultureInfo.InvariantCulture);
             Assert.AreEqual("hi?", context.Invoke("hi", new List<object> { "hi?" }));
-
-            context.AddFilters(new[] { typeof(TestFilters) });
-            Assert.AreEqual("hi? hi!", context.Invoke("hi", new List<object> { "hi?" }));
+            context.SyntaxCompatibilityLevel = SyntaxCompatibility.DotLiquid22;
+            Assert.Throws<FilterNotFoundException>(() => context.Invoke("hi", new List<object> { "hi?" }));
         }
 
         [Test]
         public void TestAddContextFilter()
         {
-            Context context = new Context(CultureInfo.InvariantCulture);
+            // This test differs from TestAddFilter only in that the Hi method within this class has a Context parameter in addition to the input string
+            Context context = new Context(CultureInfo.InvariantCulture) { SyntaxCompatibilityLevel = SyntaxCompatibility.DotLiquid20 };
             context["name"] = "King Kong";
 
             context.AddFilters(new[] { typeof(TestContextFilters) });
             Assert.AreEqual("hi? hi from King Kong!", context.Invoke("hi", new List<object> { "hi?" }));
+            context.SyntaxCompatibilityLevel = SyntaxCompatibility.DotLiquid22;
+            Assert.AreEqual("hi? hi from King Kong!", context.Invoke("hi", new List<object> { "hi?" }));
 
-            context = new Context(CultureInfo.InvariantCulture);
+            context = new Context(CultureInfo.InvariantCulture) { SyntaxCompatibilityLevel = SyntaxCompatibility.DotLiquid20 };
             Assert.AreEqual("hi?", context.Invoke("hi", new List<object> { "hi?" }));
+            context.SyntaxCompatibilityLevel = SyntaxCompatibility.DotLiquid22;
+            Assert.Throws<FilterNotFoundException>(() => context.Invoke("hi", new List<object> { "hi?" })); 
         }
 
         [Test]
@@ -932,5 +943,36 @@ public void TestToLiquidAndContextAtFirstLevel()
             Assert.IsInstanceOf<CategoryDrop>(_context["category"]);
             Assert.AreEqual(_context, ((CategoryDrop)_context["category"]).Context);
         }
+
+        [Test]
+        public void TestVariableParserV21()
+        {
+            var regex = new System.Text.RegularExpressions.Regex(Liquid.VariableParser);
+            TestVariableParser((input) => DotLiquid.Util.R.Scan(input, regex));
+        }
+
+        [Test]
+        public void TestVariableParserV22()
+        {
+            TestVariableParser((input) => GetVariableParts(input));
+        }
+
+        private void TestVariableParser(Func<string, IEnumerable<string>> variableSplitterFunc)
+        {
+            CollectionAssert.IsEmpty(variableSplitterFunc(""));
+            CollectionAssert.AreEqual(new[] { "var" }, variableSplitterFunc("var"));
+            CollectionAssert.AreEqual(new[] { "var", "method" }, variableSplitterFunc("var.method"));
+            CollectionAssert.AreEqual(new[] { "var", "[method]" }, variableSplitterFunc("var[method]"));
+            CollectionAssert.AreEqual(new[] { "var", "[method]", "[0]" }, variableSplitterFunc("var[method][0]"));
+            CollectionAssert.AreEqual(new[] { "var", "[\"method\"]", "[0]" }, variableSplitterFunc("var[\"method\"][0]"));
+            CollectionAssert.AreEqual(new[] { "var", "[method]", "[0]", "method" }, variableSplitterFunc("var[method][0].method"));
+        }
+
+        private static IEnumerable<string> GetVariableParts(string input)
+        {
+            using (var enumerator = Tokenizer.GetVariableEnumerator(input))
+                while (enumerator.MoveNext())
+                    yield return enumerator.Current;
+        }
     }
 }
diff --git a/src/DotLiquid.Tests/ExtendedFilterTests.cs b/src/DotLiquid.Tests/ExtendedFilterTests.cs
@@ -0,0 +1,51 @@
+using NUnit.Framework;
+using System;
+using System.Collections.Generic;
+using System.Globalization;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace DotLiquid.Tests
+{
+    [TestFixture]
+    public class ExtendedFilterTests
+    {
+        private Context _context;
+
+        [OneTimeSetUp]
+        public void SetUp()
+        {
+            _context = new Context(CultureInfo.InvariantCulture);
+            Template.RegisterFilter(typeof(ExtendedFilters));
+        }
+
+        [Test]
+        public void TestTitleize()
+        {
+            var context = _context;
+            Assert.AreEqual(null, ExtendedFilters.Titleize(context: context, input: null));
+            Assert.AreEqual("", ExtendedFilters.Titleize(context: context, input: ""));
+            Assert.AreEqual(" ", ExtendedFilters.Titleize(context: context, input: " "));
+            Assert.AreEqual("That Is One Sentence.", ExtendedFilters.Titleize(context: context, input: "That is one sentence."));
+
+            Helper.AssertTemplateResult(
+                expected: "Title",
+                template: "{{ 'title' | titleize }}");
+        }
+
+        [Test]
+        public void TestUpcaseFirst()
+        {
+            var context = _context;
+            Assert.AreEqual(null, ExtendedFilters.UpcaseFirst(context: context, input: null));
+            Assert.AreEqual("", ExtendedFilters.UpcaseFirst(context: context, input: ""));
+            Assert.AreEqual(" ", ExtendedFilters.UpcaseFirst(context: context, input: " "));
+            Assert.AreEqual(" My boss is Mr. Doe.", ExtendedFilters.UpcaseFirst(context: context, input: " my boss is Mr. Doe."));
+
+            Helper.AssertTemplateResult(
+                expected: "My great title",
+                template: "{{ 'my great title' | upcase_first }}");
+        }
+    }
+}
diff --git a/src/DotLiquid.Tests/HashTests.cs b/src/DotLiquid.Tests/HashTests.cs
@@ -126,18 +126,17 @@ public void TestDefaultValueConstructor()
             var hash = new Hash(0); // default value of zero
             hash["key"] = "value";
 
-            // NOTE: the next two asserts will change to true when performance changes are introduced by PR #441.
-            Assert.False(hash.Contains("unknown-key"));
-            Assert.False(hash.ContainsKey("unknown-key"));
+            Assert.True(hash.Contains("unknown-key"));
+            Assert.True(hash.ContainsKey("unknown-key"));
             Assert.AreEqual(0, hash["unknown-key"]); // ensure the default value is returned
 
             Assert.True(hash.Contains("key"));
             Assert.True(hash.ContainsKey("key"));
             Assert.AreEqual("value", hash["key"]);
 
             hash.Remove("key");
-            Assert.False(hash.Contains("key"));
-            Assert.False(hash.ContainsKey("key"));
+            Assert.True(hash.Contains("key"));
+            Assert.True(hash.ContainsKey("key"));
             Assert.AreEqual(0, hash["key"]); // ensure the default value is returned after key removed
         }
 
@@ -147,8 +146,8 @@ public void TestLambdaConstructor()
             var hash = new Hash((h, k) => { return "Lambda Value"; });
             hash["key"] = "value";
 
-            Assert.False(hash.Contains("unknown-key"));
-            Assert.False(hash.ContainsKey("unknown-key"));
+            Assert.True(hash.Contains("unknown-key"));
+            Assert.True(hash.ContainsKey("unknown-key"));
             Assert.AreEqual("Lambda Value", hash["unknown-key"]);
 
             Assert.True(hash.Contains("key"));

diff --git a/src/DotLiquid.Tests/ParsingQuirksTests.cs b/src/DotLiquid.Tests/ParsingQuirksTests.cs
@@ -65,5 +65,70 @@ public void TestLiquidTagsInQuotes()
             Helper.AssertTemplateResult("{{ {% %} }}", "{{ '{{ {% %} }}' }}");
             Helper.AssertTemplateResult("{{ {% %} }}", "{% assign x = '{{ {% %} }}' %}{{x}}");
         }
+
+        [TestCase(".")]
+        [TestCase("x.")]
+        [TestCase("$x")]
+        [TestCase("x?")]
+        [TestCase("x¿")]
+        [TestCase(".y")]
+        public void TestVariableNotTerminatedFromInvalidVariableName(string variableName)
+        {
+            var template = Template.Parse("{{ " + variableName + " }}");
+            SyntaxException ex = Assert.Throws<SyntaxException>(() => template.Render(new RenderParameters(System.Globalization.CultureInfo.InvariantCulture)
+            {
+                LocalVariables = Hash.FromAnonymousObject(new { x = "" }),
+                ErrorsOutputMode = ErrorsOutputMode.Rethrow,
+                SyntaxCompatibilityLevel = SyntaxCompatibility.DotLiquid22
+            }));
+            Assert.AreEqual(
+                expected: string.Format(Liquid.ResourceManager.GetString("VariableNotTerminatedException"), variableName),
+                actual: ex.Message);
+
+            template = Template.Parse("{{ x[" + variableName + "] }}");
+            ex = Assert.Throws<SyntaxException>(() => template.Render(new RenderParameters(System.Globalization.CultureInfo.InvariantCulture)
+            {
+                LocalVariables = Hash.FromAnonymousObject(new { x = new { x = "" } }),
+                ErrorsOutputMode = ErrorsOutputMode.Rethrow,
+                SyntaxCompatibilityLevel = SyntaxCompatibility.DotLiquid22
+            }));
+            Assert.AreEqual(
+                expected: string.Format(Liquid.ResourceManager.GetString("VariableNotTerminatedException"), variableName),
+                actual: ex.Message);
+        }
+
+        [Test]
+        public void TestNestedVariableNotTerminated()
+        {
+            var template = Template.Parse("{{ x[[] }}");
+            var ex = Assert.Throws<SyntaxException>(() => template.Render(new RenderParameters(System.Globalization.CultureInfo.InvariantCulture)
+            {
+                LocalVariables = Hash.FromAnonymousObject(new { x = new { x = "" } }),
+                ErrorsOutputMode = ErrorsOutputMode.Rethrow,
+                SyntaxCompatibilityLevel = SyntaxCompatibility.DotLiquid22
+            }));
+            Assert.AreEqual(
+                expected: string.Format(Liquid.ResourceManager.GetString("VariableNotTerminatedException"), "["),
+                actual: ex.Message);
+        }
+
+        [TestCase("[\"]")]
+        [TestCase("[\"\"")]
+        [TestCase("[']")]
+        public void TestVariableTokenizerNotTerminated(string variableName)
+        {
+            var ex = Assert.Throws<SyntaxException>(() => Tokenizer.GetVariableEnumerator(variableName).MoveNext());
+            Assert.AreEqual(
+                expected: string.Format(Liquid.ResourceManager.GetString("VariableNotTerminatedException"), variableName),
+                actual: ex.Message);
+        }
+
+        [Test]
+        public void TestShortHandSyntaxIsIgnored()
+        {
+            // These tests are based on actual handling on Ruby Liquid, not indicative of wanted behavior. Behavior for legacy dotliquid parser is in TestEmptyLiteral
+            Assert.AreEqual("}", Template.Parse("{{{}}}", SyntaxCompatibility.DotLiquid22).Render());
+            Assert.AreEqual("{##}", Template.Parse("{##}", SyntaxCompatibility.DotLiquid22).Render());
+        }
     }
 }
diff --git a/src/DotLiquid.Tests/RegexpTests.cs b/src/DotLiquid.Tests/RegexpTests.cs
@@ -81,17 +81,6 @@ public void TestQuotedWordsInTheMiddle()
             CollectionAssert.AreEqual(new[] { "arg1", "arg2", "\"arg 3\"", "arg4" }, Run("arg1 arg2 \"arg 3\" arg4", Liquid.QuotedFragment));
         }
 
-        [Test]
-        public void TestVariableParser()
-        {
-            CollectionAssert.AreEqual(new[] { "var" }, Run("var", Liquid.VariableParser));
-            CollectionAssert.AreEqual(new[] { "var", "method" }, Run("var.method", Liquid.VariableParser));
-            CollectionAssert.AreEqual(new[] { "var", "[method]" }, Run("var[method]", Liquid.VariableParser));
-            CollectionAssert.AreEqual(new[] { "var", "[method]", "[0]" }, Run("var[method][0]", Liquid.VariableParser));
-            CollectionAssert.AreEqual(new[] { "var", "[\"method\"]", "[0]" }, Run("var[\"method\"][0]", Liquid.VariableParser));
-            CollectionAssert.AreEqual(new[] { "var", "[method]", "[0]", "method" }, Run("var[method][0].method", Liquid.VariableParser));
-        }
-
         private static List<string> Run(string input, string pattern)
         {
             return R.Scan(input, new Regex(pattern));