Add string interpolation (#518)

* Add string interpolation * Add string interpolation * Add error messages related to string interpolation Add new tests * Fix bug causing conditional expressions to be not parsed correctly when direcly embedded into interpolated expression (thanks to lofcz for pointing conditional expression scenario) Add some new tests --------- Co-authored-by: Pawel Oziomek <paweloziomek@o2.pl>
scriban · Sep 5, 2023 · f823a82 · f823a82
1 parent 1328529
commit f823a82
Show file tree

Hide file tree

Showing 26 changed files with 752 additions and 43 deletions.
diff --git a/.gitignore b/.gitignore
@@ -220,3 +220,6 @@ _Pvt_Extensions
 
 # Remove artifacts produced by dotnet-releaser
 artifacts-dotnet-releaser/
+
+# ranger-turtle's notes
+notes.txt
diff --git a/src/Scriban.Tests/TestFiles/020-interpolation/020-interpolation-error1.out.txt b/src/Scriban.Tests/TestFiles/020-interpolation/020-interpolation-error1.out.txt
@@ -0,0 +1 @@
+text(5,1) : error : Opened interpolated expression not closed on the same line.
diff --git a/src/Scriban.Tests/TestFiles/020-interpolation/020-interpolation-error1.txt b/src/Scriban.Tests/TestFiles/020-interpolation/020-interpolation-error1.txt
@@ -0,0 +1,5 @@
+$"Interp: {2 + 4    # Interpolation not finished
+===
+{{
+$"Interp: {2 + 4 
+}End" }}
diff --git a/src/Scriban.Tests/TestFiles/020-interpolation/020-interpolation-error2.out.txt b/src/Scriban.Tests/TestFiles/020-interpolation/020-interpolation-error2.out.txt
@@ -0,0 +1,3 @@
+text(4,18) : error : Error while parsing binary expression: Expecting a string continuation to the right of `}` instead of `}` in: <expression> operator <expression>
+text(4,18) : error : Invalid token found `}`. Expecting <EOL>/end of line.
+text(4,18) : error : Unexpected end of file while parsing a string not terminated by a "
diff --git a/src/Scriban.Tests/TestFiles/020-interpolation/020-interpolation-error2.txt b/src/Scriban.Tests/TestFiles/020-interpolation/020-interpolation-error2.txt
@@ -0,0 +1,4 @@
+$"Interp: {2 + 4    # Interpolation not finished
+===
+{{
+$"Interp: {2 + 4}}
diff --git a/src/Scriban.Tests/TestFiles/020-interpolation/020-interpolation.out.txt b/src/Scriban.Tests/TestFiles/020-interpolation/020-interpolation.out.txt
@@ -0,0 +1,33 @@
+Scriban 5.8 can interpolate strings now
+Nested interpolation: 23new string
+UPPERCASE: 14
+UPPERCASE: 14
+Little more complex example: 2 Scriban 5
+Interpolation at the beginning: 4 is 4
+Pure string
+Another pure string
+2 8 string
+3 18 string
+4 60 string
+99 bottles
+{ is Curly Brace
+{ is Curly Brace
+{16}
+{49}
+{6 * 6}
+Interp 1: Interp 2: Interp 3
+Interp 1: Interp 2: Interp 3
+Interp 1: Concat 2: Interp 3
+Interp 1: Interp 2: Interp 3
+Concat 1: Interp 2: Interp 3
+1. Interpolation 2. Concatenation
+1. Concatenation 2. Interpolation
+1. Concatenation 2. Interpolation 3. Concatenation
+Interpolation of the strings
+Another interpolation of the strings
+Yet another interpolation of the strings
+Really complex interpolated expression: 33.
+Scriban yes 5 can interpolate strings now
+yes 5
+no
+Unary: -2
diff --git a/src/Scriban.Tests/TestFiles/020-interpolation/020-interpolation.txt b/src/Scriban.Tests/TestFiles/020-interpolation/020-interpolation.txt
@@ -0,0 +1,33 @@
+{{ $"Scriban {4 + 1}.8 can interpolate strings now" }}
+{{ $"Nested interpolation: {23 + "new string"}" }}
+{{ $"Uppercase: {4 + 10}" | string.upcase }}
+{{ string.upcase $"Uppercase: {4 + 10}" }}
+{{ $'Little more complex example: {2 + " " + string.capitalize $"scriban {5}"}' }}
+Interpolation at the beginning: {{ $"{math.times 2 2} is 4" }}
+{{ $"Pure string" }}
+{{ $'Another pure string' }}
+{{ $"{2} {2 * 4} {"string"}" }}
+{{ $'{3} {3 * 6} {"string"}' }}
+{{ $"{4} {8 * 7.5} {'string'}" }}
+{{ $"{11 * 9} bottles"}}
+{{ $"\{ is Curly Brace"}}
+{{ $'\{ is Curly Brace'}}
+{{ $"\{{4 * 4}}" }}
+{{ $'\{{7 * 7}}' }}
+{{ $"\{6 * 6}" }}
+{{ $'Interp 1: {$"Interp {2}: {$'Interp {3}'}"}' }}
+{{ $"{$'Interp 1: {$"Interp {2}: {$'Interp {3}'}"}'}" }}
+{{ $"{$'Interp 1: {$"Concat " + 2 + $": {$'Interp {3}'}"}'}" }}
+{{ $"{$'{$"Interp {1}:"} Interp {2}:'} Interp {3}" }}
+{{ $"{$'{$"Concat " + 1 + ":"} Interp {2}:'} Interp {3}" }}
+{{ $"{2 / 2}. Interpolation " + "2. Concatenation" }}
+{{ "1. Concatenation " + $"{4 / 2}. Interpolation" }}
+{{ "1. Concatenation " + $"{4 / 2}. Interpolation " + "3. Concatenation" }}
+{{ $"Interpolation" + $" of the " + $"strings"  }}
+{{ $"Another interpolation" + " of the " + $"strings" }}
+{{ "Yet another interpolation" + $" of the " + "strings" }}
+{{ $"Really complex interpolated expression: {2 * (4 + 3 + math.round 9.4) + (5 / (2 + 3))}" + "." }}
+{{ $"Scriban {( true ? $"yes {4 + 1}" : "no " + 4 )} can interpolate strings now" }}
+{{ true ? $"yes {4 + 1}" : "no" }}
+{{ false ? $"yes {4 + 1}" : $"no" }}
+{{ $"Unary: {-2}" }}
diff --git a/src/Scriban.Tests/TestFilesHelper.cs b/src/Scriban.Tests/TestFilesHelper.cs
@@ -42,6 +42,7 @@ public static IEnumerable ListAllTestFiles()
             {
                 "000-basic",
                 "010-literals",
+                "020-interpolation",
                 "100-expressions",
                 "200-statements",
                 "300-functions",

diff --git a/src/Scriban.Tests/TestLexer.cs b/src/Scriban.Tests/TestLexer.cs
@@ -722,6 +722,29 @@ public void ParseStringSingleLine()
             });
         }
 
+        [TestCase('"')]
+        [TestCase('\'')]
+        public void ParseInterpolatedStringTokens(char quoteType)
+        {
+            string strWithInterpolatedExpressions = $@"{{{{ ${quoteType}Begin {{2}} middle {{5}} end{quoteType} }}}}";
+            var tokens = ParseTokens(strWithInterpolatedExpressions);
+            Assert.AreEqual(new List<Token>
+            {
+                new Token(TokenType.CodeEnter, new TextPosition(0, 0, 0), new TextPosition(1, 0, 1)),
+                new Token(TokenType.BeginInterpString, new TextPosition(3, 0, 3), new TextPosition(11, 0, 11)),
+                new Token(TokenType.OpenInterpBrace, new TextPosition(11, 0, 11), new TextPosition(11, 0, 11)),
+                new Token(TokenType.Integer, new TextPosition(12, 0, 12), new TextPosition(12, 0, 12)),
+                new Token(TokenType.CloseInterpBrace, new TextPosition(13, 0, 13), new TextPosition(13, 0, 13)),
+                new Token(TokenType.ContinuationInterpString, new TextPosition(13, 0, 13), new TextPosition(22, 0, 22)),
+                new Token(TokenType.OpenInterpBrace, new TextPosition(22, 0, 22), new TextPosition(22, 0, 22)),
+                new Token(TokenType.Integer, new TextPosition(23, 0, 23), new TextPosition(23, 0, 23)),
+                new Token(TokenType.CloseInterpBrace, new TextPosition(24, 0, 24), new TextPosition(24, 0, 24)),
+                new Token(TokenType.EndingInterpString, new TextPosition(24, 0, 24), new TextPosition(29, 0, 29)),
+                new Token(TokenType.CodeExit, new TextPosition(31, 0, 31), new TextPosition(32, 0, 32)),
+                Token.Eof
+            }, tokens);
+        }
+
         [Test]
         public void ParseUnbalancedCloseBrace()
         {

diff --git a/src/Scriban.Tests/TestParser.cs b/src/Scriban.Tests/TestParser.cs
@@ -666,6 +666,12 @@ public static void A010_literals(string inputName)
             TestFile(inputName);
         }
 
+        [TestCaseSource("ListTestFiles", new object[] { "020-interpolation" })]
+        public static void A020_interpolation(string inputName)
+        {
+            TestFile(inputName);
+        }
+
         [TestCaseSource("ListTestFiles", new object[] { "100-expressions" })]
         public static void A100_expressions(string inputName)
         {

diff --git a/src/Scriban/Functions/StringFunctions.cs b/src/Scriban/Functions/StringFunctions.cs
@@ -63,34 +63,37 @@ public static string Escape(string text)
                     switch (c)
                     {
                         case '"':
-                            appendText = "\\\"";
+                            appendText = @"\""";
                             break;
                         case '\\':
-                            appendText = "\\\\";
+                            appendText = @"\\";
                             break;
+                        /*case '{':
+                            appendText = @"\{";
+                            break;*/
                         case '\a':
-                            appendText = "\\a";
+                            appendText = @"\a";
                             break;
                         case '\b':
-                            appendText = "\\b";
+                            appendText = @"\b";
                             break;
                         case '\t':
-                            appendText = "\\t";
+                            appendText = @"\t";
                             break;
                         case '\r':
-                            appendText = "\\r";
+                            appendText = @"\r";
                             break;
                         case '\v':
-                            appendText = "\\v";
+                            appendText = @"\v";
                             break;
                         case '\f':
-                            appendText = "\\f";
+                            appendText = @"\f";
                             break;
                         case '\n':
-                            appendText = "\\n";
+                            appendText = @"\n";
                             break;
                         default:
-                            appendText = $"\\x{(int)c:x2}";
+                            appendText = @$"\x{(int)c:x2}";
                             break;
                     }