diff --git a/.gitignore b/.gitignore index ed02a134..aa586050 100644 --- a/.gitignore +++ b/.gitignore @@ -220,3 +220,6 @@ _Pvt_Extensions # Remove artifacts produced by dotnet-releaser artifacts-dotnet-releaser/ + +# ranger-turtle's notes +notes.txt diff --git a/src/Scriban.Tests/TestFiles/020-interpolation/020-interpolation-error1.out.txt b/src/Scriban.Tests/TestFiles/020-interpolation/020-interpolation-error1.out.txt new file mode 100644 index 00000000..6c9401e1 --- /dev/null +++ b/src/Scriban.Tests/TestFiles/020-interpolation/020-interpolation-error1.out.txt @@ -0,0 +1 @@ +text(5,1) : error : Opened interpolated expression not closed on the same line. \ No newline at end of file diff --git a/src/Scriban.Tests/TestFiles/020-interpolation/020-interpolation-error1.txt b/src/Scriban.Tests/TestFiles/020-interpolation/020-interpolation-error1.txt new file mode 100644 index 00000000..75fc89ee --- /dev/null +++ b/src/Scriban.Tests/TestFiles/020-interpolation/020-interpolation-error1.txt @@ -0,0 +1,5 @@ +$"Interp: {2 + 4 # Interpolation not finished +=== +{{ +$"Interp: {2 + 4 +}End" }} \ No newline at end of file diff --git a/src/Scriban.Tests/TestFiles/020-interpolation/020-interpolation-error2.out.txt b/src/Scriban.Tests/TestFiles/020-interpolation/020-interpolation-error2.out.txt new file mode 100644 index 00000000..1ab0faa4 --- /dev/null +++ b/src/Scriban.Tests/TestFiles/020-interpolation/020-interpolation-error2.out.txt @@ -0,0 +1,3 @@ +text(4,18) : error : Error while parsing binary expression: Expecting a string continuation to the right of `}` instead of `}` in: operator +text(4,18) : error : Invalid token found `}`. Expecting /end of line. +text(4,18) : error : Unexpected end of file while parsing a string not terminated by a " \ No newline at end of file diff --git a/src/Scriban.Tests/TestFiles/020-interpolation/020-interpolation-error2.txt b/src/Scriban.Tests/TestFiles/020-interpolation/020-interpolation-error2.txt new file mode 100644 index 00000000..ffc4997e --- /dev/null +++ b/src/Scriban.Tests/TestFiles/020-interpolation/020-interpolation-error2.txt @@ -0,0 +1,4 @@ +$"Interp: {2 + 4 # Interpolation not finished +=== +{{ +$"Interp: {2 + 4}} \ No newline at end of file diff --git a/src/Scriban.Tests/TestFiles/020-interpolation/020-interpolation.out.txt b/src/Scriban.Tests/TestFiles/020-interpolation/020-interpolation.out.txt new file mode 100644 index 00000000..bba66f84 --- /dev/null +++ b/src/Scriban.Tests/TestFiles/020-interpolation/020-interpolation.out.txt @@ -0,0 +1,33 @@ +Scriban 5.8 can interpolate strings now +Nested interpolation: 23new string +UPPERCASE: 14 +UPPERCASE: 14 +Little more complex example: 2 Scriban 5 +Interpolation at the beginning: 4 is 4 +Pure string +Another pure string +2 8 string +3 18 string +4 60 string +99 bottles +{ is Curly Brace +{ is Curly Brace +{16} +{49} +{6 * 6} +Interp 1: Interp 2: Interp 3 +Interp 1: Interp 2: Interp 3 +Interp 1: Concat 2: Interp 3 +Interp 1: Interp 2: Interp 3 +Concat 1: Interp 2: Interp 3 +1. Interpolation 2. Concatenation +1. Concatenation 2. Interpolation +1. Concatenation 2. Interpolation 3. Concatenation +Interpolation of the strings +Another interpolation of the strings +Yet another interpolation of the strings +Really complex interpolated expression: 33. +Scriban yes 5 can interpolate strings now +yes 5 +no +Unary: -2 diff --git a/src/Scriban.Tests/TestFiles/020-interpolation/020-interpolation.txt b/src/Scriban.Tests/TestFiles/020-interpolation/020-interpolation.txt new file mode 100644 index 00000000..3d25c1db --- /dev/null +++ b/src/Scriban.Tests/TestFiles/020-interpolation/020-interpolation.txt @@ -0,0 +1,33 @@ +{{ $"Scriban {4 + 1}.8 can interpolate strings now" }} +{{ $"Nested interpolation: {23 + "new string"}" }} +{{ $"Uppercase: {4 + 10}" | string.upcase }} +{{ string.upcase $"Uppercase: {4 + 10}" }} +{{ $'Little more complex example: {2 + " " + string.capitalize $"scriban {5}"}' }} +Interpolation at the beginning: {{ $"{math.times 2 2} is 4" }} +{{ $"Pure string" }} +{{ $'Another pure string' }} +{{ $"{2} {2 * 4} {"string"}" }} +{{ $'{3} {3 * 6} {"string"}' }} +{{ $"{4} {8 * 7.5} {'string'}" }} +{{ $"{11 * 9} bottles"}} +{{ $"\{ is Curly Brace"}} +{{ $'\{ is Curly Brace'}} +{{ $"\{{4 * 4}}" }} +{{ $'\{{7 * 7}}' }} +{{ $"\{6 * 6}" }} +{{ $'Interp 1: {$"Interp {2}: {$'Interp {3}'}"}' }} +{{ $"{$'Interp 1: {$"Interp {2}: {$'Interp {3}'}"}'}" }} +{{ $"{$'Interp 1: {$"Concat " + 2 + $": {$'Interp {3}'}"}'}" }} +{{ $"{$'{$"Interp {1}:"} Interp {2}:'} Interp {3}" }} +{{ $"{$'{$"Concat " + 1 + ":"} Interp {2}:'} Interp {3}" }} +{{ $"{2 / 2}. Interpolation " + "2. Concatenation" }} +{{ "1. Concatenation " + $"{4 / 2}. Interpolation" }} +{{ "1. Concatenation " + $"{4 / 2}. Interpolation " + "3. Concatenation" }} +{{ $"Interpolation" + $" of the " + $"strings" }} +{{ $"Another interpolation" + " of the " + $"strings" }} +{{ "Yet another interpolation" + $" of the " + "strings" }} +{{ $"Really complex interpolated expression: {2 * (4 + 3 + math.round 9.4) + (5 / (2 + 3))}" + "." }} +{{ $"Scriban {( true ? $"yes {4 + 1}" : "no " + 4 )} can interpolate strings now" }} +{{ true ? $"yes {4 + 1}" : "no" }} +{{ false ? $"yes {4 + 1}" : $"no" }} +{{ $"Unary: {-2}" }} diff --git a/src/Scriban.Tests/TestFilesHelper.cs b/src/Scriban.Tests/TestFilesHelper.cs index 19caffde..21c9c31f 100644 --- a/src/Scriban.Tests/TestFilesHelper.cs +++ b/src/Scriban.Tests/TestFilesHelper.cs @@ -42,6 +42,7 @@ public static IEnumerable ListAllTestFiles() { "000-basic", "010-literals", + "020-interpolation", "100-expressions", "200-statements", "300-functions", diff --git a/src/Scriban.Tests/TestLexer.cs b/src/Scriban.Tests/TestLexer.cs index 7b591e61..302aaf6b 100644 --- a/src/Scriban.Tests/TestLexer.cs +++ b/src/Scriban.Tests/TestLexer.cs @@ -722,6 +722,29 @@ public void ParseStringSingleLine() }); } + [TestCase('"')] + [TestCase('\'')] + public void ParseInterpolatedStringTokens(char quoteType) + { + string strWithInterpolatedExpressions = $@"{{{{ ${quoteType}Begin {{2}} middle {{5}} end{quoteType} }}}}"; + var tokens = ParseTokens(strWithInterpolatedExpressions); + Assert.AreEqual(new List + { + new Token(TokenType.CodeEnter, new TextPosition(0, 0, 0), new TextPosition(1, 0, 1)), + new Token(TokenType.BeginInterpString, new TextPosition(3, 0, 3), new TextPosition(11, 0, 11)), + new Token(TokenType.OpenInterpBrace, new TextPosition(11, 0, 11), new TextPosition(11, 0, 11)), + new Token(TokenType.Integer, new TextPosition(12, 0, 12), new TextPosition(12, 0, 12)), + new Token(TokenType.CloseInterpBrace, new TextPosition(13, 0, 13), new TextPosition(13, 0, 13)), + new Token(TokenType.ContinuationInterpString, new TextPosition(13, 0, 13), new TextPosition(22, 0, 22)), + new Token(TokenType.OpenInterpBrace, new TextPosition(22, 0, 22), new TextPosition(22, 0, 22)), + new Token(TokenType.Integer, new TextPosition(23, 0, 23), new TextPosition(23, 0, 23)), + new Token(TokenType.CloseInterpBrace, new TextPosition(24, 0, 24), new TextPosition(24, 0, 24)), + new Token(TokenType.EndingInterpString, new TextPosition(24, 0, 24), new TextPosition(29, 0, 29)), + new Token(TokenType.CodeExit, new TextPosition(31, 0, 31), new TextPosition(32, 0, 32)), + Token.Eof + }, tokens); + } + [Test] public void ParseUnbalancedCloseBrace() { diff --git a/src/Scriban.Tests/TestParser.cs b/src/Scriban.Tests/TestParser.cs index ca61e3cc..09c0553b 100644 --- a/src/Scriban.Tests/TestParser.cs +++ b/src/Scriban.Tests/TestParser.cs @@ -666,6 +666,12 @@ public static void A010_literals(string inputName) TestFile(inputName); } + [TestCaseSource("ListTestFiles", new object[] { "020-interpolation" })] + public static void A020_interpolation(string inputName) + { + TestFile(inputName); + } + [TestCaseSource("ListTestFiles", new object[] { "100-expressions" })] public static void A100_expressions(string inputName) { diff --git a/src/Scriban/Functions/StringFunctions.cs b/src/Scriban/Functions/StringFunctions.cs index ee6171ef..463f5dbc 100644 --- a/src/Scriban/Functions/StringFunctions.cs +++ b/src/Scriban/Functions/StringFunctions.cs @@ -63,34 +63,37 @@ public static string Escape(string text) switch (c) { case '"': - appendText = "\\\""; + appendText = @"\"""; break; case '\\': - appendText = "\\\\"; + appendText = @"\\"; break; + /*case '{': + appendText = @"\{"; + break;*/ case '\a': - appendText = "\\a"; + appendText = @"\a"; break; case '\b': - appendText = "\\b"; + appendText = @"\b"; break; case '\t': - appendText = "\\t"; + appendText = @"\t"; break; case '\r': - appendText = "\\r"; + appendText = @"\r"; break; case '\v': - appendText = "\\v"; + appendText = @"\v"; break; case '\f': - appendText = "\\f"; + appendText = @"\f"; break; case '\n': - appendText = "\\n"; + appendText = @"\n"; break; default: - appendText = $"\\x{(int)c:x2}"; + appendText = @$"\x{(int)c:x2}"; break; } diff --git a/src/Scriban/Parsing/Lexer.cs b/src/Scriban/Parsing/Lexer.cs index 4015ac6f..62c4ad18 100644 --- a/src/Scriban/Parsing/Lexer.cs +++ b/src/Scriban/Parsing/Lexer.cs @@ -9,8 +9,10 @@ using System.Collections.Generic; using System.Diagnostics; using System.Linq; +using System.Numerics; using System.Runtime.CompilerServices; using Scriban.Helpers; +using Scriban.Syntax; namespace Scriban.Parsing { @@ -36,6 +38,12 @@ class Lexer : IEnumerable private bool _isExpectingFrontMatter; private readonly bool _isLiquid; + // String interpolation-related data + private readonly Stack _openingStringChars; + private bool _interpJustOpened; + private bool _interpJustClosed; + private bool AnyInterpolationOpen => _openingStringChars.Count > 0; + private readonly char _stripWhiteSpaceFullSpecialChar; private readonly char _stripWhiteSpaceRestrictedSpecialChar; private const char RawEscapeSpecialChar = '%'; @@ -87,6 +95,8 @@ public Lexer(string text, string sourcePath = null, LexerOptions? options = null _isLiquid = Options.Lang == ScriptLang.Liquid; _stripWhiteSpaceFullSpecialChar = '-'; _stripWhiteSpaceRestrictedSpecialChar = '~'; + + _openingStringChars = new Stack(); } /// @@ -515,7 +525,7 @@ private bool IsCodeExit() } // Check for either }} or ( %} if liquid active) - if (PeekChar(start) != (_isLiquidTagBlock? '%' : '}')) + if (PeekChar(start) != (_isLiquidTagBlock? '%' : '}') || AnyInterpolationOpen) { return false; } @@ -700,6 +710,19 @@ private bool ReadCode() { return true; } + if (_interpJustOpened) + { + TextPosition positionOfToken = new TextPosition(_position.Offset - 1, _position.Line, _position.Column - 1); + _token = new Token(TokenType.OpenInterpBrace, positionOfToken, positionOfToken); + _interpJustOpened = false; + return hasTokens; + } + else if (_interpJustClosed) + { + _interpJustClosed = false; + ReadInterpolatedString(); + return hasTokens; + } switch (c) { @@ -982,27 +1005,36 @@ private bool ReadCode() NextChar(); break; case '}': - if (_openBraceCount > 0) + if (AnyInterpolationOpen) { - // We match first brace open/close - _openBraceCount--; - _token = new Token(TokenType.CloseBrace, _position, _position); - NextChar(); + _interpJustClosed = true; + _token = new Token(TokenType.CloseInterpBrace, _position, _position); + break; } else { - if (Options.Mode != ScriptMode.ScriptOnly && IsCodeExit()) + if (_openBraceCount > 0) { - // We have no tokens for this ReadCode - hasTokens = false; + // We match first brace open/close + _openBraceCount--; + _token = new Token(TokenType.CloseBrace, _position, _position); + NextChar(); } else { - // Else we have a close brace but it is invalid - AddError("Unexpected } while no matching {", _position, _position); - // Remove the previous error token to still output a valid token - _token = new Token(TokenType.CloseBrace, _position, _position); - NextChar(); + if (Options.Mode != ScriptMode.ScriptOnly && IsCodeExit()) + { + // We have no tokens for this ReadCode + hasTokens = false; + } + else + { + // Else we have a close brace but it is invalid + AddError("Unexpected } while no matching {", _position, _position); + // Remove the previous error token to still output a valid token + _token = new Token(TokenType.CloseBrace, _position, _position); + NextChar(); + } } } break; @@ -1037,7 +1069,13 @@ private bool ReadCode() } bool specialIdentifier = c == '$'; - if (IsFirstIdentifierLetter(c) || specialIdentifier) + char nextChar = PeekChar(1); //Used for checking if the dollar is before the string + if (specialIdentifier && (nextChar == '"' || nextChar == '\'')) + { + ReadInterpolatedString(); + break; + } + else if (IsFirstIdentifierLetter(c) || specialIdentifier) { ReadIdentifier(specialIdentifier); break; @@ -1620,6 +1658,153 @@ private void ReadVerbatimString() _token = new Token(TokenType.VerbatimString, start, end); } + private void ReadInterpolatedString() + { + var start = _position; + var end = _position; + char startChar = c; + bool readingInterpolation = !(c == '"' || c == '\'' || c == '$'); + if (readingInterpolation) + { + startChar = _openingStringChars.Peek(); + } + if (startChar == '$') + { + NextChar(); + startChar = c; + } + NextChar(); // Skip ", ', { or } + while (true) + { + if (c == '\\') + { + end = _position; + NextChar(); + // 0 ' " \ b f n r t v u0000-uFFFF x00-xFF + switch (c) + { + case '\n': + end = _position; + NextChar(); + continue; + case '\r': + end = _position; + NextChar(); + if (c == '\n') + { + end = _position; + NextChar(); + } + continue; + case '0': + case '\'': + case '"': + case '\\': + case '{': + case 'b': + case 'f': + case 'n': + case 'r': + case 't': + case 'v': + end = _position; + NextChar(); + continue; + case 'u': + end = _position; + NextChar(); + // Must be followed 4 hex numbers (0000-FFFF) + if (c.IsHex()) // 1 + { + end = _position; + NextChar(); + if (c.IsHex()) // 2 + { + end = _position; + NextChar(); + if (c.IsHex()) // 3 + { + end = _position; + NextChar(); + if (c.IsHex()) // 4 + { + end = _position; + NextChar(); + continue; + } + } + } + } + AddError($"Unexpected hex number `{c}` following `\\u`. Expecting `\\u0000` to `\\uffff`.", _position, _position); + break; + case 'x': + end = _position; + NextChar(); + // Must be followed 2 hex numbers (00-FF) + if (c.IsHex()) + { + end = _position; + NextChar(); + if (c.IsHex()) + { + end = _position; + NextChar(); + continue; + } + } + AddError($"Unexpected hex number `{c}` following `\\x`. Expecting `\\x00` to `\\xff`", _position, _position); + break; + + } + AddError($"Unexpected escape character `{c}` in string. Only 0 ' \\ \" b f n r t v u0000-uFFFF x00-xFF are allowed", _position, _position); + } + else if (c == '\0') + { + AddError($"Unexpected end of file while parsing a string not terminated by a {startChar}", end, end); + return; + } + else if (c == '{') //If interpolation beginning is read + { + end = _position; + // if interpolation is starting + if (!readingInterpolation) + { + _token = new Token(TokenType.BeginInterpString, start, end); + _openingStringChars.Push(startChar); + } + else + { + _token = new Token(TokenType.ContinuationInterpString, start, end); + } + NextChar(); + _interpJustOpened = true; + return; + } + else if (c == startChar) + { + end = _position; + NextChar(); + break; + } + else + { + end = _position; + NextChar(); + } + } + + if (readingInterpolation) + { + _ = _openingStringChars.Pop(); + _token = new Token(TokenType.EndingInterpString, start, end); + } + else + { + start = new TextPosition(start.Offset, start.Line, start.Column); + _token = new Token(TokenType.InterpString, start, end); + } + } + private void ReadComment() { var start = _position; diff --git a/src/Scriban/Parsing/Parser.Expressions.cs b/src/Scriban/Parsing/Parser.Expressions.cs index 64844ec3..5a60a420 100644 --- a/src/Scriban/Parsing/Parser.Expressions.cs +++ b/src/Scriban/Parsing/Parser.Expressions.cs @@ -57,6 +57,8 @@ private bool TryBinaryOperator(out ScriptBinaryOperator binaryOperator, out int case TokenType.LessEqual: binaryOperator = ScriptBinaryOperator.CompareLessOrEqual; break; case TokenType.DoubleDot: binaryOperator = ScriptBinaryOperator.RangeInclude; break; case TokenType.DoubleDotLess: binaryOperator = ScriptBinaryOperator.RangeExclude; break; + case TokenType.OpenInterpBrace: binaryOperator = ScriptBinaryOperator.InterpBegin; break; + case TokenType.CloseInterpBrace: binaryOperator = ScriptBinaryOperator.InterpEnd; break; default: if (_isScientific) { @@ -87,6 +89,11 @@ private ScriptExpression ParseExpressionAsVariableOrStringOrExpression(ScriptNod return ParseVariable(); case TokenType.String: return ParseString(); + case TokenType.BeginInterpString: + case TokenType.ContinuationInterpString: + case TokenType.EndingInterpString: + case TokenType.InterpString: + return ParseInterpolatedString(); case TokenType.VerbatimString: return ParseVerbatimString(); default: @@ -165,6 +172,12 @@ private ScriptExpression ParseExpression(ScriptNode parentNode, ScriptExpression case TokenType.ImplicitString: leftOperand = ParseImplicitString(); break; + case TokenType.BeginInterpString: + case TokenType.ContinuationInterpString: + case TokenType.EndingInterpString: + case TokenType.InterpString: + leftOperand = ParseInterpolatedString(); + break; case TokenType.VerbatimString: leftOperand = ParseVerbatimString(); break; @@ -174,6 +187,9 @@ private ScriptExpression ParseExpression(ScriptNode parentNode, ScriptExpression case TokenType.OpenBrace: leftOperand = ParseObjectInitializer(); break; + case TokenType.OpenInterpBrace: + leftOperand = ParseInterpolatedExpression(); + break; case TokenType.OpenBracket: leftOperand = ParseArrayInitializer(); break; @@ -346,7 +362,7 @@ private ScriptExpression ParseExpression(ScriptNode parentNode, ScriptExpression leftOperand = new ScriptExpressionAsStatement(declaration) {Span = declaration.Span}; break; } - if(TryGetCompoundAssignmentOperator(out var scriptToken, out var tokenType) && !(scriptToken is null)) + if (TryGetCompoundAssignmentOperator(out var scriptToken, out var tokenType) && !(scriptToken is null)) { var assignExpression = Open(); assignExpression.EqualToken = scriptToken; @@ -410,10 +426,23 @@ private ScriptExpression ParseExpression(ScriptNode parentNode, ScriptExpression binaryExpression.OperatorToken.Value = binaryOperatorType.ToText(); } - // unit test: 110-binary-simple-error1.txt + string message; + if (binaryOperatorType == ScriptBinaryOperator.InterpBegin) + { + message = $"Expecting an to the right of `{{` instead of `{GetAsText(Current)}`"; + } + else if (binaryOperatorType == ScriptBinaryOperator.InterpEnd) + { + message = $"Expecting a string continuation to the right of `}}` instead of `{GetAsText(Current)}`"; + } + else + { + message = $"Expecting an to the right of the operator instead of `{GetAsText(Current)}`"; + } + // unit tests: 110-binary-simple-error1.txt and 010-interpolation-error-2.txt binaryExpression.Right = ExpectAndParseExpression(binaryExpression, functionCall ?? parentExpression, newPrecedence, - $"Expecting an to the right of the operator instead of `{GetAsText(Current)}`", + message, mode); // propagate the mode in case we are in DefaultNoNamedArgument (so that the colon in 1 + 2 + 3 : will be correctly skipped) leftOperand = Close(binaryExpression); @@ -823,7 +852,7 @@ private ScriptExpression ParseObjectInitializer() break; } - if (!expectingEndOfInitializer && (Current.Type == TokenType.Identifier || Current.Type == TokenType.String)) + if (!expectingEndOfInitializer && (Current.Type == TokenType.Identifier || Current.Type.IsStringToken())) { var positionBefore = Current; @@ -940,6 +969,16 @@ private ScriptExpression ParseParenthesis() return Close(expression); } + private ScriptExpression ParseInterpolatedExpression() + { + var expression = Open(); + ExpectAndParseTokenTo(expression.OpenBrace, TokenType.OpenInterpBrace); // Parse { + expression.Expression = ExpectAndParseExpression(expression, newPrecedence: 10); + // newPrecedence is set to 10 to support directly nested ternary expressions + return Close(expression); + } + + private ScriptToken ParseToken(TokenType tokenType) { var verbatim = Open(); @@ -1133,6 +1172,10 @@ public bool IsStartOfExpression() case TokenType.BinaryInteger: case TokenType.Float: case TokenType.String: + case TokenType.BeginInterpString: + case TokenType.ContinuationInterpString: + case TokenType.EndingInterpString: + case TokenType.InterpString: case TokenType.ImplicitString: case TokenType.VerbatimString: case TokenType.OpenParen: @@ -1234,6 +1277,9 @@ internal static int GetDefaultBinaryOperatorPrecedence(ScriptBinaryOperator op) { switch (op) { + case ScriptBinaryOperator.InterpBegin: + case ScriptBinaryOperator.InterpEnd: + return 10; case ScriptBinaryOperator.EmptyCoalescing: case ScriptBinaryOperator.NotEmptyCoalescing: return 20; diff --git a/src/Scriban/Parsing/Parser.Statements.cs b/src/Scriban/Parsing/Parser.Statements.cs index 9ecd80a0..07d0769e 100644 --- a/src/Scriban/Parsing/Parser.Statements.cs +++ b/src/Scriban/Parsing/Parser.Statements.cs @@ -210,7 +210,12 @@ private bool TryParseStatement(ScriptNode parent, bool parseEndOfStatementAfterE else { nextStatement = false; - LogError($"Unexpected token {GetAsText(Current)}"); + string message; + if (AnyInterpolation) + message = "Opened interpolated expression not closed on the same line."; + else + message = $"Unexpected token {GetAsText(Current)}"; + LogError(message); } break; } diff --git a/src/Scriban/Parsing/Parser.Terminals.cs b/src/Scriban/Parsing/Parser.Terminals.cs index bd66de9e..f7fe9d3e 100644 --- a/src/Scriban/Parsing/Parser.Terminals.cs +++ b/src/Scriban/Parsing/Parser.Terminals.cs @@ -46,6 +46,12 @@ private ScriptExpression ParseVariableOrLiteral() case TokenType.String: literal = ParseString(); break; + case TokenType.BeginInterpString: + case TokenType.ContinuationInterpString: + case TokenType.EndingInterpString: + case TokenType.InterpString: + literal = ParseInterpolatedString(); + break; case TokenType.ImplicitString: literal = ParseImplicitString(); break; @@ -396,6 +402,125 @@ private ScriptLiteral ParseString() return Close(literal); } + private ScriptLiteral ParseInterpolatedString() + { + var literal = Open(); + var text = _lexer.Text; + var builder = new StringBuilder(Current.End.Offset - Current.Start.Offset - 1); + + int begin = Current.Start.Offset + 1; + char stringQuoteTypeChar = _lexer.Text[begin]; + if (Current.Type == TokenType.BeginInterpString || Current.Type == TokenType.InterpString) + { + if (Current.Type == TokenType.BeginInterpString) + { + _interpolatedNestedStringChars.Push(stringQuoteTypeChar); + } + begin++; + } + else + { + if (Current.Type == TokenType.EndingInterpString) + { + stringQuoteTypeChar = _interpolatedNestedStringChars.Pop(); + } + } + literal.StringQuoteType = + stringQuoteTypeChar == '\'' + ? ScriptLiteralStringQuoteType.SimpleQuote + : ScriptLiteralStringQuoteType.DoubleQuote; + + literal.StringTokenType = Current.Type; + + var end = Current.End.Offset; + for (int i = begin; i < end; i++) + { + var c = text[i]; + // Handle escape characters + if (text[i] == '\\') + { + i++; + switch (text[i]) + { + case '0': + builder.Append((char)0); + break; + case '\n': + break; + case '\r': + i++; // skip next \n that was validated by the lexer + break; + case '\'': + builder.Append('\''); + break; + case '"': + builder.Append('"'); + break; + case '\\': + builder.Append('\\'); + break; + case '{': + builder.Append('{'); + break; + case 'b': + builder.Append('\b'); + break; + case 'f': + builder.Append('\f'); + break; + case 'n': + builder.Append('\n'); + break; + case 'r': + builder.Append('\r'); + break; + case 't': + builder.Append('\t'); + break; + case 'v': + builder.Append('\v'); + break; + case 'u': + { + i++; + int value = 0; + if (i < text.Length) value = text[i++].HexToInt(); + if (i < text.Length) value = (value << 4) | text[i++].HexToInt(); + if (i < text.Length) value = (value << 4) | text[i++].HexToInt(); + if (i < text.Length) value = (value << 4) | text[i].HexToInt(); + + // Is it correct? + builder.Append(ConvertFromUtf32(value)); + break; + } + case 'x': + { + i++; + int value = 0; + if (i < text.Length) value = text[i++].HexToInt(); + if (i < text.Length) value = (value << 4) | text[i].HexToInt(); + builder.Append((char)value); + break; + } + + default: + // This should not happen as the lexer is supposed to prevent this + LogError($"Unexpected escape character `{text[i]}` in string"); + break; + } + } + else + { + builder.Append(c); + } + } + + literal.Value = builder.ToString(); + + NextToken(); + return Close(literal); + } + private ScriptExpression ParseVariable() { var currentToken = Current; @@ -696,6 +821,10 @@ private static bool IsVariableOrLiteral(Token token) case TokenType.BinaryInteger: case TokenType.Float: case TokenType.String: + case TokenType.BeginInterpString: + case TokenType.ContinuationInterpString: + case TokenType.EndingInterpString: + case TokenType.InterpString: case TokenType.ImplicitString: case TokenType.VerbatimString: return true; diff --git a/src/Scriban/Parsing/Parser.cs b/src/Scriban/Parsing/Parser.cs index 2ca51ad6..2c2613b8 100644 --- a/src/Scriban/Parsing/Parser.cs +++ b/src/Scriban/Parsing/Parser.cs @@ -45,6 +45,9 @@ partial class Parser private readonly Queue _pendingStatements; private IScriptTerminal _lastTerminalWithTrivias; + private Stack _interpolatedNestedStringChars; + + /// /// Initializes a new instance of the class. /// @@ -68,6 +71,8 @@ public Parser(Lexer lexer, ParserOptions? options = null) _pendingStatements = new Queue(2); Blocks = new Stack(); + _interpolatedNestedStringChars = new Stack(); + // Initialize the iterator _tokenIt = lexer.GetEnumerator(); NextToken(); @@ -86,6 +91,8 @@ public Parser(Lexer lexer, ParserOptions? options = null) private Token Previous => _previousToken; + private bool AnyInterpolation => _interpolatedNestedStringChars.Count > 0; + public SourceSpan CurrentSpan => GetSpanForToken(Current); private ScriptMode CurrentParsingMode { get; set; } diff --git a/src/Scriban/Parsing/TokenType.cs b/src/Scriban/Parsing/TokenType.cs index b91104cf..bd4b3119 100644 --- a/src/Scriban/Parsing/TokenType.cs +++ b/src/Scriban/Parsing/TokenType.cs @@ -80,6 +80,26 @@ enum TokenType /// String, + /// + /// An interpolated string without interpolated expressions + /// + InterpString, + + /// + /// An interpolated string at the beginning + /// + BeginInterpString, + + /// + /// An interpolated string at the middle + /// + ContinuationInterpString, + + /// + /// An interpolated string at the end + /// + EndingInterpString, + /// /// An implicit string with quotes /// @@ -237,6 +257,12 @@ enum TokenType /// Token "]" CloseBracket, + /// Token "#{" + OpenInterpBrace, + + /// Token "}" + CloseInterpBrace, + /// /// Custom token /// diff --git a/src/Scriban/Parsing/TokenTypeExtensions.cs b/src/Scriban/Parsing/TokenTypeExtensions.cs index cc1062de..dd736f30 100644 --- a/src/Scriban/Parsing/TokenTypeExtensions.cs +++ b/src/Scriban/Parsing/TokenTypeExtensions.cs @@ -1,9 +1,11 @@ -// Copyright (c) Alexandre Mutel. All rights reserved. +// Copyright (c) Alexandre Mutel. All rights reserved. // Licensed under the BSD-Clause 2 license. // See license.txt file in the project root for full license information. #nullable disable +using System.Runtime.CompilerServices; + namespace Scriban.Parsing { #if SCRIBAN_PUBLIC @@ -75,8 +77,19 @@ public static string ToText(this TokenType type) TokenType.CloseBrace => "}", TokenType.OpenBracket => "[", TokenType.CloseBracket => "]", + TokenType.OpenInterpBrace => string.Empty, + TokenType.CloseInterpBrace => string.Empty, _ => null }; } + + //TODO Pattern matching from C# 7.0 could be used but permission from monsieur Mutel is needed... + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static bool IsStringToken(this TokenType token) => + token == TokenType.String || token == TokenType.InterpString || token == TokenType.BeginInterpString || token == TokenType.ContinuationInterpString || token == TokenType.EndingInterpString; + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static bool IsInterpolationStringToken(this TokenType token) => + token == TokenType.InterpString || token == TokenType.BeginInterpString || token == TokenType.ContinuationInterpString || token == TokenType.EndingInterpString; } } \ No newline at end of file diff --git a/src/Scriban/Runtime/ScriptObjectExtensions.cs b/src/Scriban/Runtime/ScriptObjectExtensions.cs index c03f8e23..e7f9ccdb 100644 --- a/src/Scriban/Runtime/ScriptObjectExtensions.cs +++ b/src/Scriban/Runtime/ScriptObjectExtensions.cs @@ -54,17 +54,17 @@ public static void AssertNotReadOnly(this IScriptObject scriptObject) /// public static void Import(this IScriptObject script, object obj, MemberFilterDelegate filter = null, MemberRenamerDelegate renamer = null) { - if (obj is IScriptObject) + if (obj is IScriptObject scriptObj) { // TODO: Add support for filter, member renamer - script.Import((IScriptObject)obj); + script.Import(scriptObj); return; } - if (obj is IDictionary) + if (obj is IDictionary dictionary) { // TODO: Add support for filter, member renamer - script.ImportDictionary((IDictionary)obj); + script.ImportDictionary(dictionary); return; } diff --git a/src/Scriban/ScribanAsync.generated.cs b/src/Scriban/ScribanAsync.generated.cs index 86cfe2b0..738996dc 100644 --- a/src/Scriban/ScribanAsync.generated.cs +++ b/src/Scriban/ScribanAsync.generated.cs @@ -1,4 +1,4 @@ -//------------------------------------------------------------------------------ +//------------------------------------------------------------------------------ // // This code was generated by a tool. // @@ -1939,6 +1939,41 @@ public async ValueTask SetValueAsync(TemplateContext context, object valueToSet) } } +#if SCRIBAN_PUBLIC + public +#else + internal +#endif + partial class ScriptInterpolatedExpression + { + public override async ValueTask EvaluateAsync(TemplateContext context) + { + // A nested expression will reset the pipe arguments for the group + context.PushPipeArguments(); + try + { + return await context.GetValueAsync(this).ConfigureAwait(false); + } + finally + { + if (context.CurrentPipeArguments != null) + { + context.PopPipeArguments(); + } + } + } + + public async ValueTask GetValueAsync(TemplateContext context) + { + return await context.EvaluateAsync(Expression).ConfigureAwait(false); + } + + public async ValueTask SetValueAsync(TemplateContext context, object valueToSet) + { + await context.SetValueAsync(Expression, valueToSet).ConfigureAwait(false); + } + } + #if SCRIBAN_PUBLIC public #else diff --git a/src/Scriban/ScribanVisitors.generated.cs b/src/Scriban/ScribanVisitors.generated.cs index 5b5e7716..e6fb33aa 100644 --- a/src/Scriban/ScribanVisitors.generated.cs +++ b/src/Scriban/ScribanVisitors.generated.cs @@ -1248,7 +1248,7 @@ public override ScriptNode Visit(ScriptKeyword node) public override ScriptNode Visit(ScriptLiteral node) { - return new ScriptLiteral() { Value = node.Value, StringQuoteType = node.StringQuoteType }; + return new ScriptLiteral() { Value = node.Value, StringQuoteType = node.StringQuoteType, StringTokenType = node.StringTokenType }; } public override ScriptNode Visit(ScriptMemberExpression node) diff --git a/src/Scriban/Syntax/Expressions/ScriptBinaryExpression.cs b/src/Scriban/Syntax/Expressions/ScriptBinaryExpression.cs index 9e8858aa..d6211260 100644 --- a/src/Scriban/Syntax/Expressions/ScriptBinaryExpression.cs +++ b/src/Scriban/Syntax/Expressions/ScriptBinaryExpression.cs @@ -187,6 +187,8 @@ public static object Evaluate(TemplateContext context, SourceSpan span, ScriptBi case ScriptBinaryOperator.LiquidContains: case ScriptBinaryOperator.LiquidStartsWith: case ScriptBinaryOperator.LiquidEndsWith: + case ScriptBinaryOperator.InterpBegin: + case ScriptBinaryOperator.InterpEnd: try { if (leftValue is string || rightValue is string || leftValue is char || rightValue is char) @@ -299,6 +301,8 @@ private static object CalculateToString(TemplateContext context, SourceSpan span switch (op) { case ScriptBinaryOperator.Add: + case ScriptBinaryOperator.InterpBegin: + case ScriptBinaryOperator.InterpEnd: return context.ObjectToString(left) + context.ObjectToString(right); case ScriptBinaryOperator.Multiply: @@ -462,6 +466,8 @@ private static object CalculateOthers(TemplateContext context, SourceSpan span, case ScriptBinaryOperator.Modulus: case ScriptBinaryOperator.RangeInclude: case ScriptBinaryOperator.RangeExclude: + case ScriptBinaryOperator.InterpBegin: + case ScriptBinaryOperator.InterpEnd: if (context.UseScientific) throw new ScriptRuntimeException(span, $"Both left and right expressions are null. Cannot perform this operation on null values."); return null; case ScriptBinaryOperator.LiquidContains: @@ -504,6 +510,8 @@ private static object CalculateOthers(TemplateContext context, SourceSpan span, case ScriptBinaryOperator.Modulus: case ScriptBinaryOperator.RangeInclude: case ScriptBinaryOperator.RangeExclude: + case ScriptBinaryOperator.InterpBegin: + case ScriptBinaryOperator.InterpEnd: if (context.UseScientific) throw new ScriptRuntimeException(span, $"The {(leftValue == null ? "left" : "right")} expression is null. Cannot perform this operation on a null value."); return null; diff --git a/src/Scriban/Syntax/Expressions/ScriptBinaryOperator.cs b/src/Scriban/Syntax/Expressions/ScriptBinaryOperator.cs index c4fd1ed8..0cec1b3c 100644 --- a/src/Scriban/Syntax/Expressions/ScriptBinaryOperator.cs +++ b/src/Scriban/Syntax/Expressions/ScriptBinaryOperator.cs @@ -70,6 +70,9 @@ enum ScriptBinaryOperator RangeInclude, RangeExclude, + InterpBegin, + InterpEnd, + Custom, @@ -139,6 +142,10 @@ public static TokenType ToTokenType(this ScriptBinaryOperator op) return TokenType.Amp; case ScriptBinaryOperator.BinaryOr: return TokenType.VerticalBar; + case ScriptBinaryOperator.InterpBegin: + return TokenType.OpenInterpBrace; + case ScriptBinaryOperator.InterpEnd: + return TokenType.CloseInterpBrace; default: throw new ArgumentOutOfRangeException(nameof(op)); } @@ -190,6 +197,10 @@ public static string ToText(this ScriptBinaryOperator op) return "<<"; case ScriptBinaryOperator.ShiftRight: return ">>"; + case ScriptBinaryOperator.InterpBegin: + return "{"; + case ScriptBinaryOperator.InterpEnd: + return "}"; case ScriptBinaryOperator.LiquidContains: return "| string.contains "; diff --git a/src/Scriban/Syntax/Expressions/ScriptInterpolatedExpression.cs b/src/Scriban/Syntax/Expressions/ScriptInterpolatedExpression.cs new file mode 100644 index 00000000..c88d71df --- /dev/null +++ b/src/Scriban/Syntax/Expressions/ScriptInterpolatedExpression.cs @@ -0,0 +1,112 @@ +// Copyright (c) Alexandre Mutel. All rights reserved. +// Licensed under the BSD-Clause 2 license. +// See license.txt file in the project root for full license information. + +#nullable disable + +using System; +using System.Collections.Generic; +using System.IO; + +namespace Scriban.Syntax +{ + [ScriptSyntax("interpolated expression", "{}")] +#if SCRIBAN_PUBLIC + public +#else + internal +#endif + partial class ScriptInterpolatedExpression : ScriptExpression, IScriptVariablePath + { + private ScriptExpression _expression; + private ScriptToken _openBrace; + private ScriptToken _closeBrace; + + public ScriptInterpolatedExpression() + { + OpenBrace = ScriptToken.OpenInterpBrace(); + CloseBrace = ScriptToken.CloseInterpBrace(); + } + + public ScriptInterpolatedExpression(ScriptExpression expression) : this() + { + Expression = expression; + } + + public static ScriptInterpolatedExpression Wrap(ScriptExpression expression, bool transferTrivia = false) + { + if (expression == null) throw new ArgumentNullException(nameof(expression)); + var nested = new ScriptInterpolatedExpression() + { + Span = expression.Span, + Expression = expression + }; + + if (!transferTrivia) return nested; + + var firstTerminal = expression.FindFirstTerminal(); + firstTerminal?.MoveLeadingTriviasTo(nested.OpenBrace); + + var lastTerminal = expression.FindLastTerminal(); + lastTerminal?.MoveTrailingTriviasTo(nested.CloseBrace, true); + + return nested; + } + + public ScriptToken OpenBrace + { + get => _openBrace; + set => ParentToThis(ref _openBrace, value); + } + + public ScriptExpression Expression + { + get => _expression; + set => ParentToThis(ref _expression, value); + } + + public ScriptToken CloseBrace + { + get => _closeBrace; + set => ParentToThis(ref _closeBrace, value); + } + + public override object Evaluate(TemplateContext context) + { + // A nested expression will reset the pipe arguments for the group + context.PushPipeArguments(); + try + { + return context.GetValue(this); + } + finally + { + if (context.CurrentPipeArguments != null) + { + context.PopPipeArguments(); + } + } + } + + public override void PrintTo(ScriptPrinter printer) + { + printer.Write(OpenBrace); + printer.Write(Expression); + printer.Write(CloseBrace); + } + public object GetValue(TemplateContext context) + { + return context.Evaluate(Expression); + } + + public void SetValue(TemplateContext context, object valueToSet) + { + context.SetValue(Expression, valueToSet); + } + + public string GetFirstPath() + { + return (Expression as IScriptVariablePath)?.GetFirstPath(); + } + } +} \ No newline at end of file diff --git a/src/Scriban/Syntax/Expressions/ScriptLiteral.cs b/src/Scriban/Syntax/Expressions/ScriptLiteral.cs index 7026ba9e..8e9746ea 100644 --- a/src/Scriban/Syntax/Expressions/ScriptLiteral.cs +++ b/src/Scriban/Syntax/Expressions/ScriptLiteral.cs @@ -4,6 +4,7 @@ #nullable disable +using Scriban.Parsing; using System; using System.Collections.Generic; using System.Globalization; @@ -34,7 +35,7 @@ public ScriptLiteral(object value) public object Value { get; set; } public ScriptLiteralStringQuoteType StringQuoteType { get; set; } - + public TokenType StringTokenType { get; set; } = TokenType.String; public override object Evaluate(TemplateContext context) { return Value; @@ -93,7 +94,7 @@ public override void PrintTo(ScriptPrinter printer) var type = Value.GetType(); if (type == typeof(string)) { - printer.Write(ToLiteral(StringQuoteType, (string) Value)); + printer.Write(ToLiteral(StringQuoteType, StringTokenType, (string) Value)); } else if (type == typeof(bool)) { @@ -147,7 +148,11 @@ public override void PrintTo(ScriptPrinter printer) } else if (type == typeof(char)) { - printer.Write(ToLiteral(ScriptLiteralStringQuoteType.SimpleQuote, Value.ToString())); + printer.Write(ToLiteral( + ScriptLiteralStringQuoteType.SimpleQuote, + StringTokenType, + Value.ToString()) + ); } else { @@ -155,7 +160,7 @@ public override void PrintTo(ScriptPrinter printer) } } - private static string ToLiteral(ScriptLiteralStringQuoteType quoteType, string input) + private static string ToLiteral(ScriptLiteralStringQuoteType quoteType, TokenType stringTokenType, string input) { char quote; switch (quoteType) @@ -174,7 +179,15 @@ private static string ToLiteral(ScriptLiteralStringQuoteType quoteType, string i } var literal = new StringBuilder(input.Length + 2); - literal.Append(quote); + + if (stringTokenType == TokenType.BeginInterpString || stringTokenType == TokenType.InterpString) + { + literal.Capacity = input.Length + 3; + literal.Append('$'); + } + + char firstChar = stringTokenType == TokenType.BeginInterpString || stringTokenType == TokenType.String || stringTokenType == TokenType.InterpString ? quote : '}'; + literal.Append(firstChar); if (quoteType == ScriptLiteralStringQuoteType.Verbatim) { @@ -195,6 +208,7 @@ private static string ToLiteral(ScriptLiteralStringQuoteType quoteType, string i case '\r': literal.Append(@"\r"); break; case '\t': literal.Append(@"\t"); break; case '\v': literal.Append(@"\v"); break; + case '{' when stringTokenType.IsInterpolationStringToken(): literal.Append(@"\{"); break; default: if (c == quote) { @@ -213,7 +227,8 @@ private static string ToLiteral(ScriptLiteralStringQuoteType quoteType, string i } } } - literal.Append(quote); + char lastChar = stringTokenType == TokenType.EndingInterpString || stringTokenType == TokenType.String || stringTokenType == TokenType.InterpString ? quote : '{'; + literal.Append(lastChar); return literal.ToString(); } diff --git a/src/Scriban/Syntax/ScriptToken.cs b/src/Scriban/Syntax/ScriptToken.cs index 063462f3..e395e41c 100644 --- a/src/Scriban/Syntax/ScriptToken.cs +++ b/src/Scriban/Syntax/ScriptToken.cs @@ -63,6 +63,8 @@ partial class ScriptToken : ScriptVerbatim public static ScriptToken CloseBrace() => new ScriptToken(TokenType.CloseBrace); public static ScriptToken OpenBracket() => new ScriptToken(TokenType.OpenBracket); public static ScriptToken CloseBracket() => new ScriptToken(TokenType.CloseBracket); + public static ScriptToken OpenInterpBrace() => new ScriptToken(TokenType.OpenInterpBrace); + public static ScriptToken CloseInterpBrace() => new ScriptToken(TokenType.CloseInterpBrace); public ScriptToken() {