Skip to content

Commit

Permalink
Merge 863efc0 into 1328529
Browse files Browse the repository at this point in the history
  • Loading branch information
@ranger-turtle
ranger-turtle committed Sep 2, 2023
2 parents 1328529 + 863efc0 commit 4ab2200
Show file tree
Hide file tree
Showing 19 changed files with 613 additions and 27 deletions.
3 changes: 3 additions & 0 deletions .gitignore
View file
Original file line number Diff line number Diff line change
Expand Up @@ -220,3 +220,6 @@ _Pvt_Extensions

# Remove artifacts produced by dotnet-releaser
artifacts-dotnet-releaser/

# ranger-turtle's notes
notes.txt
1 change: 1 addition & 0 deletions src/Scriban.Tests/TestFiles/020-interpolation/020-interpolation.out.txt
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Pure string
1 change: 1 addition & 0 deletions src/Scriban.Tests/TestFiles/020-interpolation/020-interpolation.txt
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{{ $"Pure string" }}
1 change: 1 addition & 0 deletions src/Scriban.Tests/TestFilesHelper.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@ public static IEnumerable ListAllTestFiles()
{
"000-basic",
"010-literals",
"020-interpolation",
"100-expressions",
"200-statements",
"300-functions",
Expand Down
6 changes: 6 additions & 0 deletions src/Scriban.Tests/TestParser.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -666,6 +666,12 @@ public static void A010_literals(string inputName)
TestFile(inputName);
}

[TestCaseSource("ListTestFiles", new object[] { "020-interpolation" })]
public static void A010_interpolation(string inputName)
{
TestFile(inputName);
}

[TestCaseSource("ListTestFiles", new object[] { "100-expressions" })]
public static void A100_expressions(string inputName)
{
Expand Down
209 changes: 195 additions & 14 deletions src/Scriban/Parsing/Lexer.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,8 +9,10 @@
using System.Collections.Generic;
using System.Diagnostics;
using System.Linq;
using System.Numerics;
using System.Runtime.CompilerServices;
using Scriban.Helpers;
using Scriban.Syntax;

namespace Scriban.Parsing
{
Expand All @@ -36,6 +38,12 @@ class Lexer : IEnumerable<Token>
private bool _isExpectingFrontMatter;
private readonly bool _isLiquid;

// String interpolation-related data
private readonly Stack<char> _openingStringChars;
private bool _interpJustOpened;
private bool _interpJustClosed;
private bool AnyInterpolationOpen => _openingStringChars.Count > 0;

private readonly char _stripWhiteSpaceFullSpecialChar;
private readonly char _stripWhiteSpaceRestrictedSpecialChar;
private const char RawEscapeSpecialChar = '%';
Expand Down Expand Up @@ -87,6 +95,8 @@ public Lexer(string text, string sourcePath = null, LexerOptions? options = null
_isLiquid = Options.Lang == ScriptLang.Liquid;
_stripWhiteSpaceFullSpecialChar = '-';
_stripWhiteSpaceRestrictedSpecialChar = '~';

_openingStringChars = new Stack<char>();
}

/// <summary>
Expand Down Expand Up @@ -700,6 +710,18 @@ private bool ReadCode()
{
return true;
}
if (_interpJustOpened)
{
_token = new Token(TokenType.OpenInterpBrace, new TextPosition(_position.Offset - 1, _position.Line, _position.Column - 1), _position);
_interpJustOpened = false;
return hasTokens;
}
else if (_interpJustClosed)
{
_interpJustClosed = false;
ReadInterpolatedString();
return hasTokens;
}

switch (c)
{
Expand Down Expand Up @@ -982,27 +1004,36 @@ private bool ReadCode()
NextChar();
break;
case '}':
if (_openBraceCount > 0)
if (AnyInterpolationOpen)
{
// We match first brace open/close
_openBraceCount--;
_token = new Token(TokenType.CloseBrace, _position, _position);
NextChar();
_interpJustClosed = true;
_token = new Token(TokenType.CloseInterpBrace, _position, _position);
break;
}
else
{
if (Options.Mode != ScriptMode.ScriptOnly && IsCodeExit())
if (_openBraceCount > 0)
{
// We have no tokens for this ReadCode
hasTokens = false;
// We match first brace open/close
_openBraceCount--;
_token = new Token(TokenType.CloseBrace, _position, _position);
NextChar();
}
else
{
// Else we have a close brace but it is invalid
AddError("Unexpected } while no matching {", _position, _position);
// Remove the previous error token to still output a valid token
_token = new Token(TokenType.CloseBrace, _position, _position);
NextChar();
if (Options.Mode != ScriptMode.ScriptOnly && IsCodeExit())
{
// We have no tokens for this ReadCode
hasTokens = false;
}
else
{
// Else we have a close brace but it is invalid
AddError("Unexpected } while no matching {", _position, _position);
// Remove the previous error token to still output a valid token
_token = new Token(TokenType.CloseBrace, _position, _position);
NextChar();
}
}
}
break;
Expand Down Expand Up @@ -1037,7 +1068,14 @@ private bool ReadCode()
}

bool specialIdentifier = c == '$';
if (IsFirstIdentifierLetter(c) || specialIdentifier)
char nextChar = PeekChar(1); //Used for checking if the dollar is before the string
if (specialIdentifier && (nextChar == '"' || nextChar == '\''))
{
NextChar(); //Skip '$'
ReadInterpolatedString();
break;
}
else if (IsFirstIdentifierLetter(c) || specialIdentifier)
{
ReadIdentifier(specialIdentifier);
break;
Expand Down Expand Up @@ -1620,6 +1658,149 @@ private void ReadVerbatimString()
_token = new Token(TokenType.VerbatimString, start, end);
}

private void ReadInterpolatedString()
{
var start = _position;
var end = _position;
char startChar = c;
bool readingInterpolation = !(c == '"' || c == '\'');
if (readingInterpolation)
{
startChar = _openingStringChars.Peek();
}
NextChar(); // Skip ", ', { or }
while (true)
{
if (c == '\\')
{
end = _position;
NextChar();
// 0 ' " \ b f n r t v u0000-uFFFF x00-xFF
switch (c)
{
case '\n':
end = _position;
NextChar();
continue;
case '\r':
end = _position;
NextChar();
if (c == '\n')
{
end = _position;
NextChar();
}
continue;
case '0':
case '\'':
case '"':
case '\\':
case '{':
case 'b':
case 'f':
case 'n':
case 'r':
case 't':
case 'v':
end = _position;
NextChar();
continue;
case 'u':
end = _position;
NextChar();
// Must be followed 4 hex numbers (0000-FFFF)
if (c.IsHex()) // 1
{
end = _position;
NextChar();
if (c.IsHex()) // 2
{
end = _position;
NextChar();
if (c.IsHex()) // 3
{
end = _position;
NextChar();
if (c.IsHex()) // 4
{
end = _position;
NextChar();
continue;
}
}
}
}
AddError($"Unexpected hex number `{c}` following `\\u`. Expecting `\\u0000` to `\\uffff`.", _position, _position);
break;
case 'x':
end = _position;
NextChar();
// Must be followed 2 hex numbers (00-FF)
if (c.IsHex())
{
end = _position;
NextChar();
if (c.IsHex())
{
end = _position;
NextChar();
continue;
}
}
AddError($"Unexpected hex number `{c}` following `\\x`. Expecting `\\x00` to `\\xff`", _position, _position);
break;

}
AddError($"Unexpected escape character `{c}` in string. Only 0 ' \\ \" b f n r t v u0000-uFFFF x00-xFF are allowed", _position, _position);
}
else if (c == '\0')
{
AddError($"Unexpected end of file while parsing a string not terminated by a {startChar}", end, end);
return;
}
else if (c == '{') //If interpolation beginning is read
{
end = _position;
// if interpolation is starting
if (!readingInterpolation)
{
start = new TextPosition(start.Offset - 1, start.Line, start.Column - 1); //For '$' inclusion
_token = new Token(TokenType.BeginInterpString, start, end);
_openingStringChars.Push(startChar);
}
else
{
_token = new Token(TokenType.ContinuationInterpString, start, end);
}
NextChar();
_interpJustOpened = true;
return;
}
else if (c == startChar)
{
end = _position;
NextChar();
break;
}
else
{
end = _position;
NextChar();
}
}

if (readingInterpolation)
{
_ = _openingStringChars.Pop();
_token = new Token(TokenType.EndingInterpString, start, end);
}
else
{
//string is revealed to be ordinary string and '$' symbol is discarded
_token = new Token(TokenType.String, start, end);
}
}

private void ReadComment()
{
var start = _position;
Expand Down
42 changes: 41 additions & 1 deletion src/Scriban/Parsing/Parser.Expressions.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,8 @@ private bool TryBinaryOperator(out ScriptBinaryOperator binaryOperator, out int
case TokenType.LessEqual: binaryOperator = ScriptBinaryOperator.CompareLessOrEqual; break;
case TokenType.DoubleDot: binaryOperator = ScriptBinaryOperator.RangeInclude; break;
case TokenType.DoubleDotLess: binaryOperator = ScriptBinaryOperator.RangeExclude; break;
case TokenType.OpenInterpBrace: binaryOperator = ScriptBinaryOperator.InterpBegin; break;
case TokenType.CloseInterpBrace: binaryOperator = ScriptBinaryOperator.InterpEnd; break;
default:
if (_isScientific)
{
Expand Down Expand Up @@ -87,6 +89,10 @@ private ScriptExpression ParseExpressionAsVariableOrStringOrExpression(ScriptNod
return ParseVariable();
case TokenType.String:
return ParseString();
case TokenType.BeginInterpString:
case TokenType.ContinuationInterpString:
case TokenType.EndingInterpString:
return ParseInterpolatedString();
case TokenType.VerbatimString:
return ParseVerbatimString();
default:
Expand Down Expand Up @@ -165,6 +171,11 @@ private ScriptExpression ParseExpression(ScriptNode parentNode, ScriptExpression
case TokenType.ImplicitString:
leftOperand = ParseImplicitString();
break;
case TokenType.BeginInterpString:
case TokenType.ContinuationInterpString:
case TokenType.EndingInterpString:
leftOperand = ParseInterpolatedString();
break;
case TokenType.VerbatimString:
leftOperand = ParseVerbatimString();
break;
Expand All @@ -174,6 +185,9 @@ private ScriptExpression ParseExpression(ScriptNode parentNode, ScriptExpression
case TokenType.OpenBrace:
leftOperand = ParseObjectInitializer();
break;
case TokenType.OpenInterpBrace:
leftOperand = ParseInterpolatedExpression();
break;
case TokenType.OpenBracket:
leftOperand = ParseArrayInitializer();
break;
Expand Down Expand Up @@ -823,7 +837,7 @@ private ScriptExpression ParseObjectInitializer()
break;
}

if (!expectingEndOfInitializer && (Current.Type == TokenType.Identifier || Current.Type == TokenType.String))
if (!expectingEndOfInitializer && (Current.Type == TokenType.Identifier || Current.Type.IsStringToken()))
{
var positionBefore = Current;

Expand Down Expand Up @@ -940,6 +954,26 @@ private ScriptExpression ParseParenthesis()
return Close(expression);
}

private ScriptExpression ParseInterpolatedExpression()
{
// unit test: //TODO do unit error test
var expression = Open<ScriptInterpolatedExpression>();
ExpectAndParseTokenTo(expression.OpenBrace, TokenType.OpenInterpBrace); // Parse {
expression.Expression = ExpectAndParseExpression(expression);

if (Current.Type == TokenType.CloseParen)
{
ExpectAndParseTokenTo(expression.CloseBrace, TokenType.CloseInterpBrace); // Parse }
}
else
{
// unit test: //TODO do unit error test
LogError(Current, $"Invalid token `{GetAsText(Current)}`. Expecting a closing `}}`.");
}
return Close(expression);
}


private ScriptToken ParseToken(TokenType tokenType)
{
var verbatim = Open<ScriptToken>();
Expand Down Expand Up @@ -1133,6 +1167,9 @@ public bool IsStartOfExpression()
case TokenType.BinaryInteger:
case TokenType.Float:
case TokenType.String:
case TokenType.BeginInterpString:
case TokenType.ContinuationInterpString:
case TokenType.EndingInterpString:
case TokenType.ImplicitString:
case TokenType.VerbatimString:
case TokenType.OpenParen:
Expand Down Expand Up @@ -1234,6 +1271,9 @@ internal static int GetDefaultBinaryOperatorPrecedence(ScriptBinaryOperator op)
{
switch (op)
{
case ScriptBinaryOperator.InterpBegin:
case ScriptBinaryOperator.InterpEnd:
return 10;
case ScriptBinaryOperator.EmptyCoalescing:
case ScriptBinaryOperator.NotEmptyCoalescing:
return 20;
Expand Down

0 comments on commit 4ab2200

Please sign in to comment.