Merge ec29f35 into 6955e38

CodeCoverage.runsettings

@@ -71,8 +71,11 @@
                 <Source>.*\\FacebookThreadControl.cs</Source>
                 <Source>.*\\HandoverConstants.cs</Source>
                 <Source>.*\\AttachmentPayload.cs</Source>
+                <Source>.*\\CommonRegexLexer.cs</Source>
                 <Source>.*\\CommonRegexParser.cs</Source>
+                <Source>.*\\ExpressionLexer.cs</Source>
                 <Source>.*\\ExpressionParser.cs</Source>
+                <Source>.*\\LGFileLexer.cs</Source>
                 <Source>.*\\LGFileParser.cs</Source>
               </Exclude>
             </Sources>

libraries/Microsoft.Bot.Builder.LanguageGeneration/Evaluator.cs

@@ -22,7 +22,7 @@ namespace Microsoft.Bot.Builder.LanguageGeneration
     public class Evaluator : LGFileParserBaseVisitor<object>
     {
         public const string LGType = "lgType";
-        public static readonly Regex ExpressionRecognizeRegex = new Regex(@"(?<!\\)@{(((\'([^'\r\n])*?\')|(\""([^""\r\n])*?\""))|[^\r\n{}'""])*?}", RegexOptions.Compiled);
+        public static readonly Regex ExpressionRecognizeRegex = new Regex(@"(?<!\\)@{((\'[^\r\n\']*\')|(\""[^\""\r\n]*\"")|(\`(\\\`|[^\`])*\`)|([^\r\n{}'""`]))*?}", RegexOptions.Compiled);
         private const string ReExecuteSuffix = "!";
         private readonly Stack<EvaluationTarget> evaluationTargetStack = new Stack<EvaluationTarget>();
 

libraries/Microsoft.Bot.Builder.LanguageGeneration/LGFileLexer.g4

@@ -47,7 +47,9 @@ fragment EMPTY_OBJECT: '{' WHITESPACE* '}';
 
 fragment STRING_LITERAL : ('\'' (~['\r\n])* '\'') | ('"' (~["\r\n])* '"');
 
-fragment EXPRESSION_FRAGMENT : '@' '{' (STRING_LITERAL| ~[\r\n{}'"] | EMPTY_OBJECT )*? '}';
+fragment STRING_INTERPOLATION : '`' ('\\`' | ~'`')* '`';
+
+fragment EXPRESSION_FRAGMENT : '@' '{' (STRING_LITERAL | STRING_INTERPOLATION | EMPTY_OBJECT | ~[\r\n{}'"`] )*? '}';
 
 fragment ESCAPE_CHARACTER_FRAGMENT : '\\' ~[\r\n]?;
 
@@ -77,8 +79,8 @@ LEFT_SQUARE_BRACKET
   : '[' { inTemplate && beginOfTemplateBody }? -> pushMode(STRUCTURE_NAME_MODE)
   ;
 
-IMPORT
-  : '[' ~[\r\n[\]]*? ']' '(' ~[\r\n()]*? ')' { inTemplate = false;}
+IMPORT
+  : '[' ~[\r\n[\]]*? ']' '(' ~[\r\n()]*? ')' { inTemplate = false;}
   ;
 
 INVALID_TOKEN
@@ -125,8 +127,8 @@ WS_IN_BODY
   : WHITESPACE+  {ignoreWS}? -> skip
   ;
 
-MULTILINE_PREFIX
-  : '```' { !inMultiline  && beginOfTemplateLine }? { inMultiline = true; beginOfTemplateLine = false;}-> pushMode(MULTILINE_MODE)
+MULTILINE_PREFIX
+  : '```' { !inMultiline  && beginOfTemplateLine }? { inMultiline = true; beginOfTemplateLine = false;}-> pushMode(MULTILINE_MODE)
   ;
 
 NEWLINE_IN_BODY
@@ -169,23 +171,24 @@ TEXT
   : ~[\r\n]+?  { ignoreWS = false; beginOfTemplateLine = false;}
   ;
 
-mode MULTILINE_MODE;
-
-MULTILINE_SUFFIX
-  : '```' { inMultiline = false; } -> popMode
-  ;
-
-MULTILINE_ESCAPE_CHARACTER
-  : ESCAPE_CHARACTER_FRAGMENT -> type(ESCAPE_CHARACTER)
-  ;
-
-MULTILINE_EXPRESSION
-  : EXPRESSION_FRAGMENT -> type(EXPRESSION)
-  ;
-
-MULTILINE_TEXT
-  : (('\r'? '\n') | ~[\r\n])+? -> type(TEXT)
-  ;
+
+mode MULTILINE_MODE;
+
+MULTILINE_SUFFIX
+  : '```' { inMultiline = false; } -> popMode
+  ;
+
+MULTILINE_ESCAPE_CHARACTER
+  : ESCAPE_CHARACTER_FRAGMENT -> type(ESCAPE_CHARACTER)
+  ;
+
+MULTILINE_EXPRESSION
+  : EXPRESSION_FRAGMENT -> type(EXPRESSION)
+  ;
+
+MULTILINE_TEXT
+  : (('\r'? '\n') | ~[\r\n])+? -> type(TEXT)
+  ;
 
 mode STRUCTURE_NAME_MODE;
 

libraries/Microsoft.Bot.Expressions/ExpressionFunctions.cs

@@ -2759,20 +2759,16 @@ private static int GetPropertyCount(dynamic obj)
                             var builder = new StringBuilder();
                             foreach (var arg in args)
                             {
-                                if (arg is string str)
+                                if (arg != null)
                                 {
-                                    builder.Append(str);
-                                }
-                                else
-                                {
-                                    builder.Append(string.Empty);
+                                    builder.Append(arg.ToString());
                                 }
                             }
 
                             return builder.ToString();
-                        }, VerifyStringOrNull),
+                        }),
                     ReturnType.String,
-                    ValidateString),
+                    ValidateAtLeastOne),
                 new ExpressionEvaluator(
                     ExpressionType.Length,
                     Apply(

libraries/Microsoft.Bot.Expressions/Microsoft.Bot.Expressions.csproj

@@ -51,11 +51,4 @@
       <IncludeAssets>runtime; build; native; contentfiles; analyzers</IncludeAssets>
     </PackageReference>
   </ItemGroup>
-
-  <ItemGroup>
-    <Antlr4 Update="parser\Expression.g4">
-      <DefaultCustomToolNamespace>$([MSBuild]::ValueOrDefault('$(RootNamespace).%(DefaultCustomToolNamespace)', '').TrimEnd('.'))</DefaultCustomToolNamespace>
-      <CustomToolNamespace>$([MSBuild]::ValueOrDefault(%(CustomToolNamespace), %(DefaultCustomToolNamespace)))</CustomToolNamespace>
-    </Antlr4>
-  </ItemGroup>
 </Project>

libraries/Microsoft.Bot.Expressions/parser/ExpressionEngine.cs

@@ -8,6 +8,7 @@
 using Antlr4.Runtime;
 using Antlr4.Runtime.Misc;
 using Antlr4.Runtime.Tree;
+using Microsoft.Bot.Expressions.parser;
 using Newtonsoft.Json.Linq;
 
 namespace Microsoft.Bot.Expressions
@@ -64,7 +65,7 @@ protected static IParseTree AntlrParse(string expression)
             return parser.file()?.expression();
         }
 
-        private class ExpressionTransformer : ExpressionBaseVisitor<Expression>
+        private class ExpressionTransformer : ExpressionParserBaseVisitor<Expression>
         {
             private readonly EvaluatorLookup _lookup;
 
@@ -178,15 +179,40 @@ public override Expression VisitStringAtom([NotNull] ExpressionParser.StringAtom
                 }
             }
 
+            public override Expression VisitStringInterpolationAtom([NotNull] ExpressionParser.StringInterpolationAtomContext context)
+            {
+                var children = new List<Expression>();
+                foreach (ITerminalNode node in context.stringInterpolation().children)
+                {
+                    switch (node.Symbol.Type)
+                    {
+                        case ExpressionParser.TEMPLATE:
+                            var expressionString = TrimExpression(node.GetText());
+                            children.Add(new ExpressionEngine(_lookup).Parse(expressionString));
+                            break;
+                        case ExpressionParser.TEXT_CONTENT:
+                            children.Add(Expression.ConstantExpression(node.GetText()));
+                            break;
+                        case ExpressionParser.ESCAPE_CHARACTER:
+                            children.Add(Expression.ConstantExpression(EvalEscape(node.GetText())));
+                            break;
+                        default:
+                            break;
+                    }
+                }
+
+                return MakeExpression(ExpressionType.Concat, children.ToArray());
+            }
+
             public override Expression VisitConstantAtom([NotNull] ExpressionParser.ConstantAtomContext context)
             {
                 var text = context.GetText();
-                if (string.IsNullOrWhiteSpace(text.TrimStart('[').TrimEnd(']')))
+                if (text.StartsWith("[") && text.EndsWith("]") && string.IsNullOrWhiteSpace(text.Substring(1, text.Length - 2))) 
                 {
                     return Expression.ConstantExpression(new JArray());
                 }
 
-                if (string.IsNullOrWhiteSpace(text.TrimStart('{').TrimEnd('}')))
+                if (text.StartsWith("{") && text.EndsWith("}") && string.IsNullOrWhiteSpace(text.Substring(1, text.Length - 2)))
                 {
                     return Expression.ConstantExpression(new JObject());
                 }
@@ -207,6 +233,29 @@ private IEnumerable<Expression> ProcessArgsList(ExpressionParser.ArgsListContext
                     }
                 }
             }
+
+            private string EvalEscape(string exp)
+            {
+                var commonEscapes = new List<string>() { "\\r", "\\n", "\\t" };
+                if (commonEscapes.Contains(exp))
+                {
+                    return Regex.Unescape(exp);
+                }
+
+                return exp.Substring(1);
+            }
+
+            private string TrimExpression(string expression)
+            {
+                var result = expression.Trim().TrimStart('@').Trim();
+
+                if (result.StartsWith("{") && result.EndsWith("}"))
+                {
+                    result = result.Substring(1, result.Length - 2);
+                }
+
+                return result;
+            }
         }
     }
 }

libraries/Microsoft.Bot.Expressions/parser/ExpressionLexer.g4

@@ -0,0 +1,82 @@
+lexer grammar ExpressionLexer;
+
+@lexer::members {
+  bool ignoreWS = true;      // usually we ignore whitespace, but inside stringInterpolation, whitespace is significant
+}
+
+fragment LETTER : [a-zA-Z];
+fragment DIGIT : [0-9];
+
+STRING_INTERPOLATION_START : '`' { ignoreWS = false;} -> pushMode(STRING_INTERPOLATION_MODE);
+
+// operators
+PLUS: '+';
+
+SUBSTRACT: '-';
+
+NON: '!';
+
+XOR: '^';
+
+ASTERISK: '*';
+
+SLASH: '/';
+
+PERCENT: '%';
+
+DOUBLE_EQUAL: '==';
+
+NOT_EQUAL: '!=' | '<>';
+
+SINGLE_AND: '&';
+
+DOUBLE_AND: '&&';
+
+DOUBLE_VERTICAL_CYLINDER: '||';
+
+LESS_THAN: '<';
+
+MORE_THAN: '>';
+
+LESS_OR_EQUAl: '<=';
+
+MORE_OR_EQUAL: '>=';
+
+OPEN_BRACKET: '(';
+
+CLOSE_BRACKET: ')';
+
+DOT: '.';
+
+OPEN_SQUARE_BRACKET: '[';
+
+CLOSE_SQUARE_BRACKET: ']';
+
+COMMA: ',';
+
+
+NUMBER : DIGIT + ( '.' DIGIT +)? ;
+
+WHITESPACE : (' '|'\t'|'\ufeff'|'\u00a0') {ignoreWS}? -> skip;
+
+IDENTIFIER : (LETTER | '_' | '#' | '@' | '@@' | '$' | '%') (LETTER | DIGIT | '-' | '_')* '!'?;
+
+NEWLINE : '\r'? '\n' -> skip;
+
+STRING : ('\'' (~'\'')* '\'') | ('"' (~'"')* '"');
+
+CONSTANT : ('[' WHITESPACE* ']') | ('{' WHITESPACE* '}');
+
+INVALID_TOKEN_DEFAULT_MODE : . ;
+
+mode STRING_INTERPOLATION_MODE;
+
+STRING_INTERPOLATION_END : '`' {ignoreWS = true;} -> type(STRING_INTERPOLATION_START), popMode;
+
+TEMPLATE : '@' '{' (STRING | ~[\r\n{}'"])*? '}';
+
+ESCAPE_CHARACTER : '\\' ~[\r\n]?;
+
+TEXT_CONTENT :  '\\`' | ~[\r\n];
+
+

libraries/Microsoft.Bot.Expressions/parser/ExpressionParser.g4

@@ -0,0 +1,38 @@
+parser grammar ExpressionParser;
+
+options { tokenVocab=ExpressionLexer; }
+
+file: expression EOF;
+
+expression
+    : (NON|SUBSTRACT|PLUS) expression                                         #unaryOpExp
+    | <assoc=right> expression XOR expression                                 #binaryOpExp 
+    | expression (ASTERISK|SLASH|PERCENT) expression                          #binaryOpExp
+    | expression (PLUS|SUBSTRACT) expression                                  #binaryOpExp
+    | expression (DOUBLE_EQUAL|NOT_EQUAL) expression                          #binaryOpExp
+    | expression (SINGLE_AND) expression                                      #binaryOpExp
+    | expression (LESS_THAN|LESS_OR_EQUAl|MORE_THAN|MORE_OR_EQUAL) expression #binaryOpExp
+    | expression DOUBLE_AND expression                                        #binaryOpExp
+    | expression DOUBLE_VERTICAL_CYLINDER expression                          #binaryOpExp
+    | primaryExpression                                                       #primaryExp
+    ;
+
+primaryExpression 
+    : OPEN_BRACKET expression CLOSE_BRACKET                                  #parenthesisExp
+    | CONSTANT                                                               #constantAtom
+    | NUMBER                                                                 #numericAtom
+    | STRING                                                                 #stringAtom
+    | IDENTIFIER                                                             #idAtom
+    | stringInterpolation                                                    #stringInterpolationAtom
+    | primaryExpression DOT IDENTIFIER                                       #memberAccessExp
+    | primaryExpression OPEN_BRACKET argsList? CLOSE_BRACKET                 #funcInvokeExp
+    | primaryExpression OPEN_SQUARE_BRACKET expression CLOSE_SQUARE_BRACKET  #indexAccessExp
+    ;
+
+stringInterpolation
+    : STRING_INTERPOLATION_START (ESCAPE_CHARACTER | TEMPLATE | TEXT_CONTENT)+ STRING_INTERPOLATION_START
+    ;
+
+argsList
+    : expression (COMMA expression)*
+    ;