JS: Add support for EJS and Mustache-style templating dialects

javascript/change-notes/2021-08-02-handlebars-extraction.md

@@ -0,0 +1,6 @@
+lgtm,codescanning
+* Added support for more templating languages.
+  - EJS, Mustache, Handlebars, Nunjucks, Hogan, and Swig are now supported.
+  - Template tags from the above dialects are now recognized as sinks
+    when not escaped safely for the context, leading to additional results for `js/xss` and `js/code-injection`.
+  - Files with the extension `.ejs`, `.hbs`, or `.njk` are now extracted and analyzed.

javascript/extractor/src/com/semmle/jcorn/Options.java

@@ -39,7 +39,7 @@ public void call(
         Position endLoc);
   }
 
-  private boolean allowHashBang, allowReturnOutsideFunction, allowImportExportEverywhere;
+  private boolean allowHashBang, allowReturnOutsideFunction, allowImportExportEverywhere, allowGeneratedCodeExprs;
   private boolean preserveParens, mozExtensions, jscript, esnext, v8Extensions, e4x;
   private int ecmaVersion;
   private AllowReserved allowReserved;
@@ -58,6 +58,7 @@ public Options() {
     this.allowReserved = AllowReserved.YES;
     this.allowReturnOutsideFunction = false;
     this.allowImportExportEverywhere = false;
+    this.allowGeneratedCodeExprs = true;
     this.allowHashBang = false;
     this.onToken = null;
     this.onComment = null;
@@ -75,6 +76,7 @@ public Options(Options that) {
     this.allowHashBang = that.allowHashBang;
     this.allowReturnOutsideFunction = that.allowReturnOutsideFunction;
     this.allowImportExportEverywhere = that.allowImportExportEverywhere;
+    this.allowGeneratedCodeExprs = that.allowGeneratedCodeExprs;
     this.preserveParens = that.preserveParens;
     this.mozExtensions = that.mozExtensions;
     this.jscript = that.jscript;
@@ -104,6 +106,10 @@ public boolean allowImportExportEverywhere() {
     return allowImportExportEverywhere;
   }
 
+  public boolean allowGeneratedCodeExprs() {
+    return allowGeneratedCodeExprs;
+  }
+
   public boolean preserveParens() {
     return preserveParens;
   }

javascript/extractor/src/com/semmle/jcorn/Parser.java

@@ -52,6 +52,7 @@
 import com.semmle.js.ast.ForStatement;
 import com.semmle.js.ast.FunctionDeclaration;
 import com.semmle.js.ast.FunctionExpression;
+import com.semmle.js.ast.GeneratedCodeExpr;
 import com.semmle.js.ast.IFunction;
 import com.semmle.js.ast.INode;
 import com.semmle.js.ast.IPattern;
@@ -537,7 +538,7 @@ private Token readToken_question() { // '?'
         }
         return this.finishOp(TokenType.questionquestion, 2);
       }
-      
+
     }
     return this.finishOp(TokenType.question, 1);
   }
@@ -617,6 +618,15 @@ private Token readToken_lt_gt(int code) { // '<>'
       this.skipSpace();
       return this.nextToken();
     }
+    if (next == '%' && code == '<' && this.options.allowGeneratedCodeExprs()) {
+      // `<%`, the beginning of an EJS-style template tag
+      size = 2;
+      int nextNext = charAt(this.pos + 2);
+      if (nextNext == '=' || nextNext == '-') {
+        ++size;
+      }
+      return this.finishOp(TokenType.generatedCodeDelimiterEJS, size);
+    }
     if (next == 61) size = 2;
     return this.finishOp(TokenType.relational, size);
   }
@@ -1688,6 +1698,9 @@ protected Expression parseExprAtom(DestructuringErrors refDestructuringErrors) {
       return this.parseNew();
     } else if (this.type == TokenType.backQuote) {
       return this.parseTemplate(false);
+    } else if (this.type == TokenType.generatedCodeDelimiterEJS) {
+      String openingDelimiter = (String) this.value;
+      return this.parseGeneratedCodeExpr(this.startLoc, openingDelimiter, "%>");
     } else {
       this.unexpected();
       return null;
@@ -1929,10 +1942,16 @@ MethodDefinition.Kind getMethodKind() {
   // Parse an object literal or binding pattern.
   protected Expression parseObj(boolean isPattern, DestructuringErrors refDestructuringErrors) {
     Position startLoc = this.startLoc;
+    if (!isPattern && options.allowGeneratedCodeExprs() && charAt(pos) == '{') {
+      // Parse mustache-style placeholder expression: {{ ... }} or {{{ ... }}}
+      return charAt(pos + 1) == '{'
+          ? parseGeneratedCodeExpr(startLoc, "{{{", "}}}")
+          : parseGeneratedCodeExpr(startLoc, "{{", "}}");
+    }
     boolean first = true;
     Map<String, PropInfo> propHash = new LinkedHashMap<>();
     List<Property> properties = new ArrayList<Property>();
-    this.next();
+    this.next(); // skip '{'
     while (!this.eat(TokenType.braceR)) {
       if (!first) {
         this.expect(TokenType.comma);
@@ -1949,6 +1968,42 @@ protected Expression parseObj(boolean isPattern, DestructuringErrors refDestruct
     return this.finishNode(node);
   }
 
+  /** Emit a token ranging from the current position until <code>endOfToken</code>. */
+  private Token generateTokenEndingAt(int endOfToken, TokenType tokenType) {
+    this.lastTokEnd = this.end;
+    this.lastTokStart = this.start;
+    this.lastTokEndLoc = this.endLoc;
+    this.lastTokStartLoc = this.startLoc;
+    this.start = this.pos;
+    this.startLoc = this.curPosition();
+    this.pos = endOfToken;
+    return finishToken(tokenType);
+  }
+
+  /** Parse a generated expression. The current token refers to the opening delimiter. */
+  protected Expression parseGeneratedCodeExpr(Position startLoc, String openingDelimiter, String closingDelimiter) {
+    // Emit a token for what's left of the opening delimiter, if there are any remaining characters
+    int startOfBody = startLoc.getOffset() + openingDelimiter.length();
+    if (this.pos != startOfBody) {
+      this.generateTokenEndingAt(startOfBody, TokenType.generatedCodeDelimiter);
+    }
+
+    // Emit a token for the generated code body
+    int endOfBody = this.input.indexOf(closingDelimiter, startOfBody);
+    if (endOfBody == -1) {
+    	this.unexpected(startLoc);
+    }
+    Token bodyToken = this.generateTokenEndingAt(endOfBody, TokenType.generatedCodeExpr);
+
+    // Emit a token for the closing delimiter
+    this.generateTokenEndingAt(endOfBody + closingDelimiter.length(), TokenType.generatedCodeDelimiter);
+
+    this.next(); // produce lookahead token
+
+    return finishNode(new GeneratedCodeExpr(new SourceLocation(startLoc), openingDelimiter, closingDelimiter,
+        bodyToken.getValue()));
+  }
+
   protected Property parseProperty(
       boolean isPattern,
       DestructuringErrors refDestructuringErrors,

javascript/extractor/src/com/semmle/jcorn/TokenType.java

@@ -1,9 +1,10 @@
 package com.semmle.jcorn;
 
-import com.semmle.jcorn.Parser.TokContext;
 import java.util.LinkedHashMap;
 import java.util.Map;
 
+import com.semmle.jcorn.Parser.TokContext;
+
 /// tokentype.js
 
 // ## Token types
@@ -89,6 +90,9 @@ public void updateContext(Parser parser, TokenType prevType) {
       arrow = new TokenType(new Properties("=>").beforeExpr()),
       template = new TokenType(new Properties("template")),
       invalidTemplate = new TokenType(new Properties("invalidTemplate")),
+      generatedCodeExpr = new TokenType(new Properties("generatedCodeExpr")),
+      generatedCodeDelimiter = new TokenType(new Properties("generatedCodeDelimiter")),
+      generatedCodeDelimiterEJS = new TokenType(new Properties("<%/%>")),
       ellipsis = new TokenType(new Properties("...").beforeExpr()),
       backQuote =
           new TokenType(new Properties("`").startsExpr()) {

javascript/extractor/src/com/semmle/jcorn/jsx/JSXParser.java

@@ -420,8 +420,9 @@ protected Token readToken(int code) {
         && code == 60
         && this.exprAllowed
         &&
-        // avoid getting confused on HTML comments
-        this.charAt(this.pos + 1) != '!') {
+        // avoid getting confused on HTML comments or EJS-style template tags
+        this.charAt(this.pos + 1) != '!' &&
+        this.charAt(this.pos + 1) != '%') {
       ++this.pos;
       return this.finishToken(jsxTagStart);
     }

javascript/extractor/src/com/semmle/js/ast/DefaultVisitor.java

@@ -777,4 +777,9 @@ public R visit(XMLQualifiedIdentifier nd, C c) {
   public R visit(XMLDotDotExpression nd, C c) {
     return visit((Expression) nd, c);
   }
+
+  @Override
+  public R visit(GeneratedCodeExpr nd, C c) {
+    return visit((Expression) nd, c);
+  }
 }

javascript/extractor/src/com/semmle/js/ast/GeneratedCodeExpr.java

@@ -0,0 +1,45 @@
+package com.semmle.js.ast;
+
+/**
+ * A placeholder for generated code, speculatively parsed as a primary expression.
+ *
+ * <p>For example, in this snippet,
+ *
+ * <pre>
+ * let data = {{user_data}};
+ * </pre>
+ *
+ * the expression <code>{{user_data}}</code> is assumed to be filled in by a templating engine so
+ * that it can be parsed as an expression, and a <code>GeneratedCodeExpr</code> is thus created to
+ * represent it.
+ */
+public class GeneratedCodeExpr extends Expression {
+  private String openingDelimiter;
+  private String closingDelimiter;
+  private String body;
+
+  public GeneratedCodeExpr(
+      SourceLocation loc, String openingDelimiter, String closingDelimiter, String body) {
+    super("GeneratedCodeExpr", loc);
+    this.openingDelimiter = openingDelimiter;
+    this.closingDelimiter = closingDelimiter;
+    this.body = body;
+  }
+
+  public String getOpeningDelimiter() {
+    return openingDelimiter;
+  }
+
+  public String getClosingDelimiter() {
+    return closingDelimiter;
+  }
+
+  public String getBody() {
+    return body;
+  }
+
+  @Override
+  public <C, R> R accept(Visitor<C, R> v, C c) {
+    return v.visit(this, c);
+  }
+}

javascript/extractor/src/com/semmle/js/ast/NodeCopier.java

@@ -894,4 +894,9 @@ public INode visit(XMLQualifiedIdentifier nd, Void c) {
   public INode visit(XMLDotDotExpression nd, Void c) {
     return new XMLDotDotExpression(visit(nd.getLoc()), copy(nd.getLeft()), copy(nd.getRight()));
   }
+
+  @Override
+  public INode visit(GeneratedCodeExpr nd, Void c) {
+  return new GeneratedCodeExpr(visit(nd.getLoc()), nd.getOpeningDelimiter(), nd.getClosingDelimiter(), nd.getBody());
+  }
 }

javascript/extractor/src/com/semmle/js/ast/Visitor.java

@@ -313,4 +313,6 @@ public interface Visitor<C, R> {
   public R visit(XMLQualifiedIdentifier nd, C c);
 
   public R visit(XMLDotDotExpression nd, C c);
+
+  public R visit(GeneratedCodeExpr generatedCodeExpr, C c);
 }