Skip to content

Commit

Permalink
Handle scanning and parsing template literals with invalid escapes ac…
Browse files Browse the repository at this point in the history 
…cording to the ES2018 revision that lifted restrictions on template literal escape sequences.

Scanner and Parser currently recognizes the escape sequences, but will still throw an error later during construction of the AST tree since we are not ready to change the AST tree structure yet.

-------------
Created by MOE: https://github.com/google/moe
MOE_MIGRATED_REVID=215250720
  • Loading branch information
@EatingW @lauraharker
EatingW authored and lauraharker committed Oct 2, 2018
1 parent 74a4011 commit 32436f4
Show file tree
Hide file tree
Showing 6 changed files with 262 additions and 48 deletions.
22 changes: 16 additions & 6 deletions src/com/google/javascript/jscomp/parsing/IRFactory.java
View file
Expand Up @@ -41,6 +41,7 @@
import com.google.javascript.jscomp.parsing.parser.FeatureSet.Feature;
import com.google.javascript.jscomp.parsing.parser.IdentifierToken;
import com.google.javascript.jscomp.parsing.parser.LiteralToken;
import com.google.javascript.jscomp.parsing.parser.TemplateLiteralToken;
import com.google.javascript.jscomp.parsing.parser.TokenType;
import com.google.javascript.jscomp.parsing.parser.trees.AmbientDeclarationTree;
import com.google.javascript.jscomp.parsing.parser.trees.ArrayLiteralExpressionTree;
Expand Down Expand Up @@ -1678,16 +1679,25 @@ Node processString(LiteralToken token) {
return node;
}

Node processTemplateLiteralToken(LiteralToken token) {
Node processTemplateLiteralToken(TemplateLiteralToken token) {
checkArgument(
token.type == TokenType.NO_SUBSTITUTION_TEMPLATE
|| token.type == TokenType.TEMPLATE_HEAD
|| token.type == TokenType.TEMPLATE_MIDDLE
|| token.type == TokenType.TEMPLATE_TAIL);
Node node = newStringNode(normalizeString(token, true));
node.putProp(Node.RAW_STRING_VALUE, token.value);
setSourceInfo(node, token);
return node;
if (token.hasError()) {
errorReporter.error(
"Unsupported feature: invalid template literal.",
sourceName,
lineno(token),
charno(token));
return newNode(Token.EMPTY);
} else {
Node node = newStringNode(normalizeString(token, true));
node.putProp(Node.RAW_STRING_VALUE, token.value);
setSourceInfo(node, token);
return node;
}
}

private Node processNameWithInlineJSDoc(IdentifierExpressionTree identifierExpression) {
Expand Down Expand Up @@ -2032,7 +2042,7 @@ Node processTemplateLiteral(TemplateLiteralExpressionTree tree) {
}

Node processTemplateLiteralPortion(TemplateLiteralPortionTree tree) {
return processTemplateLiteralToken(tree.value.asLiteral());
return processTemplateLiteralToken(tree.value.asTemplateLiteral());
}

Node processTemplateSubstitution(TemplateSubstitutionTree tree) {
Expand Down
51 changes: 39 additions & 12 deletions src/com/google/javascript/jscomp/parsing/parser/Parser.java
View file
Expand Up @@ -2338,9 +2338,17 @@ private TemplateLiteralExpressionTree parseTemplateLiteral(ParseTree operand) {
? getTreeStartLocation()
: operand.location.start;
Token token = nextToken();
if (!(token instanceof TemplateLiteralToken)) {
reportError(token, "Unexpected template literal token %s.", token.type.toString());
}
boolean isTaggedTemplate = operand != null;
TemplateLiteralToken templateToken = (TemplateLiteralToken) token;
if (!isTaggedTemplate) {
reportTemplateErrorIfPresent(templateToken);
}
ImmutableList.Builder<ParseTree> elements = ImmutableList.builder();
elements.add(new TemplateLiteralPortionTree(token.location, token));
if (token.type == TokenType.NO_SUBSTITUTION_TEMPLATE) {
elements.add(new TemplateLiteralPortionTree(templateToken.location, templateToken));
if (templateToken.type == TokenType.NO_SUBSTITUTION_TEMPLATE) {
return new TemplateLiteralExpressionTree(
getTreeLocation(start), operand, elements.build());
}
Expand All @@ -2349,13 +2357,15 @@ private TemplateLiteralExpressionTree parseTemplateLiteral(ParseTree operand) {
ParseTree expression = parseExpression();
elements.add(new TemplateSubstitutionTree(expression.location, expression));
while (!errorReporter.hadError()) {
token = nextTemplateLiteralToken();
if (token.type == TokenType.ERROR || token.type == TokenType.END_OF_FILE) {
templateToken = nextTemplateLiteralToken();
if (templateToken.type == TokenType.ERROR || templateToken.type == TokenType.END_OF_FILE) {
break;
}

elements.add(new TemplateLiteralPortionTree(token.location, token));
if (token.type == TokenType.TEMPLATE_TAIL) {
if (!isTaggedTemplate) {
reportTemplateErrorIfPresent(templateToken);
}
elements.add(new TemplateLiteralPortionTree(templateToken.location, templateToken));
if (templateToken.type == TokenType.TEMPLATE_TAIL) {
break;
}

Expand Down Expand Up @@ -4100,11 +4110,9 @@ private LiteralToken nextRegularExpressionLiteralToken() {
return token;
}

/**
* Consumes a template literal token and returns it.
*/
private LiteralToken nextTemplateLiteralToken() {
LiteralToken token = scanner.nextTemplateLiteralToken();
/** Consumes a template literal token and returns it. */
private TemplateLiteralToken nextTemplateLiteralToken() {
TemplateLiteralToken token = scanner.nextTemplateLiteralToken();
lastSourcePosition = token.location.end;
return token;
}
Expand Down Expand Up @@ -4210,6 +4218,25 @@ private void reportError(@FormatString String message, Object... arguments) {
errorReporter.reportError(scanner.getPosition(), message, arguments);
}

/**
* Reports an error at the specified location.
*
* @param position The position of the error.
* @param message The message to report in String.format style.
* @param arguments The arguments to fill in the message format.
*/
@FormatMethod
private void reportError(
SourcePosition position, @FormatString String message, Object... arguments) {
errorReporter.reportError(position, message, arguments);
}

private void reportTemplateErrorIfPresent(TemplateLiteralToken templateToken) {
if (templateToken.errorMessage != null) {
reportError(templateToken.errorPosition, "%s", templateToken.errorMessage);
}
}

private Parser recordFeatureUsed(Feature feature) {
features = features.with(feature);
return this;
Expand Down
163 changes: 135 additions & 28 deletions src/com/google/javascript/jscomp/parsing/parser/Scanner.java
View file
Expand Up @@ -23,6 +23,7 @@
import com.google.javascript.jscomp.parsing.parser.util.SourcePosition;
import com.google.javascript.jscomp.parsing.parser.util.SourceRange;
import java.util.ArrayList;
import javax.annotation.Nullable;

/**
* Scans javascript source code into tokens. All entrypoints assume the
Expand Down Expand Up @@ -150,7 +151,7 @@ public LiteralToken nextRegularExpressionLiteralToken() {
getTokenRange(beginToken));
}

public LiteralToken nextTemplateLiteralToken() {
public TemplateLiteralToken nextTemplateLiteralToken() {
Token token = nextToken();
if (isAtEnd() || token.type != TokenType.CLOSE_CURLY) {
reportError(getPosition(index), "Expected '}' after expression in template literal");
Expand Down Expand Up @@ -849,10 +850,10 @@ private Token scanTemplateLiteral(int beginIndex) {
TokenType.NO_SUBSTITUTION_TEMPLATE, TokenType.TEMPLATE_HEAD);
}

private LiteralToken nextTemplateLiteralTokenShared(TokenType endType,
TokenType middleType) {
private TemplateLiteralToken nextTemplateLiteralTokenShared(
TokenType endType, TokenType middleType) {
int beginIndex = index;
skipTemplateCharacters();
SkipTemplateCharactersResult skipTemplateCharactersResult = skipTemplateCharacters();
if (isAtEnd()) {
reportError(getPosition(beginIndex), "Unterminated template literal");
}
Expand All @@ -861,13 +862,28 @@ private LiteralToken nextTemplateLiteralTokenShared(TokenType endType,
switch (peekChar()) {
case '`':
nextChar();
return new LiteralToken(endType, value, getTokenRange(beginIndex - 1));
return new TemplateLiteralToken(
endType,
value,
skipTemplateCharactersResult.getErrorMessage(),
skipTemplateCharactersResult.getPosition(),
getTokenRange(beginIndex - 1));
case '$':
nextChar(); // $
nextChar(); // {
return new LiteralToken(middleType, value, getTokenRange(beginIndex - 1));
return new TemplateLiteralToken(
middleType,
value,
skipTemplateCharactersResult.getErrorMessage(),
skipTemplateCharactersResult.getPosition(),
getTokenRange(beginIndex - 1));
default: // Should have reported error already
return new LiteralToken(endType, value, getTokenRange(beginIndex - 1));
return new TemplateLiteralToken(
endType,
value,
skipTemplateCharactersResult.getErrorMessage(),
skipTemplateCharactersResult.getPosition(),
getTokenRange(beginIndex - 1));
}
}

Expand All @@ -881,33 +897,99 @@ private boolean peekStringLiteralChar(char terminator) {

private boolean skipStringLiteralChar() {
if (peek('\\')) {
return skipStringLiteralEscapeSequence(false);
return skipStringLiteralEscapeSequence();
}
nextChar();
return true;
}

private void skipTemplateCharacters() {
private SkipTemplateCharactersResult skipTemplateCharacters() {
SkipTemplateCharactersResult result = createSkipTemplateCharactersResult(null);
while (!isAtEnd()) {
switch (peekChar()) {
case '`':
return;
return result;
case '\\':
skipStringLiteralEscapeSequence(true);
// There might be multiple errors. Take the first one but continue scanning
SkipTemplateCharactersResult newError = skipTemplateLiteralEscapeSequence();
if (newError != null && !result.hasError()) {
result = newError;
}
break;
case '$':
if (peekChar(1) == '{') {
return;
return result;
}
// Fall through.
default:
nextChar();
}
}
return result;
}

@SuppressWarnings("IdentityBinaryExpression") // for "skipHexDigit() && skipHexDigit()"
private SkipTemplateCharactersResult skipTemplateLiteralEscapeSequence() {
nextChar();
if (isAtEnd()) {
reportError("Unterminated template literal escape sequence");
return null;
}
if (isLineTerminator(peekChar())) {
skipLineTerminator();
return null;
}
char next = nextChar();
switch (next) {
case '0':
if (peekOctalDigit()) {
return createSkipTemplateCharactersResult("Invalid escape sequence");
}
return null;
case '1':
case '2':
case '3':
case '4':
case '5':
case '6':
case '7':
return createSkipTemplateCharactersResult("Invalid escape sequence");
case 'x':
boolean doubleHexDigit = skipHexDigit() && skipHexDigit();
if (!doubleHexDigit) {
return createSkipTemplateCharactersResult("Hex digit expected");
}
return null;
case 'u':
if (peek('{')) {
nextChar();
if (peek('}')) {
return createSkipTemplateCharactersResult("Empty unicode escape");
}
boolean allHexDigits = true;
while (!peek('}') && allHexDigits) {
allHexDigits = allHexDigits && skipHexDigit();
}
if (!allHexDigits) {
return createSkipTemplateCharactersResult("Hex digit expected");
}
nextChar();
return null;
} else {
boolean quadHexDigit =
skipHexDigit() && skipHexDigit() && skipHexDigit() && skipHexDigit();
if (!quadHexDigit) {
return createSkipTemplateCharactersResult("Hex digit expected");
}
return null;
}
default:
return null;
}
}

@SuppressWarnings("IdentityBinaryExpression") // for "skipHexDigit() && skipHexDigit()"
private boolean skipStringLiteralEscapeSequence(boolean templateLiteral) {
private boolean skipStringLiteralEscapeSequence() {
nextChar();
if (isAtEnd()) {
reportError("Unterminated string literal escape sequence");
Expand All @@ -930,12 +1012,7 @@ private boolean skipStringLiteralEscapeSequence(boolean templateLiteral) {
case 'r':
case 't':
case 'v':
return true;
case '0':
if (templateLiteral && peekOctalDigit()) {
reportError("Invalid escape sequence");
return false;
}
return true;
case '1':
case '2':
Expand All @@ -944,13 +1021,13 @@ private boolean skipStringLiteralEscapeSequence(boolean templateLiteral) {
case '5':
case '6':
case '7':
if (templateLiteral) {
reportError("Invalid escape sequence");
return false;
}
break;
case 'x':
return skipHexDigit() && skipHexDigit();
boolean doubleHexDigit = skipHexDigit() && skipHexDigit();
if (!doubleHexDigit) {
reportError("Hex digit expected");
}
return doubleHexDigit;
case 'u':
if (peek('{')) {
nextChar();
Expand All @@ -962,20 +1039,25 @@ private boolean skipStringLiteralEscapeSequence(boolean templateLiteral) {
while (!peek('}') && allHexDigits) {
allHexDigits = allHexDigits && skipHexDigit();
}
if (!allHexDigits) {
reportError("Hex digit expected");
}
nextChar();
return allHexDigits;
} else {
return skipHexDigit() && skipHexDigit() && skipHexDigit() && skipHexDigit();
boolean quadHexDigit =
skipHexDigit() && skipHexDigit() && skipHexDigit() && skipHexDigit();
if (!quadHexDigit) {
reportError("Hex digit expected");
}
return quadHexDigit;
}
default:
break;
}

if (next == '/') {
// Don't warn for '\/' (for now) since it's common in "<\/script>"
} else if (templateLiteral) {
// Don't warn in template literals since tagged template literals
// can access the raw string value.
} else {
reportWarning("Unnecessary escape: '\\%s' is equivalent to just '%s'", next, next);
}
Expand All @@ -984,7 +1066,6 @@ private boolean skipStringLiteralEscapeSequence(boolean templateLiteral) {

private boolean skipHexDigit() {
if (!peekHexDigit()) {
reportError("Hex digit expected");
return false;
}
nextChar();
Expand Down Expand Up @@ -1143,4 +1224,30 @@ void incTypeParameterLevel() {
void decTypeParameterLevel() {
typeParameterLevel--;
}

private SkipTemplateCharactersResult createSkipTemplateCharactersResult(String message) {
return new SkipTemplateCharactersResult(message, getPosition());
}

private static class SkipTemplateCharactersResult {
@Nullable private final String errorMessage;
private final SourcePosition position;

SkipTemplateCharactersResult(String message, SourcePosition position) {
this.errorMessage = message;
this.position = position;
}

String getErrorMessage() {
return this.errorMessage;
}

SourcePosition getPosition() {
return this.position;
}

boolean hasError() {
return this.errorMessage != null;
}
}
}

0 comments on commit 32436f4

Please sign in to comment.