Skip to content

Commit

Permalink
Adds begin and end template string literal tokens, so we can better f…
Browse files Browse the repository at this point in the history 
…ormat and understand the code down stream.
  • Loading branch information
@georgejecook
georgejecook committed Jul 7, 2020
1 parent 13a2bc0 commit 3821e18
Show file tree
Hide file tree
Showing 5 changed files with 197 additions and 60 deletions.
5 changes: 5 additions & 0 deletions src/DiagnosticMessages.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -579,6 +579,11 @@ export let DiagnosticMessages = {
message: `Unterminated template string at end of file`,
code: 1113,
severity: DiagnosticSeverity.Error
}),
unterminatedTemplateExpression: () => ({
message: `Unterminated template string expression. '\${' must be followed by expression, then '}'`,
code: 1114,
severity: DiagnosticSeverity.Error
})

};
Expand Down
113 changes: 103 additions & 10 deletions src/lexer/Lexer.spec.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -447,9 +447,9 @@ describe('lexer', () => {
let { tokens } = Lexer.scan('`"`');
expect(tokens.map(t => t.kind)).to.deep.equal([
TokenKind.BackTick,
TokenKind.TemplateStringQuasi, //empty
TokenKind.EscapedCharCodeLiteral, // quote
TokenKind.TemplateStringQuasi, // empty
TokenKind.TemplateStringQuasi,
TokenKind.EscapedCharCodeLiteral,
TokenKind.TemplateStringQuasi,
TokenKind.BackTick,
TokenKind.Eof
]);
Expand Down Expand Up @@ -486,13 +486,100 @@ describe('lexer', () => {
expect(tokens.map(t => t.kind)).to.deep.equal([
TokenKind.BackTick,
TokenKind.TemplateStringQuasi,
TokenKind.TemplateStringExpressionBegin,
TokenKind.StringLiteral,
TokenKind.TemplateStringExpressionEnd,
TokenKind.TemplateStringQuasi,
TokenKind.BackTick,
TokenKind.Eof
]);
expect(tokens[1].literal).to.deep.equal(new BrsString(`hello `));
});
it('real example, which is causing issues in the formatter', () => {
let { tokens } = Lexer.scan(` function getItemXML(item)
return \`<rss version="2.0" xmlns:media="http://search.yahoo.com/mrss/">
<channel>
<title>smithsonian</title>
<item>
<title>\${item.title}</title>
<guid>\${item.vamsId}</guid>
<media:rating scheme="urn:v-chip">\${item.ratings.first.code.name}</media:rating>
</item>
</channel>
</rss>\`
end function
`);
expect(tokens.map(t => t.kind)).to.deep.equal([
TokenKind.Function,
TokenKind.Identifier,
TokenKind.LeftParen,
TokenKind.Identifier,
TokenKind.RightParen,
TokenKind.Newline,
TokenKind.Return,
TokenKind.BackTick,
TokenKind.TemplateStringQuasi,
TokenKind.EscapedCharCodeLiteral,
TokenKind.TemplateStringQuasi,
TokenKind.EscapedCharCodeLiteral,
TokenKind.TemplateStringQuasi,
TokenKind.EscapedCharCodeLiteral,
TokenKind.TemplateStringQuasi,
TokenKind.EscapedCharCodeLiteral,
TokenKind.TemplateStringQuasi,
TokenKind.EscapedCharCodeLiteral,
TokenKind.TemplateStringQuasi,
TokenKind.EscapedCharCodeLiteral,
TokenKind.TemplateStringQuasi,
TokenKind.EscapedCharCodeLiteral,
TokenKind.TemplateStringQuasi,
TokenKind.EscapedCharCodeLiteral,
TokenKind.TemplateStringQuasi,
TokenKind.TemplateStringExpressionBegin,
TokenKind.Identifier,
TokenKind.Dot,
TokenKind.Identifier,
TokenKind.TemplateStringExpressionEnd,
TokenKind.TemplateStringQuasi,
TokenKind.EscapedCharCodeLiteral,
TokenKind.TemplateStringQuasi,
TokenKind.TemplateStringExpressionBegin,
TokenKind.Identifier,
TokenKind.Dot,
TokenKind.Identifier,
TokenKind.TemplateStringExpressionEnd,
TokenKind.TemplateStringQuasi,
TokenKind.EscapedCharCodeLiteral,
TokenKind.TemplateStringQuasi,
TokenKind.EscapedCharCodeLiteral,
TokenKind.TemplateStringQuasi,
TokenKind.EscapedCharCodeLiteral,
TokenKind.TemplateStringQuasi,
TokenKind.TemplateStringExpressionBegin,
TokenKind.Identifier,
TokenKind.Dot,
TokenKind.Identifier,
TokenKind.Dot,
TokenKind.Identifier,
TokenKind.Dot,
TokenKind.Identifier,
TokenKind.Dot,
TokenKind.Identifier,
TokenKind.TemplateStringExpressionEnd,
TokenKind.TemplateStringQuasi,
TokenKind.EscapedCharCodeLiteral,
TokenKind.TemplateStringQuasi,
TokenKind.EscapedCharCodeLiteral,
TokenKind.TemplateStringQuasi,
TokenKind.EscapedCharCodeLiteral,
TokenKind.TemplateStringQuasi,
TokenKind.BackTick,
TokenKind.Newline,
TokenKind.EndFunction,
TokenKind.Newline,
TokenKind.Eof
]);
});

it('complicated example', () => {
let { tokens } = Lexer.scan(
Expand All @@ -501,32 +588,34 @@ describe('lexer', () => {
expect(tokens.map(t => t.kind)).to.eql([
TokenKind.BackTick, // `
TokenKind.TemplateStringQuasi, // hello
// ${
TokenKind.TemplateStringExpressionBegin, //$ {
TokenKind.StringLiteral, // "world"
// }
TokenKind.TemplateStringExpressionEnd, // }
TokenKind.TemplateStringQuasi, //!I am a
// ${
TokenKind.TemplateStringExpressionBegin, // ${
TokenKind.StringLiteral, // "template"
TokenKind.Plus, // +
TokenKind.StringLiteral, //"string"
// }
TokenKind.TemplateStringExpressionEnd, // }
TokenKind.TemplateStringQuasi, // and I am very
TokenKind.TemplateStringExpressionBegin, // ${
TokenKind.LeftSquareBracket, // [
TokenKind.StringLiteral, // "pleased"
TokenKind.RightSquareBracket, // ]
TokenKind.LeftSquareBracket, // [
TokenKind.IntegerLiteral, // 0
TokenKind.RightSquareBracket, // ]
// }
TokenKind.TemplateStringExpressionEnd, // }
TokenKind.TemplateStringQuasi, // to meet you
TokenKind.TemplateStringExpressionBegin, // ${
TokenKind.Identifier, // m
TokenKind.Dot, // .
TokenKind.Identifier, // top
TokenKind.Dot, // .
TokenKind.Identifier, //getChildCount,
TokenKind.LeftParen, // (
TokenKind.RightParen, // )
// }
TokenKind.TemplateStringExpressionEnd, // }
TokenKind.TemplateStringQuasi, // .The end
TokenKind.BackTick,
TokenKind.Eof
Expand Down Expand Up @@ -578,13 +667,14 @@ describe('lexer', () => {

it('Example that tripped up the expression tests', () => {
let { tokens } = Lexer.scan(
'`I am a complex example\n${a.isRunning(["a","b","c"])}\nmore ${m.finish(true)}\`'
'`I am a complex example\n${a.isRunning(["a","b","c"])}\nmore ${m.finish(true)}`'
);
expect(tokens.map(t => t.kind)).to.deep.equal([
TokenKind.BackTick,
TokenKind.TemplateStringQuasi,
TokenKind.EscapedCharCodeLiteral,
TokenKind.TemplateStringQuasi,
TokenKind.TemplateStringExpressionBegin,
TokenKind.Identifier,
TokenKind.Dot,
TokenKind.Identifier,
Expand All @@ -597,15 +687,18 @@ describe('lexer', () => {
TokenKind.StringLiteral,
TokenKind.RightSquareBracket,
TokenKind.RightParen,
TokenKind.TemplateStringExpressionEnd,
TokenKind.TemplateStringQuasi,
TokenKind.EscapedCharCodeLiteral,
TokenKind.TemplateStringQuasi,
TokenKind.TemplateStringExpressionBegin,
TokenKind.Identifier,
TokenKind.Dot,
TokenKind.Identifier,
TokenKind.LeftParen,
TokenKind.True,
TokenKind.RightParen,
TokenKind.TemplateStringExpressionEnd,
TokenKind.TemplateStringQuasi,
TokenKind.BackTick,
TokenKind.Eof
Expand Down
21 changes: 18 additions & 3 deletions src/lexer/Lexer.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -563,13 +563,26 @@ export class Lexer {
this.templateQuasiString();
this.advance();
this.advance();
this.addToken(TokenKind.TemplateStringExpressionBegin);
while (!this.isAtEnd() && !this.check('}')) {
this.start = this.current;
this.scanToken();
}
this.start = this.current + 1;
if (this.check('}')) {
this.current++;
this.addToken(TokenKind.TemplateStringExpressionEnd);
} else {

this.diagnostics.push({
...DiagnosticMessages.unexpectedConditionalCompilationString(),
range: this.rangeOf(this.source.slice(this.start, this.current))
});
}

this.start = this.current;
} else {
this.advance();
}
this.advance();
}

//get last quasi
Expand All @@ -584,7 +597,9 @@ export class Lexer {

private templateQuasiString() {
let value = this.source.slice(this.start, this.current);
this.addToken(TokenKind.TemplateStringQuasi, new BrsString(value));
if (value !== '`') { // if this is an empty string straight after an expressoin, then we'll accidentally consume the backtick
this.addToken(TokenKind.TemplateStringQuasi, new BrsString(value));
}
}

/**
Expand Down
103 changes: 56 additions & 47 deletions src/lexer/TokenKind.ts
View file
Original file line number Diff line number Diff line change
@@ -1,55 +1,64 @@
export enum TokenKind {
// parens (and friends)
LeftParen = 'LeftParen', // (
RightParen = 'RightParen', // )
LeftSquareBracket = 'LeftSquareBracket', // [
RightSquareBracket = 'RightSquareBracket', // ]
LeftCurlyBrace = 'LeftCurlyBrace', // {
RightCurlyBrace = 'RightCurlyBrace', // }
LeftParen = 'LeftParen',
RightParen = 'RightParen',
LeftSquareBracket = 'LeftSquareBracket',
RightSquareBracket = 'RightSquareBracket',
LeftCurlyBrace = 'LeftCurlyBrace',
RightCurlyBrace = 'RightCurlyBrace',


// operators
Caret = 'Caret', // ^
Minus = 'Minus', // -
Plus = 'Plus', // +
Star = 'Star', // *
Forwardslash = 'Forwardslash', // /
Mod = 'Mod', // mod
Backslash = 'Backslash', // \
Caret = 'Caret',
Minus = 'Minus',
Plus = 'Plus',
Star = 'Star',
Forwardslash = 'Forwardslash',
Mod = 'Mod',
Backslash = 'Backslash',


// postfix operators
PlusPlus = 'PlusPlus', // ++
MinusMinus = 'MinusMinus', // --
PlusPlus = 'PlusPlus',
MinusMinus = 'MinusMinus',


// bitshift
LeftShift = 'LeftShift', // <<
RightShift = 'RightShift', // >>
LeftShift = 'LeftShift',
RightShift = 'RightShift',


// assignment operators
MinusEqual = 'MinusEqual', // -=
PlusEqual = 'PlusEqual', // +=
StarEqual = 'StarEqual', // *=
ForwardslashEqual = 'ForwardslashEqual', // /=
BackslashEqual = 'BackslashEqual', // \=
LeftShiftEqual = 'LeftShiftEqual', // <<=
RightShiftEqual = 'RightShiftEqual', // >>=
MinusEqual = 'MinusEqual',
PlusEqual = 'PlusEqual',
StarEqual = 'StarEqual',
ForwardslashEqual = 'ForwardslashEqual',
BackslashEqual = 'BackslashEqual',
LeftShiftEqual = 'LeftShiftEqual',
RightShiftEqual = 'RightShiftEqual',


// comparators
Less = 'Less', // <
LessEqual = 'LessEqual', // <=
Greater = 'Greater', // >
GreaterEqual = 'GreaterEqual', // >=
Equal = 'Equal', // =
LessGreater = 'LessGreater', // BrightScript uses `<>` for "not equal"
Less = 'Less',
LessEqual = 'LessEqual',
Greater = 'Greater',
GreaterEqual = 'GreaterEqual',
Equal = 'Equal',
LessGreater = 'LessGreater',


// literals
Identifier = 'Identifier',
StringLiteral = 'StringLiteral',
TemplateStringQuasi = 'TemplateStringQuasi',
TemplateStringExpressionBegin = 'TemplateStringExpressionBegin',
TemplateStringExpressionEnd = 'TemplateStringExpressionEnd',
IntegerLiteral = 'IntegerLiteral',
FloatLiteral = 'FloatLiteral',
DoubleLiteral = 'DoubleLiteral',
LongIntegerLiteral = 'LongIntegerLiteral',
EscapedCharCodeLiteral = 'EscapedCharCodeLiteral', //this is used to capture things like `\n`, `\r\n` in template strings
EscapedCharCodeLiteral = 'EscapedCharCodeLiteral',


//types
Void = 'Void',
Expand All @@ -65,23 +74,23 @@ export enum TokenKind {
Dynamic = 'Dynamic',

// other symbols
Dot = 'Dot', // .
Comma = 'Comma', // ,
Colon = 'Colon', // :
Semicolon = 'Semicolon', // ;
At = 'At', // @
Callfunc = 'Callfunc', // @.

BackTick = 'BackTick', // `
Dot = 'Dot',
Comma = 'Comma',
Colon = 'Colon',
Semicolon = 'Semicolon',
At = 'At',
Callfunc = 'Callfunc',

// template strings
BackTick = 'BackTick',

// conditional compilation
HashIf = 'HashIf', // #if
HashElseIf = 'HashElseIf', // #elseif
HashElse = 'HashElse', // #else
HashEndIf = 'HashEndIf', // #endif
HashConst = 'HashConst', // #const
HashError = 'HashError', // #error
HashIf = 'HashIf',
HashElseIf = 'HashElseIf',
HashElse = 'HashElse',
HashEndIf = 'HashEndIf',
HashConst = 'HashConst',
HashError = 'HashError',
HashErrorMessage = 'HashErrorMessage',

// keywords
Expand All @@ -102,7 +111,7 @@ export enum TokenKind {
EndWhile = 'EndWhile',
Eval = 'Eval',
Exit = 'Exit',
ExitFor = 'ExitFor', // not technically a reserved word, but definitely a tokenKind
ExitFor = 'ExitFor',
ExitWhile = 'ExitWhile',
False = 'False',
For = 'For',
Expand Down Expand Up @@ -164,7 +173,7 @@ export enum TokenKind {
// structural
Whitespace = 'Whitespace',
Newline = 'Newline',
Eof = 'Eof'
Eof = 'Eof',
}

/**
Expand Down

0 comments on commit 3821e18

Please sign in to comment.