-
Notifications
You must be signed in to change notification settings - Fork 0
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
StringLiteral
#4
Changes from 12 commits
b6af384
6d8e517
f56c6dd
e158033
d2d3efd
53ae450
0b5d70f
a1df2c7
4e4d434
3d70d3b
ba4c745
60ceb41
bc76e30
c5e8220
ead71e2
b04fc8a
191b18c
0958d11
65a3270
282af71
d8cae0e
de468a7
459e037
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1 +1 @@ | ||
"var s = 1" | ||
"var s = 1;\nvar n = 'test';\n" |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1 @@ | ||
[] |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1 @@ | ||
"var singleQuote = 'singleQuote';\nvar doubleQuote = \"doubleQuote\"" |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,47 @@ | ||
{ | ||
"locals": { | ||
"singleQuote": [ | ||
{ | ||
"kind": "Var", | ||
"pos": 3 | ||
} | ||
], | ||
"doubleQuote": [ | ||
{ | ||
"kind": "Var", | ||
"pos": 36 | ||
} | ||
] | ||
}, | ||
"statements": [ | ||
{ | ||
"kind": "Var", | ||
"name": { | ||
"kind": "Identifier", | ||
"text": "singleQuote" | ||
}, | ||
"init": { | ||
"kind": "Literal", | ||
"value": "singleQuote" | ||
} | ||
}, | ||
{ | ||
"kind": "Var", | ||
"name": { | ||
"kind": "Identifier", | ||
"text": "doubleQuote" | ||
}, | ||
"init": { | ||
"kind": "Literal", | ||
"value": "double'Quote" | ||
} | ||
}, | ||
{ | ||
"kind": "ExpressionStatement", | ||
"expr": { | ||
"kind": "Identifier", | ||
"text": "(missing)" | ||
} | ||
} | ||
] | ||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,4 +1,4 @@ | ||
import { Token, Lexer } from './types'; | ||
import { Token, Lexer, CharCodes } from './types'; | ||
|
||
const keywords = { | ||
function: Token.Function, | ||
|
@@ -40,6 +40,9 @@ export function lex(s: string): Lexer { | |
text in keywords | ||
? keywords[text as keyof typeof keywords] | ||
: Token.Identifier; | ||
} else if (['"', "'"].includes(s.charAt(pos))) { | ||
text = scanString(); | ||
token = Token.String; | ||
} else { | ||
pos++; | ||
switch (s.charAt(pos - 1)) { | ||
|
@@ -62,6 +65,62 @@ export function lex(s: string): Lexer { | |
function scanForward(pred: (x: string) => boolean) { | ||
while (pos < s.length && pred(s.charAt(pos))) pos++; | ||
} | ||
|
||
function scanString() { | ||
const quote = s.charCodeAt(pos); | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. feels like this code needs a lot more tests There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Just made some adjustments to how I scan, parse, and emit escape characters. And separated the Also added more tests as you recommended (imteekay/mini-typescript@ead71e2 and imteekay/mini-typescript@b04fc8a). |
||
pos++; | ||
|
||
let stringValue = ''; | ||
let start = pos; | ||
|
||
while (true) { | ||
if (pos >= s.length) { | ||
// report unterminated string literal error | ||
} | ||
|
||
Comment on lines
+80
to
+82
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Still not sure how to report this kind of error in the lexer scope 🤔 There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. you'll probably have to make the lexer able to report errors the same way the other phases do There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I decided to add a new exercise to add the support for the lexer to report errors (imteekay/mini-typescript@65a3270). I also saw an interesting tweet by Maria about reporting errors for string literals. |
||
const char = s.charCodeAt(pos); | ||
|
||
if (char === quote) { | ||
stringValue += s.slice(start, pos); | ||
pos++; | ||
break; | ||
} | ||
|
||
if (char === CharCodes.backslash) { | ||
stringValue += s.slice(start, pos); | ||
stringValue += scanEscapeSequence(); | ||
start = pos; | ||
continue; | ||
} | ||
|
||
pos++; | ||
} | ||
|
||
return stringValue; | ||
} | ||
|
||
function scanEscapeSequence() { | ||
pos++; | ||
const char = s.charCodeAt(pos); | ||
pos++; | ||
|
||
switch (char) { | ||
case CharCodes.b: | ||
return '\b'; | ||
case CharCodes.t: | ||
return '\t'; | ||
case CharCodes.n: | ||
return '\n'; | ||
case CharCodes.r: | ||
return '\r'; | ||
case CharCodes.singleQuote: | ||
return "'"; | ||
case CharCodes.doubleQuote: | ||
return '"'; | ||
default: | ||
return ''; | ||
} | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I can't remember what invalid escapes are supposed to do, but I think it's returning There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Exactly, it returns the |
||
} | ||
} | ||
|
||
export function lexAll(s: string) { | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1 +1,2 @@ | ||
var s: string = 1; | ||
var n: number = 'test'; |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,2 @@ | ||
var singleQuote = 'singleQuote'; | ||
var doubleQuote = 'double\'Quote'; |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Should replace this node with the
EmptyStatement
.There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
the reason that you have empty statements is that
;
is parsed as a separator, not a terminator, so a complete program looks likeX;Y;Z
notX;Y;Z;
.(separators have become less and less popular over the history of programming languages)