Skip to content

Commit

Permalink
Fix regression in keyword tokenization
Browse files Browse the repository at this point in the history
  • Loading branch information
marijnh committed Aug 21, 2021
1 parent ae55880 commit c7b6c1f
Showing 1 changed file with 5 additions and 4 deletions.
9 changes: 5 additions & 4 deletions src/tokens.ts
Expand Up @@ -55,11 +55,12 @@ function readLiteral(input: InputStream, endQuote: number, backslashEscapes: boo
}
}

function readWord(input: InputStream) {
let result = ""
function readWord(input: InputStream): void
function readWord(input: InputStream, result: string): string
function readWord(input: InputStream, result?: string) {
for (;;) {
if (input.next != Ch.Underscore && !isAlpha(input.next)) break
result += String.fromCharCode(input.next)
if (result != null) result += String.fromCharCode(input.next)
input.advance()
}
return result
Expand Down Expand Up @@ -253,7 +254,7 @@ export function tokensFor(d: Dialect) {
} else if (next == Ch.Colon || next == Ch.Comma) {
input.acceptToken(Punctuation)
} else if (isAlpha(next)) {
let word = readWord(input)
let word = readWord(input, String.fromCharCode(next))
input.acceptToken(d.words[word.toLowerCase()] ?? Identifier)
}
})
Expand Down

0 comments on commit c7b6c1f

Please sign in to comment.