Skip to content

Commit

Permalink
Tokenize 2 *or more* line feeds as a single parabreak
Browse files Browse the repository at this point in the history
Fixes tc39#71
  • Loading branch information
gibson042 committed Jun 3, 2020
1 parent c4955d2 commit 6cabbb2
Showing 1 changed file with 11 additions and 6 deletions.
17 changes: 11 additions & 6 deletions src/tokenizer.ts
Original file line number Diff line number Diff line change
Expand Up @@ -279,12 +279,16 @@ export class Tokenizer {
case '\n':
this._newline = true;

if (str[this.pos + 1] === '\n') {
this.pos += 2;
this.enqueue({ name: 'parabreak', contents: '\n\n' }, start);
} else {
this.pos += 1;
const pos = this.pos;
let nextPos = pos + 1;
while (nextPos < str.length && str[nextPos] === '\n') {
nextPos++;
}
this.pos = nextPos;
if (nextPos === pos + 1) {
this.enqueue({ name: 'linebreak', contents: '\n' }, start);
} else {
this.enqueue({ name: 'parabreak', contents: str.slice(pos, nextPos) }, start);
}
return;
default:
Expand Down Expand Up @@ -390,8 +394,9 @@ export class Tokenizer {
this.column = 0;
++this.line;
} else if (tok.name === 'parabreak') {
let size = this.pos - startPos.offset;
this.column = 0;
this.line += 2;
this.line += size;
} else {
let width = this.pos - startPos.offset;
this.column += width;
Expand Down

0 comments on commit 6cabbb2

Please sign in to comment.