scanner.ts

/*!
 * Copyright (c) 2020 Ron Buckton (rbuckton@chronicles.org)
 *
 * This file is licensed to you under the terms of the MIT License, found in the LICENSE file
 * in the root of this repository or package.
 */

import { CancelToken } from '@esfx/async-canceltoken';
import { Cancelable } from '@esfx/cancelable';
import { toCancelToken } from './core';
import { DiagnosticMessages, Diagnostics, NullDiagnosticMessages } from "./diagnostics";
import { getNodeAccessor, getTriviaAccessor } from './nodeInternal';
import { CommentTrivia, HtmlCloseTagTrivia, HtmlOpenTagTrivia, HtmlCommentTrivia, HtmlTrivia, MultiLineCommentTrivia, SingleLineCommentTrivia, Trivia } from './nodes';
import { CharacterCodes, isProseFragmentLiteralKind, stringToToken, SyntaxKind, isHtmlTriviaKind } from "./tokens";

const enum TokenFlags {
    None = 0,
    Unterminated = 1 << 0,
    LineContinuation = 1 << 1,
    PrecedingLineTerminator = 1 << 2,
    PrecedingBlankLine = 1 << 3,
    PrecedingIndent = 1 << 4,
    PrecedingDedent = 1 << 5,
    AnyPrecedingIndent = PrecedingIndent | PrecedingDedent,
    PrecedingNonWhiteSpaceTrivia = 1 << 6,

    TriviaPrecedingLineTerminator = 1 << 7,
    TriviaPrecedingBlankLine = 1 << 8,
    TriviaPrecedingWhiteSpaceTrivia = 1 << 9,
    TriviaFlags = TriviaPrecedingLineTerminator | TriviaPrecedingBlankLine | TriviaPrecedingWhiteSpaceTrivia,

    DecodedEntity = 1 << 10, // Reset every time a character is scanned. Indicates whether the last scanned character was a decoded entity.
}

/** {@docCategory Parse} */
export class Scanner {
    public readonly text: string;
    public readonly filename: string;
    private readonly cancelToken?: CancelToken;
    private readonly len: number = 0;
    private pos: number = 0;
    private startPos: number = 0;
    private tokenPos: number = 0;
    private token: SyntaxKind = SyntaxKind.Unknown;
    private tokenValue: string = "";
    private tokenFlags: TokenFlags = TokenFlags.None;
    private trivia: Trivia[] | undefined;
    private lastTrivia: Trivia | undefined;
    private diagnostics: DiagnosticMessages;
    private insignificantIndentLength: number = 0;
    private significantIndentLength: number = 0;
    private currentIndentLength: number = 0;
    private proseStartToken: SyntaxKind | undefined;

    constructor(filename: string, text: string, diagnostics: DiagnosticMessages, cancelable?: Cancelable) {
        this.filename = filename;
        this.text = text;
        this.len = text.length;
        this.diagnostics = diagnostics;
        this.cancelToken = toCancelToken(cancelable);
    }

    public getPos(): number {
        return this.pos;
    }

    public getLen(): number {
        return this.len;
    }

    public getStartPos(): number {
        return this.startPos;
    }

    public getTokenPos(): number {
        return this.tokenPos;
    }

    public getToken(): SyntaxKind {
        return this.token;
    }

    public getTokenText(): string {
        return this.text.slice(this.tokenPos, this.pos);
    }

    public getTokenValue(): string {
        return this.tokenValue;
    }

    public getTokenIsUnterminated() {
        return (this.tokenFlags & TokenFlags.Unterminated) === TokenFlags.Unterminated;
    }

    public getDiagnostics(): DiagnosticMessages {
        return this.diagnostics;
    }

    public getTrivia() {
        return this.trivia;
    }

    /** @deprecated */
    public getHtmlTrivia() {
        return this.trivia?.filter(trivia => isHtmlTriviaKind(trivia.kind));
    }

    public isIndented() {
        return this.significantIndentLength > 0;
    }

    public isLineContinuation() {
        return (this.tokenFlags & TokenFlags.LineContinuation) === TokenFlags.LineContinuation;
    }

    public hasPrecedingIndent() {
        return (this.tokenFlags & TokenFlags.AnyPrecedingIndent) === TokenFlags.PrecedingIndent;
    }

    public hasPrecedingDedent() {
        return (this.tokenFlags & TokenFlags.AnyPrecedingIndent) === TokenFlags.PrecedingDedent;
    }

    public hasPrecedingNewLine() {
        return this.hasPrecedingLineTerminator() && !this.isLineContinuation();
    }

    public hasPrecedingLineTerminator() {
        return (this.tokenFlags & TokenFlags.PrecedingLineTerminator) === TokenFlags.PrecedingLineTerminator;
    }

    public hasPrecedingBlankLine() {
        return (this.tokenFlags & TokenFlags.PrecedingBlankLine) === TokenFlags.PrecedingBlankLine;
    }

    private hasPrecedingNonWhiteSpaceTrivia() {
        return (this.tokenFlags & TokenFlags.PrecedingNonWhiteSpaceTrivia) === TokenFlags.PrecedingNonWhiteSpaceTrivia;
    }

    private triviaHasPrecedingLineTerminator() {
        return (this.tokenFlags & TokenFlags.TriviaPrecedingLineTerminator) === TokenFlags.TriviaPrecedingLineTerminator;
    }

    private triviaHasPrecedingBlankLine() {
        return (this.tokenFlags & TokenFlags.TriviaPrecedingBlankLine) === TokenFlags.TriviaPrecedingBlankLine;
    }

    private triviaHasPrecedingWhiteSpaceTrivia() {
        return (this.tokenFlags & TokenFlags.TriviaPrecedingWhiteSpaceTrivia) === TokenFlags.TriviaPrecedingWhiteSpaceTrivia;
    }

    private isStartOfFile() {
        return this.startPos === 0;
    }

    private setTokenAsUnterminated() {
        this.tokenFlags |= TokenFlags.Unterminated;
    }

    private setHasPrecedingLineTerminator() {
        this.tokenFlags |= TokenFlags.PrecedingLineTerminator | TokenFlags.TriviaPrecedingLineTerminator;
        this.currentIndentLength = 0;
    }

    private setHasPrecedingBlankLine() {
        this.tokenFlags |= TokenFlags.PrecedingBlankLine | TokenFlags.TriviaPrecedingBlankLine;
        if (this.significantIndentLength > 0) {
            this.setHasPrecedingDedent();
        }
        this.insignificantIndentLength = 0;
        this.significantIndentLength = 0;
        this.currentIndentLength = 0;
    }

    private setHasPrecedingIndent() {
        this.tokenFlags = this.tokenFlags & ~TokenFlags.PrecedingDedent | TokenFlags.PrecedingIndent;
    }

    private setHasPrecedingDedent() {
        this.tokenFlags = this.tokenFlags & ~TokenFlags.PrecedingIndent | TokenFlags.PrecedingDedent;
    }

    private setIsLineContinuation() {
        this.tokenFlags |= TokenFlags.LineContinuation;
    }

    private setHasPrecedingNonWhiteSpaceTrivia() {
        this.tokenFlags |= TokenFlags.PrecedingNonWhiteSpaceTrivia;
    }

    private resetHasPrecedingNonWhiteSpaceTrivia() {
        this.tokenFlags &= ~TokenFlags.PrecedingNonWhiteSpaceTrivia;
    }

    private setTriviaHasPrecedingBlankLine() {
        this.tokenFlags |= TokenFlags.TriviaPrecedingBlankLine;
    }

    private setTriviaHasPrecedingWhiteSpaceTrivia() {
        this.tokenFlags |= TokenFlags.TriviaPrecedingWhiteSpaceTrivia;
    }

    private resetTriviaHasPrecedingWhiteSpaceTrivia() {
        this.tokenFlags &= ~TokenFlags.TriviaPrecedingWhiteSpaceTrivia;
    }

    private resetTriviaFlags() {
        this.tokenFlags &= ~TokenFlags.TriviaFlags;
    }

    public resetIndent() {
        this.insignificantIndentLength = this.currentIndentLength;
        this.significantIndentLength = 0;
        this.tokenFlags &= ~(TokenFlags.PrecedingIndent | TokenFlags.PrecedingDedent | TokenFlags.LineContinuation);
    }

    public speculate<T>(callback: () => T, isLookahead: boolean): T {
        const savedPos = this.pos;
        const savedStartPos = this.startPos;
        const savedTokenPos = this.tokenPos;
        const savedToken = this.token;
        const savedTokenValue = this.tokenValue;
        const savedTokenFlags = this.tokenFlags;
        const savedTrivia = this.trivia;
        const savedLastTrivia = this.lastTrivia;
        const savedDiagnostics = this.diagnostics;
        const savedInitialIndentLength = this.insignificantIndentLength;
        const savedSignificantIndentLength = this.significantIndentLength;
        const savedCurrentIndentLength = this.currentIndentLength;

        this.diagnostics = NullDiagnosticMessages.instance;
        const result = callback();
        this.diagnostics = savedDiagnostics;
        if (!result || isLookahead) {
            this.pos = savedPos;
            this.startPos = savedStartPos;
            this.tokenPos = savedTokenPos;
            this.token = savedToken;
            this.tokenValue = savedTokenValue;
            this.tokenFlags = savedTokenFlags;
            this.trivia = savedTrivia;
            this.lastTrivia = savedLastTrivia;
            this.insignificantIndentLength = savedInitialIndentLength;
            this.significantIndentLength = savedSignificantIndentLength;
            this.currentIndentLength = savedCurrentIndentLength;
        }
        return result;
    }

    public scan(): SyntaxKind {
        this.cancelToken?.throwIfSignaled();
        this.startPos = this.pos;
        this.tokenValue = "";
        this.tokenFlags = 0;
        this.trivia = undefined;
        while (true) {
            this.tokenPos = this.pos;
            if (this.pos >= this.len) {
                if (this.significantIndentLength > 0) {
                    this.setHasPrecedingDedent();
                }

                this.insignificantIndentLength = 0;
                this.significantIndentLength = 0;
                this.currentIndentLength = 0;
                return this.token = SyntaxKind.EndOfFileToken;
            }

            if (this.proseStartToken) {
                return this.token = this.scanProse();
            }

            let ch = this.text.charCodeAt(this.pos++);

            // scan trivia
            switch (ch) {
                // newline trivia
                case CharacterCodes.CarriageReturn:
                    if (this.pos < this.len && this.text.charCodeAt(this.pos) === CharacterCodes.LineFeed) {
                        this.pos++;
                    }
                    // falls through

                case CharacterCodes.LineFeed:
                    if (this.hasPrecedingLineTerminator() && !this.hasPrecedingNonWhiteSpaceTrivia() && !this.hasPrecedingBlankLine()) {
                        this.setHasPrecedingBlankLine();
                    }

                    if (this.triviaHasPrecedingLineTerminator() && !this.triviaHasPrecedingBlankLine()) {
                        this.setTriviaHasPrecedingBlankLine();
                    }

                    this.resetHasPrecedingNonWhiteSpaceTrivia();
                    this.resetTriviaHasPrecedingWhiteSpaceTrivia();
                    this.setHasPrecedingLineTerminator();
                    continue;

                case CharacterCodes.Space:
                case CharacterCodes.Tab:
                    // significant whitespace trivia
                    if (this.hasPrecedingLineTerminator()) this.currentIndentLength++;
                    // falls through

                case CharacterCodes.VerticalTab:
                case CharacterCodes.FormFeed:
                    // whitespace trivia
                    this.setTriviaHasPrecedingWhiteSpaceTrivia();
                    continue;

                case CharacterCodes.LessThan: {
                    // html trivia
                    const ch = this.peekChar(0);
                    if (ch === CharacterCodes.Slash ||
                        ch === CharacterCodes.GreaterThan ||
                        (ch === CharacterCodes.LowerU ? this.peekChar(1) !== CharacterCodes.Plus : isHtmlTagNameStart(ch))) {
                        this.scanHtmlTrivia();
                        this.setHasPrecedingNonWhiteSpaceTrivia();
                        continue;
                    }

                    // `<!--`
                    if (ch === CharacterCodes.Exclamation &&
                        this.peekChar(1) === CharacterCodes.Minus &&
                        this.peekChar(2) === CharacterCodes.Minus) {
                        this.scanHtmlComment();
                        this.setHasPrecedingNonWhiteSpaceTrivia();
                        continue;
                    }
                    break;
                }

                case CharacterCodes.Slash:
                    // comment trivia
                    if (this.pos < this.len) {
                        switch (this.text.charCodeAt(this.pos)) {
                            case CharacterCodes.Slash:
                                // single-line comment
                                this.scanSingleLineCommentTrivia();
                                this.setHasPrecedingNonWhiteSpaceTrivia();
                                continue;

                            case CharacterCodes.Asterisk:
                                // multi-line comment
                                this.scanMultiLineCommentTrivia();
                                this.setHasPrecedingNonWhiteSpaceTrivia();
                                continue;
                        }
                    }
                    break;
            }

            // check for changes in indentation
            if (this.isStartOfFile() || this.hasPrecedingLineTerminator()) {
                if (this.isStartOfFile() || this.hasPrecedingBlankLine()) {
                    // Reset indentation at the start of a file or following a blank line.
                    // Any whitespace here will be considered insignificant and won't count
                    // towards indentation.
                    this.insignificantIndentLength = this.currentIndentLength;
                    this.significantIndentLength = 0;
                }
                else if (this.currentIndentLength <= this.insignificantIndentLength) {
                    // The current indentation is equal to or less than the insignificant indentation.
                    if (this.significantIndentLength > 0) {
                        // If we have a significant indent, reset the significant indentation and mark this
                        // as a dedent.
                        this.significantIndentLength = 0;
                        this.setHasPrecedingDedent();
                    }
                }
                else {
                    // The current indentation is greater than the insignificant indentation length we
                    // recorded at the start of the file or since the last blank line.
                    if (this.significantIndentLength === 0) {
                        // If we have no significant indentation, record this as the new significant
                        // indentation length and mark that we saw an indent.
                        this.significantIndentLength = this.currentIndentLength;
                        this.setHasPrecedingIndent();
                    }
                    else if (this.currentIndentLength > this.significantIndentLength) {
                        // If the new indentation is greater than the significant indentation length,
                        // then this is a line continuation.
                        this.setIsLineContinuation();
                    }
                }
            }

            this.finishLastTrivia();

            if (ch === CharacterCodes.Ampersand) {
                ch = this.scanCharacterEntity();
            }

            // scan token
            switch (ch) {
                case CharacterCodes.At:
                    return this.token = SyntaxKind.AtToken;

                case CharacterCodes.NumberSign:
                    return this.tokenValue = this.scanLine(), this.token = SyntaxKind.LinkReference;

                case CharacterCodes.DoubleQuote:
                case CharacterCodes.SingleQuote:
                    return this.tokenValue = this.scanString(ch, this.token = SyntaxKind.StringLiteral), this.token;

                case CharacterCodes.Backtick:
                    return this.tokenValue = this.scanString(ch, this.token = SyntaxKind.TerminalLiteral), this.token;

                case CharacterCodes.Bar:
                    return this.tokenValue = this.scanString(ch, this.token = SyntaxKind.Identifier), this.token;

                case CharacterCodes.LessThan:
                    if (this.expectCharacter(/*decodeEntity*/ true, CharacterCodes.Exclamation)) {
                        return this.token = SyntaxKind.LessThanExclamationToken;
                    }
                    else if (this.expectCharacter(/*decodeEntity*/ true, CharacterCodes.Minus)) {
                        return this.token = SyntaxKind.LessThanMinusToken;
                    }
                    else {
                        return this.tokenValue = this.scanString(CharacterCodes.GreaterThan, this.token = SyntaxKind.UnicodeCharacterLiteral), this.token;
                    }

                case CharacterCodes.NotEqualTo:
                    return this.token = SyntaxKind.NotEqualToToken;

                case CharacterCodes.ElementOf:
                    return this.token = SyntaxKind.ElementOfToken;

                case CharacterCodes.NotAnElementOf:
                    return this.token = SyntaxKind.NotAnElementOfToken;

                case CharacterCodes.GreaterThan:
                    return this.skipWhiteSpace(), this.proseStartToken = this.token = SyntaxKind.GreaterThanToken;

                case CharacterCodes.OpenParen:
                    return this.token = SyntaxKind.OpenParenToken;

                case CharacterCodes.CloseParen:
                    return this.token = SyntaxKind.CloseParenToken;

                case CharacterCodes.OpenBracket:
                    if (this.expectCharacter(/*decodeEntity*/ true, CharacterCodes.GreaterThan)) {
                        return this.skipWhiteSpace(), this.proseStartToken = this.token = SyntaxKind.OpenBracketGreaterThanToken;
                    }

                    return this.token = SyntaxKind.OpenBracketToken;

                case CharacterCodes.CloseBracket:
                    return this.token = SyntaxKind.CloseBracketToken;

                case CharacterCodes.OpenBrace:
                    return this.token = SyntaxKind.OpenBraceToken;

                case CharacterCodes.CloseBrace:
                    return this.token = SyntaxKind.CloseBraceToken;

                case CharacterCodes.Plus:
                    return this.token = SyntaxKind.PlusToken;

                case CharacterCodes.Tilde:
                    return this.token = SyntaxKind.TildeToken;

                case CharacterCodes.Comma:
                    return this.token = SyntaxKind.CommaToken;

                case CharacterCodes.Colon:
                    if (this.expectCharacter(/*decodeEntity*/ true, CharacterCodes.Colon)) {
                        if (this.expectCharacter(/*decodeEntity*/ true, CharacterCodes.Colon)) {
                            return this.token = SyntaxKind.ColonColonColonToken;
                        }

                        return this.token = SyntaxKind.ColonColonToken;
                    }

                    return this.token = SyntaxKind.ColonToken;

                case CharacterCodes.Question:
                    return this.token = SyntaxKind.QuestionToken;

                case CharacterCodes.Equals:
                    if (this.expectCharacter(/*decodeEntity*/ true, CharacterCodes.Equals)) {
                        return this.token = SyntaxKind.EqualsEqualsToken;
                    }

                    return this.token = SyntaxKind.EqualsToken;

                case CharacterCodes.Exclamation:
                    if (this.expectCharacter(/*decodeEntity*/ true, CharacterCodes.Equals)) {
                        return this.token = SyntaxKind.ExclamationEqualsToken;
                    }

                    return this.token = SyntaxKind.Unknown;

                case CharacterCodes.Number0:
                case CharacterCodes.Number1:
                case CharacterCodes.Number2:
                case CharacterCodes.Number3:
                case CharacterCodes.Number4:
                case CharacterCodes.Number5:
                case CharacterCodes.Number6:
                case CharacterCodes.Number7:
                case CharacterCodes.Number8:
                case CharacterCodes.Number9:
                    return this.tokenValue = this.scanNumber(ch, /*decodeEntities*/ true), this.token = SyntaxKind.NumberLiteral;

                case CharacterCodes.UpperU:
                case CharacterCodes.LowerU: {
                    const savedPos = this.pos;
                    const savedTokenFlags = this.tokenFlags;
                    if (this.pos < this.len && this.expectCharacter(/*decodeEntity*/ true, CharacterCodes.Plus)) {
                        if (this.scanHexDigits(/*decodeEntities*/ true, 1, 6, /*forbidExcessDigits*/ true) !== -1) {
                            // NOTE: code points are validated in checker
                            return this.token = SyntaxKind.UnicodeCharacterLiteral;
                        }

                        this.pos = savedPos;
                        this.tokenFlags = savedTokenFlags;
                    }
                }
                // fall-through

                default:
                    if (isIdentifierStart(ch)) {
                        let startPos = this.tokenPos;
                        if (this.tokenFlags & TokenFlags.DecodedEntity) {
                            this.tokenValue = String.fromCharCode(ch);
                            startPos = this.pos;
                        }
                        else {
                            this.tokenValue = "";
                        }

                        while (this.pos < this.len) {
                            const lastPos = this.pos;
                            const ch = this.scanCharacter(/*decodeEntity*/ true);
                            if (!isIdentifierPart(ch)) {
                                this.pos = lastPos;
                                break;
                            }
                            if (this.tokenFlags & TokenFlags.DecodedEntity) {
                                this.tokenValue += this.text.slice(startPos, lastPos);
                                this.tokenValue += String.fromCharCode(ch);
                                startPos = this.pos;
                            }
                        }
                        this.tokenValue += this.text.slice(startPos, this.pos);
                        return this.token = this.getIdentifierToken();
                    }

                    this.getDiagnostics().report(this.pos, Diagnostics.Invalid_character);
                    return this.token = SyntaxKind.Unknown;
            }
        }
    }

    public scanRange<T>(pos: number, cb: () => T) {
        if (pos < 0) throw new RangeError();
        pos = Math.min(pos, this.len);
        return this.speculate(() => {
            // if pos - 1 is whitespace, walk back the whitespace to rescan indentation
            if (pos > 0 && pos < this.len) {
                let start = pos - 1;
                while (isWhiteSpace(this.text.charCodeAt(start))) {
                    start--;
                }
                if (isLineTerminator(this.text.charCodeAt(start))) {
                    pos = start + 1;
                }
            }
            this.pos = pos;
            this.startPos = pos;
            this.tokenPos = pos;
            this.tokenValue = "";
            this.tokenFlags = 0;
            this.trivia = undefined;
            this.insignificantIndentLength = 0;
            this.significantIndentLength = 0;
            this.currentIndentLength = 0;
            return cb();
        }, /*isLookahead*/ true);
    }

    private peekChar(offset: number) {
        return this.pos + offset < this.text.length ?
            this.text.charCodeAt(this.pos + offset) :
            -1;
    }

    private scanLine(): string {
        const start = this.pos;
        while (this.pos < this.len) {
            const ch = this.text.charCodeAt(this.pos);
            if (isLineTerminator(ch)) {
                break;
            }

            this.pos++;
        }

        return this.text.substring(start, this.pos);
    }

    private skipLineTerminator() {
        const ch = this.text.charCodeAt(this.pos);
        if (ch === CharacterCodes.CarriageReturn || ch === CharacterCodes.LineFeed) {
            if (ch === CharacterCodes.CarriageReturn && this.text.charCodeAt(this.pos + 1) === CharacterCodes.LineFeed) {
                this.pos += 2;
            }
            else {
                this.pos++;
            }
        }
    }

    private skipWhiteSpace(): void {
        while (true) {
            const ch = this.text.charCodeAt(this.pos);
            switch (ch) {
                case CharacterCodes.Space:
                case CharacterCodes.Tab:
                case CharacterCodes.VerticalTab:
                case CharacterCodes.FormFeed:
                    this.pos++;
                    continue;

                default:
                    return;
            }
        }
    }

    private scanProse() {
        const previousToken = this.token;
        const isMultiLine = this.proseStartToken === SyntaxKind.GreaterThanToken;
        const atStartOfProse = previousToken === this.proseStartToken;
        const previousTokenWasFragment = isProseFragmentLiteralKind(previousToken);

        let start = this.pos;
        let tokenValue: string = "";
        while (true) {
            if (this.pos >= this.len) {
                tokenValue += this.text.substring(start, this.pos);
                this.tokenValue = tokenValue;
                this.proseStartToken = undefined;
                if (!isMultiLine) {
                    this.setTokenAsUnterminated();
                    this.getDiagnostics().report(this.pos, Diagnostics._0_expected, "]");
                }
                return atStartOfProse ? SyntaxKind.ProseFull : SyntaxKind.ProseTail;
            }

            const ch = this.text.charCodeAt(this.pos);
            if (previousTokenWasFragment) {
                if (ch === CharacterCodes.Backtick) {
                    return this.pos++ , this.tokenValue = this.scanString(ch, this.token = SyntaxKind.TerminalLiteral), this.token;
                }
                else if (ch === CharacterCodes.Bar) {
                    return this.pos++ , this.tokenValue = this.scanString(ch, this.token = SyntaxKind.Identifier), this.token;
                }
            }

            if (isMultiLine) {
                if (isLineTerminator(ch)) {
                    tokenValue += this.text.substring(start, this.pos);
                    if (this.nextLineIsProse()) {
                        tokenValue += "\n";
                        continue;
                    }

                    this.tokenValue = tokenValue;
                    this.proseStartToken = undefined;
                    return atStartOfProse ? SyntaxKind.ProseFull : SyntaxKind.ProseTail;
                }
            }
            else {
                if (ch === CharacterCodes.CloseBracket) {
                    tokenValue += this.text.substring(start, this.pos);
                    this.tokenValue = tokenValue;
                    this.proseStartToken = undefined;
                    return atStartOfProse ? SyntaxKind.ProseFull : SyntaxKind.ProseTail;
                }
            }

            if (ch === CharacterCodes.Backtick || ch === CharacterCodes.Bar) {
                tokenValue += this.text.substring(start, this.pos);
                this.tokenValue = tokenValue;
                return atStartOfProse ? SyntaxKind.ProseHead : SyntaxKind.ProseMiddle;
            }

            if (ch === CharacterCodes.Ampersand) {
                tokenValue += this.text.substring(start, this.pos);
                this.pos++;
                tokenValue += String.fromCharCode(this.scanCharacterEntity());
                start = this.pos;
                continue;
            }

            this.pos++;
        }
    }

    private nextLineIsProse() {
        return this.speculate(() => {
            this.skipLineTerminator();
            this.skipWhiteSpace();
            if (this.pos < this.len) {
                if (this.expectCharacter(/*decodeEntity*/ true, CharacterCodes.GreaterThan)) {
                    this.skipWhiteSpace();
                    return true;
                }
            }
            return false;
        }, /*isLookahead*/ false);
    }

    private finishLastTrivia() {
        if (this.lastTrivia) {
            getTriviaAccessor().setFollowingFields(
                this.lastTrivia,
                this.triviaHasPrecedingLineTerminator(),
                this.triviaHasPrecedingBlankLine(),
                this.triviaHasPrecedingWhiteSpaceTrivia());
            this.lastTrivia = undefined;
        }
    }

    private recordTrivia(trivia: Trivia) {
        this.finishLastTrivia();
        getTriviaAccessor().setPrecedingFields(
            trivia,
            this.triviaHasPrecedingLineTerminator(),
            this.triviaHasPrecedingBlankLine(),
            this.triviaHasPrecedingWhiteSpaceTrivia());
        getNodeAccessor().setTextRange(trivia, this.tokenPos, this.pos);
        this.trivia ||= [];
        this.trivia.push(trivia);
        this.lastTrivia = trivia;
        this.resetTriviaFlags();
    }

    private scanSingleLineCommentTrivia() {
        // //comment
        //  ^
        //  current pos

        const end = findSingleLineCommentTriviaEnd(this.text, this.pos + 1, this.len);
        this.pos = end;

        this.recordTrivia(new SingleLineCommentTrivia());
    }

    private scanMultiLineCommentTrivia() {
        // /*comment*/
        //  ^
        //  current pos

        const { closed, end } = findMultiLineCommentTriviaEnd(this.text, this.pos + 1, this.len);
        this.pos = end;

        if (!closed) {
            this.getDiagnostics().report(this.pos, Diagnostics._0_expected, "*/");
        }

        this.recordTrivia(new MultiLineCommentTrivia());
    }

    private scanHtmlTrivia() {
        // <tag>
        //  ^
        //  current pos

        const isClosingTag = this.text.charCodeAt(this.pos) === CharacterCodes.Slash;
        const end = findHtmlTriviaEnd(this.text, this.pos, this.len);
        const tagNamePos = isClosingTag ? this.pos + 1 : this.pos;
        const tagNameEnd = findHtmlTagNameEnd(this.text, tagNamePos, end);
        this.pos = end;

        const tagName = this.text.slice(tagNamePos, tagNameEnd === -1 ? end : tagNameEnd);
        this.recordTrivia(isClosingTag ? new HtmlCloseTagTrivia(tagName) : new HtmlOpenTagTrivia(tagName));
    }

    private scanHtmlComment() {
        // <!--
        //  ^
        //  current pos

        const end = findHtmlCommentEnd(this.text, this.pos + 3, this.len);
        this.pos = end;

        this.recordTrivia(new HtmlCommentTrivia());
    }

    private expectCharacter(decodeEntity: boolean, ch: number) {
        const savedPos = this.pos;
        const savedTokenFlags = this.tokenFlags;
        if (this.scanCharacter(decodeEntity) !== ch) {
            this.pos = savedPos;
            this.tokenFlags = savedTokenFlags;
            return false;
        }
        return true;
    }

    private matchCharacter(decodeEntity: boolean, cb: (ch: number) => boolean) {
        const savedPos = this.pos;
        const savedTokenFlags = this.tokenFlags;
        const ch = this.scanCharacter(decodeEntity);
        if (!cb(ch)) {
            this.pos = savedPos;
            this.tokenFlags = savedTokenFlags;
            return -1;
        }
        return ch;
    }

    private scanCharacter(decodeEntity: boolean) {
        const ch = this.text.charCodeAt(this.pos++);
        if (decodeEntity && ch === CharacterCodes.Ampersand) {
            return this.scanCharacterEntity();
        }
        this.tokenFlags &= ~TokenFlags.DecodedEntity;
        return ch;
    }

    private scanCharacterEntity() {
        let value = 0;
        let start: number;
        let pos: number;
        if (this.text.charCodeAt(this.pos) === CharacterCodes.Hash) {
            const hex =
                this.text.charCodeAt(this.pos + 1) === CharacterCodes.LowerX ||
                this.text.charCodeAt(this.pos + 1) === CharacterCodes.UpperX;
            start = this.pos + (hex ? 2 : 1);
            pos = start;
            while (pos < this.len) {
                const ch = this.text.charCodeAt(pos);
                const digit = hex ? charToHexDigit(ch) : charToDecimalDigit(ch);
                if (digit === -1) break;
                value = value * (hex ? 16 : 10) + digit;
                pos++;
            }
        }
        else {
            start = pos = this.pos;
            while (pos < this.len) {
                if (!isAlphaNum(this.text.charCodeAt(pos))) break;
                pos++;
            }
            const entity = this.text.slice(start, pos);
            if (htmlCharacterEntities.hasOwnProperty(entity)) {
                value = htmlCharacterEntities[entity];
            }
        }
        if (pos > start && this.text.charCodeAt(pos) === CharacterCodes.Semicolon) {
            this.pos = pos + 1;
            this.tokenFlags |= TokenFlags.DecodedEntity;
            return value;
        }
        this.tokenFlags &= ~TokenFlags.DecodedEntity;
        return CharacterCodes.Ampersand;
    }

    private scanString(quote: number, kind: SyntaxKind.StringLiteral | SyntaxKind.TerminalLiteral | SyntaxKind.Identifier | SyntaxKind.UnicodeCharacterLiteral): string {
        const diagnostic = kind === SyntaxKind.Identifier ? Diagnostics.Unterminated_identifier_literal : Diagnostics.Unterminated_string_literal;
        const decodeEscapeSequences = kind !== SyntaxKind.TerminalLiteral;

        let result = "";
        let start = this.pos;
        while (true) {
            if (this.pos >= this.len) {
                result += this.text.slice(start, this.pos);
                this.setTokenAsUnterminated();
                this.getDiagnostics().reportRange(this.tokenPos, this.pos, diagnostic || Diagnostics.Unterminated_string_literal);
                break;
            }

            const lastPos = this.pos;
            let ch = this.scanCharacter(decodeEscapeSequences);
            if (this.tokenFlags & TokenFlags.DecodedEntity) {
                // Reference-decoded characters span multiple indexes, breaking naive assumptions.
                // Read in everything preceding the reference, and set the new position to the
                // index following it.
                result += this.text.slice(start, lastPos);
                start = this.pos;
            }

            if (ch === quote) {
                // If this is a terminal consisting solely of one backtick character (e.g. ```),
                // we capture the backtick.
                if (quote === CharacterCodes.Backtick && result === "" && this.pos < this.len) {
                    const peekPos = this.pos;

                    ch = this.scanCharacter(decodeEscapeSequences);
                    if (ch === CharacterCodes.Backtick) {
                        result = "`";
                        break;
                    }

                    this.pos = peekPos;
                }

                result += this.text.slice(start, this.pos - 1);
                break;
            }
            else if (quote === CharacterCodes.GreaterThan && ch === CharacterCodes.LessThan) {
                this.pos = lastPos;
                result += this.text.slice(start, lastPos);
                this.setTokenAsUnterminated();
                this.getDiagnostics().reportRange(this.tokenPos, lastPos, diagnostic || Diagnostics.Unterminated_string_literal);
                break;
            }
            else if (decodeEscapeSequences && ch === CharacterCodes.Backslash) {
                // terminals cannot have escape sequences
                result += this.text.slice(start, this.pos - 1);
                result += this.scanEscapeSequence(lastPos);
                start = this.pos;
                continue;
            }
            else if (isLineTerminator(ch)) {
                this.pos--;
                result += this.text.slice(start, this.pos);
                this.setTokenAsUnterminated();
                this.getDiagnostics().reportRange(this.tokenPos, this.pos, diagnostic || Diagnostics.Unterminated_string_literal);
                break;
            }
        }

        return result;
    }

    private scanEscapeSequence(start: number): string {
        if (this.pos >= this.len) {
            this.getDiagnostics().reportRange(start, this.pos, Diagnostics.Invalid_escape_sequence);
            return "";
        }

        let ch = this.text.charCodeAt(this.pos++);
        switch (ch) {
            case CharacterCodes.Number0:
                return "\0";

            case CharacterCodes.LowerB:
                return "\b";

            case CharacterCodes.LowerT:
                return "\t";

            case CharacterCodes.LowerN:
                return "\n";

            case CharacterCodes.LowerV:
                return "\v";

            case CharacterCodes.LowerF:
                return "\f";

            case CharacterCodes.LowerR:
                return "\r";

            case CharacterCodes.SingleQuote:
                return "\'";

            case CharacterCodes.DoubleQuote:
                return "\"";

            case CharacterCodes.LowerX:
            case CharacterCodes.LowerU:
                ch = this.scanHexDigits(/*decodeEntities*/ false, ch === CharacterCodes.LowerX ? 2 : 4);
                if (ch >= 0) {
                    return String.fromCharCode(ch);
                }
                else {
                    this.getDiagnostics().reportRange(start, this.pos, Diagnostics.Invalid_escape_sequence);
                    return "";
                }

            // when encountering a LineContinuation (i.e. a backslash and a line terminator sequence),
            // the line terminator is interpreted to be "the empty code unit sequence".
            case CharacterCodes.CarriageReturn:
                if (this.pos < this.len && this.text.charCodeAt(this.pos) === CharacterCodes.LineFeed) {
                    this.pos++;
                }

            // fall through

            case CharacterCodes.LineFeed:
            case CharacterCodes.LineSeparator:
            case CharacterCodes.ParagraphSeparator:
                return ""

            default:
                return String.fromCharCode(ch);
        }
    }

    private scanHexDigit(decodeEntity: boolean) {
        const initialPos = this.pos;
        const ch = this.scanCharacter(decodeEntity);
        const value = charToHexDigit(ch);
        if (value === -1) {
            this.pos = initialPos;
        }
        return value;
    }

    private scanHexDigits(decodeEntities: boolean, minCount: number, maxCount = minCount, forbidExcessDigits?: boolean): number {
        const start = this.pos;
        let digits = 0;
        let value = 0;
        while (digits < maxCount) {
            const digit = this.scanHexDigit(decodeEntities);
            if (digit === -1) break;
            value = value * 16 + digit;
            digits++;
        }
        if (digits < minCount || digits === maxCount && forbidExcessDigits && this.scanHexDigit(/*decodeEntity*/ true) !== -1) {
            this.pos = start;
            return -1;
        }
        return value;
    }

    private scanNumber(ch: number, decodeEntities: boolean): string {
        let text = "";
        let startPos = this.tokenPos;
        let lastPos = this.pos;

        if (this.tokenFlags & TokenFlags.DecodedEntity) {
            text += this.text.slice(startPos, lastPos);
            text += String.fromCharCode(ch);
            startPos = this.pos;
        }

        while (this.pos < this.len) {
            ch = this.matchCharacter(decodeEntities, isDecimalDigit);
            if (ch < 0) break;
            if (this.tokenFlags & TokenFlags.DecodedEntity) {
                text += this.text.slice(startPos, lastPos);
                text += String.fromCharCode(ch);
                startPos = this.pos;
            }

            lastPos = this.pos;
        }

        if (this.pos < this.len && this.expectCharacter(decodeEntities, CharacterCodes.Dot)) {
            if (this.tokenFlags & TokenFlags.DecodedEntity) {
                text += this.text.slice(startPos, lastPos);
                text += ".";
                startPos = this.pos;
            }

            lastPos = this.pos;
            while (this.pos < this.len) {
                const ch = this.matchCharacter(decodeEntities, isDecimalDigit);
                if (ch < 0) break;
                if (this.tokenFlags & TokenFlags.DecodedEntity) {
                    text += this.text.slice(startPos, lastPos);
                    text += String.fromCharCode(ch);
                    startPos = this.pos;
                }

                lastPos = this.pos;
            }
        }

        if (this.pos < this.len) {
            ch = this.matchCharacter(decodeEntities, isUpperOrLowerE);
            if (ch >= 0) {
                if (this.tokenFlags & TokenFlags.DecodedEntity) {
                    text += this.text.slice(startPos, lastPos);
                    text += String.fromCharCode(ch);
                    startPos = this.pos;
                }

                lastPos = this.pos;

                if (this.pos < this.len) {
                    ch = this.matchCharacter(decodeEntities, isPlusOrMinus);
                    if (ch >= 0) {
                        if (this.tokenFlags & TokenFlags.DecodedEntity) {
                            text += this.text.slice(startPos, lastPos);
                            text += String.fromCharCode(ch);
                            startPos = this.pos;
                        }
                    }

                    lastPos = this.pos;
                }

                if (this.pos < this.len) {
                    ch = this.matchCharacter(decodeEntities, isDecimalDigit);
                    if (ch >= 0) {
                        if (this.tokenFlags & TokenFlags.DecodedEntity) {
                            text += this.text.slice(startPos, lastPos);
                            text += String.fromCharCode(ch);
                            startPos = this.pos;
                        }

                        lastPos = this.pos;

                        while (this.pos < this.len) {
                            ch = this.matchCharacter(decodeEntities, isDecimalDigit);
                            if (ch < 0) break;
                            if (this.tokenFlags & TokenFlags.DecodedEntity) {
                                text += this.text.slice(startPos, lastPos);
                                text += String.fromCharCode(ch);
                                startPos = this.pos;
                            }

                            lastPos = this.pos;
                        }
                    }
                    else {
                        this.getDiagnostics().report(this.pos, Diagnostics.Digit_expected);
                    }
                }
                else {
                    this.getDiagnostics().report(this.pos, Diagnostics.Digit_expected);
                }
            }
        }

        text += this.text.slice(startPos, this.pos);
        return +text + "";
    }

    private getIdentifierToken(): SyntaxKind {
        const len = this.tokenValue.length;
        if (len >= 2 && len <= 9) {
            const ch = this.tokenValue.charCodeAt(0);
            if (ch >= CharacterCodes.LowerA && ch <= CharacterCodes.LowerT) {
                const token = stringToToken(this.tokenValue);
                if (token !== undefined) {
                    return this.token = token;
                }
            }
        }

        return this.token = SyntaxKind.Identifier;
    }
}

function isLineTerminator(ch: number): boolean {
    return ch === CharacterCodes.CarriageReturn || ch === CharacterCodes.LineFeed;
}

function isSignificantWhiteSpace(ch: number) {
    switch (ch) {
        case CharacterCodes.Space:
        case CharacterCodes.Tab:
            return true;
    }
    return false;
}

function isInsignificantWhiteSpace(ch: number) {
    switch (ch) {
        case CharacterCodes.VerticalTab:
        case CharacterCodes.FormFeed:
            return true;
    }
    return false;
}

/** @internal */
export function isWhiteSpace(ch: number) {
    return isSignificantWhiteSpace(ch) || isInsignificantWhiteSpace(ch);
}

function isUpperAlpha(ch: number): boolean {
    return ch >= CharacterCodes.UpperA && ch <= CharacterCodes.UpperZ;
}

function isLowerAlpha(ch: number): boolean {
    return ch >= CharacterCodes.LowerA && ch <= CharacterCodes.LowerZ;
}

function isAlpha(ch: number): boolean {
    return isUpperAlpha(ch) || isLowerAlpha(ch);
}

function isDecimalDigit(ch: number): boolean {
    return ch >= CharacterCodes.Number0 && ch <= CharacterCodes.Number9;
}

function isAlphaNum(ch: number): boolean {
    return isAlpha(ch) || isDecimalDigit(ch);
}

function isUpperOrLowerE(ch: number) {
    return ch === CharacterCodes.UpperE || ch === CharacterCodes.LowerE;
}

function isPlusOrMinus(ch: number) {
    return ch === CharacterCodes.Plus || ch === CharacterCodes.Minus;
}

function charToHexDigit(ch: number) {
    if (ch >= CharacterCodes.Number0 && ch <= CharacterCodes.Number9) {
        return ch - CharacterCodes.Number0;
    }
    else if (ch >= CharacterCodes.UpperA && ch <= CharacterCodes.UpperF) {
        return ch - CharacterCodes.UpperA + 10;
    }
    else if (ch >= CharacterCodes.LowerA && ch <= CharacterCodes.LowerF) {
        return ch - CharacterCodes.LowerA + 10;
    }
    return -1;
}

function charToDecimalDigit(ch: number) {
    if (ch >= CharacterCodes.Number0 && ch <= CharacterCodes.Number9) {
        return ch - CharacterCodes.Number0;
    }
    return -1;
}

/** @internal */
export function isHexDigit(ch: number): boolean {
    return ch >= CharacterCodes.UpperA && ch <= CharacterCodes.UpperF
        || ch >= CharacterCodes.LowerA && ch <= CharacterCodes.LowerF
        || ch >= CharacterCodes.Number0 && ch <= CharacterCodes.Number9;
}

/** @internal */
export function isIdentifierStart(ch: number): boolean {
    return isAlpha(ch)
        || ch === CharacterCodes.Underscore;
}

/** @internal */
export function isIdentifierPart(ch: number): boolean {
    return isAlphaNum(ch)
        || ch === CharacterCodes.Underscore;
}

function isHtmlTagNameStart(ch: number): boolean {
    return isLowerAlpha(ch);
}

function isHtmlTagNamePart(ch: number): boolean {
    return isIdentifierPart(ch)
        || ch === CharacterCodes.Minus;
}

function findHtmlCommentEnd(text: string, pos: number, end: number) {
    while (pos + 2 < end) {
        if (text.charCodeAt(pos) === CharacterCodes.Minus &&
            text.charCodeAt(pos + 1) === CharacterCodes.Minus &&
            text.charCodeAt(pos + 2) === CharacterCodes.GreaterThan) {
            return pos + 3;
        }
        pos++;
    }
    return end;
}

function findHtmlTriviaEnd(text: string, pos: number, end: number) {
    while (pos < end) {
        const ch = text.charCodeAt(pos++);
        if (ch === CharacterCodes.GreaterThan) {
            return pos;
        }
    }
    return end;
}

function findHtmlTagNameEnd(text: string, pos: number, end: number) {
    if (pos < end && isHtmlTagNameStart(text.charCodeAt(pos))) {
        pos++;
        while (pos < end && isHtmlTagNamePart(text.charCodeAt(pos))) {
            pos++;
        }
        return pos;
    }
    return -1;
}

export function skipTrivia(text: string, pos: number, end: number, htmlTrivia?: HtmlTrivia[], commentTrivia?: CommentTrivia[]) {
    scan: while (pos < end) {
        const ch = text.charCodeAt(pos);
        switch (ch) {
            case CharacterCodes.CarriageReturn:
            case CharacterCodes.LineFeed:
            case CharacterCodes.Space:
            case CharacterCodes.Tab:
            case CharacterCodes.VerticalTab:
            case CharacterCodes.FormFeed:
                pos++;
                continue;
            case CharacterCodes.Slash:
                if (pos + 1 < end) {
                    switch (text.charCodeAt(pos + 1)) {
                        case CharacterCodes.Slash: {
                            const commentEnd = findSingleLineCommentTriviaEnd(text, pos + 2, end);
                            if (commentTrivia) {
                                const comment = new SingleLineCommentTrivia();
                                getNodeAccessor().setTextRange(comment, pos, commentEnd);
                                commentTrivia.push(comment);
                            }
                            pos = commentEnd;
                            continue;
                        }
                        case CharacterCodes.Asterisk: {
                            const { end: commentEnd } = findMultiLineCommentTriviaEnd(text, pos + 2, end);
                            if (commentTrivia) {
                                const comment = new MultiLineCommentTrivia();
                                getNodeAccessor().setTextRange(comment, pos, commentEnd);
                                commentTrivia.push(comment);
                            }
                            pos = commentEnd;
                            continue;
                        }
                    }
                }
                break scan;
            case CharacterCodes.LessThan:
                if (pos + 1 < end) {
                    const ch = text.charCodeAt(pos + 1);
                    if (ch === CharacterCodes.Slash ||
                        (ch === CharacterCodes.LowerU ? !(pos + 2 < end && text.charCodeAt(pos + 2) === CharacterCodes.Plus) :
                            (ch >= CharacterCodes.LowerA && ch <= CharacterCodes.LowerZ))) {
                        const triviaEnd = findHtmlTriviaEnd(text, pos + 1, end);
                        if (htmlTrivia) {
                            const tagNamePos = ch === CharacterCodes.Slash ? pos + 2 : pos + 1;
                            const tagNameEnd = findHtmlTagNameEnd(text, tagNamePos, triviaEnd);
                            if (tagNameEnd !== -1) {
                                const tagName = text.slice(tagNamePos, tagNameEnd);
                                const tag = ch === CharacterCodes.Slash ? new HtmlCloseTagTrivia(tagName) : new HtmlOpenTagTrivia(tagName);
                                getNodeAccessor().setTextRange(tag, pos, triviaEnd);
                                htmlTrivia.push(tag);
                            }
                        }
                        pos = triviaEnd;
                        continue;
                    }
                    if (pos + 3 < end &&
                        ch === CharacterCodes.Exclamation &&
                        text.charCodeAt(pos + 2) === CharacterCodes.Minus &&
                        text.charCodeAt(pos + 3) === CharacterCodes.Minus) {
                        const triviaEnd = findHtmlCommentEnd(text, pos + 3, end);
                        if (htmlTrivia) {
                            const trivia = new HtmlCommentTrivia();
                            getNodeAccessor().setTextRange(trivia, pos, triviaEnd);
                            htmlTrivia.push(trivia);
                        }
                        pos = triviaEnd;
                        continue;
                    }
                }
                break scan;
            default:
                break scan;
        }
    }
    return pos;
}

export function scanHtmlTrivia(text: string, pos: number, end: number): HtmlTrivia[] | undefined {
    const trivia: HtmlTrivia[] = [];
    skipTrivia(text, pos, end, trivia);
    return trivia.length > 0 ? trivia : undefined;
}

function findMultiLineCommentTriviaEnd(text: string, pos: number, end: number) {
    while (pos < end) {
        const ch = text.charCodeAt(pos);
        if (ch === CharacterCodes.Asterisk && pos + 1 < end && text.charCodeAt(pos + 1) === CharacterCodes.Slash) {
            return { closed: true, end: pos + 2 };
        }
        pos++;
    }
    return { closed: false, end };
}

function findSingleLineCommentTriviaEnd(text: string, pos: number, end: number) {
    while (pos < end) {
        const ch = text.charCodeAt(pos);
        if (ch === CharacterCodes.CarriageReturn || ch === CharacterCodes.LineFeed) {
            return pos;
        }
        pos++;
    }
    return end;
}

export function decodeHtmlEntities(text: string) {
    let ch: number;
    let pos = 0;
    let startPos = pos;
    let entityPos: number;
    let entityStart: number;
    let result: string | undefined;
    while (pos < text.length) {
        const lastPos = pos;
        ch = text.charCodeAt(pos++);
        if (ch === CharacterCodes.Ampersand) {
            if (text.charCodeAt(pos) === CharacterCodes.Hash) {
                const hex =
                    text.charCodeAt(pos + 1) === CharacterCodes.LowerX ||
                    text.charCodeAt(pos + 1) === CharacterCodes.UpperX;
                entityStart = pos + (hex ? 2 : 1);
                entityPos = entityStart;
                ch = 0;
                while (entityPos < text.length) {
                    const digit = hex ?
                        charToHexDigit(text.charCodeAt(entityPos)) :
                        charToDecimalDigit(text.charCodeAt(entityPos));
                    if (digit === -1) break;
                    ch = ch * (hex ? 16 : 10) + digit;
                    entityPos++;
                }
            }
            else {
                entityStart = entityPos = pos;
                while (entityPos < text.length) {
                    if (!isAlphaNum(text.charCodeAt(entityPos))) break;
                    entityPos++;
                }
                const entity = text.slice(entityStart, entityPos);
                if (htmlCharacterEntities.hasOwnProperty(entity)) {
                    ch = htmlCharacterEntities[entity];
                }
            }
            if (entityPos > entityStart && text.charCodeAt(entityPos) === CharacterCodes.Semicolon) {
                result ??= "";
                result += text.slice(startPos, lastPos);
                result += String.fromCharCode(ch);
                pos = entityPos + 1;
                startPos = pos;
            }
        }
    }
    if (result !== undefined) {
        return result + text.slice(startPos);
    }
    return text;
}

const htmlCharacterEntities: Record<string, number> = {
    quot: 0x0022,
    amp: 0x0026,
    apos: 0x0027,
    lt: 0x003C,
    gt: 0x003E,
    nbsp: 0x00A0,
    iexcl: 0x00A1,
    cent: 0x00A2,
    pound: 0x00A3,
    curren: 0x00A4,
    yen: 0x00A5,
    brvbar: 0x00A6,
    sect: 0x00A7,
    uml: 0x00A8,
    copy: 0x00A9,
    ordf: 0x00AA,
    laquo: 0x00AB,
    not: 0x00AC,
    shy: 0x00AD,
    reg: 0x00AE,
    macr: 0x00AF,
    deg: 0x00B0,
    plusmn: 0x00B1,
    sup2: 0x00B2,
    sup3: 0x00B3,
    acute: 0x00B4,
    micro: 0x00B5,
    para: 0x00B6,
    middot: 0x00B7,
    cedil: 0x00B8,
    sup1: 0x00B9,
    ordm: 0x00BA,
    raquo: 0x00BB,
    frac14: 0x00BC,
    frac12: 0x00BD,
    frac34: 0x00BE,
    iquest: 0x00BF,
    Agrave: 0x00C0,
    Aacute: 0x00C1,
    Acirc: 0x00C2,
    Atilde: 0x00C3,
    Auml: 0x00C4,
    Aring: 0x00C5,
    AElig: 0x00C6,
    Ccedil: 0x00C7,
    Egrave: 0x00C8,
    Eacute: 0x00C9,
    Ecirc: 0x00CA,
    Euml: 0x00CB,
    Igrave: 0x00CC,
    Iacute: 0x00CD,
    Icirc: 0x00CE,
    Iuml: 0x00CF,
    ETH: 0x00D0,
    Ntilde: 0x00D1,
    Ograve: 0x00D2,
    Oacute: 0x00D3,
    Ocirc: 0x00D4,
    Otilde: 0x00D5,
    Ouml: 0x00D6,
    times: 0x00D7,
    Oslash: 0x00D8,
    Ugrave: 0x00D9,
    Uacute: 0x00DA,
    Ucirc: 0x00DB,
    Uuml: 0x00DC,
    Yacute: 0x00DD,
    THORN: 0x00DE,
    szlig: 0x00DF,
    agrave: 0x00E0,
    aacute: 0x00E1,
    acirc: 0x00E2,
    atilde: 0x00E3,
    auml: 0x00E4,
    aring: 0x00E5,
    aelig: 0x00E6,
    ccedil: 0x00E7,
    egrave: 0x00E8,
    eacute: 0x00E9,
    ecirc: 0x00EA,
    euml: 0x00EB,
    igrave: 0x00EC,
    iacute: 0x00ED,
    icirc: 0x00EE,
    iuml: 0x00EF,
    eth: 0x00F0,
    ntilde: 0x00F1,
    ograve: 0x00F2,
    oacute: 0x00F3,
    ocirc: 0x00F4,
    otilde: 0x00F5,
    ouml: 0x00F6,
    divide: 0x00F7,
    oslash: 0x00F8,
    ugrave: 0x00F9,
    uacute: 0x00FA,
    ucirc: 0x00FB,
    uuml: 0x00FC,
    yacute: 0x00FD,
    thorn: 0x00FE,
    yuml: 0x00FF,
    OElig: 0x0152,
    oelig: 0x0153,
    Scaron: 0x0160,
    scaron: 0x0161,
    Yuml: 0x0178,
    fnof: 0x0192,
    circ: 0x02C6,
    tilde: 0x02DC,
    Alpha: 0x0391,
    Beta: 0x0392,
    Gamma: 0x0393,
    Delta: 0x0394,
    Epsilon: 0x0395,
    Zeta: 0x0396,
    Eta: 0x0397,
    Theta: 0x0398,
    Iota: 0x0399,
    Kappa: 0x039A,
    Lambda: 0x039B,
    Mu: 0x039C,
    Nu: 0x039D,
    Xi: 0x039E,
    Omicron: 0x039F,
    Pi: 0x03A0,
    Rho: 0x03A1,
    Sigma: 0x03A3,
    Tau: 0x03A4,
    Upsilon: 0x03A5,
    Phi: 0x03A6,
    Chi: 0x03A7,
    Psi: 0x03A8,
    Omega: 0x03A9,
    alpha: 0x03B1,
    beta: 0x03B2,
    gamma: 0x03B3,
    delta: 0x03B4,
    epsilon: 0x03B5,
    zeta: 0x03B6,
    eta: 0x03B7,
    theta: 0x03B8,
    iota: 0x03B9,
    kappa: 0x03BA,
    lambda: 0x03BB,
    mu: 0x03BC,
    nu: 0x03BD,
    xi: 0x03BE,
    omicron: 0x03BF,
    pi: 0x03C0,
    rho: 0x03C1,
    sigmaf: 0x03C2,
    sigma: 0x03C3,
    tau: 0x03C4,
    upsilon: 0x03C5,
    phi: 0x03C6,
    chi: 0x03C7,
    psi: 0x03C8,
    omega: 0x03C9,
    thetasym: 0x03D1,
    upsih: 0x03D2,
    piv: 0x03D6,
    ensp: 0x2002,
    emsp: 0x2003,
    thinsp: 0x2009,
    zwnj: 0x200C,
    zwj: 0x200D,
    lrm: 0x200E,
    rlm: 0x200F,
    ndash: 0x2013,
    mdash: 0x2014,
    lsquo: 0x2018,
    rsquo: 0x2019,
    sbquo: 0x201A,
    ldquo: 0x201C,
    rdquo: 0x201D,
    bdquo: 0x201E,
    dagger: 0x2020,
    Dagger: 0x2021,
    bull: 0x2022,
    hellip: 0x2026,
    permil: 0x2030,
    prime: 0x2032,
    Prime: 0x2033,
    lsaquo: 0x2039,
    rsaquo: 0x203A,
    oline: 0x203E,
    frasl: 0x2044,
    euro: 0x20AC,
    image: 0x2111,
    weierp: 0x2118,
    real: 0x211C,
    trade: 0x2122,
    alefsym: 0x2135,
    larr: 0x2190,
    uarr: 0x2191,
    rarr: 0x2192,
    darr: 0x2193,
    harr: 0x2194,
    crarr: 0x21B5,
    lArr: 0x21D0,
    uArr: 0x21D1,
    rArr: 0x21D2,
    dArr: 0x21D3,
    hArr: 0x21D4,
    forall: 0x2200,
    part: 0x2202,
    exist: 0x2203,
    empty: 0x2205,
    nabla: 0x2207,
    isin: 0x2208,
    notin: 0x2209,
    ni: 0x220B,
    prod: 0x220F,
    sum: 0x2211,
    minus: 0x2212,
    lowast: 0x2217,
    radic: 0x221A,
    prop: 0x221D,
    infin: 0x221E,
    ang: 0x2220,
    and: 0x2227,
    or: 0x2228,
    cap: 0x2229,
    cup: 0x222A,
    int: 0x222B,
    there4: 0x2234,
    sim: 0x223C,
    cong: 0x2245,
    asymp: 0x2248,
    ne: 0x2260,
    equiv: 0x2261,
    le: 0x2264,
    ge: 0x2265,
    sub: 0x2282,
    sup: 0x2283,
    nsub: 0x2284,
    sube: 0x2286,
    supe: 0x2287,
    oplus: 0x2295,
    otimes: 0x2297,
    perp: 0x22A5,
    sdot: 0x22C5,
    lceil: 0x2308,
    rceil: 0x2309,
    lfloor: 0x230A,
    rfloor: 0x230B,
    lang: 0x2329,
    rang: 0x232A,
    loz: 0x25CA,
    spades: 0x2660,
    clubs: 0x2663,
    hearts: 0x2665,
    diams: 0x2666
};