tokenizer.ts

import { TagToken, HTMLToken, HashToken, QuotedToken, LiquidTagToken, OutputToken, ValueToken, Token, RangeToken, FilterToken, TopLevelToken, PropertyAccessToken, OperatorToken, LiteralToken, IdentifierToken, NumberToken } from '../tokens'
import { Trie, createTrie, ellipsis, literalValues, assert, TokenizationError, TYPES, QUOTE, BLANK, IDENTIFIER } from '../util'
import { Operators, Expression } from '../render'
import { NormalizedFullOptions, defaultOptions } from '../liquid-options'
import { FilterArg } from './filter-arg'
import { matchOperator } from './match-operator'
import { whiteSpaceCtrl } from './whitespace-ctrl'

export class Tokenizer {
  p = 0
  N: number
  private rawBeginAt = -1
  private opTrie: Trie

  constructor (
    public input: string,
    operators: Operators = defaultOptions.operators,
    public file?: string
  ) {
    this.N = input.length
    this.opTrie = createTrie(operators)
  }

  readExpression () {
    return new Expression(this.readExpressionTokens())
  }

  * readExpressionTokens (): IterableIterator<Token> {
    while (this.p < this.N) {
      const operator = this.readOperator()
      if (operator) {
        yield operator
        continue
      }
      const operand = this.readValue()
      if (operand) {
        yield operand
        continue
      }
      return
    }
  }
  readOperator (): OperatorToken | undefined {
    this.skipBlank()
    const end = matchOperator(this.input, this.p, this.opTrie)
    if (end === -1) return
    return new OperatorToken(this.input, this.p, (this.p = end), this.file)
  }
  readFilters (): FilterToken[] {
    const filters = []
    while (true) {
      const filter = this.readFilter()
      if (!filter) return filters
      filters.push(filter)
    }
  }
  readFilter (): FilterToken | null {
    this.skipBlank()
    if (this.end()) return null
    assert(this.peek() === '|', () => `expected "|" before filter`)
    this.p++
    const begin = this.p
    const name = this.readIdentifier()
    if (!name.size()) return null
    const args = []
    this.skipBlank()
    if (this.peek() === ':') {
      do {
        ++this.p
        const arg = this.readFilterArg()
        arg && args.push(arg)
        this.skipBlank()
        assert(this.end() || this.peek() === ',' || this.peek() === '|', () => `unexpected character ${this.snapshot()}`)
      } while (this.peek() === ',')
    } else if (this.peek() === '|' || this.end()) {
      // do nothing
    } else {
      throw new Error('expected ":" after filter name')
    }
    return new FilterToken(name.getText(), args, this.input, begin, this.p, this.file)
  }

  readFilterArg (): FilterArg | undefined {
    const key = this.readValue()
    if (!key) return
    this.skipBlank()
    if (this.peek() !== ':') return key
    ++this.p
    const value = this.readValue()
    return [key.getText(), value]
  }

  readTopLevelTokens (options: NormalizedFullOptions = defaultOptions): TopLevelToken[] {
    const tokens: TopLevelToken[] = []
    while (this.p < this.N) {
      const token = this.readTopLevelToken(options)
      tokens.push(token)
    }
    whiteSpaceCtrl(tokens, options)
    return tokens
  }

  readTopLevelToken (options: NormalizedFullOptions): TopLevelToken {
    const { tagDelimiterLeft, outputDelimiterLeft } = options
    if (this.rawBeginAt > -1) return this.readEndrawOrRawContent(options)
    if (this.match(tagDelimiterLeft)) return this.readTagToken(options)
    if (this.match(outputDelimiterLeft)) return this.readOutputToken(options)
    return this.readHTMLToken([tagDelimiterLeft, outputDelimiterLeft])
  }

  readHTMLToken (stopStrings: string[]): HTMLToken {
    const begin = this.p
    while (this.p < this.N) {
      if (stopStrings.some(str => this.match(str))) break
      ++this.p
    }
    return new HTMLToken(this.input, begin, this.p, this.file)
  }

  readTagToken (options: NormalizedFullOptions = defaultOptions): TagToken {
    const { file, input } = this
    const begin = this.p
    if (this.readToDelimiter(options.tagDelimiterRight) === -1) {
      throw this.mkError(`tag ${this.snapshot(begin)} not closed`, begin)
    }
    const token = new TagToken(input, begin, this.p, options, file)
    if (token.name === 'raw') this.rawBeginAt = begin
    return token
  }

  readToDelimiter (delimiter: string) {
    while (this.p < this.N) {
      if ((this.peekType() & QUOTE)) {
        this.readQuoted()
        continue
      }
      ++this.p
      if (this.rmatch(delimiter)) return this.p
    }
    return -1
  }

  readOutputToken (options: NormalizedFullOptions = defaultOptions): OutputToken {
    const { file, input } = this
    const { outputDelimiterRight } = options
    const begin = this.p
    if (this.readToDelimiter(outputDelimiterRight) === -1) {
      throw this.mkError(`output ${this.snapshot(begin)} not closed`, begin)
    }
    return new OutputToken(input, begin, this.p, options, file)
  }

  readEndrawOrRawContent (options: NormalizedFullOptions): HTMLToken | TagToken {
    const { tagDelimiterLeft, tagDelimiterRight } = options
    const begin = this.p
    let leftPos = this.readTo(tagDelimiterLeft) - tagDelimiterLeft.length
    while (this.p < this.N) {
      if (this.readIdentifier().getText() !== 'endraw') {
        leftPos = this.readTo(tagDelimiterLeft) - tagDelimiterLeft.length
        continue
      }
      while (this.p <= this.N) {
        if (this.rmatch(tagDelimiterRight)) {
          const end = this.p
          if (begin === leftPos) {
            this.rawBeginAt = -1
            return new TagToken(this.input, begin, end, options, this.file)
          } else {
            this.p = leftPos
            return new HTMLToken(this.input, begin, leftPos, this.file)
          }
        }
        if (this.rmatch(tagDelimiterLeft)) break
        this.p++
      }
    }
    throw this.mkError(`raw ${this.snapshot(this.rawBeginAt)} not closed`, begin)
  }

  readLiquidTagTokens (options: NormalizedFullOptions = defaultOptions): LiquidTagToken[] {
    const tokens: LiquidTagToken[] = []
    while (this.p < this.N) {
      const token = this.readLiquidTagToken(options)
      if (token.name) tokens.push(token)
    }
    return tokens
  }

  readLiquidTagToken (options: NormalizedFullOptions): LiquidTagToken {
    const { file, input } = this
    const begin = this.p
    let end = this.N
    if (this.readToDelimiter('\n') !== -1) end = this.p
    return new LiquidTagToken(input, begin, end, options, file)
  }

  mkError (msg: string, begin: number) {
    return new TokenizationError(msg, new IdentifierToken(this.input, begin, this.N, this.file))
  }

  snapshot (begin: number = this.p) {
    return JSON.stringify(ellipsis(this.input.slice(begin), 16))
  }

  /**
   * @deprecated use #readIdentifier instead
   */
  readWord () {
    return this.readIdentifier()
  }

  readIdentifier (): IdentifierToken {
    this.skipBlank()
    const begin = this.p
    while (this.peekType() & IDENTIFIER) ++this.p
    return new IdentifierToken(this.input, begin, this.p, this.file)
  }

  readTagName (): string {
    this.skipBlank()
    // Handle inline comment tags
    if (this.input[this.p] === '#') return this.input.slice(this.p, ++this.p)
    return this.readIdentifier().getText()
  }

  readHashes (jekyllStyle?: boolean) {
    const hashes = []
    while (true) {
      const hash = this.readHash(jekyllStyle)
      if (!hash) return hashes
      hashes.push(hash)
    }
  }

  readHash (jekyllStyle?: boolean): HashToken | undefined {
    this.skipBlank()
    if (this.peek() === ',') ++this.p
    const begin = this.p
    const name = this.readIdentifier()
    if (!name.size()) return
    let value

    this.skipBlank()
    const sep = jekyllStyle ? '=' : ':'
    if (this.peek() === sep) {
      ++this.p
      value = this.readValue()
    }
    return new HashToken(this.input, begin, this.p, name, value, this.file)
  }

  remaining () {
    return this.input.slice(this.p)
  }

  advance (i = 1) {
    this.p += i
  }

  end () {
    return this.p >= this.N
  }

  readTo (end: string): number {
    while (this.p < this.N) {
      ++this.p
      if (this.rmatch(end)) return this.p
    }
    return -1
  }

  readValue (): ValueToken | undefined {
    const value = this.readQuoted() || this.readRange()
    if (value) return value

    if (this.peek() === '[') {
      this.p++
      const prop = this.readQuoted()
      if (!prop) return
      if (this.peek() !== ']') return
      this.p++
      return new PropertyAccessToken(prop, [], this.p)
    }

    const variable = this.readIdentifier()
    if (!variable.size()) return

    let isNumber = variable.isNumber(true)
    const props: (QuotedToken | IdentifierToken)[] = []
    while (true) {
      if (this.peek() === '[') {
        isNumber = false
        this.p++
        const prop = this.readValue() || new IdentifierToken(this.input, this.p, this.p, this.file)
        this.readTo(']')
        props.push(prop)
      } else if (this.peek() === '.' && this.peek(1) !== '.') { // skip range syntax
        this.p++
        const prop = this.readIdentifier()
        if (!prop.size()) break
        if (!prop.isNumber()) isNumber = false
        props.push(prop)
      } else break
    }
    if (!props.length && literalValues.hasOwnProperty(variable.content)) {
      return new LiteralToken(this.input, variable.begin, variable.end, this.file)
    }
    if (isNumber) return new NumberToken(variable, props[0] as IdentifierToken)
    return new PropertyAccessToken(variable, props, this.p)
  }

  readRange (): RangeToken | undefined {
    this.skipBlank()
    const begin = this.p
    if (this.peek() !== '(') return
    ++this.p
    const lhs = this.readValueOrThrow()
    this.p += 2
    const rhs = this.readValueOrThrow()
    ++this.p
    return new RangeToken(this.input, begin, this.p, lhs, rhs, this.file)
  }

  readValueOrThrow (): ValueToken {
    const value = this.readValue()
    assert(value, () => `unexpected token ${this.snapshot()}, value expected`)
    return value!
  }

  readQuoted (): QuotedToken | undefined {
    this.skipBlank()
    const begin = this.p
    if (!(this.peekType() & QUOTE)) return
    ++this.p
    let escaped = false
    while (this.p < this.N) {
      ++this.p
      if (this.input[this.p - 1] === this.input[begin] && !escaped) break
      if (escaped) escaped = false
      else if (this.input[this.p - 1] === '\\') escaped = true
    }
    return new QuotedToken(this.input, begin, this.p, this.file)
  }

  * readFileNameTemplate (options: NormalizedFullOptions): IterableIterator<TopLevelToken> {
    const { outputDelimiterLeft } = options
    const htmlStopStrings = [',', ' ', outputDelimiterLeft]
    const htmlStopStringSet = new Set(htmlStopStrings)
    // break on ',' and ' ', outputDelimiterLeft only stops HTML token
    while (this.p < this.N && !htmlStopStringSet.has(this.peek())) {
      yield this.match(outputDelimiterLeft)
        ? this.readOutputToken(options)
        : this.readHTMLToken(htmlStopStrings)
    }
  }

  match (word: string) {
    for (let i = 0; i < word.length; i++) {
      if (word[i] !== this.input[this.p + i]) return false
    }
    return true
  }

  rmatch (pattern: string) {
    for (let i = 0; i < pattern.length; i++) {
      if (pattern[pattern.length - 1 - i] !== this.input[this.p - 1 - i]) return false
    }
    return true
  }

  peekType (n = 0) {
    return TYPES[this.input.charCodeAt(this.p + n)]
  }

  peek (n = 0) {
    return this.input[this.p + n]
  }

  skipBlank () {
    while (this.peekType() & BLANK) ++this.p
  }
}