Permalink
Browse files

feat: add ignoreUnclosed option to nextToken (#1191)

  • Loading branch information...
shellscape authored and ai committed Sep 25, 2018
1 parent c009b4a commit b08719bddcdfe02f67e71833748ee21753938837
Showing with 31 additions and 4 deletions.
  1. +7 −4 lib/tokenize.es6
  2. +24 −0 test/tokenize.test.js
@@ -45,10 +45,13 @@ export default function tokenizer (input, options = {}) {
return returned.length === 0 && pos >= length
}
function nextToken () {
function nextToken (opts = {}) {
if (returned.length) return returned.pop()
if (pos >= length) return
let defaults = { ignoreUnclosed: false }
opts = Object.assign({}, defaults, opts)
code = css.charCodeAt(pos)
if (
code === NEWLINE || code === FEED ||
@@ -109,7 +112,7 @@ export default function tokenizer (input, options = {}) {
escaped = false
next = css.indexOf(')', next + 1)
if (next === -1) {
if (ignore) {
if (ignore || opts.ignoreUnclosed) {
next = pos
break
} else {
@@ -154,7 +157,7 @@ export default function tokenizer (input, options = {}) {
escaped = false
next = css.indexOf(quote, next + 1)
if (next === -1) {
if (ignore) {
if (ignore || opts.ignoreUnclosed) {
next = pos + 1
break
} else {
@@ -247,7 +250,7 @@ export default function tokenizer (input, options = {}) {
if (code === SLASH && css.charCodeAt(pos + 1) === ASTERISK) {
next = css.indexOf('*/', pos + 2) + 1
if (next === 0) {
if (ignore) {
if (ignore || opts.ignoreUnclosed) {
next = css.length
} else {
unclosed('comment')
@@ -269,3 +269,27 @@ it('tokenizes hexadecimal escape', () => {
['space', ' ']
])
})
it('ignore unclosed per token request', () => {
function tokn (css, opts) {
let processor = tokenizer(new Input(css), opts)
let tokens = []
while (!processor.endOfFile()) {
tokens.push(processor.nextToken({ ignoreUnclosed: true }))
}
return tokens
}
let css = `How's it going (`
let tokens = tokn(css, {})
let expected = [['word', 'How', 1, 1, 1, 3],
['string', "'s", 1, 4, 1, 5],
['space', ' '],
['word', 'it', 1, 7, 1, 8],
['space', ' '],
['word', 'going', 1, 10, 1, 14],
['space', ' '],
['(', '(', 1, 16]]
expect(tokens).toEqual(expected)
})

0 comments on commit b08719b

Please sign in to comment.