diff --git a/moo.js b/moo.js index 9b0251c..e942a1b 100644 --- a/moo.js +++ b/moo.js @@ -271,6 +271,19 @@ this.stack = [] this.setState(state) this.reset() + if (Transform) Transform.call(this, {readableObjectMode: true}) + } + + if (typeof module !== 'undefined' && module.exports) { + var Transform = require('stream').Transform + require('util').inherits(Lexer, Transform) + + Lexer.prototype._transform = function(chunk, encoding, cb) { + this.feed(chunk.toString()) + var token + while (token = this.next()) this.push(token) + cb() + } } Lexer.prototype.setState = function(state) { diff --git a/test/test.js b/test/test.js index 3283785..0d280e9 100644 --- a/test/test.js +++ b/test/test.js @@ -469,6 +469,73 @@ describe('errors', () => { }) +describe('streams', () => { + const lexer = compile({ + word: /[a-z]+/, + space: {match: /\s+/, lineBreaks: true}, + }) + const {Readable, Writable} = require('stream') + + const inputs = ['this is\n', 'a test'] + const tokens = [ + {type: 'word', value: 'this'}, + {type: 'space', value: ' '}, + {type: 'word', value: 'is'}, + {type: 'space', value: '\n'}, + {type: 'word', value: 'a'}, + {type: 'space', value: ' '}, + {type: 'word', value: 'test'}, + ] + + test('can be written and read', () => new Promise((resolve, reject) => { + let index = 0 + expect.assertions(tokens.length) + + const s = lexer.clone() + s.write(inputs[0]) + s.end(inputs[1]) + + s.on('data', tok => { + try { + expect(tok).toMatchObject(tokens[index++]) + } catch (e) {reject(e)} + }) + .on('error', reject) + .on('end', resolve) + })) + + test('can be piped to/from', () => new Promise((resolve, reject) => { + let input = 0 + const rs = new Readable({ + read() { + try { + this.push(input < inputs.length ? + Buffer.from(inputs[input++], 'ascii') : null) + } catch (e) {console.log('read', e) || reject(e)} + } + }) + + let index = 0 + expect.assertions(tokens.length) + const ws = new Writable({ + objectMode: true, + write(tok, _, cb) { + try { + expect(tok).toMatchObject(tokens[index++]) + cb() + } catch (e) {cb(e)} + } + }) + + rs + .on('error', reject).pipe(lexer.clone()) + .on('error', reject).pipe(ws) + .on('error', reject) + .on('finish', resolve) + })) +}) + + describe('example: python', () => { test('kurt tokens', () => {