-
Notifications
You must be signed in to change notification settings - Fork 1
/
index.js
90 lines (77 loc) · 2.38 KB
/
index.js
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
// @ts-check
/**
* moo-ignore module.
* @module moo-ignore
* @see module:moo
*/
const moo = require("moo");
/**
* Moo rules as described in https://www.npmjs.com/package/moo
* @typedef {Object} Rules
*/
/**
* Has all the properties and methods of a moo lexer. Additionally has the `ignore` method and the `ignoreSet` property
* @typedef {Function} MooLexer
* @property {Set} moolexer.ignoreSet - Set of tokens to ignore
* @property {Function} moolexer.ignore - Method to add/set tokens to ignore
*/
/**
* A function that provides a nearley.js compatible lexer
* @static
* @param {Array<Rules>} tokens - The Array of Moo rules specifying the lexer
* @param {Array<String>} ignoreTokens - Array of Strings containing the token types to ignore
* @returns {MooLexer} moolexer - The Moo lexer for the specified tokens ignoring tokens in ignoreTokens
*/
function makeLexer(tokens, ignoreTokens, options = {}) {
let lexer;
let oldnext;
let newTokens = {};
lexer = moo.compile(tokens);
oldnext = lexer.next;
/**
* Sets or adds the ignoreSet to the token types specified in the arguments
* @memberof MooLexer
* @param {...String} types
*/
lexer.ignore = function(...types) {
if (this.ignoreSet) { // ignoreSet was built at construction time
this.ignoreSet = new Set([...this.ignoreSet, ...types]); // union
}
else this.ignoreSet = new Set(types); // build it now
}
lexer.next = function () {
while (true) {
let token = oldnext.call(this);
// moo oldnext iterator returns undefined when finished
if (token == undefined || !this.ignoreSet.has(token.type)) {
return token;
}
//console.error("ignoring token "+token.type);
}
};
debugger;
if (options.eof) {
let oldReset = lexer.reset;
lexer.reset = function(input) {
if (tokens.EOF) input += tokens.EOF; //Check that exists
return oldReset.call(this, input)
}
}
if (ignoreTokens) {
lexer.ignoreSet = new Set(ignoreTokens);
}
return lexer;
}
/**
* This module exports an object having the `makeLexer` constructor and the `moo` object (as in `const moo = require("moo")`):
*/
module.exports = {
/**
* The lexer constructor
*/
makeLexer,
/**
* The moo object (as built in `const moo = require("moo")`)
*/
moo
};