Prepare E2E and Unit tests for lexer

lib/analyzeDefinition.js

@@ -10,6 +10,16 @@ const findRegexpBeginning = require('find-regexp-beginning')
  * @returns {{ uid: string, generic: boolean, type: string, strategy: string, characters: null|string[], strategy: string, [value]: string, [regex]: RegExp, [valid]: RegExp|null, [indices]: object }}
  */
 function analyzeDefinition (definition, uid) {
+  // Check if definition is an object
+  if (!definition || typeof definition !== 'object') {
+    throw new Error('Missing definition')
+  }
+
+  // Check if definition has UID
+  if (uid === void 0) {
+    throw new Error('Missing UID.')
+  }
+
   // Check if definition has a type
   if (!definition.type) {
     throw new Error('Missing definition type.')

lib/buildCharactersCondition.js

@@ -6,7 +6,7 @@
  */
 function buildCharactersCondition (characters) {
   // When there is no characters, it's always true
-  if (characters.length === 0) {
+  if (!characters || characters.length === 0) {
     return 'true'
   }
 

lib/buildNamedRegularExpression.js

@@ -1,65 +1,5 @@
 const namedRegexp = require('named-js-regexp')
 
-/**
- * Wrapper for regular expression
- * with named properties
- */
-class NamedRegExp {
-  /**
-   * @param {RegExp} regex
-   * @param {object} indices
-   */
-  constructor (regex, indices) {
-    this.regex = regex
-    this.indices = indices
-  }
-
-  /**
-   * Execute regular expression
-   *
-   * @param {string} str
-   * @returns {Array|{index:number, input:string}}
-   */
-  exec (str) {
-    // Execute regular expression
-    const v = this.regex.exec(str)
-
-    // When not passed, just return null
-    if (v === null) {
-      return null
-    }
-
-    // Build object for groups
-    v.groups = {}
-
-    // Add any groups are specified
-    for (let key in this.indices) {
-      v.groups[key] = v[this.indices[key]]
-    }
-
-    return v
-  }
-
-  /**
-   * Test regular expression against text
-   *
-   * @param {string} str
-   * @returns {boolean}
-   */
-  test (str) {
-    return this.regex.test(str)
-  }
-
-  /**
-   * Parse regular expression to string
-   *
-   * @returns {string}
-   */
-  toString () {
-    return this.regex.toString()
-  }
-}
-
 /**
  * As `named-js-regexp` library is changing RegExp object,
  * it's much slower than regular expressions.
@@ -85,7 +25,10 @@ function buildNamedRegularExpression (text, flags) {
   const regex = eval(enhancedRegex.toString())
 
   // Build object which simulates regular expressions
-  return new NamedRegExp(regex, indices)
+  return {
+    regex: regex,
+    indices: indices
+  }
 }
 
 module.exports = buildNamedRegularExpression

lib/buildTokenDefinitionCode.js

@@ -28,7 +28,7 @@ function buildRegexTokenDefinitionCode (definition) {
 function buildCharacterDefinitionCode (definition) {
   return `{
     type: ${JSON.stringify(definition.type)},
-    data: ${buildDataDefinitionCode(definition, null, 'chr')},
+    data: ${buildDataDefinitionCode(definition, null, 'code[0]')},
     start: index,
     end: index + 1
   }`

lib/utils.js

@@ -26,7 +26,7 @@ function beautify (code) {
  * Build internal variable name for specified definition
  *
  * @param {string} name
- * @param {{ uid: string }} definition
+ * @param {{ uid: string|int }} definition
  * @returns {string}
  */
 function buildVariableName (name, definition) {

tests/e2e/compileLexerSpec.js

@@ -0,0 +1,115 @@
+const expect = require('expect.js')
+const compile = require('../../lib/compileLexer')
+
+describe('E2E: Compile lexer', () => {
+  it('should correctly parse with simple definition', () => {
+    const tokenize = compile([
+      { type: 'WS', value: ' ' }
+    ])
+
+    expect(tokenize('   ')).to.eql({
+      tokens: [
+        { type: 'WS', data: { value: ' ' }, start: 0, end: 1 },
+        { type: 'WS', data: { value: ' ' }, start: 1, end: 2 },
+        { type: 'WS', data: { value: ' ' }, start: 2, end: 3 }
+      ]
+    })
+  })
+
+  it('should correctly parse with regex definition', () => {
+    const tokenize = compile([
+      { type: 'WS', value: ' ' },
+      { type: 'LT', regex: '[a-z]+', regexFlags: 'i' },
+    ])
+
+    expect(tokenize('  Sth ')).to.eql({
+      tokens: [
+        { type: 'WS', data: { value: ' ' }, start: 0, end: 1 },
+        { type: 'WS', data: { value: ' ' }, start: 1, end: 2 },
+        { type: 'LT', data: { value: 'Sth' }, start: 2, end: 5 },
+        { type: 'WS', data: { value: ' ' }, start: 5, end: 6 }
+      ]
+    })
+  })
+
+  it('should correctly parse with regex (named groups) definition', () => {
+    const tokenize = compile([
+      { type: 'WS', value: ' ' },
+      { type: 'LT', regex: '(?<first>[a-z])(?<later>[a-z]*)', regexFlags: 'i' },
+    ])
+
+    expect(tokenize('  Sth ')).to.eql({
+      tokens: [
+        { type: 'WS', data: { value: ' ' }, start: 0, end: 1 },
+        { type: 'WS', data: { value: ' ' }, start: 1, end: 2 },
+        { type: 'LT', data: { first: 'S', later: 'th' }, start: 2, end: 5 },
+        { type: 'WS', data: { value: ' ' }, start: 5, end: 6 }
+      ]
+    })
+  })
+
+  it('should correctly parse with text definition', () => {
+    const tokenize = compile([
+      { type: 'WS', regex: '[ ]+' },
+      { type: 'FN', value: '@fn' },
+    ])
+
+    expect(tokenize('  @fn ')).to.eql({
+      tokens: [
+        { type: 'WS', data: { value: '  ' }, start: 0, end: 2 },
+        { type: 'FN', data: { value: '@fn' }, start: 2, end: 5 },
+        { type: 'WS', data: { value: ' ' }, start: 5, end: 6 }
+      ]
+    })
+  })
+
+  it('should correctly parse with validated regex definition', () => {
+    const tokenize = compile([
+      { type: 'WS', regex: '[ ]+' },
+      { type: 'FN', regex: '@fn', valid: '@fn( |$)' },
+      { type: 'WORD', regex: '[^ ]*' },
+    ])
+
+    expect(tokenize('  @fnx ')).to.eql({
+      tokens: [
+        { type: 'WS', data: { value: '  ' }, start: 0, end: 2 },
+        { type: 'WORD', data: { value: '@fnx' }, start: 2, end: 6 },
+        { type: 'WS', data: { value: ' ' }, start: 6, end: 7 }
+      ]
+    })
+
+    expect(tokenize('  @fn x')).to.eql({
+      tokens: [
+        { type: 'WS', data: { value: '  ' }, start: 0, end: 2 },
+        { type: 'FN', data: { value: '@fn' }, start: 2, end: 5 },
+        { type: 'WS', data: { value: ' ' }, start: 5, end: 6 },
+        { type: 'WORD', data: { value: 'x' }, start: 6, end: 7 }
+      ]
+    })
+  })
+
+  it('should fail because of validated regex definition', () => {
+    const tokenize = compile([
+      { type: 'WS', regex: '[ ]+' },
+      { type: 'FN', regex: '@fn', valid: '@fn( |$)' }
+    ])
+
+    expect(tokenize('  @fnx ')).to.eql({
+      error: 'Unrecognized token',
+      index: 2,
+      line: 1,
+      column: 3
+    })
+  })
+
+  it('should fail because of no definitions', () => {
+    const tokenize = compile([])
+
+    expect(tokenize('  @fnx ')).to.eql({
+      error: 'Unrecognized token',
+      index: 0,
+      line: 1,
+      column: 1
+    })
+  })
+})