continued development

- update gitignore
- add support for the Plural directive
- implement missing test cases
- minor refactoring
- lexer now detects syntax errors other than non matching production rules

.gitignore

@@ -1,5 +1,5 @@
 build
-lib
+dist
 coverage
 node_modules
 .*.bak

README.md

@@ -3,6 +3,12 @@
 
 # ypo-lexer-ypo
 
+ypo-lexer-ypo is a lexer for the YPO file format that procudes a token stream for consumption by for example the ypo parser.
+The lexer is rather dumb and does not validate the correct order of the input tokens.
+
+You might want to have a look at [ypo-cli](https://github.com/coldrye-es/ypo-cli.git) instead and see how
+it can help you with your translations.
+
 
 ## Releases
 
@@ -25,7 +31,7 @@ See the [contributing guidelines](https://github.com/coldrye-es/ypo-lexer-ypo/bl
 
 ### Contributors
 
- - [Carsten Klein](https://github.com/silkentrance) **Maintainer**
+See [contributors](https://github.com/coldrye-es/ypo-lexer-ypo/graphs/contributors) for more information.
 
 
 ### Building

package.json

@@ -49,23 +49,24 @@
     "babel-preset-es2015": ">=6.3.x",
     "babel-preset-stage-0": ">=6.3.x",
     "babel-runtime": ">=6.2.4",
-    "esaver": ">=0.0.4",
-    "esdoc-es7-plugin": ">=0.0.3",
-    "esdoc-importpath-plugin": ">=0.0.1",
+    "chai": ">=3.5.0",
+    "esdoc-es7-plugin": ">=0.0",
+    "esdoc-importpath-plugin": ">=0.0",
     "esmake": "coldrye-es/esmake",
     "glob": ">=6.0.1",
-    "ypo-parser-common": ">=0.0.4"
-  },
+    "sinon": ">=1.17.3",
+    "sinon-chai": ">=2.8.0",
+    "ypo-parser-common": ">=0.0"
+  },  
   "globalDevDependencies": {
     "babel-cli": ">=6.3.x",
     "babel-core": ">=6.3.x",
     "babel-eslint": ">=4.1.1",
     "babel-istanbul": ">=0.3.20",
-    "babel-traverse": ">=6.3.19",
-    "conventional-changelog": ">=1.0.1",
+    "conventional-changelog-cli": ">=1.1.1",
     "esdoc": ">=0.3.1",
-    "eslint": ">=1.3.1",
-    "eslint-config-coldrye": ">=0.0.9",
+    "eslint": "<2.3.0",
+    "eslint-config-coldrye": ">=0.0",
     "mocha": ">=2.2.5"
   }
 }

src/lexer.es

@@ -16,53 +16,67 @@
  */
 
 
-import AbstractLexer from 'ypo-parser-common/lexer';
-import {AbstractOption} from 'ypo-parser-common/option';
-import Authorship from 'ypo-parser-common/authorship';
-import Comment from 'ypo-parser-common/comment';
-import EmptyLine from 'ypo-parser-common/emptyline';
+import AbstractTokenizer from 'ypo-parser-common/tokenizer';
+import {AbstractOption} from 'ypo-parser-common/directives/option';
+import Authorship from 'ypo-parser-common/directives/authorship';
+import Comment from 'ypo-parser-common/directives/comment';
+import Context from 'ypo-parser-common/directives/context';
+import Plural from 'ypo-parser-common/directives/plural';
+import TranslationId from 'ypo-parser-common/directives/translationid';
+import EmptyLine from 'ypo-parser-common/text/emptyline';
+import Line from 'ypo-parser-common/text/line';
 import Location from 'ypo-parser-common/location';
 import ParseError from 'ypo-parser-common/exceptions';
-import Text from 'ypo-parser-common/text';
-import TranslationId from 'ypo-parser-common/translationid';
+import {isNonEmptyString} from 'ypo-parser-common/utils';
 
 import
 {
-    EMPTY_LINE, COMMENT, OPTION,
-    AUTHORSHIP, TRANSLATION_ID, TEXT
-} from './constants';
+    RULE_EMPTY_LINE, RULE_COMMENT, RULE_CONTEXT, RULE_PLURAL, RULE_OPTION,
+    RULE_AUTHORSHIP, RULE_TRANSLATION_ID, RULE_TEXT
+} from './rules';
 
 
 /**
  * @private
  */
-export const TOKENS = [
-    {rule:EMPTY_LINE, factory:EmptyLine.createNode},
-    {rule:COMMENT, factory:Comment.createNode},
-    {rule:OPTION, factory:AbstractOption.createNode},
-    {rule:AUTHORSHIP, factory:Authorship.createNode},
-    {rule:TRANSLATION_ID, factory:TranslationId.createNode},
-    {rule:TEXT, factory:Text.createNode}
+export const PRODUCTIONS = [
+    {rule:RULE_EMPTY_LINE, klass:EmptyLine},
+    {rule:RULE_COMMENT, klass:Comment},
+    {rule:RULE_CONTEXT, klass:Context},
+    {rule:RULE_PLURAL, klass:Plural},
+    {rule:RULE_OPTION, factory:AbstractOption.createNewInstance},
+    {rule:RULE_AUTHORSHIP, klass:Authorship},
+    {rule:RULE_TRANSLATION_ID, klass:TranslationId},
+    {rule:RULE_TEXT, klass:Line}
 ]
 
 
 /**
- * The class Lexer models a parser that produces a stream of tokens from a
+ * The class YpoLexer models a parser that produces a stream of tokens from a
  * stream of input lines.
+ *
+ * Note that the lexer does not make any assumptions on the correct order of
+ * the tokens. It is up to the parser to validate that order.
  */
-export default class Lexer extends AbstractLexer
+export default class YpoLexer extends AbstractTokenizer
 {
     /**
-     * Generates a stream of tokens from lines read from the input stream.
-     *
-     * @param {string} file - the absolute and resolved file name
-     * @param {Generator|Array<string>|Array<Buffer>} lines - the input line (buffer) stream or array
-     * @throws {ParseError}
-     * @returns {AbstractToken} - the yielded token instances
+     * @override
      */
     * tokenize(file, lines)
     {
-        assertParams(file, lines)
+        if (!isNonEmptyString(file))
+        {
+            throw new TypeError('file must be a non empty string');
+        }
+
+        if (
+            typeof lines == 'undefined'
+            || !Array.isArray(lines) && typeof lines.next != 'function'
+        )
+        {
+            throw new TypeError('lines must be an iterable');
+        }
 
         let lineno = 1;
         for (let line of lines)
@@ -72,60 +86,80 @@ export default class Lexer extends AbstractLexer
             // make sure that we are dealing with a string object here
             line = line.toString();
 
-            let token, rule, factory, match;
-            for (token of TOKENS)
-            {
-                rule = token.rule;
-                factory = token.factory;
-                match = rule.regex.exec(line);
-
-                if (match)
-                {
-                    break;
-                }
-            }
+            yield this.nextToken(line, location);
 
-            if (!match)
-            {
-                throw new ParseError(
-                    'error parsing file', {location: location}
-                );
-            }
+            lineno++;
+        }
+    }
+
+    nextToken(line, location)
+    {
+        let production, match;
+        for (production of PRODUCTIONS)
+        {
+            match = production.rule.regex.exec(line);
 
-            const params = new Array(rule.groups.length);
-            for (let gidx = 0; gidx < rule.groups.length; gidx++)
+            if (match)
             {
-                params[gidx] = match[1 + gidx];
+                break;
             }
+        }
 
-            yield factory(location, ...params);
+        if (!match)
+        {
+            this.doFail(null, location, line);
+        }
 
-            lineno++;
+        const options = {};
+        for (let gidx = 0; gidx < production.rule.groups.length; gidx++)
+        {
+            options[production.rule.groups[gidx]] = match[1 + gidx];
         }
-    }
-}
 
+        let result;
 
-/**
- * @private
- * @param {string} file - the absolute and resolved file name
- * @param {Generator|Array<string>} lines - the input line (buffer) stream or array
- * @returns {void}
- * @throws TypeError in case of invalid arguments
- */
-function assertParams(file, lines)
-{
-    if (typeof file != 'string' || file.length == 0)
-    {
-        throw new TypeError('MSG_file must be a non empty string');
+        try
+        {
+            if (production.klass)
+            {
+                result = new production.klass(location, options);
+            }
+            else
+            {
+                result = production.factory.apply(null, [location, options]);
+            }
+        }
+        catch (error)
+        {
+            this.doFail(error, location, line, production.klass);
+        }
+
+        return result;
     }
 
-    if (
-        typeof lines == 'undefined'
-        || !Array.isArray(lines) && typeof lines.next != 'function'
-    )
+    doFail(cause, location, line, klass)
     {
-        throw new TypeError('MSG_lines must be an iterable');
+        // handle expected syntax errors
+        if (
+            !klass
+            || klass instanceof AbstractOption
+            || klass instanceof Authorship
+            || klass instanceof Context
+            || klass instanceof Plural
+            || klass instanceof TranslationId
+        )
+        {
+            throw new ParseError(
+                'syntax error', {location, line, cause}
+            );
+        }
+        else
+        {
+            /*istanbul ignore next*/
+            throw new ParseError(
+                'internal error', {location, line, cause}
+            );
+        }
     }
 }