Unify how to reference generators, models, and transformers from the …

…command line

The patch unifies how generators, models and transformers are referenced when using
the command line tools. Until now, generators were referenced using file names of
the implementing modules, while models and transformers were referenced in
package.module.{class|function} format. From now on, generators are also referenced
using the latter approach.

Furthermore, the patch fixes a typo in HTMLCustomGenerator.

README.rst

@@ -122,8 +122,8 @@ a try, run the processor first::
 
 Then, use the generator to produce test cases::
 
-    grammarinator-generate examples/fuzzer/HTMLCustomGenerator.py -r htmlDocument \
-    -o examples/tests/test_%d.html -t HTMLGenerator.html_space_transformer -n 100 -d 20
+    grammarinator-generate HTMLCustomGenerator.HTMLCustomGenerator -r htmlDocument \
+    -o examples/tests/test_%d.html -t HTMLGenerator.html_space_transformer -n 100 -d 20 --sys-path examples/fuzzer/
 
 .. _example: examples/
 

examples/fuzzer/HTMLCustomGenerator.py

@@ -30,7 +30,7 @@ class HTMLCustomGenerator(HTMLGenerator):
     # Customize the function generated from the htmlTagName parser rule to produce valid tag names.
     def htmlTagName(self):
         current = self.create_node(UnparserRule(name='htmlTagName'))
-        name = random.choice(tags[self.tag_stack[-1]]['children'] or self.tag_names if self.tag_stack else self.tag_names)
+        name = random.choice(tags[self.tag_stack[-1]]['children'] or tag_names if self.tag_stack else tag_names)
         self.tag_stack.append(name)
         current += UnlexerRule(src=name)
         self.tag_stack.append(name)

examples/grammars/HTMLParser.g4

@@ -27,8 +27,8 @@
 */
 
 // TEST-PROCESS: {grammar}Parser.g4 {grammar}Lexer.g4 -o {tmpdir}
-// TEST-GENERATE: {grammar}Generator -r htmlDocument -t HTMLGenerator.html_space_transformer -n 5 -o {tmpdir}/{grammar}G%d.html
-// TEST-GENERATE: ../fuzzer/{grammar}CustomGenerator.py -r htmlDocument -t HTMLGenerator.html_space_transformer -n 5 -o {tmpdir}/{grammar}C%d.html
+// TEST-GENERATE: {grammar}Generator.{grammar}Generator -r htmlDocument -t {grammar}Generator.html_space_transformer -n 5 -o {tmpdir}/{grammar}G%d.html
+// TEST-GENERATE: {grammar}CustomGenerator.{grammar}CustomGenerator -r htmlDocument -t {grammar}Generator.html_space_transformer -n 5 -o {tmpdir}/{grammar}C%d.html --sys-path ../fuzzer/
 
 parser grammar HTMLParser;
 

grammarinator/cli.py

@@ -57,3 +57,14 @@ def add_jobs_argument(parser):
 def add_disable_cleanup_argument(parser):
     parser.add_argument('--disable-cleanup', dest='cleanup', default=True, action='store_false',
                         help='disable the removal of intermediate files.')
+
+
+def add_sys_path_argument(parser):
+    parser.add_argument('--sys-path', metavar='DIR', action='append', default=[],
+                        help='add directory to the search path for Python modules (may be specified multiple times)')
+
+
+def process_sys_path_argument(args):
+    for path in args.sys_path:
+        if path not in sys.path:
+            sys.path.append(path)

grammarinator/generate.py

@@ -11,15 +11,14 @@
 import json
 import os
 import random
-import sys
 
 from argparse import ArgumentParser, ArgumentTypeError
 from math import inf
 from multiprocessing import Pool
 from os.path import abspath, basename, dirname, isdir, join, splitext
 from shutil import rmtree
 
-from .cli import add_jobs_argument, add_log_level_argument, add_sys_recursion_limit_argument, add_version_argument, logger, process_log_level_argument, process_sys_recursion_limit_argument
+from .cli import add_jobs_argument, add_log_level_argument, add_sys_path_argument, add_sys_recursion_limit_argument, add_version_argument, logger, process_log_level_argument, process_sys_path_argument, process_sys_recursion_limit_argument
 from .model import CooldownModel
 from .runtime import Tree
 
@@ -45,24 +44,22 @@ def size(self):
 
 class Generator(object):
 
-    def __init__(self, generator_path, rule, out_format,
+    def __init__(self, generator, rule, out_format,
                  model=None, max_depth=inf, cooldown=1.0,
                  population=None, generate=True, mutate=True, recombine=True, keep_trees=False,
                  tree_transformers=None, test_transformers=None,
                  cleanup=True, encoding='utf-8'):
 
         def import_entity(name):
+            if not name:
+                return None
             steps = name.split('.')
             return getattr(importlib.import_module('.'.join(steps[0:-1])), steps[-1])
 
         def get_boolean(value):
             return value in ['True', True, 1]
 
-        if dirname(generator_path) not in sys.path:
-            sys.path.append(dirname(generator_path))
-
-        generator = splitext(basename(generator_path))[0]
-        self.generator_cls = import_entity('.'.join([generator, generator]))
+        self.generator_cls = import_entity(generator)
         self.model_cls = import_entity(model or 'grammarinator.model.DefaultModel')
         self.rule = rule or self.generator_cls.default_rule.__name__
 
@@ -219,7 +216,7 @@ def restricted_float(value):
         """)
     # Settings for generating from grammar.
     parser.add_argument('generator', metavar='FILE',
-                        help='generator created by grammarinator-process.')
+                        help='reference to the generator created by grammarinator-process  (in package.module.class format).')
     parser.add_argument('--model', default='grammarinator.model.DefaultModel',
                         help='reference to the decision model (in package.module.class format) (default: %(default)s).')
     parser.add_argument('-r', '--rule', metavar='NAME',
@@ -259,6 +256,7 @@ def restricted_float(value):
     parser.add_argument('--random-seed', type=int, metavar='NUM',
                         help='initialize random number generator with fixed seed (not set by default; noneffective if parallelization is enabled).')
     add_jobs_argument(parser)
+    add_sys_path_argument(parser)
     add_sys_recursion_limit_argument(parser)
     add_log_level_argument(parser)
     add_version_argument(parser)
@@ -268,14 +266,15 @@ def restricted_float(value):
         random.seed(args.random_seed)
 
     process_log_level_argument(args)
+    process_sys_path_argument(args)
     process_sys_recursion_limit_argument(args)
 
     if args.population:
         if not isdir(args.population):
             parser.error('Population must point to an existing directory.')
         args.population = abspath(args.population)
 
-    with Generator(generator_path=args.generator, rule=args.rule, out_format=args.out,
+    with Generator(generator=args.generator, rule=args.rule, out_format=args.out,
                    model=args.model, max_depth=args.max_depth, cooldown=args.cooldown,
                    population=args.population, generate=args.generate, mutate=args.mutate, recombine=args.recombine, keep_trees=args.keep_trees,
                    tree_transformers=args.tree_transformers, test_transformers=args.test_transformers,

grammarinator/parse.py

@@ -17,7 +17,7 @@
 
 from antlr4 import CommonTokenStream, error, FileStream, ParserRuleContext, TerminalNode, Token
 
-from .cli import add_antlr_argument, add_disable_cleanup_argument, add_jobs_argument, add_log_level_argument, add_sys_recursion_limit_argument, add_version_argument, logger, process_antlr_argument, process_log_level_argument, process_sys_recursion_limit_argument
+from .cli import add_antlr_argument, add_disable_cleanup_argument, add_jobs_argument, add_log_level_argument, add_sys_path_argument, add_sys_recursion_limit_argument, add_version_argument, logger, process_antlr_argument, process_log_level_argument, process_sys_path_argument, process_sys_recursion_limit_argument
 from .parser_builder import build_grammars
 from .pkgdata import default_antlr_path
 from .runtime import Tree, UnlexerRule, UnparserRule
@@ -173,6 +173,7 @@ def execute():
     add_disable_cleanup_argument(parser)
     add_jobs_argument(parser)
     add_antlr_argument(parser)
+    add_sys_path_argument(parser)
     add_sys_recursion_limit_argument(parser)
     add_log_level_argument(parser)
     add_version_argument(parser)
@@ -186,6 +187,7 @@ def execute():
         args.parser_dir = join(args.out, 'grammars')
 
     process_log_level_argument(args)
+    process_sys_path_argument(args)
     process_sys_recursion_limit_argument(args)
     process_antlr_argument(args)
 

tests/grammars/Alternatives.g4

@@ -17,8 +17,8 @@
  */
 
 // TEST-PROCESS: {grammar}.g4 -o {tmpdir}
-// TEST-GENERATE: {grammar}Generator -r start -j 1 -n 5 -o {tmpdir}/{grammar}S%d.txt -c 0.9
-// TEST-GENERATE: {grammar}Generator -r start -j 2 -n 5 -o {tmpdir}/{grammar}M%d.txt -c 0.9
+// TEST-GENERATE: {grammar}Generator.{grammar}Generator -r start -j 1 -n 5 -o {tmpdir}/{grammar}S%d.txt -c 0.9
+// TEST-GENERATE: {grammar}Generator.{grammar}Generator -r start -j 2 -n 5 -o {tmpdir}/{grammar}M%d.txt -c 0.9
 // TEST-ANTLR: {grammar}.g4 -o {tmpdir}
 // TEST-PARSE: -p {grammar}Parser -l {grammar}Lexer -r start {tmpdir}/{grammar}S%d.txt
 // TEST-PARSE: -p {grammar}Parser -l {grammar}Lexer -r start {tmpdir}/{grammar}M%d.txt

tests/grammars/Curly.g4

@@ -13,7 +13,7 @@
  */
 
 // TEST-PROCESS: {grammar}.g4 -o {tmpdir}
-// TEST-GENERATE: {grammar}Generator -r start -o {tmpdir}/{grammar}%d.txt
+// TEST-GENERATE: {grammar}Generator.{grammar}Generator -r start -o {tmpdir}/{grammar}%d.txt
 // TEST-ANTLR: {grammar}.g4 -o {tmpdir}
 
 grammar Curly;

tests/grammars/Custom.g4

@@ -25,10 +25,10 @@
  */
 
 // TEST-PROCESS: {grammar}.g4 -o {tmpdir}
-// TEST-GENERATE: {grammar}Generator -r start -j 1 -n 5 -o {tmpdir}/{grammar}GS%d.txt
-// TEST-GENERATE: {grammar}Generator -r start -j 2 -n 5 -o {tmpdir}/{grammar}GM%d.txt
-// TEST-GENERATE: {grammar}SubclassGenerator -r start -j 1 -n 5 -o {tmpdir}/{grammar}SS%d.txt
-// TEST-GENERATE: {grammar}SubclassGenerator -r start -j 2 -n 5 -o {tmpdir}/{grammar}SM%d.txt
+// TEST-GENERATE: {grammar}Generator.{grammar}Generator -r start -j 1 -n 5 -o {tmpdir}/{grammar}GS%d.txt
+// TEST-GENERATE: {grammar}Generator.{grammar}Generator -r start -j 2 -n 5 -o {tmpdir}/{grammar}GM%d.txt
+// TEST-GENERATE: {grammar}SubclassGenerator.{grammar}SubclassGenerator -r start -j 1 -n 5 -o {tmpdir}/{grammar}SS%d.txt
+// TEST-GENERATE: {grammar}SubclassGenerator.{grammar}SubclassGenerator -r start -j 2 -n 5 -o {tmpdir}/{grammar}SM%d.txt
 
 grammar Custom;
 

tests/grammars/DefaultRule.g4

@@ -12,7 +12,7 @@
  */
 
 // TEST-PROCESS: {grammar}.g4 -o {tmpdir}
-// TEST-GENERATE: {grammar}Generator -o {tmpdir}/{grammar}%d.txt
+// TEST-GENERATE: {grammar}Generator.{grammar}Generator -o {tmpdir}/{grammar}%d.txt
 // TEST-ANTLR: {grammar}.g4 -o {tmpdir}
 // TEST-PARSE: -p {grammar}Parser -l {grammar}Lexer -r start {tmpdir}/{grammar}%d.txt
 

tests/grammars/Eof.g4

@@ -13,7 +13,7 @@
  */
 
 // TEST-PROCESS: {grammar}.g4 -o {tmpdir}
-// TEST-GENERATE: {grammar}Generator -r start -o {tmpdir}/{grammar}%d.txt
+// TEST-GENERATE: {grammar}Generator.{grammar}Generator -r start -o {tmpdir}/{grammar}%d.txt
 // TEST-ANTLR: {grammar}.g4 -o {tmpdir}
 // TEST-PARSE: -p {grammar}Parser -l {grammar}Lexer -r start {tmpdir}/{grammar}%d.txt
 

tests/grammars/Hello.g4

@@ -15,7 +15,7 @@
  */
 
 // TEST-PROCESS: {grammar}.g4 -o {tmpdir} --pep8
-// TEST-GENERATE: {grammar}Generator -r start -o {tmpdir}/{grammar}%d.txt
+// TEST-GENERATE: {grammar}Generator.{grammar}Generator -r start -o {tmpdir}/{grammar}%d.txt
 // TEST-ANTLR: {grammar}.g4 -o {tmpdir}
 // TEST-PARSE: -p {grammar}Parser -l {grammar}Lexer -r start {tmpdir}/{grammar}%d.txt
 

tests/grammars/ImagToken.g4

@@ -12,7 +12,7 @@
  */
 
 // TEST-PROCESS: {grammar}.g4 -o {tmpdir}
-// TEST-GENERATE: {grammar}Generator -r start -o {tmpdir}/{grammar}%d.txt
+// TEST-GENERATE: {grammar}Generator.{grammar}Generator -r start -o {tmpdir}/{grammar}%d.txt
 // TEST-ANTLR: {grammar}.g4 -o {tmpdir}
 
 grammar ImagToken;

tests/grammars/Importer.g4

@@ -14,7 +14,7 @@
  */
 
 // TEST-PROCESS: {grammar}.g4 -o {tmpdir} --lib import
-// TEST-GENERATE: {grammar}Generator -r start -o {tmpdir}/{grammar}%d.txt
+// TEST-GENERATE: {grammar}Generator.{grammar}Generator -r start -o {tmpdir}/{grammar}%d.txt
 // TEST-ANTLR: {grammar}.g4 -o {tmpdir} -lib import
 // TEST-PARSE: -p {grammar}Parser -l {grammar}Lexer -r start {tmpdir}/{grammar}%d.txt
 

tests/grammars/LabeledAlternatives.g4

@@ -17,8 +17,8 @@
  */
 
 // TEST-PROCESS: {grammar}.g4 -o {tmpdir}
-// TEST-GENERATE: {grammar}Generator -r start -j 1 -n 5 -o {tmpdir}/{grammar}S%d.txt
-// TEST-GENERATE: {grammar}Generator -r start -j 2 -n 5 -o {tmpdir}/{grammar}M%d.txt
+// TEST-GENERATE: {grammar}Generator.{grammar}Generator -r start -j 1 -n 5 -o {tmpdir}/{grammar}S%d.txt
+// TEST-GENERATE: {grammar}Generator.{grammar}Generator -r start -j 2 -n 5 -o {tmpdir}/{grammar}M%d.txt
 // TEST-ANTLR: {grammar}.g4 -o {tmpdir}
 // TEST-PARSE: -p {grammar}Parser -l {grammar}Lexer -r start {tmpdir}/{grammar}S%d.txt
 // TEST-PARSE: -p {grammar}Parser -l {grammar}Lexer -r start {tmpdir}/{grammar}M%d.txt

tests/grammars/NoAction.g4

@@ -19,7 +19,7 @@
  */
 
 // TEST-PROCESS: {grammar}.g4 -o {tmpdir} --no-action
-// TEST-GENERATE: {grammar}Generator -r start -o {tmpdir}/{grammar}%d.txt
+// TEST-GENERATE: {grammar}Generator.{grammar}Generator -r start -o {tmpdir}/{grammar}%d.txt
 // TEST-ANTLR: {grammar}.g4 -o {tmpdir}
 
 grammar NoAction;

tests/grammars/Quantifiers.g4

@@ -16,8 +16,8 @@
  */
 
 // TEST-PROCESS: {grammar}.g4 -o {tmpdir}
-// TEST-GENERATE: {grammar}Generator -r start -j 1 -n 5 -o {tmpdir}/{grammar}S%d.txt
-// TEST-GENERATE: {grammar}Generator -r start -j 2 -n 5 -o {tmpdir}/{grammar}M%d.txt
+// TEST-GENERATE: {grammar}Generator.{grammar}Generator -r start -j 1 -n 5 -o {tmpdir}/{grammar}S%d.txt
+// TEST-GENERATE: {grammar}Generator.{grammar}Generator -r start -j 2 -n 5 -o {tmpdir}/{grammar}M%d.txt
 // TEST-ANTLR: {grammar}.g4 -o {tmpdir}
 // TEST-PARSE: -p {grammar}Parser -l {grammar}Lexer -r start {tmpdir}/{grammar}S%d.txt
 // TEST-PARSE: -p {grammar}Parser -l {grammar}Lexer -r start {tmpdir}/{grammar}M%d.txt

tests/grammars/Recursive.g4

@@ -20,8 +20,8 @@
  */
 
 // TEST-PROCESS: {grammar}.g4 -o {tmpdir}
-// TEST-GENERATE: {grammar}Generator -r start -j 1 -n 5 -d 5 -o {tmpdir}/{grammar}S%d.txt
-// TEST-GENERATE: {grammar}Generator -r start -j 2 -n 5 -d 5 -o {tmpdir}/{grammar}M%d.txt
+// TEST-GENERATE: {grammar}Generator.{grammar}Generator -r start -j 1 -n 5 -d 5 -o {tmpdir}/{grammar}S%d.txt
+// TEST-GENERATE: {grammar}Generator.{grammar}Generator -r start -j 2 -n 5 -d 5 -o {tmpdir}/{grammar}M%d.txt
 // TEST-ANTLR: {grammar}.g4 -o {tmpdir}
 // TEST-PARSE: -p {grammar}Parser -l {grammar}Lexer -r start {tmpdir}/{grammar}S%d.txt
 // TEST-PARSE: -p {grammar}Parser -l {grammar}Lexer -r start {tmpdir}/{grammar}M%d.txt

tests/grammars/Semantic.g4

@@ -17,8 +17,8 @@
  */
 
 // TEST-PROCESS: {grammar}.g4 -o {tmpdir}
-// TEST-GENERATE: {grammar}Generator -r start -j 1 -n 5 -o {tmpdir}/{grammar}S%d.txt
-// TEST-GENERATE: {grammar}Generator -r start -j 2 -n 5 -o {tmpdir}/{grammar}M%d.txt
+// TEST-GENERATE: {grammar}Generator.{grammar}Generator -r start -j 1 -n 5 -o {tmpdir}/{grammar}S%d.txt
+// TEST-GENERATE: {grammar}Generator.{grammar}Generator -r start -j 2 -n 5 -o {tmpdir}/{grammar}M%d.txt
 // TEST-ANTLR: {grammar}.g4 -o {tmpdir}
 
 grammar Semantic;

tests/grammars/SeparateParser.g4

@@ -13,7 +13,7 @@
  */
 
 // TEST-PROCESS: {grammar}Parser.g4 {grammar}Lexer.g4 -o {tmpdir}
-// TEST-GENERATE: {grammar}Generator -r start -o {tmpdir}/{grammar}%d.txt
+// TEST-GENERATE: {grammar}Generator.{grammar}Generator -r start -o {tmpdir}/{grammar}%d.txt
 // TEST-ANTLR: {grammar}Parser.g4 {grammar}Lexer.g4 -o {tmpdir}
 // TEST-PARSE: -p {grammar}Parser -l {grammar}Lexer -r start {tmpdir}/{grammar}%d.txt
 

tests/grammars/SuperClass.g4

@@ -16,8 +16,8 @@
  */
 
 // TEST-PROCESS: {grammar}.g4 -o {tmpdir}
-// TEST-GENERATE: {grammar}Generator -r start -j 1 -n 5 -o {tmpdir}/{grammar}S%d.txt
-// TEST-GENERATE: {grammar}Generator -r start -j 2 -n 5 -o {tmpdir}/{grammar}M%d.txt
+// TEST-GENERATE: {grammar}Generator.{grammar}Generator -r start -j 1 -n 5 -o {tmpdir}/{grammar}S%d.txt
+// TEST-GENERATE: {grammar}Generator.{grammar}Generator -r start -j 2 -n 5 -o {tmpdir}/{grammar}M%d.txt
 // TEST-ANTLR: {grammar}.g4 -o {tmpdir}
 
 grammar SuperClass;

tests/grammars/Whitespace.g4

@@ -15,7 +15,7 @@
  */
 
 // TEST-PROCESS: {grammar}.g4 -o {tmpdir}
-// TEST-GENERATE: {grammar}Generator -r start -t grammarinator.runtime.simple_space_transformer -o {tmpdir}/{grammar}%d.txt
+// TEST-GENERATE: {grammar}Generator.{grammar}Generator -r start -t grammarinator.runtime.simple_space_transformer -o {tmpdir}/{grammar}%d.txt
 // TEST-ANTLR: {grammar}.g4 -o {tmpdir}
 // TEST-PARSE: -p {grammar}Parser -l {grammar}Lexer -r start {tmpdir}/{grammar}%d.txt