vsc-tools: refactor python module structure

* Add python setuptools setup.py script.
* Move python code to the vsc folder.
* Move templates folder to the vsc folder.
* Install vscgen tool to simplify codegenerator use via cmd.
* Update github actions (use python 3.10 and vscgen).

Signed-off-by: Mikhail Tsukerman <mikhail.tcukerman@daimler.com>
Signed-off-by: Magnus Feuer <magnus.feuer@mercedes-benz.com>

.github/workflows/buildcheck.yml

@@ -11,7 +11,7 @@ jobs:
       - name: Setup Python
         uses: actions/setup-python@v1
         with:
-          python-version: 3.10.5
+          python-version: 3.10.6
 
       - name: Install dependencies
         run: |
@@ -21,13 +21,16 @@ jobs:
           pip install anytree
           pip install jinja2
 
+      - name: Install vsc module
+        run: |
+          python setup.py develop
+
       - name: Run unit test
         run: |
           cd tests
           pytest -v
-          
+
       - name: Run example tool on latest VSC
         run: |
           git clone https://github.com/GENIVI/vehicle_service_catalog/
-          python model/vsc_generator.py vehicle_service_catalog/comfort-service.yml simple_overview.tpl
-
+          vscgen vehicle_service_catalog/comfort-service.yml simple_overview.tpl

README.md

@@ -33,6 +33,12 @@ target=C:\SomeDir\Where\Your\Account\Can\Write\To
 
 * If you use a custom pip installation directory, set the `PYTHONPATH` environment variable to the directory that you set in the `pip.ini` file.
 
+### Setup with `virtualenv`
+
+```sh
+python3 -m venv venv
+```
+
 ### Setup with `pipenv`
 [pipenv](https://pypi.org/project/pipenv/) is a tool that manages a virtual environment and install the package and its dependencies, making the process much simpler and predictable, since the `Pipfile` states the dependencies, while `Pipfile.lock` freezes the exact version in use.
 
@@ -43,9 +49,9 @@ curl https://pyenv.run | bash  # download and install
 exec $SHELL                    # restart your shell using the new $PATH
 ```
 
-Make sure Python version 3.8.5 is installed:
+Make sure Python version 3.10.6 is installed:
 ```sh
-pyenv install 3.8.5  # install the versions required by Pipfile
+pyenv install 3.10.6  # install the versions required by Pipfile
 ```
 
 Install this project and its dependencies in the local `.venv` folder in this project, then use it (`pipenv shell`):

requirements.txt

@@ -1,4 +1,13 @@
-anytree>=2.8.0
-Jinja2>=3.0.0
-pytest>=2.7.2
-PyYAML>=5.4.1
+anytree==2.8.0
+attrs==22.1.0
+iniconfig==1.1.1
+Jinja2==3.1.2
+MarkupSafe==2.1.1
+packaging==21.3
+pluggy==1.0.0
+py==1.11.0
+pyparsing==3.0.9
+pytest==7.1.2
+PyYAML==6.0
+six==1.16.0
+tomli==2.0.1

setup.py

@@ -0,0 +1,17 @@
+#!/usr/bin/env python
+
+from distutils.core import setup
+from importlib.metadata import entry_points
+
+setup(name='vsc',
+      version='0.1',
+      description='Vehicle service catalog tools',
+      author='',
+      author_email='',
+      url='https://github.com/covesa/vsc-tools',
+      packages=['vsc','tests'],
+      entry_points='''
+            [console_scripts]
+            vscgen=vsc.scripts.generator:vsc_generator_run
+      '''
+      )

vsc/__init__.py

@@ -0,0 +1,3 @@
+# SPDX-FileCopyrightText: Copyright (c) 2022 MBition GmbH.
+# All rights reserved.
+# SPDX-License-Identifier: MPL-2.0

vsc/generators/__init__.py

@@ -0,0 +1,3 @@
+# SPDX-FileCopyrightText: Copyright (c) 2022 MBition GmbH.
+# All rights reserved.
+# SPDX-License-Identifier: MPL-2.0

vsc/model/__init__.py

@@ -0,0 +1,3 @@
+# SPDX-FileCopyrightText: Copyright (c) 2022 MBition GmbH.
+# All rights reserved.
+# SPDX-License-Identifier: MPL-2.0

model/vsc_generator.py → vsc/model/vsc_generator.py

@@ -12,30 +12,30 @@
 
 # This performs the following related functions:
 
-# 1. As input we expect an AST as provided by the parser module
+# 1. As input.yaml we expect an AST as provided by the parser module
 #
 # 2. It uses jinja2 templating library to generate code or configuration
 #    according to given templates.
 
 # It's useful to have these classes in our namespace directly
-from vsc_parser import AST, Argument, Enum, Error, Event, Include, Member, Method, Namespace, Option, Property, Service, Struct, Typedef
+from vsc.model.vsc_parser import AST, Argument, Enum, Error, Event, Include, Member, Method, Namespace, Option, Property, Service, Struct, Typedef
+
+# For other features from parser module
+from vsc.model.vsc_parser import get_ast_from_file
+
+from vsc.templates import TemplatePath
 
-import vsc_parser # For other features from parser module
-import anytree
-import getopt
 import jinja2
-import os
 import sys
 
 # Set up Jinja
 jinja_env = jinja2.Environment(
         # Use the subdirectory 'templates' relative to this file's location
-        loader =
-        jinja2.FileSystemLoader(os.path.dirname(os.path.realpath(__file__)) + '/../templates'),
+        loader=jinja2.FileSystemLoader(TemplatePath),
 
-        # Templates with these extension gets automatic autoescape for HTML
+        # Templates with these extension gets automatic auto escape for HTML
         # It's more annoying for code generation, so passing empty list for now.
-        autoescape = jinja2.select_autoescape([])
+        autoescape=jinja2.select_autoescape([])
         )
 
 # We want the control blocks in the template to NOT result in any extra
@@ -48,19 +48,22 @@
 
 default_templates = {}
 
+
 # Exception:
 class GeneratorError(BaseException):
     def __init__(self, m):
         self.msg = m
 
 # ---------- GENERATION FUNCTIONS ------------
 
+
 # Get template with given name (search path should be handled by the loader)
 def get_template(filename):
     return jinja_env.get_template(filename)
 
 # Frontend to overloaded gen() function:
 
+
 def gen(node : AST, second = None):
     # Processing of lists of objects?
     if type(node) == list or type(node) == tuple:
@@ -168,29 +171,29 @@ def gen_dict_with_template_file(variables : dict, templatefile):
         Typedef=Typedef)
 
 # This code file can be used in two ways.  Either calling this file as a
-# program using the main entry points here, and specifying input parameter.
+# program using the main entry points here, and specifying input.yaml parameter.
 # Alternatively, for more advanced usage, the file might be included as a
 # module in a custom generator implementation.  That implementation is
 # likely to call some of the funcctions that were defined above.
 
 # For the first case, here follows the main entry points and configuration:
 
 def usage():
-    print("usage: generator.py <input-yaml-file (path)> <output-template-file (name only, not path)>")
+    print("usage: generator.py <input.yaml-file (path)> <output-template-file (name only, not path)>")
     sys.exit(1)
 
 # This is a default definition for our current generation tests.
 # It may need to be changed, or defined differently in a custom generator
 default_templates = {
-        'Service' : 'Service-simple_doc.html',
-        'Argument' : 'Argument-simple_doc.html'
+        'Service': 'Service-simple_doc.html',
+        'Argument': 'Argument-simple_doc.html'
         }
 
 # If run as a script, generate a single YAML file using a single root template
 if __name__ == "__main__":
     if not len(sys.argv) == 3:
         usage()
-    ast = vsc_parser.get_ast_from_file(sys.argv[1])
+    ast = get_ast_from_file(sys.argv[1])
     templatename = sys.argv[2]
     print(gen(ast, templatename))
 

model/vsc_parser.py → vsc/model/vsc_parser.py

@@ -69,6 +69,7 @@ class Service(AST): pass
 class Struct(AST): pass
 class Typedef(AST): pass
 
+
 # Some identifiers used to indicate optionality
 MAN = 0 # MAN(datory)
 REC = 1 # REC(ommended)
@@ -243,7 +244,7 @@ def get_recommended_yaml_value(tree, nodename):
 # ----------------------------------------------------------------------------
 
 # Here we construct a graph of nodes using typed classes (AST
-# subclasses) that are also AnyTree Nodes, from an input tree that
+# subclasses) that are also AnyTree Nodes, from an input.yaml tree that
 # represents the raw YAML text but read into a tree in the standard
 # dict/list format from the YAML parser.
 

vsc/scripts/__init__.py

@@ -0,0 +1,3 @@
+# SPDX-FileCopyrightText: Copyright (c) 2022 MBition GmbH.
+# All rights reserved.
+# SPDX-License-Identifier: MPL-2.0

vsc/scripts/generator.py

@@ -0,0 +1,26 @@
+# SPDX-FileCopyrightText: Copyright (c) 2022 MBition GmbH.
+# All rights reserved.
+# SPDX-License-Identifier: MPL-2.0
+
+from vsc.model.vsc_generator import gen
+from vsc.model.vsc_parser import get_ast_from_file
+import argparse
+
+def vsc_generator_run():
+    parser = argparse.ArgumentParser(description='Runs vehicle service catalog code generator.')
+    parser.add_argument('input', metavar='input', type=str,
+                        help='input.yaml-file (path)')
+    parser.add_argument('template', metavar='template', type=str,                        
+                        help='output-template-file (name only, not path)')
+
+    args = parser.parse_args()
+
+    ast = get_ast_from_file(args.input)
+
+    templatename = args.template
+
+    print(gen(ast, templatename))
+
+
+if __name__ == "__main__":
+    vsc_generator_run()

vsc/templates/__init__.py

@@ -0,0 +1,7 @@
+# SPDX-FileCopyrightText: Copyright (c) 2022 MBition GmbH.
+# All rights reserved.
+# SPDX-License-Identifier: MPL-2.0
+
+import os
+
+TemplatePath = os.path.dirname(os.path.realpath(__file__))