From 3b004dfeb3d3c3a9a2b683974aaa90be2e1be61b Mon Sep 17 00:00:00 2001 From: Michael Tsukerman Date: Thu, 1 Sep 2022 15:19:38 +0200 Subject: [PATCH] vsc-tools: refactor python module structure * Add python setuptools setup.py script. * Move python code to the vsc folder. * Move templates folder to the vsc folder. * Install vscgen tool to simplify codegenerator use via cmd. * Update github actions (use python 3.10 and vscgen). Signed-off-by: Mikhail Tsukerman Signed-off-by: Magnus Feuer --- .github/workflows/buildcheck.yml | 11 ++++--- README.md | 10 ++++-- requirements.txt | 17 +++++++--- setup.py | 17 ++++++++++ templates/.gitkeep | 0 vsc/__init__.py | 3 ++ {generators => vsc/generators}/.gitkeep | 0 vsc/generators/__init__.py | 3 ++ vsc/model/__init__.py | 3 ++ {model => vsc/model}/vsc_generator.py | 33 ++++++++++--------- {model => vsc/model}/vsc_parser.py | 3 +- vsc/scripts/__init__.py | 3 ++ vsc/scripts/generator.py | 26 +++++++++++++++ .../templates}/AST-simple_doc.tpl | 0 .../templates}/Argument-simple_doc.html | 0 .../templates}/Service-simple_doc.html | 0 vsc/templates/__init__.py | 7 ++++ {templates => vsc/templates}/dtdl.md | 0 {templates => vsc/templates}/dtdl.tpl | 0 {templates => vsc/templates}/protobuf.md | 0 {templates => vsc/templates}/protobuf.tpl | 0 .../templates}/sds-bamm-aspect-model.md | 0 .../templates}/sds-bamm-aspect-model.tpl | 0 .../templates}/sds-bamm-macros.tpl | 0 .../templates}/simple_overview.tpl | 0 {templates => vsc/templates}/test.tpl | 0 26 files changed, 110 insertions(+), 26 deletions(-) create mode 100644 setup.py delete mode 100644 templates/.gitkeep create mode 100644 vsc/__init__.py rename {generators => vsc/generators}/.gitkeep (100%) create mode 100644 vsc/generators/__init__.py create mode 100644 vsc/model/__init__.py rename {model => vsc/model}/vsc_generator.py (90%) rename {model => vsc/model}/vsc_parser.py (99%) create mode 100644 vsc/scripts/__init__.py create mode 100644 vsc/scripts/generator.py rename {templates => vsc/templates}/AST-simple_doc.tpl (100%) rename {templates => vsc/templates}/Argument-simple_doc.html (100%) rename {templates => vsc/templates}/Service-simple_doc.html (100%) create mode 100644 vsc/templates/__init__.py rename {templates => vsc/templates}/dtdl.md (100%) rename {templates => vsc/templates}/dtdl.tpl (100%) rename {templates => vsc/templates}/protobuf.md (100%) rename {templates => vsc/templates}/protobuf.tpl (100%) rename {templates => vsc/templates}/sds-bamm-aspect-model.md (100%) rename {templates => vsc/templates}/sds-bamm-aspect-model.tpl (100%) rename {templates => vsc/templates}/sds-bamm-macros.tpl (100%) rename {templates => vsc/templates}/simple_overview.tpl (100%) rename {templates => vsc/templates}/test.tpl (100%) diff --git a/.github/workflows/buildcheck.yml b/.github/workflows/buildcheck.yml index 896e7a7..0b32bf3 100644 --- a/.github/workflows/buildcheck.yml +++ b/.github/workflows/buildcheck.yml @@ -11,7 +11,7 @@ jobs: - name: Setup Python uses: actions/setup-python@v1 with: - python-version: 3.10.5 + python-version: 3.10.6 - name: Install dependencies run: | @@ -21,13 +21,16 @@ jobs: pip install anytree pip install jinja2 + - name: Install vsc module + run: | + python setup.py develop + - name: Run unit test run: | cd tests pytest -v - + - name: Run example tool on latest VSC run: | git clone https://github.com/GENIVI/vehicle_service_catalog/ - python model/vsc_generator.py vehicle_service_catalog/comfort-service.yml simple_overview.tpl - + vscgen vehicle_service_catalog/comfort-service.yml simple_overview.tpl diff --git a/README.md b/README.md index c182de2..8fa1663 100644 --- a/README.md +++ b/README.md @@ -33,6 +33,12 @@ target=C:\SomeDir\Where\Your\Account\Can\Write\To * If you use a custom pip installation directory, set the `PYTHONPATH` environment variable to the directory that you set in the `pip.ini` file. +### Setup with `virtualenv` + +```sh +python3 -m venv venv +``` + ### Setup with `pipenv` [pipenv](https://pypi.org/project/pipenv/) is a tool that manages a virtual environment and install the package and its dependencies, making the process much simpler and predictable, since the `Pipfile` states the dependencies, while `Pipfile.lock` freezes the exact version in use. @@ -43,9 +49,9 @@ curl https://pyenv.run | bash # download and install exec $SHELL # restart your shell using the new $PATH ``` -Make sure Python version 3.8.5 is installed: +Make sure Python version 3.10.6 is installed: ```sh -pyenv install 3.8.5 # install the versions required by Pipfile +pyenv install 3.10.6 # install the versions required by Pipfile ``` Install this project and its dependencies in the local `.venv` folder in this project, then use it (`pipenv shell`): diff --git a/requirements.txt b/requirements.txt index b08f7d8..8ce56dc 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,13 @@ -anytree>=2.8.0 -Jinja2>=3.0.0 -pytest>=2.7.2 -PyYAML>=5.4.1 +anytree==2.8.0 +attrs==22.1.0 +iniconfig==1.1.1 +Jinja2==3.1.2 +MarkupSafe==2.1.1 +packaging==21.3 +pluggy==1.0.0 +py==1.11.0 +pyparsing==3.0.9 +pytest==7.1.2 +PyYAML==6.0 +six==1.16.0 +tomli==2.0.1 diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..51cbef5 --- /dev/null +++ b/setup.py @@ -0,0 +1,17 @@ +#!/usr/bin/env python + +from distutils.core import setup +from importlib.metadata import entry_points + +setup(name='vsc', + version='0.1', + description='Vehicle service catalog tools', + author='', + author_email='', + url='https://github.com/covesa/vsc-tools', + packages=['vsc','tests'], + entry_points=''' + [console_scripts] + vscgen=vsc.scripts.generator:vsc_generator_run + ''' + ) diff --git a/templates/.gitkeep b/templates/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/vsc/__init__.py b/vsc/__init__.py new file mode 100644 index 0000000..b421226 --- /dev/null +++ b/vsc/__init__.py @@ -0,0 +1,3 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022 MBition GmbH. +# All rights reserved. +# SPDX-License-Identifier: MPL-2.0 diff --git a/generators/.gitkeep b/vsc/generators/.gitkeep similarity index 100% rename from generators/.gitkeep rename to vsc/generators/.gitkeep diff --git a/vsc/generators/__init__.py b/vsc/generators/__init__.py new file mode 100644 index 0000000..b421226 --- /dev/null +++ b/vsc/generators/__init__.py @@ -0,0 +1,3 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022 MBition GmbH. +# All rights reserved. +# SPDX-License-Identifier: MPL-2.0 diff --git a/vsc/model/__init__.py b/vsc/model/__init__.py new file mode 100644 index 0000000..b421226 --- /dev/null +++ b/vsc/model/__init__.py @@ -0,0 +1,3 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022 MBition GmbH. +# All rights reserved. +# SPDX-License-Identifier: MPL-2.0 diff --git a/model/vsc_generator.py b/vsc/model/vsc_generator.py similarity index 90% rename from model/vsc_generator.py rename to vsc/model/vsc_generator.py index 7b0484e..8adb06c 100644 --- a/model/vsc_generator.py +++ b/vsc/model/vsc_generator.py @@ -12,30 +12,30 @@ # This performs the following related functions: -# 1. As input we expect an AST as provided by the parser module +# 1. As input.yaml we expect an AST as provided by the parser module # # 2. It uses jinja2 templating library to generate code or configuration # according to given templates. # It's useful to have these classes in our namespace directly -from vsc_parser import AST, Argument, Enum, Error, Event, Include, Member, Method, Namespace, Option, Property, Service, Struct, Typedef +from vsc.model.vsc_parser import AST, Argument, Enum, Error, Event, Include, Member, Method, Namespace, Option, Property, Service, Struct, Typedef + +# For other features from parser module +from vsc.model.vsc_parser import get_ast_from_file + +from vsc.templates import TemplatePath -import vsc_parser # For other features from parser module -import anytree -import getopt import jinja2 -import os import sys # Set up Jinja jinja_env = jinja2.Environment( # Use the subdirectory 'templates' relative to this file's location - loader = - jinja2.FileSystemLoader(os.path.dirname(os.path.realpath(__file__)) + '/../templates'), + loader=jinja2.FileSystemLoader(TemplatePath), - # Templates with these extension gets automatic autoescape for HTML + # Templates with these extension gets automatic auto escape for HTML # It's more annoying for code generation, so passing empty list for now. - autoescape = jinja2.select_autoescape([]) + autoescape=jinja2.select_autoescape([]) ) # We want the control blocks in the template to NOT result in any extra @@ -48,6 +48,7 @@ default_templates = {} + # Exception: class GeneratorError(BaseException): def __init__(self, m): @@ -55,12 +56,14 @@ def __init__(self, m): # ---------- GENERATION FUNCTIONS ------------ + # Get template with given name (search path should be handled by the loader) def get_template(filename): return jinja_env.get_template(filename) # Frontend to overloaded gen() function: + def gen(node : AST, second = None): # Processing of lists of objects? if type(node) == list or type(node) == tuple: @@ -168,7 +171,7 @@ def gen_dict_with_template_file(variables : dict, templatefile): Typedef=Typedef) # This code file can be used in two ways. Either calling this file as a -# program using the main entry points here, and specifying input parameter. +# program using the main entry points here, and specifying input.yaml parameter. # Alternatively, for more advanced usage, the file might be included as a # module in a custom generator implementation. That implementation is # likely to call some of the funcctions that were defined above. @@ -176,21 +179,21 @@ def gen_dict_with_template_file(variables : dict, templatefile): # For the first case, here follows the main entry points and configuration: def usage(): - print("usage: generator.py ") + print("usage: generator.py ") sys.exit(1) # This is a default definition for our current generation tests. # It may need to be changed, or defined differently in a custom generator default_templates = { - 'Service' : 'Service-simple_doc.html', - 'Argument' : 'Argument-simple_doc.html' + 'Service': 'Service-simple_doc.html', + 'Argument': 'Argument-simple_doc.html' } # If run as a script, generate a single YAML file using a single root template if __name__ == "__main__": if not len(sys.argv) == 3: usage() - ast = vsc_parser.get_ast_from_file(sys.argv[1]) + ast = get_ast_from_file(sys.argv[1]) templatename = sys.argv[2] print(gen(ast, templatename)) diff --git a/model/vsc_parser.py b/vsc/model/vsc_parser.py similarity index 99% rename from model/vsc_parser.py rename to vsc/model/vsc_parser.py index c85a363..2b76222 100644 --- a/model/vsc_parser.py +++ b/vsc/model/vsc_parser.py @@ -69,6 +69,7 @@ class Service(AST): pass class Struct(AST): pass class Typedef(AST): pass + # Some identifiers used to indicate optionality MAN = 0 # MAN(datory) REC = 1 # REC(ommended) @@ -243,7 +244,7 @@ def get_recommended_yaml_value(tree, nodename): # ---------------------------------------------------------------------------- # Here we construct a graph of nodes using typed classes (AST -# subclasses) that are also AnyTree Nodes, from an input tree that +# subclasses) that are also AnyTree Nodes, from an input.yaml tree that # represents the raw YAML text but read into a tree in the standard # dict/list format from the YAML parser. diff --git a/vsc/scripts/__init__.py b/vsc/scripts/__init__.py new file mode 100644 index 0000000..b421226 --- /dev/null +++ b/vsc/scripts/__init__.py @@ -0,0 +1,3 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022 MBition GmbH. +# All rights reserved. +# SPDX-License-Identifier: MPL-2.0 diff --git a/vsc/scripts/generator.py b/vsc/scripts/generator.py new file mode 100644 index 0000000..17cd32a --- /dev/null +++ b/vsc/scripts/generator.py @@ -0,0 +1,26 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022 MBition GmbH. +# All rights reserved. +# SPDX-License-Identifier: MPL-2.0 + +from vsc.model.vsc_generator import gen +from vsc.model.vsc_parser import get_ast_from_file +import argparse + +def vsc_generator_run(): + parser = argparse.ArgumentParser(description='Runs vehicle service catalog code generator.') + parser.add_argument('input', metavar='input', type=str, + help='input.yaml-file (path)') + parser.add_argument('template', metavar='template', type=str, + help='output-template-file (name only, not path)') + + args = parser.parse_args() + + ast = get_ast_from_file(args.input) + + templatename = args.template + + print(gen(ast, templatename)) + + +if __name__ == "__main__": + vsc_generator_run() \ No newline at end of file diff --git a/templates/AST-simple_doc.tpl b/vsc/templates/AST-simple_doc.tpl similarity index 100% rename from templates/AST-simple_doc.tpl rename to vsc/templates/AST-simple_doc.tpl diff --git a/templates/Argument-simple_doc.html b/vsc/templates/Argument-simple_doc.html similarity index 100% rename from templates/Argument-simple_doc.html rename to vsc/templates/Argument-simple_doc.html diff --git a/templates/Service-simple_doc.html b/vsc/templates/Service-simple_doc.html similarity index 100% rename from templates/Service-simple_doc.html rename to vsc/templates/Service-simple_doc.html diff --git a/vsc/templates/__init__.py b/vsc/templates/__init__.py new file mode 100644 index 0000000..0ee37bf --- /dev/null +++ b/vsc/templates/__init__.py @@ -0,0 +1,7 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022 MBition GmbH. +# All rights reserved. +# SPDX-License-Identifier: MPL-2.0 + +import os + +TemplatePath = os.path.dirname(os.path.realpath(__file__)) diff --git a/templates/dtdl.md b/vsc/templates/dtdl.md similarity index 100% rename from templates/dtdl.md rename to vsc/templates/dtdl.md diff --git a/templates/dtdl.tpl b/vsc/templates/dtdl.tpl similarity index 100% rename from templates/dtdl.tpl rename to vsc/templates/dtdl.tpl diff --git a/templates/protobuf.md b/vsc/templates/protobuf.md similarity index 100% rename from templates/protobuf.md rename to vsc/templates/protobuf.md diff --git a/templates/protobuf.tpl b/vsc/templates/protobuf.tpl similarity index 100% rename from templates/protobuf.tpl rename to vsc/templates/protobuf.tpl diff --git a/templates/sds-bamm-aspect-model.md b/vsc/templates/sds-bamm-aspect-model.md similarity index 100% rename from templates/sds-bamm-aspect-model.md rename to vsc/templates/sds-bamm-aspect-model.md diff --git a/templates/sds-bamm-aspect-model.tpl b/vsc/templates/sds-bamm-aspect-model.tpl similarity index 100% rename from templates/sds-bamm-aspect-model.tpl rename to vsc/templates/sds-bamm-aspect-model.tpl diff --git a/templates/sds-bamm-macros.tpl b/vsc/templates/sds-bamm-macros.tpl similarity index 100% rename from templates/sds-bamm-macros.tpl rename to vsc/templates/sds-bamm-macros.tpl diff --git a/templates/simple_overview.tpl b/vsc/templates/simple_overview.tpl similarity index 100% rename from templates/simple_overview.tpl rename to vsc/templates/simple_overview.tpl diff --git a/templates/test.tpl b/vsc/templates/test.tpl similarity index 100% rename from templates/test.tpl rename to vsc/templates/test.tpl