Merge pull request #365 from simphony/refactor-generator

Refactor generator - Part 1

scripts/cli/generator.py

@@ -0,0 +1,81 @@
+from __future__ import print_function
+
+import yaml
+import click
+
+from scripts.cuba_enum_generator import CUBAEnumGenerator
+from scripts.keywords_generator import KeywordsGenerator
+from scripts.meta_class_generator import MetaClassGenerator
+
+
+@click.group()
+def cli():
+    """ Auto-generate code from simphony-metadata yaml description. """
+
+
+@cli.command()
+@click.argument('yaml_file', type=click.File('rb'))
+@click.argument('out_path', type=click.Path())
+@click.option('-O', '--overwrite', is_flag=True, default=False,
+              help='Overwrite OUT_PATH')
+def meta_class(yaml_file, out_path, overwrite):
+    """ Create the Simphony Metadata classes
+
+    yaml_file:
+        path to the simphony_metadata yaml file
+
+    out_path:
+        path to the directory where the output files should be placed
+
+    overwrite:
+        Allow overwrite of the file.
+    """
+    simphony_metadata_dict = yaml.safe_load(yaml_file)
+
+    generator = MetaClassGenerator()
+    generator.generate(simphony_metadata_dict, out_path, overwrite)
+
+
+@cli.command()
+@click.argument('cuba_input', type=click.File('rb'))
+@click.argument('cuds_input', type=click.File('rb'))
+@click.argument('output', type=click.File('wb'))
+def cuba_enum(cuba_input, cuds_input, output):
+    """ Create the CUBA Enum
+
+    cuba_input:
+        Path to the cuba.yml
+
+    cuds_input:
+        Path to the simphony_metadata.yml
+
+    output:
+        Path to the output cuba.py file
+    """
+    cuba_dict = yaml.safe_load(cuba_input)
+    simphony_metadata_dict = yaml.safe_load(cuds_input)
+
+    generator = CUBAEnumGenerator()
+    generator.generate(cuba_dict, simphony_metadata_dict, output)
+
+
+@cli.command()
+@click.argument('cuba_input', type=click.File('rb'))
+@click.argument('cuds_input', type=click.File('rb'))
+@click.argument('output', type=click.File('wb'))
+def keywords(cuba_input, cuds_input, output):
+    """ Create a dictionary of keywords.
+
+    cuba_input:
+        Path to the cuba.yml
+
+    cuds_input:
+        Path to the simphony_metadata.yml
+
+    output:
+        Path to the output keywords.py file
+    """
+    cuba_dict = yaml.safe_load(cuba_input)
+    simphony_metadata_dict = yaml.safe_load(cuds_input)
+    generator = KeywordsGenerator()
+    generator.generate(cuba_dict, simphony_metadata_dict, output)

scripts/cuba_enum_generator.py

@@ -0,0 +1,33 @@
+from scripts.single_meta_class_generator import CUBA_DATA_CONTAINER_EXCLUDE
+
+
+class CUBAEnumGenerator(object):
+    """Generator class for cuba.py enumeration.
+    """
+
+    def generate(self, cuba_dict, simphony_metadata_dict, output):
+        """Generates the cuba file from the yaml-extracted dictionary
+        of cuba and simphony_metadata files. Writes the generated code
+        in the file object output
+        """
+        lines = [
+            '# code auto-generated by the\n',
+            '# simphony-metadata/scripts/generate.py script.\n',
+            '# cuba.yml VERSION: {}\n'.format(cuba_dict['VERSION']),
+            'from enum import Enum, unique\n',
+            '\n',
+            '\n',
+            '@unique\n',
+            'class CUBA(Enum):\n',
+            '\n']
+        template = '    {} = "{}"\n'
+
+        all_keys = set(
+            cuba_dict['CUBA_KEYS']) | set(simphony_metadata_dict['CUDS_KEYS'])
+
+        for keyword in sorted(list(all_keys)):
+            if keyword in CUBA_DATA_CONTAINER_EXCLUDE:
+                continue
+            lines.append(template.format(keyword, keyword))
+
+        output.writelines(lines)

scripts/keywords_generator.py

@@ -0,0 +1,59 @@
+from scripts.utils import to_camel_case
+
+
+class KeywordsGenerator(object):
+    """Generator for the keywords.py file."""
+    def generate(self, cuba_dict, simphony_metadata_dict, output):
+        """ Create a dictionary of keywords from the cuba and simphony_metadata
+        yaml-extracted dictionaries. Writes the generated code in the file
+        object output.
+        """
+        lines = [
+            '# code auto-generated by the\n',
+            '# simphony-metadata/scripts/generate.py script.\n',
+            '# cuba.yml VERSION: {}\n'.format(cuba_dict['VERSION']),
+            '# simphony_metadata.yml VERSION: {}\n'.format(
+                    simphony_metadata_dict['VERSION']),
+            'from collections import namedtuple\n',
+            '\n',
+            'import numpy\n',
+            'import uuid  # noqa\n',
+            '\n',
+            '\n',
+            'ATTRIBUTES = [\n'
+            '    "name", "definition", "key", "shape", "dtype"]\n'  # noqa
+            'Keyword = namedtuple("Keyword", ATTRIBUTES)\n',
+            '\n',
+            '\n',
+            'KEYWORDS = {\n']
+        data_types = {
+            'uuid': 'uuid.UUID',
+            'string': 'numpy.str',
+            'double': 'numpy.float64',
+            'integer': 'numpy.int32',
+            'boolean': 'bool'}
+        template = (
+            "    '{key}': Keyword(\n"
+            "        name='{name}',\n"
+            "        definition='{definition}',  # noqa\n"
+            "        key='{key}',\n"
+            "        shape={shape},\n"
+            "        dtype={type}),\n")
+        for keyword, content in sorted(cuba_dict['CUBA_KEYS'].items(),
+                                       key=lambda x: x[0]):
+            content['type'] = data_types[content['type']]
+            content['name'] = to_camel_case(keyword)
+            content['key'] = keyword
+            lines.extend(template.format(**content))
+
+        for keyword, content in sorted(
+                simphony_metadata_dict['CUDS_KEYS'].items(),
+                key=lambda x: x[0]):
+            content['type'] = "None"
+            content['name'] = to_camel_case(keyword)
+            content['key'] = keyword
+            content['shape'] = [1]
+            lines.extend(template.format(**content))
+        lines.append('}\n')
+
+        output.writelines(lines)

scripts/meta_class_generator.py

@@ -0,0 +1,105 @@
+from __future__ import print_function
+
+import shutil
+import os
+import warnings
+import re
+
+from scripts.single_meta_class_generator import (
+    SingleMetaClassGenerator, IMPORT_PATHS)
+from scripts.utils import make_temporary_directory, to_camel_case
+
+
+class MetaClassGenerator(object):
+    def generate(self, simphony_metadata_dict, out_path, overwrite=True):
+        """
+        Create the Simphony Metadata classes in the directory specified by
+        out_path, starting from the yaml-extracted data in
+        simphony_metadata_dict.
+
+        If the optional overwrite flag is True, the directory will be
+        emptied first.
+        """
+
+        if os.path.exists(out_path):
+            if overwrite:
+                shutil.rmtree(out_path)
+            else:
+                raise OSError('Destination already exists: {!r}'.format(
+                    out_path))
+
+        all_generators = {}
+
+        # Temporary directory that stores the output
+        with make_temporary_directory() as temp_dir:
+
+            for key, class_data in simphony_metadata_dict['CUDS_KEYS'].items():
+                # Catch inconsistent definitions that would choke the generator
+                parent = class_data['parent']
+                if (parent and parent.replace('CUBA.', '')
+                        not in simphony_metadata_dict['CUDS_KEYS']):
+                    message = ('{0} is SKIPPED because its parent {1} '
+                               'is not defined in CUDS_KEYS')
+                    warnings.warn(message.format(key, class_data['parent']))
+                    continue
+
+                if key.lower() in ('validation', 'api'):
+                    message = 'Name crashes with utility modules: '+key.lower()
+                    raise ValueError(message)
+
+                # Create the generator object, on init, it identifies its own
+                # required/optional user-defined attributes and
+                # system-managed attributes
+                all_generators[key] = SingleMetaClassGenerator(key, class_data)
+
+            for key, gen in all_generators.items():
+                # Collect parents and attributes inherited from parents
+                gen.collect_parents_to_mro(all_generators)
+                gen.collect_attributes_from_parents(all_generators)
+
+                # Target .py file
+                filename = os.path.join(
+                    temp_dir,
+                    "{}.py".format(gen.original_key.lower()))
+
+                # Now write the code
+                with open(filename, 'wb') as generated_file:
+                    gen.generate(file_out=generated_file)
+
+                # Print to the api.py
+                with open(os.path.join(temp_dir, "api.py"), 'ab') as api_file:
+                    print(
+                        'from .{} import {}   # noqa'.format(
+                            key.lower(),
+                            to_camel_case(key)
+                        ),
+                        sep='\n',
+                        file=api_file
+                    )
+
+            # Create an empty __init__.py
+            init_path = os.path.join(temp_dir, '__init__.py')
+            open(init_path, 'a').close()
+
+            # Create validation.py
+            validation_path = os.path.join(temp_dir, 'validation.py')
+
+            from . import validation
+            # validation.py for validation codes.
+            validation_py_path = os.path.splitext(validation.__file__)[0]+'.py'
+
+            with open(validation_path, 'wb') as dst_file, \
+                    open(validation_py_path, 'rb') as src_file:
+
+                # Replace import path for KEYWORDS
+                def read_lines(src_file):
+                    while True:
+                        line = src_file.next()
+                        yield re.sub(r'(\s*).+import KEYWORDS',
+                                     "\\1"+IMPORT_PATHS['KEYWORDS'], line)
+
+                # Copy the rest of the file
+                print(*read_lines(src_file), file=dst_file, sep='')
+
+            # Copy everything to the output directory
+            shutil.copytree(temp_dir, out_path)