Merge pull request #384 from simphony/use-simphony-metaparser

Using simphony metaparser instead of direct yaml parsing.

.travis.yml

@@ -1,36 +1,37 @@
 language: python
 sudo: false
 python:
-- '2.7'
+  - '2.7'
 virtualenv:
   system_site_packages: false
 addons:
   apt:
     packages:
-    - python-pip
-    - libhdf5-serial-dev
+      - python-pip
+      - libhdf5-serial-dev
 cache:
   directories:
-  - "$HOME/.cache"
-  - "$HOME/.ccache"
+    - "$HOME/.cache"
+    - "$HOME/.ccache"
 before_install:
-- ccache -s
-- export PATH=/usr/lib/ccache:${PATH}
-- pip install --upgrade pip
+  - ccache -s
+  - export PATH=/usr/lib/ccache:${PATH}
+  - pip install --upgrade pip
 install:
-- pip install numexpr
-- pip install -r dev_requirements.txt
-- python setup.py develop
-- pushd examples/plugin
-- python setup.py develop
-- popd
+  - pip install -r requirements.txt
+  - pip install -r dev_requirements.txt
+  - python setup.py develop
+  - pushd examples/plugin
+  - python setup.py develop
+  - popd
+  - python setup.py build_meta
 script:
-- flake8 .
-- py.test -v
-- sphinx-build -W doc/source doc/build/sphinx
+  - flake8 .
+  - py.test -v
+  - sphinx-build -W doc/source doc/build/sphinx
 after_success:
-- pip install codecov
-- codecov
+  - pip install codecov
+  - codecov
 notifications:
   slack:
     secure: a3aAlsxs2uPa1ggrDQouECqMGfR9DaZQBOVUmdGeOK6LHq+Pta/KbgUbiwA42c3QbR9WfI+J3oRuC9ZrZDo6d/DmobTCBTutWsTPs/LIJIZVfdbvYyJWzzeFTE9ca4iwHiQiZlG3d6EVQYVEf3cWuE89o0YEmRLmxAy99MPAtWM=

appveyor-install.cmd

@@ -2,6 +2,7 @@
 call setenv /x64
 
 rem install python packages
+pip install -r requirements.txt
 pip install -r dev_requirements.txt
 
 rem install simphony-common 

requirements.txt

@@ -0,0 +1,2 @@
+numexpr
+git+https://github.com/simphony/simphony-metaparser.git@master#egg=simphony_metaparser

scripts/api_generator.py

@@ -1,23 +1,31 @@
 from __future__ import print_function
 
-import os
-from scripts.utils import to_camel_case
+from simphony_metaparser.utils import traverse
+
+from scripts.utils import cuba_key_to_meta_class_module_name, \
+    cuba_key_to_meta_class_name
 
 
 class APIGenerator(object):
-    def generate(self, simphony_metadata_dict, out_path):
+    def generate(self, ontology, output):
         """
         Generates the api.py with the appropriate imports.
         The imports are alphabetically ordered.
+
+        ontology: Ontology
+            The node of the ontology
+
+        output: file
+            The file on which to write the result
         """
 
-        for key in sorted(simphony_metadata_dict['CUDS_KEYS'].keys()):
-            with open(os.path.join(out_path, "api.py"), 'ab') as api_file:
-                print(
-                    'from .{} import {}   # noqa'.format(
-                        key.lower(),
-                        to_camel_case(key)
-                    ),
-                    sep='\n',
-                    file=api_file
+        lines = []
+        for cuds_item, _ in traverse(ontology.root_cuds_item):
+            lines.append('from .{} import {}   # noqa\n'.format(
+                    cuba_key_to_meta_class_module_name(cuds_item.name),
+                    cuba_key_to_meta_class_name(cuds_item.name)
                 )
+            )
+
+        for line in sorted(lines):
+            output.write(line)

scripts/cli/generator.py

@@ -1,10 +1,11 @@
 from __future__ import print_function
 
-import yaml
 import click
 import os
 import shutil
 
+from simphony_metaparser.yamldirparser import YamlDirParser
+
 from scripts.api_generator import APIGenerator
 from scripts.cuba_enum_generator import CUBAEnumGenerator
 from scripts.keywords_generator import KeywordsGenerator
@@ -33,6 +34,7 @@ def cli(yaml_dir, module_root_path, overwrite):
     """
 
     meta_class_output = os.path.join(module_root_path, "cuds", "meta")
+    api_output = os.path.join(module_root_path, "cuds", "meta", "api.py")
     keyword_output = os.path.join(module_root_path, "core", "keywords.py")
     cuba_output = os.path.join(module_root_path, "core", "cuba.py")
 
@@ -55,28 +57,23 @@ def cli(yaml_dir, module_root_path, overwrite):
     except OSError:
         pass
 
-    cuba_input = os.path.join(yaml_dir, "cuba.yml")
-    cuds_input = os.path.join(yaml_dir, "simphony_metadata.yml")
-
-    with open(cuba_input) as f:
-        cuba_dict = yaml.safe_load(f)
-
-    with open(cuds_input) as f:
-        simphony_metadata_dict = yaml.safe_load(f)
+    parser = YamlDirParser()
+    ontology = parser.parse(yaml_dir)
 
     generator = KeywordsGenerator()
     with open(keyword_output, "wb") as f:
-        generator.generate(cuba_dict, simphony_metadata_dict, f)
+        generator.generate(ontology, f)
 
     generator = CUBAEnumGenerator()
     with open(cuba_output, "wb") as f:
-        generator.generate(cuba_dict, simphony_metadata_dict, f)
+        generator.generate(ontology, f)
 
     meta_class_generator = MetaClassGenerator()
-    meta_class_generator.generate(simphony_metadata_dict, meta_class_output)
+    meta_class_generator.generate(ontology, meta_class_output)
 
     api_generator = APIGenerator()
-    api_generator.generate(simphony_metadata_dict, meta_class_output)
+    with open(api_output, "wb") as f:
+        api_generator.generate(ontology, f)
 
     validation_generator = ValidationGenerator()
     validation_generator.generate(meta_class_output)

scripts/cuba_enum_generator.py

@@ -1,28 +1,33 @@
+from simphony_metaparser.utils import traverse, without_cuba_prefix
+
+
 class CUBAEnumGenerator(object):
     """Generator class for cuba.py enumeration.
     """
 
-    def generate(self, cuba_dict, simphony_metadata_dict, output):
+    def generate(self, ontology, output):
         """Generates the cuba file from the yaml-extracted dictionary
         of cuba and simphony_metadata files. Writes the generated code
         in the file object output
         """
         lines = [
             '# code auto-generated by the\n',
             '# simphony-metadata/scripts/generate.py script.\n',
-            '# cuba.yml VERSION: {}\n'.format(cuba_dict['VERSION']),
+            '# cuba.yml\n',
             'from enum import Enum, unique\n',
             '\n',
             '\n',
             '@unique\n',
             'class CUBA(Enum):\n'
             ]
-        template = '    {keyword} = "{keyword}"\n'
+        template = '    {cuba_name} = "{cuba_name}"\n'
 
-        all_keys = set(
-            cuba_dict['CUBA_KEYS']) | set(simphony_metadata_dict['CUDS_KEYS'])
+        all_keys = set(d.name for d in ontology.data_types)
+        all_keys.update([d.name for d, _ in traverse(ontology.root_cuds_item)])
 
         for keyword in sorted(list(all_keys)):
-            lines.append(template.format(keyword=keyword))
+            lines.append(
+                template.format(
+                    cuba_name=without_cuba_prefix(keyword)))
 
         output.writelines(lines)

scripts/keywords_generator.py

@@ -1,19 +1,18 @@
+from simphony_metaparser.utils import traverse, without_cuba_prefix
+
 from . import utils
 
 
 class KeywordsGenerator(object):
     """Generator for the keywords.py file."""
-    def generate(self, cuba_dict, simphony_metadata_dict, output):
+    def generate(self, ontology, output):
         """ Create a dictionary of keywords from the cuba and simphony_metadata
         yaml-extracted dictionaries. Writes the generated code in the file
         object output.
         """
         lines = [
             '# code auto-generated by the\n',
             '# simphony-metadata/scripts/generate.py script.\n',
-            '# cuba.yml VERSION: {}\n'.format(cuba_dict['VERSION']),
-            '# simphony_metadata.yml VERSION: {}\n'.format(
-                    simphony_metadata_dict['VERSION']),
             'from collections import namedtuple\n',
             '\n',
             'import numpy\n',
@@ -40,25 +39,35 @@ def generate(self, cuba_dict, simphony_metadata_dict, output):
             "        shape={shape},\n"
             "        length={length},\n"
             "        dtype={type}),\n")
-        for keyword, content in sorted(cuba_dict['CUBA_KEYS'].items(),
-                                       key=lambda x: x[0]):
-            template_ctx = dict(content)
-            template_ctx['type'] = data_types[content['type']]
-            template_ctx['name'] = utils.to_camel_case(keyword)
-            template_ctx['key'] = keyword
-            template_ctx['shape'] = content.get("shape", [1])
-            template_ctx['length'] = content.get("length", None)
+        for data_type in sorted(ontology.data_types,
+                                key=lambda x: x.name):
+            template_ctx = dict(
+                type=data_types[data_type.type],
+                name=utils.cuba_key_to_meta_class_name(
+                    data_type.name),
+                definition=data_type.definition,
+                key=without_cuba_prefix(data_type.name),
+                shape=data_type.shape,
+                length=data_type.length,
+            )
             lines.extend(template.format(**template_ctx))
 
-        for keyword, content in sorted(
-                simphony_metadata_dict['CUDS_KEYS'].items(),
-                key=lambda x: x[0]):
-            template_ctx = dict(content)
-            template_ctx['type'] = "None"
-            template_ctx['name'] = utils.to_camel_case(keyword)
-            template_ctx['key'] = keyword
-            template_ctx['shape'] = content.get("shape", [1])
-            template_ctx['length'] = "None"
+        for cuds_item in sorted(
+                [d for d, _ in traverse(ontology.root_cuds_item)],
+                key=lambda x: x.name):
+
+            try:
+                definition = cuds_item.property_entries["definition"].default
+            except KeyError:
+                definition = ""
+
+            template_ctx = dict(
+                type="None",
+                name=utils.cuba_key_to_meta_class_name(cuds_item.name),
+                definition=definition,
+                key=without_cuba_prefix(cuds_item.name),
+                shape=[1],
+                length="None")
             lines.extend(template.format(**template_ctx))
         lines.append('}\n')
 

scripts/meta_class_generator.py

@@ -2,32 +2,32 @@
 
 import os
 
+from simphony_metaparser.utils import traverse
+
 from .single_meta_class_generator import SingleMetaClassGenerator
 from . import utils
 
 
 class MetaClassGenerator(object):
-    def generate(self, simphony_metadata_dict, out_path):
+    def generate(self, ontology, output_dir):
         """
         Create the Simphony Metadata classes in the directory specified by
-        out_path, starting from the yaml-extracted data in
-        simphony_metadata_dict.
-
-        If the optional overwrite flag is True, the directory will be
-        emptied first.
+        output_dir, from the ontology elements.
         """
 
-        for key, _ in simphony_metadata_dict['CUDS_KEYS'].items():
-            gen = SingleMetaClassGenerator(key, simphony_metadata_dict)
+        for item, _ in traverse(ontology.root_cuds_item):
+            gen = SingleMetaClassGenerator()
 
             filename = os.path.join(
-                out_path,
-                "{}.py".format(utils.cuba_key_to_meta_class_module_name(key)))
+                output_dir,
+                "{}.py".format(utils.cuba_key_to_meta_class_module_name(
+                    item.name)))
 
             # Now write the code
             with open(filename, 'wb') as generated_file:
-                gen.generate(out=generated_file)
+                gen.generate(item, generated_file)
 
         # Create an empty __init__.py
-        init_path = os.path.join(out_path, '__init__.py')
-        open(init_path, 'a').close()
+        init_path = os.path.join(output_dir, '__init__.py')
+        with open(init_path, 'a'):
+            pass