Merge ca32e12 into 8a50c14

src/otoole/__init__.py

@@ -2,6 +2,8 @@
 from pkg_resources import get_distribution, DistributionNotFound
 import os
 from yaml import SafeLoader, load
+from datapackage import Package
+from sqlalchemy import create_engine
 
 try:
     import importlib.resources as resources
@@ -39,3 +41,22 @@ def read_packaged_file(filename: str, module_name: str = None):
             contents = _read_file(open_file, ending)
 
     return contents
+
+
+def read_datapackage(filepath: str, sql: bool = False):
+    """Open an OSeMOSYS datapackage
+
+    Arguments
+    ---------
+    filepath : str
+    sql : bool, default=False
+    """
+    if sql:
+        engine = create_engine('sqlite:///{}'.format(filepath))
+        package = Package(storage='sql', engine=engine)
+    else:
+        package = Package(filepath)
+
+    package.infer()
+
+    return package

src/otoole/cli.py

@@ -41,22 +41,32 @@
 from otoole.preprocess import convert_file_to_package, create_datafile, create_datapackage, generate_csv_from_excel
 from otoole.preprocess.create_datapackage import convert_datapackage_to_sqlite
 from otoole.results.convert import convert_cplex_file
+from otoole.validate import main as validate
 from otoole.visualise import create_res
 
 
+def validate_model(args):
+    file_format = args.format
+    validate(file_format, args.filepath)
+
+
 def cplex2cbc(args):
     convert_cplex_file(args.cplex_file, args.output_file, args.start_year,
                        args.end_year, args.output_format)
 
 
 def conversion_matrix(args):
-    """
-    from\to     ex cs dp sq df
-    excel       -- yy
-    csv         nn -- yy nn nn
-    datapackage nn ?? -- yy yy
-    sql         nn       -- yy
-    datafile    nn ?? yy    --
+    """Convert from one format to another
+
+    Implemented conversion functions::
+
+        from\to     ex cs dp sq df
+        --------------------------
+        excel       -- yy
+        csv         nn -- yy nn nn
+        datapackage nn ?? -- yy yy
+        sql         nn       -- yy
+        datafile    nn ?? yy    --
 
     """
 
@@ -139,6 +149,12 @@ def get_parser():
     cplex_parser.add_argument('output_format', choices=['csv', 'cbc'], default='cbc')
     cplex_parser.set_defaults(func=cplex2cbc)
 
+    # Parser for validation
+    valid_parser = subparsers.add_parser('validate', help='Validate an OSeMOSYS model')
+    valid_parser.add_argument('format', help='The format of the OSeMOSYS model to validate')
+    valid_parser.add_argument('filepath', help='Path to the OSeMOSYS model to validate')
+    valid_parser.set_defaults(func=validate_model)
+
     # Parser for visualisation
     viz_parser = subparsers.add_parser('viz', help='Visualise the model')
     viz_subparsers = viz_parser.add_subparsers()

src/otoole/validate.py

@@ -33,7 +33,7 @@
 import re
 from typing import Dict, List
 
-from otoole import read_packaged_file
+from otoole import read_datapackage, read_packaged_file
 
 logger = logging.getLogger(__name__)
 
@@ -42,6 +42,11 @@ def read_validation_config():
     return read_packaged_file('validate.yaml', 'otoole')
 
 
+def check_for_duplicates(codes: List) -> bool:
+    duplicate_values = len(codes) != len(set(codes))
+    return duplicate_values
+
+
 def create_schema(config: Dict = None):
     """Populate the dict of schema with codes from the validation config
 
@@ -57,6 +62,13 @@ def create_schema(config: Dict = None):
         for name in schema:
             if isinstance(name['valid'], str):
                 name['valid'] = list(config['codes'][name['valid']].keys())
+                logger.debug("create_schema: %s", name['valid'])
+            elif isinstance(name['valid'], list):
+                pass
+            else:
+                raise ValueError("Entry {} is not correct".format(name['name']))
+            if check_for_duplicates(name['valid']):
+                raise ValueError("There are duplicate values in codes for {}", name['name'])
     return config['schema']
 
 
@@ -69,6 +81,7 @@ def compose_expression(schema: List) -> str:
     """
     expression = "^"
     for x in schema:
+        logger.debug("compose_expression: %s", x['valid'])
         valid_entries = "|".join(x['valid'])
         expression += "({})".format(valid_entries)
     return expression
@@ -86,6 +99,7 @@ def validate(expression: str, name: str) -> bool:
     -------
     bool
     """
+    logger.debug("Running validation for %s", name)
 
     valid = False
 
@@ -98,7 +112,7 @@ def validate(expression: str, name: str) -> bool:
         msg = "{} is invalid"
         valid = False
 
-    logger.debug(msg.format(name))
+    logger.info(msg.format(name))
     return valid
 
 
@@ -124,3 +138,25 @@ def validate_fuel_name(name: str) -> bool:
     valid = validate(expression, name)
 
     return valid
+
+
+def validate_resource(package, schema, resource):
+
+    logger.debug(schema)
+
+    expression = compose_expression(schema)
+    resources = package.get_resource(resource).read(keyed=True)
+    for row in resources:
+        validate(expression, row['VALUE'])
+
+
+def main(file_format: str, filepath: str):
+
+    if file_format == 'datapackage':
+        package = read_datapackage(filepath)
+    elif file_format == 'sql':
+        package = read_datapackage(filepath, sql=True)
+
+    schema = create_schema()
+    validate_resource(package, schema['technology_name'], 'TECHNOLOGY')
+    validate_resource(package, schema['fuel_name'], 'FUEL')