Merge branch 'features/postprocessing'

setup.py

@@ -76,13 +76,31 @@ def read(*names, **kwargs):
         "scipy",
         "shapely",
         "tsam",
+        "click",
+        "toml"
     ],
     extras_require={
         # eg:
         #   'rst': ['docutils>=0.11'],
         #   ':python_version=="2.6"': ['argparse'],
     },
-    dependency_links=[
+    entry_points={
+        'console_scripts': [
+            'ota = oemof.tabular.cli:main',
+        ]
+    },
+    dependency_links=(
+        [
+            (
+                "git+https://git@github.com/gnn/pyproj.git"
+                "@69a26ce46634749f602518a375849999cb5e41e0"
+                "#egg=pyproj-1.9.5.1.dev0"
+            )
+        ]
+        if sys.version_info.major >= 3 and sys.version_info.minor >= 7
+        else []
+    )
+    + [
         (
             "git+https://git@github.com/oemof/oemof.git"
             "@releases/v0_3_0"

src/oemof/tabular/__main__.py

@@ -0,0 +1,14 @@
+"""
+Entrypoint module, in case you use `python -mtabular`.
+
+
+Why does this file exist, and why __main__? For more info, read:
+
+- https://www.python.org/dev/peps/pep-0338/
+- https://docs.python.org/2/using/cmdline.html#cmdoption-m
+- https://docs.python.org/3/using/cmdline.html#cmdoption-m
+"""
+from tabular.cli import main
+
+if __name__ == "__main__":
+    main()

src/oemof/tabular/cli.py

@@ -0,0 +1,62 @@
+"""
+Module that contains the command line app.
+
+"""
+import click
+from datapackage import Package, exceptions
+import pandas as pd
+
+
+def _test(ctx, package):
+    """
+    """
+    p = Package(package)
+    for r in p.resources:
+        try:
+            r.read(keyed=True)
+            fks = r.descriptor['schema'].get('foreignKeys', [])
+            for fk in fks:
+                if fk.get('fields') == 'profile':
+                    profile = p.get_resource(fk['reference']['resource'])
+                    field_names = set([
+                        field['name']
+                        for field in profile.descriptor['schema']['fields']
+                    ])
+                    field_names.remove('timeindex')
+                    profile_names = set(
+                        pd.DataFrame(r.read(keyed=True))['profile'])
+
+                    diff = field_names.symmetric_difference(profile_names)
+                    if len(diff) > 0:
+                        for d in diff:
+                            print(
+                                ("Foreign key error for {} in "
+                                 " resource {} and {}.".format(d, r.name,
+                                                               profile.name))
+                            )
+
+        except:
+            raise exceptions.DataPackageException(
+                (
+                    "Could not read resource {} from datpackage "
+                    "with name {}".format(r.name, p.descriptor['name'])
+                 )
+            )
+    print("Successfully tested datapackage {}.".format(
+            p.descriptor['name']))
+
+
+@click.group(chain=True)
+
+@click.pass_context
+def cli(ctx, **kwargs):
+    ctx.obj = kwargs
+
+@cli.command()
+@click.argument("package", type=str, default="datapackage.json")
+@click.pass_context
+def test(ctx, package):
+    _test(ctx, package)
+
+def main():
+    cli(obj={})

src/oemof/tabular/datapackage/aggregation.py

@@ -41,7 +41,7 @@ def temporal_skip(datapackage, n, path="/tmp", name=None, *args):
     else:
         copied_package_name = name
 
-    copy_path = os.path.join(path, p.descriptor["name"])
+    copy_path = os.path.join(path, copied_package_name)
 
     copied_root = copy_datapackage(
         datapackage, os.path.abspath(copy_path), subset="data"

src/oemof/tabular/datapackage/building.py

@@ -12,6 +12,7 @@
 from datapackage import Package, Resource, infer
 from geojson import Feature, FeatureCollection, dump, load
 from shapely.geometry import shape
+import toml
 import pandas as pd
 import paramiko
 
@@ -255,8 +256,8 @@ def _ftp(remotepath, localpath, hostname, username=None, passwd=""):
 def _sftp(
     remotepath,
     localpath,
-    hostname="atlite.openmod.net",
-    username="atlite",
+    hostname="",
+    username="rutherford",
     password=""
 ):
     """ Download data with SFTP
@@ -275,6 +276,7 @@ def _sftp(
 
     client = paramiko.SSHClient()
     client.load_host_keys(os.path.expanduser("~/.ssh/known_hosts"))
+
     client.connect(hostname=hostname, username=username, password=password)
 
     sftp = client.open_sftp()
@@ -395,47 +397,35 @@ def member(x):
     return filepath
 
 
-def timeindex(year=None, periods=None, freq=None):
-    """ Create pandas datetimeindexself. If no parameters are not specified,
-    config file will be used to set the values.
+def timeindex(year, periods=8760, freq='H'):
+    """ Create pandas datetimeindex.
 
     Parameters
     ----------
     year: string
         Year of the index
     periods: string
-        Number of periods
+        Number of periods, default: 8760
     freq: string
-        Freq of the datetimeindex
+        Freq of the datetimeindex, default: 'H'
     """
-    config = get_config()
-
-    if not year:
-        year = str(config.get("year", "2050"))
-
-    if not periods:
-        periods = config.get("periods", 8760)
-
-    if not freq:
-        freq = "H"
 
     idx = pd.date_range(start=year, periods=periods, freq=freq)
 
     return idx
 
 
-def initialize_datapackage(config):
+def initialize(config, directory='.'):
     """ Initialize datapackage by reading config file and creating required
     directories (data/elements, data/sequences etc.) if directories are
     not specified in the config file, the default directory setup up
     will be used.
 
     """
-    directories = {
-        "elements": "data/elements",
+    sub_directories = {
+        "elements":  "data/elements",
         "sequences": "data/sequences",
         "geometries": "data/geometries",
-        "cache": "cache"
     }
 
     if not config:
@@ -447,16 +437,17 @@ def initialize_datapackage(config):
                 "Default path `{}` of config file not found!".format(default)
             )
 
-    if config.get("directories"):
-        directories = config["directories"]
+    sub_directories.update(config.get("sub-directories", {}))
 
-    for directory in directories.values():
+    for subdir in sub_directories.values():
         try:
-            os.makedirs(directory)
+            os.makedirs(os.path.join(directory, subdir))
         except OSError as e:
             if e.errno != errno.EEXIST:
                 raise
 
+    return sub_directories
+
 def input_filepath(file, directory="archive/"):
     """
     """
@@ -480,17 +471,16 @@ def input_filepath(file, directory="archive/"):
     return file_path
 
 
-def get_config(file="config.json"):
-    """ Read config json file
+def read_build_config(file="build.toml"):
+    """ Read config build file in toml format
 
     Parameters
     ----------
     file: string
         String with name of config file
     """
     try:
-        with open(file, "r") as stream:
-            config = json.load(stream)
+            config = toml.load(file)
 
             # create paths
             if config.get("directories"):
@@ -675,7 +665,7 @@ def write_elements(filename, elements, directory="data/elements",
         elements.index.name = "name"
 
     if not replace:
-        existing_elements = read_elements(filename)
+        existing_elements = read_elements(filename, directory=directory)
         elements = pd.concat(
             [existing_elements, elements], verify_integrity=True
         )
@@ -723,11 +713,11 @@ def write_sequences(filename, sequences, directory= "data/sequences",
         sequences = pd.concat(
             [existing_sequences, sequences], axis=1, verify_integrity=True
         )
-
-        if len(sequences.index.difference(timeindex())) > 0:
-            raise ValueError(
-                "Wrong timeindex for sequence {}.".format(filename)
-            )
+        # TODO: Adapt to new build config file
+        # if len(sequences.index.difference(timeindex())) > 0:
+        #     raise ValueError(
+        #         "Wrong timeindex for sequence {}.".format(filename)
+        #     )
 
     sequences = sequences.reindex(sorted(sequences.columns), axis=1)
 

src/oemof/tabular/datapackage/processing.py

@@ -21,6 +21,8 @@ def copy_datapackage(source, destination, subset=None):
         datapackage.json
     destination: str
         Destination of copied datapackage
+    name (optional): str
+        Name of datapackage
     only_data: str
         Name of directory to only copy subset of datapackage (for example
         only the 'data' directory)
@@ -69,7 +71,7 @@ def copy_datapackage(source, destination, subset=None):
     return destination
 
 
-def clean_datapackage(path=None, directories=["data", "cache", "resources"]):
+def clean(path=None, directories=["data", "cache", "resources"]):
     """
     Parameters
     ----------

src/oemof/tabular/examples/datapackages/investment/datapackage.json

@@ -36,7 +36,7 @@
                     {
                         "format": "default",
                         "name": "capacity",
-                        "type": "string"
+                        "type": "number"
                     },
                     {
                         "format": "default",
@@ -113,7 +113,7 @@
                     {
                         "format": "default",
                         "name": "capacity",
-                        "type": "string"
+                        "type": "number"
                     },
                     {
                         "format": "default",
@@ -223,7 +223,7 @@
                     {
                         "format": "default",
                         "name": "capacity",
-                        "type": "string"
+                        "type": "number"
                     },
                     {
                         "format": "default",
@@ -303,12 +303,12 @@
                     {
                         "format": "default",
                         "name": "storage_capacity",
-                        "type": "string"
+                        "type": "number"
                     },
                     {
                         "format": "default",
                         "name": "capacity",
-                        "type": "string"
+                        "type": "number"
                     },
                     {
                         "format": "default",
@@ -380,7 +380,7 @@
                     {
                         "format": "default",
                         "name": "capacity",
-                        "type": "string"
+                        "type": "number"
                     },
                     {
                         "format": "default",
@@ -524,4 +524,4 @@
     ],
     "profile": "tabular-data-package",
     "name": "renpass-invest-example"
-}
+}