Manage configuration through .omie folder and early documentation

CHANGELOG.rst

@@ -2,7 +2,7 @@
 Changelog
 =========
 
-0.0.0 (2019-09-11)
+0.0.0 (2019-09-15)
 ------------------
 
-* First release on PyPI.
+* First release on GitHub.

README.rst

@@ -83,6 +83,156 @@ Installation
 
     pip install aomie
 
+Usage
+=====
+
+Import the aomie library into your Python libraries, scripts or applications as usual::
+
+    import aomie
+
+amoie includes a succint command line interface that make OMIE data handling extremely easy.
+
+From the command line help:
+
+.. raw:: html
+
+        <div>
+        <PRE>
+        <B>AOMIE</B>                         User Commands                        <B>AOMIE</B>
+
+        Usage: omie [OPTIONS] COMMAND [ARGS]...
+
+          aomie: OMIE electricity market data handling tool
+
+        Options:
+          --version               Display version.
+          -f, --config-file TEXT  Configuration file name and location.
+          -d, --display-config    Display configuration
+          -c, --config KEY VALUE  Overrides a config key/value pair.
+          -v, --verbose           Enables verbose mode.
+          --help                  Show this message and exit.
+
+        Commands:
+          download  Download OMIE data.
+          extract   Extract data from zip files.
+          fetch     Download, extract and insert files into database.
+          insert    Insert data into SQLite database.
+
+        </PRE></div>
+
+
+Some usage examples follow.
+
+A typical aomie starts by jointly setting the required configuration parameters through
+a toml configuration file
+
+::
+
+    omie -f myconfig.toml
+
+
+The configuration settings included in myconfig.toml are now avalaible to omie commands
+without having to explicitly call the toml config file again, e.g. to download data just type
+
+::
+
+    omie download
+
+Obviously you can use a different config file at any time
+
+::
+
+    omie -f otherconfig.toml download
+
+or just change some of the configuration settings
+
+::
+
+    omie -c end 200512
+
+To check the current configuration settings type
+
+::
+
+    omie -d
+
+This will return something like this
+
+.. raw:: html
+
+        <div>
+        <PRE>
+        &lt;Config OrderedDict([('servername', 'www.omel.es'),
+                     ('fichero', 'pdbf'),
+                     ('start', 200501),
+                     ('end', 200512),
+                     ('path', '_data3/'),
+                     ('dbname', 'test2.db3'),
+                     ('filter_unit',
+                      ['BES5',
+                       'CTN4',
+                       'PGR5',
+                       'ECT2'])])&gt;
+        </PRE></div>
+
+Once the zip files have been downloaded we can extract them like this
+
+::
+
+    omie extract
+
+To complete the workflow by inserting the extracted data into a SQLite database type
+
+::
+
+    omie insert
+
+The aomie commmand fetch bundles all the key data handling tasks. To run these
+tasks in a single step just type
+
+::
+
+    omie -f myconfig.toml -c end 200512 fetch
+
+Given the convenience of the fetch command, other commands that just perform one of
+the steps in omie workflow may seem redundant. Note however that omie data
+handling tasks covering long time horizons may involve downloading and processing
+hundreds of MBs that are disk and time consuming, and you may therefore prefer to proceed
+cautiously step by step.
+
+More information can be found in the command line help, e.g. to learn more about
+aomie commands such as download type
+
+::
+
+    omie download --help
+
+to display this
+
+.. raw:: html
+
+        <div>
+        <PRE>
+        <B>AOMIE</B>                         User Commands                        <B>AOMIE</B>
+
+        Usage: omie download [OPTIONS]
+
+          Download OMIE files to local destination.
+
+        Options:
+          -e, --extract  Extract downloaded files.
+          --help         Show this message and exit.
+
+        </PRE></div>
+
+
+From this help we learn that we can download and extract in a single step by typing
+
+::
+
+    omie download -e
+
+
 Documentation
 =============
 

src/aomie/cli.py

@@ -15,6 +15,10 @@
   Also see (1) from http://click.pocoo.org/5/setuptools/#setuptools-integration
 """
 import sys
+import os
+import shutil
+from collections import OrderedDict
+from pprint import pformat
 
 import click
 import toml
@@ -23,50 +27,140 @@
 from aomie.handling import (download_files, extract_files, insert_files,
                             fetch_files)
 
+TML = '.omie\omie.toml'
+
+MSGNOCFG = ('No amoie configuration available.\n'
+            'Type omie --help for information about '
+            'amoie configuration.')
+
+
+class Config(object):
+
+    def __init__(self, source=None, path=None):
+        self.verbose = False
+        self.source = source
+        if path:
+            pth = os.path.abspath(path)
+        else:
+            pth = os.getcwd()
+        tml = os.path.join(pth, TML)
+        if source:
+            click.echo(f'Reading configuration source from: \n\t{source}')
+            src = os.path.abspath(source)
+            os.makedirs(os.path.dirname(tml), exist_ok=True)
+            shutil.copy(src, tml)
+            self.home = tml
+        try:
+            self.config = toml.load(tml, _dict=OrderedDict)
+        except FileNotFoundError:
+            pass
+
+    def set_config(self, key, value):
+        self.config[key] = value
+        tml = os.path.join(os.getcwd(), TML)
+        with open(tml, 'w') as f:
+            toml.dump(self.config, f)
+            # f.write(pformat(self.config))
+        if self.verbose:
+            click.echo('Updating configuration:\n\tconfig[%s] = %s' %
+                       (key, value), file=sys.stderr)
+
+    def __repr__(self):
+        try:
+            return '<Config %s>' % pformat(self.config)
+        except AttributeError:
+            return '<Config %s>' % dict()
+
+
+pass_config = click.make_pass_decorator(Config, ensure=True)
+
 
 @click.group(invoke_without_command=True)
-@click.pass_context
 @click.option('--version', is_flag=True, default=False,
               help='Display version.')
-def cli(ctx, version):
+@click.option('--config-file', '-f', required=False,
+              help='Configuration file name and location.')
+@click.option('--display-config', '-d', is_flag=True, default=False,
+              help='Display configuration')
+@click.option('--config', '-c', nargs=2, multiple=True,
+              metavar='KEY VALUE', help='Overrides a config key/value pair.')
+@click.option('--verbose', '-v', is_flag=True,
+              help='Enables verbose mode.')
+@click.pass_context
+def cli(ctx, config_file, config, verbose, version, display_config):
     """aomie: OMIE electricity market data handling tool"""
-    if ctx.invoked_subcommand is None and not version:
-        click.echo(ctx.get_help())
+    """Create a config object to handle data handling specs."""
+    # Remember the config object as the context object.  From
+    # this point onwards other commands can refer to it by using the
+    # @pass_config decorator.
     if version:
-        click.echo(__version__)
+        click.echo(f'aomie {__version__}')
+        return
+    source = None
+    if config_file:
+        source = os.path.abspath(config_file)
+    ctx.obj = Config(source)
+    ctx.obj.verbose = verbose
+    for key, value in config:
+        ctx.obj.set_config(key, value)
+    if display_config:
+        print(ctx.obj)
 
 
 @cli.command(short_help='Download OMIE data.')
-@click.argument('config')
 @click.option('--extract', '-e', is_flag=True, default=False,
               help='Extract downloaded files.')
+@pass_config
 def download(config, extract):
     """Download OMIE files to local destination."""
-    cfg = toml.load(config)
+    try:
+        cfg = config.config
+    except AttributeError:
+        click.echo(MSGNOCFG)
+        return
+    pth = os.path.abspath(cfg['path'])
+    click.echo(f'Downloading files to:\n\t{pth}')
     download_files(**cfg)
     if extract:
         extract_files(**cfg)
 
 
 @cli.command(short_help='Extract data from zip files.')
-@click.argument('config')
+@pass_config
 def extract(config):
     """Extract zip files."""
-    cfg = toml.load(config)
+    try:
+        cfg = config.config
+    except AttributeError:
+        click.echo(MSGNOCFG)
+        return
+    pth = os.path.abspath(cfg['path'])
+    click.echo(f'Extracting files at {pth}')
     extract_files(**cfg)
 
 
 @cli.command(short_help='Insert data into SQLite database.')
-@click.argument('config')
+@pass_config
 def insert(config):
     """Insert data into SQLite database."""
-    cfg = toml.load(config)
+    try:
+        cfg = config.config
+    except AttributeError:
+        click.echo(MSGNOCFG)
+        return
+    pthfrom = os.path.abspath(cfg['path'])
+    pthto = os.path.abspath(cfg['dbname'])
+    click.echo(f'Inserting files from:\n\t {pthfrom}\nto:\n\t {pthto}')
     insert_files(**cfg)
 
 
 @cli.command(short_help='Download, extract and insert files into database.')
-@click.argument('config')
+@pass_config
 def fetch(config):
     """Download, extract and insert into database."""
-    cfg = toml.load(config)
+    try:
+        cfg = config.config
+    except AttributeError:
+        click.echo(MSGNOCFG)
+        return
     fetch_files(**cfg)

src/aomie/handling.py

@@ -31,21 +31,22 @@ def _move_files(source, destination):
                          dtype=dtype)
 
 
-def download_files(servername, fichero, path, listfile, **kwargs):
+def download_files(servername, fichero, path, start, end, **kwargs):
     fileroot = f'/datosPub/{fichero}/'
+    os.makedirs(os.path.dirname(path), exist_ok=True)
 
-    with open(listfile) as f:
-        items = f.read().splitlines()
-
+    items=[*map('pdbf_{}.zip'.format,
+                pd.date_range(start=pd.to_datetime(str(start),format='%Y%m'),
+                              end=pd.to_datetime(str(end), format='%Y%m'),
+                              freq='MS').strftime('%Y%m'))]
     for item in items:
         filename = fileroot + item
         localname = path + item
         remoteaddr = 'http://%s%s' % (servername, filename)
         print(remoteaddr)
         urllib.request.urlretrieve(remoteaddr, localname)
 
-    print()
-    print('Downloaded %s files to %s' % (str(len(items)), path[:-1]))
+    print('\nDownloaded %s files to %s' % (str(len(items)), path[:-1]))
 
 
 def extract_files(path, **kwargs):
@@ -62,8 +63,7 @@ def extract_files(path, **kwargs):
 
 def gather_files(sources, destination, remove=False):
     """Move files in sources to destination."""
-    if not os.path.exists(destination):
-        os.makedirs(destination)
+    os.makedirs(destination, exist_ok=True)
     for s in sources:
         _move_files(s, destination)
         if remove:
@@ -102,8 +102,8 @@ def insert_files(path, filter_unit, dbname, fichero, delatstart=True,
             print(pd.read_sql(f'select * from {fichero}', conn))
 
 
-def fetch_files(path, fichero, filter_unit, dbname, servername, listfile,
+def fetch_files(path, fichero, filter_unit, dbname, servername, start, end,
                 **kwargs):
-    download_files(servername, fichero, path, listfile)
+    download_files(servername, fichero, path, start, end)
     extract_files(path)
     insert_files(path, filter_unit, dbname, fichero)