remove general package

eggshell/config.py

@@ -0,0 +1,92 @@
+"""
+Configuration
+-------------
+
+WPS servers often need to specify a number of paths for processes to find data, shapefiles, caches and determine where
+outputs are stored. To make sure all birds use the same architecture, eggshell provides a :class:`Paths` class to help
+with this.
+"""
+
+import os
+from pywps import configuration
+import logging
+LOGGER = logging.getLogger("PYWPS")
+
+
+class Paths(object):
+    """This class facilitates the configuration of WPS birds."""
+    def __init__(self, module):
+        """Instantiate class relative to the given module.
+
+        :param module: Imported module relative to which paths will be defined.
+        """
+        self._base = module.__path__[0]
+
+    @property
+    def cache(self):
+        """Return the path to the server cache directory."""
+        out = configuration.get_config_value("cache", "cache_path")
+        if not out:
+            LOGGER.warn("No cache path configured. Using default value.")
+            out = os.path.join(configuration.get_config_value("server", "outputpath"), "cache")
+        return out
+
+    @property
+    def data(self):
+        """Return the path to the data directory."""
+        return os.path.join(self._base, 'data')
+
+    @property
+    def masks(self):
+        """Return the path to the masks directory."""
+        # TODO: currently this folder is not used
+        return os.path.join(self.data, 'masks')
+
+    @property
+    def outputpath(self):
+        """Return the server directory for process outputs."""
+        return configuration.get_config_value("server", "outputpath")
+
+    @property
+    def outputurl(self):
+        """Return the server URL for process outputs."""
+        return configuration.get_config_value("server", "outputurl").rstrip('/')
+
+    @property
+    def Rsrc_dir(self):
+        """Return the path to the R source directory."""
+        return os.path.join(self._base, 'Rsrc')
+
+    @property
+    def shapefiles(self):
+        """Return the path to the geographic data directory."""
+        return os.path.join(self.data, 'shapefiles')
+
+    @property
+    def static(self):
+        """Return the path to the static content directory."""
+        return os.path.join(self._base, 'static')
+
+    @property
+    def testdata(self):
+        """Return the path to the test data directory."""
+        return os.path.join(self._base, 'tests/testdata')
+
+
+# Should these go into the class or they're too specialized ?
+def esgfsearch_distrib():
+    """TODO"""
+    distrib = configuration.get_config_value("extra", "esgfsearch_distrib")
+    if distrib is None:
+        LOGGER.warn("No ESGF Search distrib option configured. Using default value.")
+        distrib = True
+    return distrib
+
+
+def esgfsearch_url():
+    """Return the server configuration value for the ESGF search node URL."""
+    url = configuration.get_config_value("extra", "esgfsearch_url")
+    if not url:
+        LOGGER.warn("No ESGF Search URL configured. Using default value.")
+        url = 'https://esgf-data.dkrz.de/esg-search'
+    return url

eggshell/exceptions.py

@@ -0,0 +1,2 @@
+class CalculationException(Exception):
+    pass

eggshell/log.py

@@ -0,0 +1,31 @@
+"""
+Logging
+-------
+
+Progress and errors in WPS processes are logged by the server. The initialization of the log file for each process
+is done using the :func:`init_process_logger`.
+
+"""
+
+import logging
+
+
+def init_process_logger(filename=None):
+    """Connect and initialize the logging mechanism to a given file.
+
+    :param str filename: Logging file name. Defaults to log.txt
+    """
+    filename = filename or 'log.txt'
+    # create console handler and set level to debug
+    ch = logging.FileHandler(filename=filename, mode="a", delay=False)
+    ch.setLevel(logging.DEBUG)
+
+    # create formatter
+    formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
+
+    # add formatter to ch
+    ch.setFormatter(formatter)
+
+    # add ch to root logger
+    logger = logging.getLogger()
+    logger.addHandler(ch)

eggshell/utils.py

@@ -0,0 +1,151 @@
+# -*- coding: utf-8 -*-
+
+"""Utitility functions."""
+
+import os
+import tempfile
+import tarfile
+import zipfile
+
+import logging
+
+LOGGER = logging.getLogger("EGGSHELL")
+
+
+def archive(resources, format=None, output_dir=None, mode=None):
+    """
+    Compresses a list of files into an archive.
+
+    :param resources: list of files to be stored in archive
+    :param format: archive format. Options: tar (default), zip
+    :param output_dir: path to output folder (default: tempory folder)
+    :param mode: for format='tar':
+                  'w' or 'w:'  open for writing without compression
+                  'w:gz'       open for writing with gzip compression
+                  'w:bz2'      open for writing with bzip2 compression
+                  'w|'         open an uncompressed stream for writing
+                  'w|gz'       open a gzip compressed stream for writing
+                  'w|bz2'      open a bzip2 compressed stream for writing
+
+                  for foramt='zip':
+                  read "r", write "w" or append "a"
+
+    :return str: archive path/filname.ext
+    """
+    format = format or 'tar'
+    output_dir = output_dir or tempfile.gettempdir()
+    mode = mode or 'w'
+
+    if format not in ['tar', 'zip']:
+        raise Exception('archive format {} not supported (only zip and tar)'.format(format))
+
+    LOGGER.info('compressing files to archive, format={}'.format(format))
+
+    # convert to list if necessary
+    if not isinstance(resources, list):
+        resources = list([resources])
+    resources = [x for x in resources if x is not None]
+
+    _, archive = tempfile.mkstemp(dir=output_dir, suffix='.{}'.format(format))
+
+    try:
+        if format == 'tar':
+            with tarfile.open(archive, mode) as tar:
+                for f in resources:
+                    tar.add(f, arcname=os.path.basename(f))
+        elif format == 'zip':
+            with zipfile.ZipFile(archive, mode=mode) as zf:
+                for f in resources:
+                    zf.write(f, os.path.basename(f))
+    except Exception as e:
+        raise Exception('failed to create {} archive: {}'.format(format, e))
+    return archive
+
+
+def extract_archive(resources, output_dir=None):
+    """
+    extracts archives (tar/zip)
+
+    :param resources: list of archive files (if netCDF files are in list,
+                     they are passed and returnd as well in the return).
+    :param output_dir: define a directory to store the results (default: tempory folder).
+
+    :return list: [list of extracted files]
+    """
+    output_dir = output_dir or tempfile.gettempdir()
+
+    if not isinstance(resources, list):
+        resources = list([resources])
+    files = []
+
+    for archive in resources:
+        try:
+            LOGGER.debug("archive=%s", archive)
+            ext = os.path.basename(archive).split('.')[-1]
+
+            if ext == 'nc':
+                files.append(os.path.join(output_dir, archive))
+            elif ext == 'tar':
+                with tarfile.open(archive, mode='r') as tar:
+                    tar.extractall()
+                    files.extend([os.path.join(output_dir, f) for f in tar.getnames()])
+            elif ext == 'zip':
+                with zipfile.open(archive, mode='r') as zf:
+                    zf.extractall()
+                    files.extend([os.path.join(output_dir, f) for f in zf.filelist])
+            else:
+                LOGGER.warn('file extention {} unknown'.format(ext))
+        except Exception:
+            LOGGER.error('failed to extract sub archive {}'.format(archive))
+    return files
+
+def get_coordinates(resource, variable=None, unrotate=False):
+    """
+    reads out the coordinates of a variable
+
+    :param resource: netCDF resource file
+    :param variable: variable name
+    :param unrotate: If True the coordinates will be returned for unrotated pole
+
+    :returns list, list: latitudes , longitudes
+    """
+    if type(resource) != list:
+        resource = [resource]
+
+    if variable is None:
+        variable = get_variable(resource)
+
+    if unrotate is False:
+        try:
+            if len(resource) > 1:
+                ds = MFDataset(resource)
+            else:
+                ds = Dataset(resource[0])
+
+            var = ds.variables[variable]
+            dims = list(var.dimensions)
+            if 'time' in dims: dims.remove('time')
+            # TODO: find position of lat and long in list and replace dims[0] dims[1]
+            lats = ds.variables[dims[0]][:]
+            lons = ds.variables[dims[1]][:]
+            ds.close()
+            LOGGER.info('got coordinates without pole rotation')
+        except Exception:
+            msg = 'failed to extract coordinates'
+            LOGGER.exception(msg)
+    else:
+        lats, lons = unrotate_pole(resource)
+        LOGGER.info('got coordinates with pole rotation')
+    return lats, lons
+
+
+def rename_complexinputs(complexinputs):
+    """
+    TODO: this method is just a dirty workaround to rename input files according to the url name.
+    """
+    resources = []
+    for inpt in complexinputs:
+        new_name = inpt.url.split('/')[-1]
+        os.rename(inpt.file, new_name)
+        resources.append(os.path.abspath(new_name))
+    return resources