diff --git a/doc/changes.rst b/doc/changes.rst index e2058c8b..678b0b16 100644 --- a/doc/changes.rst +++ b/doc/changes.rst @@ -5,7 +5,12 @@ Change Log 1.9.0 (unreleased) ------------------ -* No changes yet. +* Shorten Python version printed in dependency headers. +* :mod:`desiutil.test.test_plots` was not cleaning up after itself. +* Support new DESI+Anaconda software stack infrastructure (`PR #43`_). + +.. _`PR #43`: https://github.com/desihub/desiutil/pull/43 + 1.8.0 (2016-09-10) ------------------ diff --git a/doc/desiInstall.rst b/doc/desiInstall.rst index 2658ce69..65d9b971 100644 --- a/doc/desiInstall.rst +++ b/doc/desiInstall.rst @@ -45,7 +45,16 @@ file:: # This section can override details of Module file installation. # [Module Processing] + # + # nersc_module_dir overrides the Module file install directory for + # ALL NERSC hosts. + # nersc_module_dir = /project/projectdirs/desi/test/modules + # + # cori_module_dir overrides the Module file install directory only + # on cori. + # + cori_module_dir = /global/common/cori/contrib/desi/test/modules Finally, desiInstall both reads and sets several environment variables. @@ -66,7 +75,7 @@ Environment variables that strongly affect the behavior of desiInstall. :command:`svn`. Environment variables that are *set* by desiInstall for use by -``python setup.py`` or ``make``. +:command:`python setup.py` or :command:`make`. :envvar:`INSTALL_DIR` This variable is *set* by desiInstall to the directory that will contain @@ -96,7 +105,8 @@ manipulated by setting up Modules, or loading Module files. It may be manipulated by :mod:`desiutil.modules`. :envvar:`MODULESHOME` This variable points to the Modules infrastructure. If it is not set, - it typically means that the system has no Modules infrastructure. + it typically means that the system has no Modules infrastructure. This + is needed to find the executable program that reads Module files. :envvar:`PYTHONPATH` Obviously this is important for any Python package! :envvar:`PYTHONPATH` may be manipulated by :mod:`desiutil.modules`. @@ -106,6 +116,59 @@ manipulated by setting up Modules, or loading Module files. .. _desiutil: https://github.com/desihub/desiutil +Directory Structure Assumed by the Install +========================================== + +desiInstall is primarily intended to run in a production environment that +supports Module files. In practice, this means NERSC, though it can also +install on any other system that has a Modules infrastructure installed. + +*desiInstall does not install a Modules infrastructure for you.* You have to +do this yourself, if your system does not already have this. + +For the purposes of this section, we define ``$product_root`` as the +directory that desiInstall will be writing to. This directory could be the +same as :envvar:`DESI_PRODUCT_ROOT`, but for standard NERSC installs it +defaults to a pre-defined value. ``$product_root`` may contain the following +directories: + +code/ + This contains the installed code, the result of :command:`python setup.py install` + or :command:`make install`. The code is always placed in a ``product/version`` + directory. So for example, the full path to desiInstall might be + ``$product_root/code/desiutil/1.8.0/bin/desiInstall``. +conda/ + At NERSC, this contains the Anaconda_ infrastructure. desiInstall does + not manipulate this directory in any way, though it may *use* the + :command:`python` executable installed here. +modulefiles/ + This contains the the Module files installed by desiInstall. A Module + file is almost always named ``product/version``. For example, the + Module file for desiutil might be ``$product_root/modulefiles/desiutil/1.8.0``. + +.. _Anaconda: https://www.continuum.io + +Within a ``$product_root/code/product/version`` directory, you might see the +following: + +bin/ + Contains command-line executables, including Python or Shell scripts. +data/ + Rarely, packages need data files that cannot be incorporated into the + package structure itself, so it will be installed here. desimodel_ is + an example of this. +etc/ + Miscellaneous metadata and configuration. In most packages this only + contains a template Module file. +lib/pythonX.Y/site-packages/ + Contains installed Python code. ``X.Y`` would be ``2.7`` or ``3.5``. +py/ + Sometimes we need to install a git checkout rather than an installed package. + If so, the Python code will live in *this* directory not the ``lib/`` + directory, and the product's Module file will be adjusted accordingly. + +.. _desimodel: https://github.com/desihub/desimodel + Stages of the Install ===================== @@ -131,14 +194,14 @@ Product Existence After the product name and version have been determined, desiInstall constructs the full URL pointing to the product/version and runs the code necessary to verify that the product/version really exists. Typically, this -will be ``svn ls``, unless a GitHub install is detected. +will be :command:`svn ls`, unless a GitHub install is detected. Download Code ------------- -The code is downloaded, using ``svn export`` for standard (tag) installs, or -``svn checkout`` for trunk or branch installs. For GitHub installs, desiInstall -will look for a release tarball, or do a ``git clone`` for tag or master/branch +The code is downloaded, using :command:`svn export` for standard (tag) installs, or +:command:`svn checkout` for trunk or branch installs. For GitHub installs, desiInstall +will look for a release tarball, or do a :command:`git clone` for tag or master/branch installs. desiInstall will set the environment variable :envvar:`WORKING_DIR` to point to the directory containing this downloaded code. @@ -153,14 +216,14 @@ plain is simply copied to the final install directory. py If a setup.py file is detected, desiInstall will attempt to execute - ``python setup.py install``. This build type can be suppressed with the + :command:`python setup.py install`. This build type can be suppressed with the command line option ``--compile-c``. make If a Makefile is detected, desiInstall will attempt to execute - ``make install``. + :command:`make install`. src If a Makefile is not present, but a src/ directory is, - desiInstall will attempt to execute ``make -C src all``. This build type + desiInstall will attempt to execute :command:`make -C src all`. This build type *is* mutually exclusive with 'make', but is not mutually exclusive with the other types. @@ -171,12 +234,24 @@ Determine Install Directory --------------------------- The install directory is where the code will live permanently. If the -install is taking place at NERSC, the install directory will be placed in -``/project/projectdirs/desi/software/${NERSC_HOST}``. +install is taking place at NERSC, the top-level install directory is +predetermined based on the value of :envvar:`NERSC_HOST`. + +edison + ``/global/common/edison/contrib/desi`` +cori + ``/global/common/cori/contrib/desi`` +datatran + ``/global/project/projectdirs/desi/software/datatran`` +scigate + ``/global/project/projectdirs/desi/software/scigate`` At other locations, the user must set the environment variable :envvar:`DESI_PRODUCT_ROOT` to point to the equivalent directory. +The actual install directory is determined by appending ``/code/product/verson`` +to the combining the top-level directory listed above. + If the install directory already exists, desiInstall will exit, unless the ``--force`` parameter is supplied on the command line. @@ -213,7 +288,7 @@ desiInstall will scan :envvar:`WORKING_DIR` to determine the details that need to be added to the module file. The final module file will then be written into the DESI module directory at NERSC or the module directory associated with :envvar:`DESI_PRODUCT_ROOT`. If ``--default`` is specified on the command -line, an approproate .version file will be created. +line, an appropriate .version file will be created. Load Module ----------- @@ -239,7 +314,7 @@ Copy All Files The entire contents of :envvar:`WORKING_DIR` will be copied to :envvar:`INSTALL_DIR`. If this is a trunk or branch install and a src/ directory is detected, -desiInstall will attempt to run ``make -C src all`` in :envvar:`INSTALL_DIR`. +desiInstall will attempt to run :command:`make -C src all` in :envvar:`INSTALL_DIR`. For trunk or branch installs, no further processing is performed past this point. @@ -248,19 +323,19 @@ Create site-packages If the build-type 'py' is detected, a site-packages directory will be created in :envvar:`INSTALL_DIR`. If necessary, this directory will be -added to Python's ``sys.path``. +added to Python's :data:`sys.path`. Run setup.py ------------ -If the build-type 'py' is detected, ``python setup.py install`` will be run +If the build-type 'py' is detected, :command:`python setup.py install` will be run at this point. Build C/C++ Code ---------------- -If the build-type 'make' is detected, ``make install`` will be run in -:envvar:`WORKING_DIR`. If the build-type 'src' is detected, ``make -C src all`` +If the build-type 'make' is detected, :command:`make install` will be run in +:envvar:`WORKING_DIR`. If the build-type 'src' is detected, :command:`make -C src all` will be run in :envvar:`INSTALL_DIR`. Cross Install diff --git a/etc/desiutil.module b/etc/desiutil.module index f8bd0d42..8d43cb0b 100644 --- a/etc/desiutil.module +++ b/etc/desiutil.module @@ -52,11 +52,11 @@ module-whatis "Sets up $product/$version in your environment." # will need to set the DESI_PRODUCT_ROOT environment variable # if {{[info exists env(DESI_PRODUCT_ROOT)]}} {{ - set PRODUCT_ROOT $env(DESI_PRODUCT_ROOT) + set code_root $env(DESI_PRODUCT_ROOT)/code }} else {{ - set PRODUCT_ROOT /project/projectdirs/desi/software/$env(NERSC_HOST) + set code_root {product_root} }} -set PRODUCT_DIR $PRODUCT_ROOT/$product/$version +set PRODUCT_DIR $code_root/$product/$version # # This line creates an environment variable pointing to the install # directory of your product. diff --git a/py/desiutil/depend.py b/py/desiutil/depend.py index 2c2d627e..23c0e47c 100644 --- a/py/desiutil/depend.py +++ b/py/desiutil/depend.py @@ -134,15 +134,18 @@ def iterdep(header): 'redmonster', 'specter', 'speclite', 'specsim', ] -def add_dependencies(header, module_names=None): +def add_dependencies(header, module_names=None, long_python=False): '''Adds DEPNAMnn, DEPVERnn keywords to header for imported modules Args: - header : dict-like object, e.g. astropy.io.fits.Header + header : dict-like object, *e.g.* :class:`astropy.io.fits.Header`. Options: module_names : list of module names to check if None, checks desiutil.depend.possible_dependencies + long_python : If ``True`` use the full, verbose ``sys.version`` + string for the Python version. Otherwise, use a short + version, *e.g.*, ``3.5.2``. Only adds the dependency keywords if the module has already been previously loaded in this python session. Uses module.__version__ @@ -151,7 +154,10 @@ def add_dependencies(header, module_names=None): import sys import importlib - setdep(header, 'python', sys.version.replace('\n', ' ')) + py_version = ".".join(map(str, sys.version_info[0:3])) + if long_python: + py_version = sys.version.replace('\n', ' ') + setdep(header, 'python', py_version) if module_names is None: module_names = possible_dependencies diff --git a/py/desiutil/install.py b/py/desiutil/install.py index 75ad902c..ee12bec3 100644 --- a/py/desiutil/install.py +++ b/py/desiutil/install.py @@ -24,7 +24,7 @@ try: from cStringIO import StringIO except ImportError: - from io import StringIO + from io import BytesIO as StringIO if PY3: from configparser import ConfigParser as SafeConfigParser else: @@ -57,6 +57,7 @@ 'fiberassign': 'https://github.com/desihub/fiberassign', 'fiberassign_sqlite': 'https://github.com/desihub/fiberassign_sqlite', 'imaginglss': 'https://github.com/desihub/imaginglss', + 'redmonster': 'https://github.com/desihub/redmonster', 'specex': 'https://github.com/desihub/specex', 'speclite': 'https://github.com/dkirkby/speclite', 'specsim': 'https://github.com/desihub/specsim', @@ -139,6 +140,8 @@ class DesiInstall(object): this holds the object that reads it. cross_install_host : :class:`str` The NERSC host on which to perform cross-installs. + default_nersc_dir : :class:`dict` + The default code and Modules install directory for every NERSC host. executable : :class:`str` The command used to invoke the script. fullproduct : :class:`str` @@ -155,9 +158,7 @@ class DesiInstall(object): nersc : :class:`str` Holds the value of :envvar:`NERSC_HOST`, or ``None`` if not defined. nersc_hosts : :func:`tuple` - The list of NERSC hosts names to be used for cross-installs. - nersc_module_dir : :class:`str` - The directory that contains Module directories at NERSC. + The list of NERSC host names to be used for cross-installs. options : :class:`argparse.Namespace` The parsed command-line options. product_url : :class:`str` @@ -168,7 +169,10 @@ class DesiInstall(object): """ cross_install_host = 'edison' nersc_hosts = ('cori', 'edison', 'datatran', 'scigate') - nersc_module_dir = '/project/projectdirs/desi/software/modules' + default_nersc_dir = {'edison': '/global/common/edison/contrib/desi', + 'cori': '/global/common/cori/contrib/desi', + 'datatran': '/global/project/projectdirs/desi/software/datatran', + 'scigate': '/global/project/projectdirs/desi/software/scigate'} def __init__(self, test=False): """Bare-bones initialization. @@ -207,7 +211,8 @@ def get_options(self, test_args=None): log = logging.getLogger(__name__ + '.DesiInstall.get_options') check_env = {'MODULESHOME': None, 'DESI_PRODUCT_ROOT': None, - 'USER': None} + 'USER': None, + 'LANG': None} for e in check_env: try: check_env[e] = environ[e] @@ -613,20 +618,18 @@ def set_install_dir(self): The directory selected for installation. """ log = logging.getLogger(__name__ + '.DesiInstall.set_install_dir') - self.nersc = None try: self.nersc = environ['NERSC_HOST'] except KeyError: - pass + self.nersc = None if self.options.root is None or not isdir(self.options.root): if self.nersc is not None: - self.options.root = join('/project/projectdirs/desi/software', - self.nersc) + self.options.root = self.default_nersc_dir[self.nersc] else: message = "DESI_PRODUCT_ROOT is missing or not set." log.critical(message) raise DesiInstallException(message) - self.install_dir = join(self.options.root, self.baseproduct, + self.install_dir = join(self.options.root, 'code', self.baseproduct, self.baseversion) if isdir(self.install_dir) and not self.options.test: if self.options.force: @@ -698,6 +701,30 @@ def module_dependencies(self): self.module(m_command, d) return self.deps + @property + def nersc_module_dir(self): + """The directory that contains Module directories at NERSC. + """ + if not hasattr(self, 'nersc'): + return None + if self.nersc is None: + return None + else: + nersc_module = join(self.default_nersc_dir[self.nersc], + 'modulefiles') + if not hasattr(self, 'config'): + return nersc_module + if self.config is not None: + if self.config.has_option("Module Processing", + 'nersc_module_dir'): + nersc_module = self.config.get("Module Processing", + 'nersc_module_dir') + if self.config.has_option("Module Processing", + '{0}_module_dir'.format(self.nersc)): + nersc_module = self.config.get("Module Processing", + '{0}_module_dir'.format(self.nersc)) + return nersc_module + def install_module(self): """Process the module file. @@ -706,7 +733,7 @@ def install_module(self): :class:`str` The text of the processed module file. """ - log = logging.getLogger(__name__ + '.DesiInstall.process_module') + log = logging.getLogger(__name__ + '.DesiInstall.install_module') dev = False if 'py' in self.build_type: if self.is_trunk or self.is_branch: @@ -719,6 +746,7 @@ def install_module(self): self.working_dir, dev)) self.module_keywords = configure_module(self.baseproduct, self.baseversion, + join(self.options.root, 'code'), working_dir=self.working_dir, dev=dev) if self.options.moduledir == '': @@ -728,15 +756,8 @@ def install_module(self): if self.nersc is None: self.options.moduledir = join(self.options.root, 'modulefiles') else: - if self.config is not None: - if self.config.has_option("Module Processing", - 'nersc_module_dir'): - nersc_module = self.config.get("Module Processing", - 'nersc_module_dir') - else: - nersc_module = self.nersc_module_dir - log.debug("nersc_module_dir set to {0}.".format(nersc_module)) - self.options.moduledir = join(nersc_module, self.nersc) + self.options.moduledir = self.nersc_module_dir + log.debug("nersc_module_dir set to {0}.".format(self.options.moduledir)) if not self.options.test: if not isdir(self.options.moduledir): log.info("Creating Modules directory {0}.".format( @@ -917,7 +938,8 @@ def install(self): # r"matching '[^']+'") lines = [l for l in err.split('\n') if len(l) > 0 and manifestre.search(l) is None and - 'astropy_helpers' not in l] + 'astropy_helpers' not in l and + 'astropy-helpers' not in l] if len(lines) > 0: message = ("Error during installation: " + "{0}".format("\n".join(lines))) @@ -983,14 +1005,14 @@ def cross_install(self): for nh in nersc_hosts: if nh == cross_install_host: continue - dst = join('/project/projectdirs/desi/software', nh, + dst = join(self.default_nersc_dir[nh], 'code', self.baseproduct) if not islink(dst): src = join('..', cross_install_host, self.baseproduct) links.append((src, dst)) - dst = join('/project/projectdirs/desi/software/modules', - nh, self.baseproduct) + dst = join(self.default_nersc_dir[nh], 'modulefiles', + self.baseproduct) if not islink(dst): src = join('..', cross_install_host, self.baseproduct) diff --git a/py/desiutil/modules.py b/py/desiutil/modules.py index 4852facf..bfb45bbe 100644 --- a/py/desiutil/modules.py +++ b/py/desiutil/modules.py @@ -143,7 +143,7 @@ def desiutil_module_method(self, command, *arguments): return desiutil_module -def configure_module(product, version, working_dir=None, dev=False): +def configure_module(product, version, product_root, working_dir=None, dev=False): """Decide what needs to go in the Module file. Parameters @@ -152,6 +152,8 @@ def configure_module(product, version, working_dir=None, dev=False): Name of the product. version : :class:`str` Version of the product. + product_root : :class:`str` + Directory that contains the installed code. working_dir : :class:`str`, optional The directory to examine. If not set, the current working directory will be used. @@ -176,6 +178,7 @@ def configure_module(product, version, working_dir=None, dev=False): module_keywords = { 'name': product, 'version': version, + 'product_root': product_root, 'needs_bin': '# ', 'needs_python': '# ', 'needs_trunk_py': '# ', diff --git a/py/desiutil/test/t/desiInstall_configuration.ini b/py/desiutil/test/t/desiInstall_configuration.ini index 29d31033..2861d9b5 100644 --- a/py/desiutil/test/t/desiInstall_configuration.ini +++ b/py/desiutil/test/t/desiInstall_configuration.ini @@ -20,3 +20,17 @@ nersc_hosts = cori,edison,datatran [Known Products] my_new_product = https://github.com/me/my_new_product # desiutil = https://github.com/you/new_path_to_desiutil +# +# This section can override details of Module file installation. +# +[Module Processing] +# +# nersc_module_dir overrides the Module file install directory for +# ALL NERSC hosts. +# +nersc_module_dir = /project/projectdirs/desi/test/modules +# +# cori_module_dir overrides the Module file install directory only +# on cori. +# +cori_module_dir = /global/common/cori/contrib/desi/test/modules diff --git a/py/desiutil/test/test_depend.py b/py/desiutil/test/test_depend.py index 2222193c..67aa0330 100644 --- a/py/desiutil/test/test_depend.py +++ b/py/desiutil/test/test_depend.py @@ -6,9 +6,11 @@ print_function, unicode_literals) import unittest +import sys from collections import OrderedDict -from ..depend import setdep, getdep, hasdep, iterdep -from ..depend import Dependencies, add_dependencies +from ..depend import (setdep, getdep, hasdep, iterdep, Dependencies, + add_dependencies) +from .. import __version__ as desiutil_version try: from astropy.io import fits @@ -16,6 +18,7 @@ except ImportError: test_fits_header = False + class TestDepend(unittest.TestCase): """Test desiutil.depend """ @@ -96,6 +99,8 @@ def test_fits_header(self): getdep(hdr, 'foo') def test_update(self): + """Test updates of dependencies. + """ hdr = dict() setdep(hdr, 'blat', '1.0') self.assertEqual(getdep(hdr, 'blat'), '1.0') @@ -141,11 +146,17 @@ def test_class(self): self.assertEqual(x[name], getdep(hdr, name)) def test_add_dependencies(self): - """Test add_dependencies function.""" - import desiutil + """Test add_dependencies function. + """ + hdr = OrderedDict() + add_dependencies(hdr, long_python=True) + self.assertEqual(getdep(hdr, 'python'), + sys.version.replace('\n', ' ')) hdr = OrderedDict() add_dependencies(hdr) - self.assertEqual(getdep(hdr, 'desiutil'), desiutil.__version__) + self.assertEqual(getdep(hdr, 'python'), + ".".join(map(str, sys.version_info[0:3]))) + self.assertEqual(getdep(hdr, 'desiutil'), desiutil_version) import numpy add_dependencies(hdr) self.assertEqual(getdep(hdr, 'numpy'), numpy.__version__) diff --git a/py/desiutil/test/test_install.py b/py/desiutil/test/test_install.py index b5bb6188..2db9e783 100644 --- a/py/desiutil/test/test_install.py +++ b/py/desiutil/test/test_install.py @@ -299,12 +299,13 @@ def test_set_install_dir(self): 'desiutil', 'master']) self.desiInstall.get_product_version() install_dir = self.desiInstall.set_install_dir() - self.assertEqual(install_dir, join(self.data_dir, 'desiutil', + self.assertEqual(install_dir, join(self.data_dir, 'code', 'desiutil', 'master')) # Test for presence of existing directory. - tmpdir = join(self.data_dir, 'desiutil') + tmpdir = join(self.data_dir, 'code') mkdir(tmpdir) - mkdir(join(tmpdir, 'master')) + mkdir(join(tmpdir, 'desiutil')) + mkdir(join(tmpdir, 'desiutil', 'master')) options = self.desiInstall.get_options(['--root', self.data_dir, 'desiutil', 'master']) self.desiInstall.get_product_version() @@ -312,14 +313,14 @@ def test_set_install_dir(self): install_dir = self.desiInstall.set_install_dir() self.assertEqual(str(cm.exception), "Install directory, {0}, already exists!".format( - join(tmpdir, 'master'))) + join(tmpdir, 'desiutil', 'master'))) options = self.desiInstall.get_options(['--root', self.data_dir, '--force', 'desiutil', 'master']) self.assertTrue(self.desiInstall.options.force) self.desiInstall.get_product_version() install_dir = self.desiInstall.set_install_dir() - self.assertFalse(isdir(join(tmpdir, 'master'))) + self.assertFalse(isdir(join(tmpdir, 'desiutil', 'master'))) if isdir(tmpdir): rmtree(tmpdir) # Test NERSC installs. Unset DESI_PRODUCT_ROOT for this to work. @@ -328,12 +329,12 @@ def test_set_install_dir(self): del environ['DESI_PRODUCT_ROOT'] except KeyError: old_root = None - environ['NERSC_HOST'] = 'FAKE' + environ['NERSC_HOST'] = 'edison' options = self.desiInstall.get_options(['desiutil', 'master']) self.desiInstall.get_product_version() install_dir = self.desiInstall.set_install_dir() self.assertEqual(install_dir, join( - '/project/projectdirs/desi/software/FAKE', + self.desiInstall.default_nersc_dir['edison'], 'code', 'desiutil', 'master')) if old_root is not None: environ['DESI_PRODUCT_ROOT'] = old_root @@ -359,6 +360,28 @@ def test_start_modules(self): status = self.desiInstall.start_modules() self.assertTrue(callable(self.desiInstall.module)) + def test_nersc_module_dir(self): + """Test the nersc_module_dir property. + """ + self.assertIsNone(self.desiInstall.nersc_module_dir) + self.desiInstall.nersc = None + self.assertIsNone(self.desiInstall.nersc_module_dir) + for n in ('edison', 'cori', 'datatran', 'scigate'): + self.desiInstall.nersc = n + self.assertEqual(self.desiInstall.nersc_module_dir, + join(self.desiInstall.default_nersc_dir[n], + "modulefiles")) + options = self.desiInstall.get_options(['--configuration', + join(self.data_dir, + 'desiInstall_configuration.ini'), + 'my_new_product', '1.2.3']) + self.desiInstall.nersc = 'edison' + self.assertEqual(self.desiInstall.nersc_module_dir, + '/project/projectdirs/desi/test/modules') + self.desiInstall.nersc = 'cori' + self.assertEqual(self.desiInstall.nersc_module_dir, + '/global/common/cori/contrib/desi/test/modules') + def test_cleanup(self): """Test the cleanup stage of the install. """ diff --git a/py/desiutil/test/test_io.py b/py/desiutil/test/test_io.py index a5530fb5..88c43663 100644 --- a/py/desiutil/test/test_io.py +++ b/py/desiutil/test/test_io.py @@ -9,14 +9,14 @@ import sys import numpy as np from astropy.table import Table -#import pdb -import desiutil.io +from ..io import combine_dicts, decode_table, encode_table, yamlify try: basestring except NameError: # For Python 3 basestring = str + class TestIO(unittest.TestCase): """Test desiutil.io """ @@ -30,15 +30,16 @@ def tearDownClass(cls): pass def test_endecode_table(self): - #- Test encoding / decoding round-trip with numpy structured array + """Test encoding / decoding round-trip with numpy structured array. + """ data = np.zeros(4, dtype=[(str('x'), 'U4'), (str('y'), 'f8')]) data['x'] = 'ab' #- purposefully have fewer characters than width data['y'] = np.arange(len(data)) - t1 = desiutil.io.encode_table(data) + t1 = encode_table(data) self.assertEqual(t1['x'].dtype.kind, 'S') self.assertEqual(t1['y'].dtype.kind, data['y'].dtype.kind) self.assertTrue(np.all(t1['y'] == data['y'])) - t2 = desiutil.io.decode_table(t1, native=False) + t2 = decode_table(t1, native=False) self.assertEqual(t2['x'].dtype.kind, 'U') self.assertEqual(t2['x'].dtype, data['x'].dtype) self.assertEqual(t2['y'].dtype.kind, data['y'].dtype.kind) @@ -47,22 +48,22 @@ def test_endecode_table(self): #- have to give an encoding with self.assertRaises(UnicodeError): - tx = desiutil.io.encode_table(data, encoding=None) + tx = encode_table(data, encoding=None) del t1.meta['ENCODING'] with self.assertRaises(UnicodeError): - tx = desiutil.io.decode_table(t1, encoding=None, native=False) + tx = decode_table(t1, encoding=None, native=False) #- Test encoding / decoding round-trip with Table data = Table() data['x'] = np.asarray(['a', 'bb', 'ccc'], dtype='U') data['y'] = np.arange(len(data['x'])) - t1 = desiutil.io.encode_table(data) + t1 = encode_table(data) self.assertEqual(t1['x'].dtype.kind, 'S') self.assertEqual(t1['y'].dtype.kind, data['y'].dtype.kind) self.assertTrue(np.all(t1['y'] == data['y'])) - t2 = desiutil.io.decode_table(t1, native=False) + t2 = decode_table(t1, native=False) self.assertEqual(t2['x'].dtype.kind, 'U') self.assertEqual(t2['y'].dtype.kind, data['y'].dtype.kind) self.assertTrue(np.all(t2['x'] == data['x'])) @@ -70,28 +71,28 @@ def test_endecode_table(self): #- Non-default encoding with non-ascii unicode data['x'][0] = 'ยต' - t1 = desiutil.io.encode_table(data, encoding='utf-8') + t1 = encode_table(data, encoding='utf-8') self.assertEqual(t1.meta['ENCODING'], 'utf-8') - t2 = desiutil.io.decode_table(t1, encoding=None, native=False) + t2 = decode_table(t1, encoding=None, native=False) self.assertEqual(t2.meta['ENCODING'], 'utf-8') self.assertTrue(np.all(t2['x'] == data['x'])) with self.assertRaises(UnicodeEncodeError): - tx = desiutil.io.encode_table(data, encoding='ascii') + tx = encode_table(data, encoding='ascii') with self.assertRaises(UnicodeDecodeError): - tx = desiutil.io.decode_table(t1, encoding='ascii', native=False) + tx = decode_table(t1, encoding='ascii', native=False) #- Table can specify encoding if option encoding=None data['x'][0] = 'p' data.meta['ENCODING'] = 'utf-8' - t1 = desiutil.io.encode_table(data, encoding=None) + t1 = encode_table(data, encoding=None) self.assertEqual(t1.meta['ENCODING'], 'utf-8') - t2 = desiutil.io.decode_table(t1, native=False, encoding=None) + t2 = decode_table(t1, native=False, encoding=None) self.assertEqual(t2.meta['ENCODING'], 'utf-8') #- conflicting encodings print warning but still proceed - t1 = desiutil.io.encode_table(data, encoding='ascii') + t1 = encode_table(data, encoding='ascii') self.assertEqual(t1.meta['ENCODING'], 'ascii') - t2 = desiutil.io.decode_table(t1, encoding='utf-8', native=False) + t2 = decode_table(t1, encoding='utf-8', native=False) self.assertEqual(t2.meta['ENCODING'], 'utf-8') #- native=True should retain native str type @@ -99,7 +100,7 @@ def test_endecode_table(self): data['x'] = np.asarray(['a', 'bb', 'ccc'], dtype='S') data['y'] = np.arange(len(data['x'])) native_str_kind = np.str_('a').dtype.kind - tx = desiutil.io.decode_table(data, native=True) + tx = decode_table(data, native=True) self.assertIsInstance(tx['x'][0], str) #- Test roundtype with 2D array and unsigned ints @@ -107,16 +108,16 @@ def test_endecode_table(self): data['y'] = np.arange(len(data)) data['x'][0] = ['a', 'bb', 'c'] data['x'][1] = ['x', 'yy', 'z'] - t1 = desiutil.io.encode_table(data) + t1 = encode_table(data) self.assertEqual(t1['x'].dtype.kind, 'S') self.assertEqual(t1['y'].dtype.kind, data['y'].dtype.kind) self.assertTrue(np.all(t1['y'] == data['y'])) - t2 = desiutil.io.decode_table(t1, native=False) + t2 = decode_table(t1, native=False) self.assertEqual(t2['x'].dtype.kind, 'U') self.assertEqual(t2['x'].dtype, data['x'].dtype) self.assertEqual(t2['y'].dtype.kind, data['y'].dtype.kind) self.assertTrue(np.all(t2['x'] == data['x'])) - self.assertTrue(np.all(t2['y'] == data['y'])) + self.assertTrue(np.all(t2['y'] == data['y'])) def test_yamlify(self): """Test yamlify @@ -129,7 +130,7 @@ def test_yamlify(self): else: self.assertIsInstance(fdict['name'], unicode) # Run - ydict = desiutil.io.yamlify(fdict) + ydict = yamlify(fdict) self.assertIsInstance(ydict['flt32'], float) self.assertIsInstance(ydict['array'], list) for key in ydict.keys(): @@ -137,12 +138,12 @@ def test_yamlify(self): self.assertIsInstance(key, str) def test_combinedicts(self): - """ Test combining dicts + """Test combining dicts """ # Merge two dicts with a common key dict1 = {'a': {'b':2, 'c': 3}} dict2 = {'a': {'d': 4}} - dict3 = desiutil.io.combine_dicts(dict1, dict2) + dict3 = combine_dicts(dict1, dict2) self.assertEqual(dict3, {'a': {'b':2, 'c':3, 'd':4}}) # Shouldn't modify originals self.assertEqual(dict1, {'a': {'b':2, 'c': 3}}) @@ -150,7 +151,7 @@ def test_combinedicts(self): # Merge two dicts with different keys dict1 = {'a': 2} dict2 = {'b': 4} - dict3 = desiutil.io.combine_dicts(dict1, dict2) + dict3 = combine_dicts(dict1, dict2) self.assertEqual(dict3, {'a':2, 'b':4}) self.assertEqual(dict1, {'a': 2}) self.assertEqual(dict2, {'b': 4}) @@ -158,21 +159,22 @@ def test_combinedicts(self): dict1 = {'a': 2} dict2 = {'a': 4} with self.assertRaises(ValueError): - dict3 = desiutil.io.combine_dicts(dict1, dict2) + dict3 = combine_dicts(dict1, dict2) # Overlapping leafs with a scalar/dict mix raise an error dict1 = {'a': {'b':3}} dict2 = {'a': {'b':2, 'c': 3}} with self.assertRaises(ValueError): - desiutil.io.combine_dicts(dict1, dict2) + combine_dicts(dict1, dict2) with self.assertRaises(ValueError): - desiutil.io.combine_dicts(dict2, dict1) + combine_dicts(dict2, dict1) # Deep merge dict1 = {'a': {'b': {'x':1, 'y':2}}} dict2 = {'a': {'b': {'p':3, 'q':4}}} - dict3 = desiutil.io.combine_dicts(dict1, dict2) + dict3 = combine_dicts(dict1, dict2) self.assertEqual(dict3, {'a': {'b': {'x':1, 'y':2, 'p':3, 'q':4}}}) self.assertEqual(dict1, {'a': {'b': {'x':1, 'y':2}}}) self.assertEqual(dict2, {'a': {'b': {'p':3, 'q':4}}}) + if __name__ == '__main__': unittest.main() diff --git a/py/desiutil/test/test_modules.py b/py/desiutil/test/test_modules.py index fc17970f..498897ab 100644 --- a/py/desiutil/test/test_modules.py +++ b/py/desiutil/test/test_modules.py @@ -150,6 +150,7 @@ def test_configure_module(self): results = { 'name': 'foo', 'version': 'bar', + 'product_root': '/my/product/root', 'needs_bin': '', 'needs_python': '', 'needs_trunk_py': '# ', @@ -160,7 +161,8 @@ def test_configure_module(self): } for t in test_dirs: mkdir(join(self.data_dir, t)) - conf = configure_module('foo', 'bar', working_dir=self.data_dir) + conf = configure_module('foo', 'bar', '/my/product/root', + working_dir=self.data_dir) for key in results: self.assertEqual(conf[key], results[key]) # @@ -168,7 +170,8 @@ def test_configure_module(self): # results['needs_python'] = '# ' results['needs_trunk_py'] = '' - conf = configure_module('foo', 'bar', working_dir=self.data_dir, + conf = configure_module('foo', 'bar', '/my/product/root', + working_dir=self.data_dir, dev=True) for key in results: self.assertEqual(conf[key], results[key]) @@ -190,11 +193,13 @@ def test_configure_module(self): results['needs_trunk_py'] = '# ' results['needs_ld_lib'] = '# ' results['needs_idl'] = '# ' - conf = configure_module('foo', 'bar', working_dir=self.data_dir) + conf = configure_module('foo', 'bar', '/my/product/root', + working_dir=self.data_dir) results['needs_python'] = '# ' results['needs_trunk_py'] = '' results['trunk_py_dir'] = '' - conf = configure_module('foo', 'bar', working_dir=self.data_dir, + conf = configure_module('foo', 'bar', '/my/product/root', + working_dir=self.data_dir, dev=True) for key in results: self.assertEqual(conf[key], results[key]) diff --git a/py/desiutil/test/test_plots.py b/py/desiutil/test/test_plots.py index 3c7846cf..7fb9aea7 100644 --- a/py/desiutil/test/test_plots.py +++ b/py/desiutil/test/test_plots.py @@ -6,33 +6,34 @@ print_function, unicode_literals) # The line above will help with 2to3 support. import unittest -import sys +import os import numpy as np -#import pdb # Set non-interactive backend for Travis import matplotlib matplotlib.use('agg') import matplotlib.pyplot as plt -from desiutil.plots import plot_slices +from ..plots import plot_slices try: basestring except NameError: # For Python 3 basestring = str + class TestPlots(unittest.TestCase): """Test desiutil.plots """ @classmethod def setUpClass(cls): - pass + cls.plot_file = 'test.png' @classmethod def tearDownClass(cls): - pass + if os.path.exists(cls.plot_file): + os.remove(cls.plot_file) def test_slices(self): """Test plot_slices @@ -44,7 +45,7 @@ def test_slices(self): ax = plot_slices(x,y,0.,1.,0.) ax.set_ylabel('N sigma') ax.set_xlabel('x') - plt.savefig('test.png') + plt.savefig(self.plot_file) if __name__ == '__main__':