From e4ba1f64525037a806753f6a5f568b18feb59822 Mon Sep 17 00:00:00 2001 From: Benjamin Alan Weaver Date: Fri, 16 Sep 2016 14:45:46 -0700 Subject: [PATCH 01/14] use shorter Python version by default --- doc/changes.rst | 2 +- py/desiutil/depend.py | 12 +++++++++--- py/desiutil/test/test_depend.py | 21 ++++++++++++++++----- 3 files changed, 26 insertions(+), 9 deletions(-) diff --git a/doc/changes.rst b/doc/changes.rst index e2058c8b..f2a50d8f 100644 --- a/doc/changes.rst +++ b/doc/changes.rst @@ -5,7 +5,7 @@ Change Log 1.9.0 (unreleased) ------------------ -* No changes yet. +* Shorten Python version printed in dependency headers. 1.8.0 (2016-09-10) ------------------ diff --git a/py/desiutil/depend.py b/py/desiutil/depend.py index 2c2d627e..23c0e47c 100644 --- a/py/desiutil/depend.py +++ b/py/desiutil/depend.py @@ -134,15 +134,18 @@ def iterdep(header): 'redmonster', 'specter', 'speclite', 'specsim', ] -def add_dependencies(header, module_names=None): +def add_dependencies(header, module_names=None, long_python=False): '''Adds DEPNAMnn, DEPVERnn keywords to header for imported modules Args: - header : dict-like object, e.g. astropy.io.fits.Header + header : dict-like object, *e.g.* :class:`astropy.io.fits.Header`. Options: module_names : list of module names to check if None, checks desiutil.depend.possible_dependencies + long_python : If ``True`` use the full, verbose ``sys.version`` + string for the Python version. Otherwise, use a short + version, *e.g.*, ``3.5.2``. Only adds the dependency keywords if the module has already been previously loaded in this python session. Uses module.__version__ @@ -151,7 +154,10 @@ def add_dependencies(header, module_names=None): import sys import importlib - setdep(header, 'python', sys.version.replace('\n', ' ')) + py_version = ".".join(map(str, sys.version_info[0:3])) + if long_python: + py_version = sys.version.replace('\n', ' ') + setdep(header, 'python', py_version) if module_names is None: module_names = possible_dependencies diff --git a/py/desiutil/test/test_depend.py b/py/desiutil/test/test_depend.py index 2222193c..67aa0330 100644 --- a/py/desiutil/test/test_depend.py +++ b/py/desiutil/test/test_depend.py @@ -6,9 +6,11 @@ print_function, unicode_literals) import unittest +import sys from collections import OrderedDict -from ..depend import setdep, getdep, hasdep, iterdep -from ..depend import Dependencies, add_dependencies +from ..depend import (setdep, getdep, hasdep, iterdep, Dependencies, + add_dependencies) +from .. import __version__ as desiutil_version try: from astropy.io import fits @@ -16,6 +18,7 @@ except ImportError: test_fits_header = False + class TestDepend(unittest.TestCase): """Test desiutil.depend """ @@ -96,6 +99,8 @@ def test_fits_header(self): getdep(hdr, 'foo') def test_update(self): + """Test updates of dependencies. + """ hdr = dict() setdep(hdr, 'blat', '1.0') self.assertEqual(getdep(hdr, 'blat'), '1.0') @@ -141,11 +146,17 @@ def test_class(self): self.assertEqual(x[name], getdep(hdr, name)) def test_add_dependencies(self): - """Test add_dependencies function.""" - import desiutil + """Test add_dependencies function. + """ + hdr = OrderedDict() + add_dependencies(hdr, long_python=True) + self.assertEqual(getdep(hdr, 'python'), + sys.version.replace('\n', ' ')) hdr = OrderedDict() add_dependencies(hdr) - self.assertEqual(getdep(hdr, 'desiutil'), desiutil.__version__) + self.assertEqual(getdep(hdr, 'python'), + ".".join(map(str, sys.version_info[0:3]))) + self.assertEqual(getdep(hdr, 'desiutil'), desiutil_version) import numpy add_dependencies(hdr) self.assertEqual(getdep(hdr, 'numpy'), numpy.__version__) From 8bbfd49ef7200ec2b561fd390db94c71fa5eeb4c Mon Sep 17 00:00:00 2001 From: Benjamin Alan Weaver Date: Fri, 16 Sep 2016 15:09:38 -0700 Subject: [PATCH 02/14] clean up io, plots tests --- doc/changes.rst | 1 + py/desiutil/test/test_io.py | 60 ++++++++++++++++++---------------- py/desiutil/test/test_plots.py | 13 ++++---- 3 files changed, 39 insertions(+), 35 deletions(-) diff --git a/doc/changes.rst b/doc/changes.rst index f2a50d8f..50101b8b 100644 --- a/doc/changes.rst +++ b/doc/changes.rst @@ -6,6 +6,7 @@ Change Log ------------------ * Shorten Python version printed in dependency headers. +* :mod:`desiutil.test.test_plots` was not cleaning up after itself. 1.8.0 (2016-09-10) ------------------ diff --git a/py/desiutil/test/test_io.py b/py/desiutil/test/test_io.py index a5530fb5..88c43663 100644 --- a/py/desiutil/test/test_io.py +++ b/py/desiutil/test/test_io.py @@ -9,14 +9,14 @@ import sys import numpy as np from astropy.table import Table -#import pdb -import desiutil.io +from ..io import combine_dicts, decode_table, encode_table, yamlify try: basestring except NameError: # For Python 3 basestring = str + class TestIO(unittest.TestCase): """Test desiutil.io """ @@ -30,15 +30,16 @@ def tearDownClass(cls): pass def test_endecode_table(self): - #- Test encoding / decoding round-trip with numpy structured array + """Test encoding / decoding round-trip with numpy structured array. + """ data = np.zeros(4, dtype=[(str('x'), 'U4'), (str('y'), 'f8')]) data['x'] = 'ab' #- purposefully have fewer characters than width data['y'] = np.arange(len(data)) - t1 = desiutil.io.encode_table(data) + t1 = encode_table(data) self.assertEqual(t1['x'].dtype.kind, 'S') self.assertEqual(t1['y'].dtype.kind, data['y'].dtype.kind) self.assertTrue(np.all(t1['y'] == data['y'])) - t2 = desiutil.io.decode_table(t1, native=False) + t2 = decode_table(t1, native=False) self.assertEqual(t2['x'].dtype.kind, 'U') self.assertEqual(t2['x'].dtype, data['x'].dtype) self.assertEqual(t2['y'].dtype.kind, data['y'].dtype.kind) @@ -47,22 +48,22 @@ def test_endecode_table(self): #- have to give an encoding with self.assertRaises(UnicodeError): - tx = desiutil.io.encode_table(data, encoding=None) + tx = encode_table(data, encoding=None) del t1.meta['ENCODING'] with self.assertRaises(UnicodeError): - tx = desiutil.io.decode_table(t1, encoding=None, native=False) + tx = decode_table(t1, encoding=None, native=False) #- Test encoding / decoding round-trip with Table data = Table() data['x'] = np.asarray(['a', 'bb', 'ccc'], dtype='U') data['y'] = np.arange(len(data['x'])) - t1 = desiutil.io.encode_table(data) + t1 = encode_table(data) self.assertEqual(t1['x'].dtype.kind, 'S') self.assertEqual(t1['y'].dtype.kind, data['y'].dtype.kind) self.assertTrue(np.all(t1['y'] == data['y'])) - t2 = desiutil.io.decode_table(t1, native=False) + t2 = decode_table(t1, native=False) self.assertEqual(t2['x'].dtype.kind, 'U') self.assertEqual(t2['y'].dtype.kind, data['y'].dtype.kind) self.assertTrue(np.all(t2['x'] == data['x'])) @@ -70,28 +71,28 @@ def test_endecode_table(self): #- Non-default encoding with non-ascii unicode data['x'][0] = 'ยต' - t1 = desiutil.io.encode_table(data, encoding='utf-8') + t1 = encode_table(data, encoding='utf-8') self.assertEqual(t1.meta['ENCODING'], 'utf-8') - t2 = desiutil.io.decode_table(t1, encoding=None, native=False) + t2 = decode_table(t1, encoding=None, native=False) self.assertEqual(t2.meta['ENCODING'], 'utf-8') self.assertTrue(np.all(t2['x'] == data['x'])) with self.assertRaises(UnicodeEncodeError): - tx = desiutil.io.encode_table(data, encoding='ascii') + tx = encode_table(data, encoding='ascii') with self.assertRaises(UnicodeDecodeError): - tx = desiutil.io.decode_table(t1, encoding='ascii', native=False) + tx = decode_table(t1, encoding='ascii', native=False) #- Table can specify encoding if option encoding=None data['x'][0] = 'p' data.meta['ENCODING'] = 'utf-8' - t1 = desiutil.io.encode_table(data, encoding=None) + t1 = encode_table(data, encoding=None) self.assertEqual(t1.meta['ENCODING'], 'utf-8') - t2 = desiutil.io.decode_table(t1, native=False, encoding=None) + t2 = decode_table(t1, native=False, encoding=None) self.assertEqual(t2.meta['ENCODING'], 'utf-8') #- conflicting encodings print warning but still proceed - t1 = desiutil.io.encode_table(data, encoding='ascii') + t1 = encode_table(data, encoding='ascii') self.assertEqual(t1.meta['ENCODING'], 'ascii') - t2 = desiutil.io.decode_table(t1, encoding='utf-8', native=False) + t2 = decode_table(t1, encoding='utf-8', native=False) self.assertEqual(t2.meta['ENCODING'], 'utf-8') #- native=True should retain native str type @@ -99,7 +100,7 @@ def test_endecode_table(self): data['x'] = np.asarray(['a', 'bb', 'ccc'], dtype='S') data['y'] = np.arange(len(data['x'])) native_str_kind = np.str_('a').dtype.kind - tx = desiutil.io.decode_table(data, native=True) + tx = decode_table(data, native=True) self.assertIsInstance(tx['x'][0], str) #- Test roundtype with 2D array and unsigned ints @@ -107,16 +108,16 @@ def test_endecode_table(self): data['y'] = np.arange(len(data)) data['x'][0] = ['a', 'bb', 'c'] data['x'][1] = ['x', 'yy', 'z'] - t1 = desiutil.io.encode_table(data) + t1 = encode_table(data) self.assertEqual(t1['x'].dtype.kind, 'S') self.assertEqual(t1['y'].dtype.kind, data['y'].dtype.kind) self.assertTrue(np.all(t1['y'] == data['y'])) - t2 = desiutil.io.decode_table(t1, native=False) + t2 = decode_table(t1, native=False) self.assertEqual(t2['x'].dtype.kind, 'U') self.assertEqual(t2['x'].dtype, data['x'].dtype) self.assertEqual(t2['y'].dtype.kind, data['y'].dtype.kind) self.assertTrue(np.all(t2['x'] == data['x'])) - self.assertTrue(np.all(t2['y'] == data['y'])) + self.assertTrue(np.all(t2['y'] == data['y'])) def test_yamlify(self): """Test yamlify @@ -129,7 +130,7 @@ def test_yamlify(self): else: self.assertIsInstance(fdict['name'], unicode) # Run - ydict = desiutil.io.yamlify(fdict) + ydict = yamlify(fdict) self.assertIsInstance(ydict['flt32'], float) self.assertIsInstance(ydict['array'], list) for key in ydict.keys(): @@ -137,12 +138,12 @@ def test_yamlify(self): self.assertIsInstance(key, str) def test_combinedicts(self): - """ Test combining dicts + """Test combining dicts """ # Merge two dicts with a common key dict1 = {'a': {'b':2, 'c': 3}} dict2 = {'a': {'d': 4}} - dict3 = desiutil.io.combine_dicts(dict1, dict2) + dict3 = combine_dicts(dict1, dict2) self.assertEqual(dict3, {'a': {'b':2, 'c':3, 'd':4}}) # Shouldn't modify originals self.assertEqual(dict1, {'a': {'b':2, 'c': 3}}) @@ -150,7 +151,7 @@ def test_combinedicts(self): # Merge two dicts with different keys dict1 = {'a': 2} dict2 = {'b': 4} - dict3 = desiutil.io.combine_dicts(dict1, dict2) + dict3 = combine_dicts(dict1, dict2) self.assertEqual(dict3, {'a':2, 'b':4}) self.assertEqual(dict1, {'a': 2}) self.assertEqual(dict2, {'b': 4}) @@ -158,21 +159,22 @@ def test_combinedicts(self): dict1 = {'a': 2} dict2 = {'a': 4} with self.assertRaises(ValueError): - dict3 = desiutil.io.combine_dicts(dict1, dict2) + dict3 = combine_dicts(dict1, dict2) # Overlapping leafs with a scalar/dict mix raise an error dict1 = {'a': {'b':3}} dict2 = {'a': {'b':2, 'c': 3}} with self.assertRaises(ValueError): - desiutil.io.combine_dicts(dict1, dict2) + combine_dicts(dict1, dict2) with self.assertRaises(ValueError): - desiutil.io.combine_dicts(dict2, dict1) + combine_dicts(dict2, dict1) # Deep merge dict1 = {'a': {'b': {'x':1, 'y':2}}} dict2 = {'a': {'b': {'p':3, 'q':4}}} - dict3 = desiutil.io.combine_dicts(dict1, dict2) + dict3 = combine_dicts(dict1, dict2) self.assertEqual(dict3, {'a': {'b': {'x':1, 'y':2, 'p':3, 'q':4}}}) self.assertEqual(dict1, {'a': {'b': {'x':1, 'y':2}}}) self.assertEqual(dict2, {'a': {'b': {'p':3, 'q':4}}}) + if __name__ == '__main__': unittest.main() diff --git a/py/desiutil/test/test_plots.py b/py/desiutil/test/test_plots.py index 3c7846cf..7fb9aea7 100644 --- a/py/desiutil/test/test_plots.py +++ b/py/desiutil/test/test_plots.py @@ -6,33 +6,34 @@ print_function, unicode_literals) # The line above will help with 2to3 support. import unittest -import sys +import os import numpy as np -#import pdb # Set non-interactive backend for Travis import matplotlib matplotlib.use('agg') import matplotlib.pyplot as plt -from desiutil.plots import plot_slices +from ..plots import plot_slices try: basestring except NameError: # For Python 3 basestring = str + class TestPlots(unittest.TestCase): """Test desiutil.plots """ @classmethod def setUpClass(cls): - pass + cls.plot_file = 'test.png' @classmethod def tearDownClass(cls): - pass + if os.path.exists(cls.plot_file): + os.remove(cls.plot_file) def test_slices(self): """Test plot_slices @@ -44,7 +45,7 @@ def test_slices(self): ax = plot_slices(x,y,0.,1.,0.) ax.set_ylabel('N sigma') ax.set_xlabel('x') - plt.savefig('test.png') + plt.savefig(self.plot_file) if __name__ == '__main__': From 5e5a2e4b4d11835cd9de15f1ea8a18e56d1fdaad Mon Sep 17 00:00:00 2001 From: Benjamin Alan Weaver Date: Tue, 20 Sep 2016 15:57:44 -0700 Subject: [PATCH 03/14] update module install paths --- doc/desiInstall.rst | 9 +++++++++ py/desiutil/install.py | 45 ++++++++++++++++++++++++++++-------------- 2 files changed, 39 insertions(+), 15 deletions(-) diff --git a/doc/desiInstall.rst b/doc/desiInstall.rst index 2658ce69..9cfcd47c 100644 --- a/doc/desiInstall.rst +++ b/doc/desiInstall.rst @@ -45,7 +45,16 @@ file:: # This section can override details of Module file installation. # [Module Processing] + # + # nersc_module_dir overrides the Module file install directory for + # ALL NERSC hosts. + # nersc_module_dir = /project/projectdirs/desi/test/modules + # + # cori_module_dir overrides the Module file install directory only + # on cori. + # + cori_module_dir = /global/common/cori/contrib/desi/test/modules Finally, desiInstall both reads and sets several environment variables. diff --git a/py/desiutil/install.py b/py/desiutil/install.py index 75ad902c..3d745fe1 100644 --- a/py/desiutil/install.py +++ b/py/desiutil/install.py @@ -139,6 +139,8 @@ class DesiInstall(object): this holds the object that reads it. cross_install_host : :class:`str` The NERSC host on which to perform cross-installs. + default_module_dir : :class:`dict` + The default Modules install directory for every NERSC host. executable : :class:`str` The command used to invoke the script. fullproduct : :class:`str` @@ -155,9 +157,7 @@ class DesiInstall(object): nersc : :class:`str` Holds the value of :envvar:`NERSC_HOST`, or ``None`` if not defined. nersc_hosts : :func:`tuple` - The list of NERSC hosts names to be used for cross-installs. - nersc_module_dir : :class:`str` - The directory that contains Module directories at NERSC. + The list of NERSC host names to be used for cross-installs. options : :class:`argparse.Namespace` The parsed command-line options. product_url : :class:`str` @@ -168,8 +168,11 @@ class DesiInstall(object): """ cross_install_host = 'edison' nersc_hosts = ('cori', 'edison', 'datatran', 'scigate') - nersc_module_dir = '/project/projectdirs/desi/software/modules' - + # nersc_module_dir = '/project/projectdirs/desi/software/modules' + default_module_dir = {'edison': '/global/common/edison/contrib/desi/modulefiles', + 'cori': '/global/common/cori/contrib/desi/modulefiles', + 'datatran': '/global/project/projectdirs/desi/software/datatran/modulefiles', + 'scigate': '/global/project/projectdirs/desi/software/datatran/modulefiles'} def __init__(self, test=False): """Bare-bones initialization. @@ -698,6 +701,25 @@ def module_dependencies(self): self.module(m_command, d) return self.deps + @property + def nersc_module_dir(self): + """The directory that contains Module directories at NERSC. + """ + if self.nersc is None: + return None + else: + nersc_module = self.default_module_dir[self.nersc] + if self.config is not None: + if self.config.has_option("Module Processing", + 'nersc_module_dir'): + nersc_module = self.config.get("Module Processing", + 'nersc_module_dir') + if self.config.has_option("Module Processing", + '{0}_module_dir'.format(self.nersc)): + nersc_module = self.config.get("Module Processing", + '{0}_module_dir'.format(self.nersc)) + return nersc_module + def install_module(self): """Process the module file. @@ -706,7 +728,7 @@ def install_module(self): :class:`str` The text of the processed module file. """ - log = logging.getLogger(__name__ + '.DesiInstall.process_module') + log = logging.getLogger(__name__ + '.DesiInstall.install_module') dev = False if 'py' in self.build_type: if self.is_trunk or self.is_branch: @@ -728,15 +750,8 @@ def install_module(self): if self.nersc is None: self.options.moduledir = join(self.options.root, 'modulefiles') else: - if self.config is not None: - if self.config.has_option("Module Processing", - 'nersc_module_dir'): - nersc_module = self.config.get("Module Processing", - 'nersc_module_dir') - else: - nersc_module = self.nersc_module_dir - log.debug("nersc_module_dir set to {0}.".format(nersc_module)) - self.options.moduledir = join(nersc_module, self.nersc) + self.options.moduledir = self.nersc_module_dir + log.debug("nersc_module_dir set to {0}.".format(self.options.moduledir)) if not self.options.test: if not isdir(self.options.moduledir): log.info("Creating Modules directory {0}.".format( From b4752873e4edff8cc1f7f196dcd3de0b65f96aec Mon Sep 17 00:00:00 2001 From: Benjamin Alan Weaver Date: Tue, 20 Sep 2016 17:18:13 -0700 Subject: [PATCH 04/14] working on new NERSC paths --- doc/desiInstall.rst | 46 ++++++++++++------- py/desiutil/install.py | 34 ++++++++------ .../test/t/desiInstall_configuration.ini | 14 ++++++ py/desiutil/test/test_install.py | 26 ++++++++++- 4 files changed, 87 insertions(+), 33 deletions(-) diff --git a/doc/desiInstall.rst b/doc/desiInstall.rst index 9cfcd47c..75d38494 100644 --- a/doc/desiInstall.rst +++ b/doc/desiInstall.rst @@ -75,7 +75,7 @@ Environment variables that strongly affect the behavior of desiInstall. :command:`svn`. Environment variables that are *set* by desiInstall for use by -``python setup.py`` or ``make``. +:command:`python setup.py` or :command:`make`. :envvar:`INSTALL_DIR` This variable is *set* by desiInstall to the directory that will contain @@ -140,14 +140,14 @@ Product Existence After the product name and version have been determined, desiInstall constructs the full URL pointing to the product/version and runs the code necessary to verify that the product/version really exists. Typically, this -will be ``svn ls``, unless a GitHub install is detected. +will be :command:`svn ls`, unless a GitHub install is detected. Download Code ------------- -The code is downloaded, using ``svn export`` for standard (tag) installs, or -``svn checkout`` for trunk or branch installs. For GitHub installs, desiInstall -will look for a release tarball, or do a ``git clone`` for tag or master/branch +The code is downloaded, using :command:`svn export` for standard (tag) installs, or +:command:`svn checkout` for trunk or branch installs. For GitHub installs, desiInstall +will look for a release tarball, or do a :command:`git clone` for tag or master/branch installs. desiInstall will set the environment variable :envvar:`WORKING_DIR` to point to the directory containing this downloaded code. @@ -162,14 +162,14 @@ plain is simply copied to the final install directory. py If a setup.py file is detected, desiInstall will attempt to execute - ``python setup.py install``. This build type can be suppressed with the + :command:`python setup.py install`. This build type can be suppressed with the command line option ``--compile-c``. make If a Makefile is detected, desiInstall will attempt to execute - ``make install``. + :command:`make install`. src If a Makefile is not present, but a src/ directory is, - desiInstall will attempt to execute ``make -C src all``. This build type + desiInstall will attempt to execute :command:`make -C src all`. This build type *is* mutually exclusive with 'make', but is not mutually exclusive with the other types. @@ -180,12 +180,24 @@ Determine Install Directory --------------------------- The install directory is where the code will live permanently. If the -install is taking place at NERSC, the install directory will be placed in -``/project/projectdirs/desi/software/${NERSC_HOST}``. +install is taking place at NERSC, the top-level install directory is +predetermined based on the value of :envvar:`NERSC_HOST`. + +edison + ``/global/common/edison/contrib/desi`` +cori + ``/global/common/cori/contrib/desi`` +datatran + ``/global/project/projectdirs/desi/software/datatran`` +scigate + ``/global/project/projectdirs/desi/software/scigate`` At other locations, the user must set the environment variable :envvar:`DESI_PRODUCT_ROOT` to point to the equivalent directory. +The actual install directory is determined by appending ``/code/product/verson`` +to the combining the top-level directory listed above. + If the install directory already exists, desiInstall will exit, unless the ``--force`` parameter is supplied on the command line. @@ -222,7 +234,7 @@ desiInstall will scan :envvar:`WORKING_DIR` to determine the details that need to be added to the module file. The final module file will then be written into the DESI module directory at NERSC or the module directory associated with :envvar:`DESI_PRODUCT_ROOT`. If ``--default`` is specified on the command -line, an approproate .version file will be created. +line, an appropriate .version file will be created. Load Module ----------- @@ -246,9 +258,11 @@ install and manipulate data that is bundled *with* the package. Copy All Files -------------- +**I am proposing to ONLY perform this copy for trunk/branch/master installs.** + The entire contents of :envvar:`WORKING_DIR` will be copied to :envvar:`INSTALL_DIR`. If this is a trunk or branch install and a src/ directory is detected, -desiInstall will attempt to run ``make -C src all`` in :envvar:`INSTALL_DIR`. +desiInstall will attempt to run :command:`make -C src all` in :envvar:`INSTALL_DIR`. For trunk or branch installs, no further processing is performed past this point. @@ -257,19 +271,19 @@ Create site-packages If the build-type 'py' is detected, a site-packages directory will be created in :envvar:`INSTALL_DIR`. If necessary, this directory will be -added to Python's ``sys.path``. +added to Python's :data:`sys.path`. Run setup.py ------------ -If the build-type 'py' is detected, ``python setup.py install`` will be run +If the build-type 'py' is detected, :command:`python setup.py install` will be run at this point. Build C/C++ Code ---------------- -If the build-type 'make' is detected, ``make install`` will be run in -:envvar:`WORKING_DIR`. If the build-type 'src' is detected, ``make -C src all`` +If the build-type 'make' is detected, :command:`make install` will be run in +:envvar:`WORKING_DIR`. If the build-type 'src' is detected, :command:`make -C src all` will be run in :envvar:`INSTALL_DIR`. Cross Install diff --git a/py/desiutil/install.py b/py/desiutil/install.py index 3d745fe1..cdb099a4 100644 --- a/py/desiutil/install.py +++ b/py/desiutil/install.py @@ -139,8 +139,8 @@ class DesiInstall(object): this holds the object that reads it. cross_install_host : :class:`str` The NERSC host on which to perform cross-installs. - default_module_dir : :class:`dict` - The default Modules install directory for every NERSC host. + default_nersc_dir : :class:`dict` + The default code and Modules install directory for every NERSC host. executable : :class:`str` The command used to invoke the script. fullproduct : :class:`str` @@ -168,11 +168,11 @@ class DesiInstall(object): """ cross_install_host = 'edison' nersc_hosts = ('cori', 'edison', 'datatran', 'scigate') - # nersc_module_dir = '/project/projectdirs/desi/software/modules' - default_module_dir = {'edison': '/global/common/edison/contrib/desi/modulefiles', - 'cori': '/global/common/cori/contrib/desi/modulefiles', - 'datatran': '/global/project/projectdirs/desi/software/datatran/modulefiles', - 'scigate': '/global/project/projectdirs/desi/software/datatran/modulefiles'} + default_nersc_dir = {'edison': '/global/common/edison/contrib/desi', + 'cori': '/global/common/cori/contrib/desi', + 'datatran': '/global/project/projectdirs/desi/software/datatran', + 'scigate': '/global/project/projectdirs/desi/software/scigate'} + def __init__(self, test=False): """Bare-bones initialization. @@ -616,15 +616,14 @@ def set_install_dir(self): The directory selected for installation. """ log = logging.getLogger(__name__ + '.DesiInstall.set_install_dir') - self.nersc = None try: self.nersc = environ['NERSC_HOST'] except KeyError: - pass + self.nersc = None if self.options.root is None or not isdir(self.options.root): if self.nersc is not None: - self.options.root = join('/project/projectdirs/desi/software', - self.nersc) + self.options.root = join(self.default_nersc_dir[self.nersc], + 'code') else: message = "DESI_PRODUCT_ROOT is missing or not set." log.critical(message) @@ -705,10 +704,15 @@ def module_dependencies(self): def nersc_module_dir(self): """The directory that contains Module directories at NERSC. """ + if not hasattr(self, 'nersc'): + return None if self.nersc is None: return None else: - nersc_module = self.default_module_dir[self.nersc] + nersc_module = join(self.default_nersc_dir[self.nersc], + 'modulefiles') + if not hasattr(self, 'config'): + return nersc_module if self.config is not None: if self.config.has_option("Module Processing", 'nersc_module_dir'): @@ -998,14 +1002,14 @@ def cross_install(self): for nh in nersc_hosts: if nh == cross_install_host: continue - dst = join('/project/projectdirs/desi/software', nh, + dst = join(self.default_nersc_dir[nh], 'code', self.baseproduct) if not islink(dst): src = join('..', cross_install_host, self.baseproduct) links.append((src, dst)) - dst = join('/project/projectdirs/desi/software/modules', - nh, self.baseproduct) + dst = join(self.default_nersc_dir[nh], 'modulefiles', + self.baseproduct) if not islink(dst): src = join('..', cross_install_host, self.baseproduct) diff --git a/py/desiutil/test/t/desiInstall_configuration.ini b/py/desiutil/test/t/desiInstall_configuration.ini index 29d31033..2861d9b5 100644 --- a/py/desiutil/test/t/desiInstall_configuration.ini +++ b/py/desiutil/test/t/desiInstall_configuration.ini @@ -20,3 +20,17 @@ nersc_hosts = cori,edison,datatran [Known Products] my_new_product = https://github.com/me/my_new_product # desiutil = https://github.com/you/new_path_to_desiutil +# +# This section can override details of Module file installation. +# +[Module Processing] +# +# nersc_module_dir overrides the Module file install directory for +# ALL NERSC hosts. +# +nersc_module_dir = /project/projectdirs/desi/test/modules +# +# cori_module_dir overrides the Module file install directory only +# on cori. +# +cori_module_dir = /global/common/cori/contrib/desi/test/modules diff --git a/py/desiutil/test/test_install.py b/py/desiutil/test/test_install.py index b5bb6188..3da18ecd 100644 --- a/py/desiutil/test/test_install.py +++ b/py/desiutil/test/test_install.py @@ -328,12 +328,12 @@ def test_set_install_dir(self): del environ['DESI_PRODUCT_ROOT'] except KeyError: old_root = None - environ['NERSC_HOST'] = 'FAKE' + environ['NERSC_HOST'] = 'edison' options = self.desiInstall.get_options(['desiutil', 'master']) self.desiInstall.get_product_version() install_dir = self.desiInstall.set_install_dir() self.assertEqual(install_dir, join( - '/project/projectdirs/desi/software/FAKE', + self.desiInstall.default_nersc_dir['edison'], 'code', 'desiutil', 'master')) if old_root is not None: environ['DESI_PRODUCT_ROOT'] = old_root @@ -359,6 +359,28 @@ def test_start_modules(self): status = self.desiInstall.start_modules() self.assertTrue(callable(self.desiInstall.module)) + def test_nersc_module_dir(self): + """Test the nersc_module_dir property. + """ + self.assertIsNone(self.desiInstall.nersc_module_dir) + self.desiInstall.nersc = None + self.assertIsNone(self.desiInstall.nersc_module_dir) + for n in ('edison', 'cori', 'datatran', 'scigate'): + self.desiInstall.nersc = n + self.assertEqual(self.desiInstall.nersc_module_dir, + join(self.desiInstall.default_nersc_dir[n], + "modulefiles")) + options = self.desiInstall.get_options(['--configuration', + join(self.data_dir, + 'desiInstall_configuration.ini'), + 'my_new_product', '1.2.3']) + self.desiInstall.nersc = 'edison' + self.assertEqual(self.desiInstall.nersc_module_dir, + '/project/projectdirs/desi/test/modules') + self.desiInstall.nersc = 'cori' + self.assertEqual(self.desiInstall.nersc_module_dir, + '/global/common/cori/contrib/desi/test/modules') + def test_cleanup(self): """Test the cleanup stage of the install. """ From 4cfbc442d3bab71b3dbe991c4f62e24ec78fa5ea Mon Sep 17 00:00:00 2001 From: Benjamin Alan Weaver Date: Wed, 21 Sep 2016 14:58:48 -0700 Subject: [PATCH 05/14] requests content is bytes in Python 3 --- py/desiutil/install.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/py/desiutil/install.py b/py/desiutil/install.py index cdb099a4..60498229 100644 --- a/py/desiutil/install.py +++ b/py/desiutil/install.py @@ -24,7 +24,7 @@ try: from cStringIO import StringIO except ImportError: - from io import StringIO + from io import BytesIO as StringIO if PY3: from configparser import ConfigParser as SafeConfigParser else: From d73c38cc6806f67dda3431325f1b161011705910 Mon Sep 17 00:00:00 2001 From: Benjamin Alan Weaver Date: Wed, 21 Sep 2016 15:41:14 -0700 Subject: [PATCH 06/14] set product_root at install time, not hard-coded --- etc/desiutil.module | 2 +- py/desiutil/install.py | 1 + py/desiutil/modules.py | 5 ++++- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/etc/desiutil.module b/etc/desiutil.module index f8bd0d42..7a237153 100644 --- a/etc/desiutil.module +++ b/etc/desiutil.module @@ -54,7 +54,7 @@ module-whatis "Sets up $product/$version in your environment." if {{[info exists env(DESI_PRODUCT_ROOT)]}} {{ set PRODUCT_ROOT $env(DESI_PRODUCT_ROOT) }} else {{ - set PRODUCT_ROOT /project/projectdirs/desi/software/$env(NERSC_HOST) + set PRODUCT_ROOT {product_root} }} set PRODUCT_DIR $PRODUCT_ROOT/$product/$version # diff --git a/py/desiutil/install.py b/py/desiutil/install.py index 60498229..1ad86419 100644 --- a/py/desiutil/install.py +++ b/py/desiutil/install.py @@ -745,6 +745,7 @@ def install_module(self): self.working_dir, dev)) self.module_keywords = configure_module(self.baseproduct, self.baseversion, + self.options.root, working_dir=self.working_dir, dev=dev) if self.options.moduledir == '': diff --git a/py/desiutil/modules.py b/py/desiutil/modules.py index 4852facf..bfb45bbe 100644 --- a/py/desiutil/modules.py +++ b/py/desiutil/modules.py @@ -143,7 +143,7 @@ def desiutil_module_method(self, command, *arguments): return desiutil_module -def configure_module(product, version, working_dir=None, dev=False): +def configure_module(product, version, product_root, working_dir=None, dev=False): """Decide what needs to go in the Module file. Parameters @@ -152,6 +152,8 @@ def configure_module(product, version, working_dir=None, dev=False): Name of the product. version : :class:`str` Version of the product. + product_root : :class:`str` + Directory that contains the installed code. working_dir : :class:`str`, optional The directory to examine. If not set, the current working directory will be used. @@ -176,6 +178,7 @@ def configure_module(product, version, working_dir=None, dev=False): module_keywords = { 'name': product, 'version': version, + 'product_root': product_root, 'needs_bin': '# ', 'needs_python': '# ', 'needs_trunk_py': '# ', From a57db5277ded84e5bc31539cecc1501bc96f9a04 Mon Sep 17 00:00:00 2001 From: Benjamin Alan Weaver Date: Wed, 21 Sep 2016 16:28:18 -0700 Subject: [PATCH 07/14] adjust product root paths and add directory documentation --- doc/desiInstall.rst | 60 ++++++++++++++++++++++++++++++-- etc/desiutil.module | 6 ++-- py/desiutil/install.py | 2 +- py/desiutil/test/test_modules.py | 13 ++++--- 4 files changed, 70 insertions(+), 11 deletions(-) diff --git a/doc/desiInstall.rst b/doc/desiInstall.rst index 75d38494..ffee5ce4 100644 --- a/doc/desiInstall.rst +++ b/doc/desiInstall.rst @@ -105,7 +105,8 @@ manipulated by setting up Modules, or loading Module files. It may be manipulated by :mod:`desiutil.modules`. :envvar:`MODULESHOME` This variable points to the Modules infrastructure. If it is not set, - it typically means that the system has no Modules infrastructure. + it typically means that the system has no Modules infrastructure. This + is needed to find the executable program that reads Module files. :envvar:`PYTHONPATH` Obviously this is important for any Python package! :envvar:`PYTHONPATH` may be manipulated by :mod:`desiutil.modules`. @@ -115,6 +116,61 @@ manipulated by setting up Modules, or loading Module files. .. _desiutil: https://github.com/desihub/desiutil +Directory Structure Assumed by the Install +========================================== + +desiInstall is primarily intended to run in a production environment that +supports Module files. In practice, this means NERSC, though it can also +install on any other system that has a Modules infrastructure installed. + +*desiInstall does not install a Modules infrastructure for you.* You have to +do this yourself, if your system does not already have this. + +For the purposes of this section, we define ``$product_root`` as the +directory that desiInstall will be writing to. This directory could be the +same as :envvar:`DESI_PRODUCT_ROOT`, but for standard NERSC installs it +defaults to a pre-defined value. ``$product_root`` may contain the following +directories: + +code/ + This contains the installed code, the result of :command:`python setup.py install` + or :command:`make install`. The code is always placed in a ``product/version`` + directory. So for example, the full path to desiInstall might be + ``$product_root/code/desiutil/1.8.0/bin/desiInstall``. +conda/ + At NERSC, this contains the Anaconda_ infrastructure. desiInstall does + not manipulate this directory in any way, though it may *use* the + :command:`python` executable installed here. + **Note**: currently we have ``conda/conda_version``. Do we want + ``conda/desi-conda-{base,extra}/version`` to match the Module files? +modulefiles/ + This contains the the Module files installed by desiInstall. A Module + file is almost always named ``product/version``. For example, the + Module file for desiutil might be ``$product_root/modulefiles/desiutil/1.8.0``. + +.. _Anaconda: https://www.continuum.io + +Within a ``$product_root/code/product/version`` directory, you might see the +following: + +bin/ + Contains command-line executables, including Python or Shell scripts. +data/ + Rarely, packages need data files that cannot be incorporated into the + package structure itself, so it will be installed here. desimodel_ is + an example of this. +etc/ + Miscellaneous metadata and configuration. In most packages this only + contains a template Module file. +lib/pythonX.Y/site-packages/ + Contains installed Python code. ``X.Y`` would be ``2.7`` or ``3.5``. +py/ + Sometimes we need to install a git checkout rather than an installed package. + If so, the Python code will live in *this* directory not the ``lib/`` + directory, and the product's Module file will be adjusted accordingly. + +.. _desimodel: https://github.com/desihub/desimodel + Stages of the Install ===================== @@ -258,8 +314,6 @@ install and manipulate data that is bundled *with* the package. Copy All Files -------------- -**I am proposing to ONLY perform this copy for trunk/branch/master installs.** - The entire contents of :envvar:`WORKING_DIR` will be copied to :envvar:`INSTALL_DIR`. If this is a trunk or branch install and a src/ directory is detected, desiInstall will attempt to run :command:`make -C src all` in :envvar:`INSTALL_DIR`. diff --git a/etc/desiutil.module b/etc/desiutil.module index 7a237153..8d43cb0b 100644 --- a/etc/desiutil.module +++ b/etc/desiutil.module @@ -52,11 +52,11 @@ module-whatis "Sets up $product/$version in your environment." # will need to set the DESI_PRODUCT_ROOT environment variable # if {{[info exists env(DESI_PRODUCT_ROOT)]}} {{ - set PRODUCT_ROOT $env(DESI_PRODUCT_ROOT) + set code_root $env(DESI_PRODUCT_ROOT)/code }} else {{ - set PRODUCT_ROOT {product_root} + set code_root {product_root} }} -set PRODUCT_DIR $PRODUCT_ROOT/$product/$version +set PRODUCT_DIR $code_root/$product/$version # # This line creates an environment variable pointing to the install # directory of your product. diff --git a/py/desiutil/install.py b/py/desiutil/install.py index 1ad86419..7d666c9d 100644 --- a/py/desiutil/install.py +++ b/py/desiutil/install.py @@ -628,7 +628,7 @@ def set_install_dir(self): message = "DESI_PRODUCT_ROOT is missing or not set." log.critical(message) raise DesiInstallException(message) - self.install_dir = join(self.options.root, self.baseproduct, + self.install_dir = join(self.options.root, 'code', self.baseproduct, self.baseversion) if isdir(self.install_dir) and not self.options.test: if self.options.force: diff --git a/py/desiutil/test/test_modules.py b/py/desiutil/test/test_modules.py index fc17970f..498897ab 100644 --- a/py/desiutil/test/test_modules.py +++ b/py/desiutil/test/test_modules.py @@ -150,6 +150,7 @@ def test_configure_module(self): results = { 'name': 'foo', 'version': 'bar', + 'product_root': '/my/product/root', 'needs_bin': '', 'needs_python': '', 'needs_trunk_py': '# ', @@ -160,7 +161,8 @@ def test_configure_module(self): } for t in test_dirs: mkdir(join(self.data_dir, t)) - conf = configure_module('foo', 'bar', working_dir=self.data_dir) + conf = configure_module('foo', 'bar', '/my/product/root', + working_dir=self.data_dir) for key in results: self.assertEqual(conf[key], results[key]) # @@ -168,7 +170,8 @@ def test_configure_module(self): # results['needs_python'] = '# ' results['needs_trunk_py'] = '' - conf = configure_module('foo', 'bar', working_dir=self.data_dir, + conf = configure_module('foo', 'bar', '/my/product/root', + working_dir=self.data_dir, dev=True) for key in results: self.assertEqual(conf[key], results[key]) @@ -190,11 +193,13 @@ def test_configure_module(self): results['needs_trunk_py'] = '# ' results['needs_ld_lib'] = '# ' results['needs_idl'] = '# ' - conf = configure_module('foo', 'bar', working_dir=self.data_dir) + conf = configure_module('foo', 'bar', '/my/product/root', + working_dir=self.data_dir) results['needs_python'] = '# ' results['needs_trunk_py'] = '' results['trunk_py_dir'] = '' - conf = configure_module('foo', 'bar', working_dir=self.data_dir, + conf = configure_module('foo', 'bar', '/my/product/root', + working_dir=self.data_dir, dev=True) for key in results: self.assertEqual(conf[key], results[key]) From 42c69c096684ee210d3fe1c2b83d5597bbe4d000 Mon Sep 17 00:00:00 2001 From: Benjamin Alan Weaver Date: Wed, 21 Sep 2016 16:58:14 -0700 Subject: [PATCH 08/14] fix some tests --- py/desiutil/install.py | 3 +-- py/desiutil/test/test_install.py | 11 ++++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/py/desiutil/install.py b/py/desiutil/install.py index 7d666c9d..33cef62a 100644 --- a/py/desiutil/install.py +++ b/py/desiutil/install.py @@ -622,8 +622,7 @@ def set_install_dir(self): self.nersc = None if self.options.root is None or not isdir(self.options.root): if self.nersc is not None: - self.options.root = join(self.default_nersc_dir[self.nersc], - 'code') + self.options.root = self.default_nersc_dir[self.nersc] else: message = "DESI_PRODUCT_ROOT is missing or not set." log.critical(message) diff --git a/py/desiutil/test/test_install.py b/py/desiutil/test/test_install.py index 3da18ecd..2db9e783 100644 --- a/py/desiutil/test/test_install.py +++ b/py/desiutil/test/test_install.py @@ -299,12 +299,13 @@ def test_set_install_dir(self): 'desiutil', 'master']) self.desiInstall.get_product_version() install_dir = self.desiInstall.set_install_dir() - self.assertEqual(install_dir, join(self.data_dir, 'desiutil', + self.assertEqual(install_dir, join(self.data_dir, 'code', 'desiutil', 'master')) # Test for presence of existing directory. - tmpdir = join(self.data_dir, 'desiutil') + tmpdir = join(self.data_dir, 'code') mkdir(tmpdir) - mkdir(join(tmpdir, 'master')) + mkdir(join(tmpdir, 'desiutil')) + mkdir(join(tmpdir, 'desiutil', 'master')) options = self.desiInstall.get_options(['--root', self.data_dir, 'desiutil', 'master']) self.desiInstall.get_product_version() @@ -312,14 +313,14 @@ def test_set_install_dir(self): install_dir = self.desiInstall.set_install_dir() self.assertEqual(str(cm.exception), "Install directory, {0}, already exists!".format( - join(tmpdir, 'master'))) + join(tmpdir, 'desiutil', 'master'))) options = self.desiInstall.get_options(['--root', self.data_dir, '--force', 'desiutil', 'master']) self.assertTrue(self.desiInstall.options.force) self.desiInstall.get_product_version() install_dir = self.desiInstall.set_install_dir() - self.assertFalse(isdir(join(tmpdir, 'master'))) + self.assertFalse(isdir(join(tmpdir, 'desiutil', 'master'))) if isdir(tmpdir): rmtree(tmpdir) # Test NERSC installs. Unset DESI_PRODUCT_ROOT for this to work. From 043628ee8fb9f1059d45a3f5042d042a759bd8a3 Mon Sep 17 00:00:00 2001 From: Benjamin Alan Weaver Date: Thu, 22 Sep 2016 16:29:44 -0700 Subject: [PATCH 09/14] add warning about LANG --- py/desiutil/install.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/py/desiutil/install.py b/py/desiutil/install.py index 33cef62a..bf54813b 100644 --- a/py/desiutil/install.py +++ b/py/desiutil/install.py @@ -210,7 +210,8 @@ def get_options(self, test_args=None): log = logging.getLogger(__name__ + '.DesiInstall.get_options') check_env = {'MODULESHOME': None, 'DESI_PRODUCT_ROOT': None, - 'USER': None} + 'USER': None, + 'LANG': None} for e in check_env: try: check_env[e] = environ[e] From aa0dc57bd074bb2acca74b88c455eaf463838757 Mon Sep 17 00:00:00 2001 From: Benjamin Alan Weaver Date: Thu, 22 Sep 2016 16:37:35 -0700 Subject: [PATCH 10/14] ignore another warning about astropy-helpers --- py/desiutil/install.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/py/desiutil/install.py b/py/desiutil/install.py index bf54813b..d7336702 100644 --- a/py/desiutil/install.py +++ b/py/desiutil/install.py @@ -937,7 +937,8 @@ def install(self): # r"matching '[^']+'") lines = [l for l in err.split('\n') if len(l) > 0 and manifestre.search(l) is None and - 'astropy_helpers' not in l] + 'astropy_helpers' not in l and + 'astropy-helpers' not in l] if len(lines) > 0: message = ("Error during installation: " + "{0}".format("\n".join(lines))) From f6aff89b8c76eec39d93d76e8bdf1deb78b30707 Mon Sep 17 00:00:00 2001 From: Benjamin Alan Weaver Date: Fri, 23 Sep 2016 14:36:36 -0700 Subject: [PATCH 11/14] add redmonster to known products --- py/desiutil/install.py | 1 + 1 file changed, 1 insertion(+) diff --git a/py/desiutil/install.py b/py/desiutil/install.py index d7336702..30a1d647 100644 --- a/py/desiutil/install.py +++ b/py/desiutil/install.py @@ -57,6 +57,7 @@ 'fiberassign': 'https://github.com/desihub/fiberassign', 'fiberassign_sqlite': 'https://github.com/desihub/fiberassign_sqlite', 'imaginglss': 'https://github.com/desihub/imaginglss', + 'redmonster': 'https://github.com/desihub/redmonster', 'specex': 'https://github.com/desihub/specex', 'speclite': 'https://github.com/dkirkby/speclite', 'specsim': 'https://github.com/desihub/specsim', From 6471bcda0d9e0dfe56ce5c51151b15e80bf975f4 Mon Sep 17 00:00:00 2001 From: Benjamin Alan Weaver Date: Fri, 23 Sep 2016 14:53:07 -0700 Subject: [PATCH 12/14] append code to root directory --- py/desiutil/install.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/py/desiutil/install.py b/py/desiutil/install.py index 30a1d647..ee12bec3 100644 --- a/py/desiutil/install.py +++ b/py/desiutil/install.py @@ -746,7 +746,7 @@ def install_module(self): self.working_dir, dev)) self.module_keywords = configure_module(self.baseproduct, self.baseversion, - self.options.root, + join(self.options.root, 'code'), working_dir=self.working_dir, dev=dev) if self.options.moduledir == '': From fd7e76cbe9908a254aaf015819b63f13debf32d3 Mon Sep 17 00:00:00 2001 From: Benjamin Alan Weaver Date: Tue, 27 Sep 2016 14:11:50 -0700 Subject: [PATCH 13/14] add line to changes.rst --- doc/changes.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/doc/changes.rst b/doc/changes.rst index 50101b8b..678b0b16 100644 --- a/doc/changes.rst +++ b/doc/changes.rst @@ -7,6 +7,10 @@ Change Log * Shorten Python version printed in dependency headers. * :mod:`desiutil.test.test_plots` was not cleaning up after itself. +* Support new DESI+Anaconda software stack infrastructure (`PR #43`_). + +.. _`PR #43`: https://github.com/desihub/desiutil/pull/43 + 1.8.0 (2016-09-10) ------------------ From 3435d103cb4fe7540256c300609b61539908ea90 Mon Sep 17 00:00:00 2001 From: Benjamin Alan Weaver Date: Tue, 27 Sep 2016 14:21:14 -0700 Subject: [PATCH 14/14] remove unresolved question from desiInstall docs --- doc/desiInstall.rst | 2 -- 1 file changed, 2 deletions(-) diff --git a/doc/desiInstall.rst b/doc/desiInstall.rst index ffee5ce4..65d9b971 100644 --- a/doc/desiInstall.rst +++ b/doc/desiInstall.rst @@ -141,8 +141,6 @@ conda/ At NERSC, this contains the Anaconda_ infrastructure. desiInstall does not manipulate this directory in any way, though it may *use* the :command:`python` executable installed here. - **Note**: currently we have ``conda/conda_version``. Do we want - ``conda/desi-conda-{base,extra}/version`` to match the Module files? modulefiles/ This contains the the Module files installed by desiInstall. A Module file is almost always named ``product/version``. For example, the