diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 604dc93..121f2e0 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.2.2dev +current_version = 1.1.0dev commit = True tag = False tag_name = {new_version} diff --git a/.travis.yml b/.travis.yml index 000c31f..b1f8b57 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,37 +4,62 @@ cache: pip: true timeout: 1000 -sudo: false - python: - '2.7' - '3.5' - '3.6' +- '3.7' os: - linux -matrix: +jobs: fast_finish: true + allow_failures: + - python: '2.7' + - os: osx + include: + - name: "Python 3.7 on macOS" + os: osx + osx_image: + - xcode11.2 # Python 3.7.7 running on macOS 10.14.6 + language: shell # 'language: python' is an error on Travis CI macOS + - name: "Python 3.6 on macOS" + os: osx + osx_image: + - xcode9.4 # Python 3.6.5 running on macOS 10.13 + language: shell notifications: email: false # repo branches to test branches: -- master + only: + - /.*/ + +before_install: + # installing lmod modules on linux + - if [ "$TRAVIS_OS_NAME" = "linux" ]; then sudo apt-get update -y ; fi + - if [ "$TRAVIS_OS_NAME" = "linux" ]; then sudo apt-get install -y lmod; fi + - if [ "$TRAVIS_OS_NAME" = "linux" ]; then source /usr/share/lmod/lmod/init/profile; fi + # installing lmod modules on osx + - if [ "$TRAVIS_OS_NAME" = "osx" ]; then brew update ; fi + - if [ "$TRAVIS_OS_NAME" = "osx" ]; then brew install lmod ; fi + - if [ "$TRAVIS_OS_NAME" = "osx" ]; then source /usr/local/opt/lmod/init/profile; fi + install: -- pip install -U pip wheel --quiet -- pip install --upgrade setuptools --quiet -- pip install -r requirements.txt --quiet -- pip install pytest -- pip install pytest-coverage -- pip install coveralls -#- python setup.py install +- pip3 install -U pip wheel --quiet +- pip3 install --upgrade setuptools --quiet +- pip3 install -r requirements.txt --quiet +- pip3 install pytest +- pip3 install pytest-coverage +- pip3 install coveralls +#- python setup.py install script: -- pytest python/sdss_install/tests --cov python/sdss_install --cov-report html +- pytest --public-svn-only python/sdss_install/tests --cov python/sdss_install --cov-report html after_success: - coveralls diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 9ed10bf..1ebee29 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,32 +1,40 @@ -.. _sdss_install-changelog: - -========== -Change Log -========== - -This document records the main changes to the sdss_install code. - -.. _changelog-0.1.0: -0.1.0 (unreleased) ------------------- - -Added -^^^^^ -* A thing we added. - -.. x.y.z (unreleased) -.. ------------------ -.. -.. A short description -.. -.. Added -.. ^^^^^ -.. * TBD -.. -.. Changed -.. ^^^^^^^ -.. * TBD -.. -.. Fixed -.. ^^^^^ -.. * TBD + +Changelog +========= + +This document records the main changes to the ``sdss_install`` code. + +* :release:`1.1.0 ` +* :support:`7` Added sphinx documentation for readthedocs +* :feature:`48` new ``--github-url`` option to optionally set a different public git url +* :support:`48` Updated changelog +* :feature:`12` Added new configuration bash sript to aid setup during sdss_install bootstrap +* :feature:`12` Added new environment variables to customize GIT and SVN product and modulefile locations +* :feature:`48` Use of custom `sdss_install.yml` config file for loading of parameters and environment variables +* :feature:`45` New ``--skip-git-verdirs`` option to turn off use of version sub-directories for git repos +* :support:`48` Initial test suite for better failure tracking and robustness +* :support:`48` Set up testing on Travis-CI +* :bug:`48 major` synchronized package version with tag versions + +* :release:`1.0.6 <2019-12-09>` +* :feature:`47` Added new command line option ``--https`` enables the user to use sdss_install with either https or ssh. +* :bug:`46` - no option for git install from https + +* :release:`1.0.5 <2019-11-13>` +* :feature:`39 backported` Added support for installation of external dependencies + +* :release:`1.0.4 <2019-07-02>` +* :feature:`42 backported` Added new ``sdss_install_version`` script to check version of the currently checked out Git branch or tag. +* :feature:`25 backported` Added support for Mac Brew installed modules + +* :release:`1.0.3 <2019-05-16>` +* :feature:`- backported` Added custom `Module(s)` class for improved system handling of module files + +* :release:`1.0.2 <2019-05-13>` +* :feature:`- backported` Implemented subprocessing of git commands for version validation +* :feature:`29 backported` Grab latest tag of sdss_install during install for robustness +* :bug:`21` Removed GraphQL implementation of version validation + +* :release:`1.0.1 <2019-05-13>` +* :feature:`-` Initial release of `sdss_install` +* :feature:`-` GraphQL product/version validation of Github repos diff --git a/bin/add_to_config.py b/bin/add_to_config.py new file mode 100644 index 0000000..68850ac --- /dev/null +++ b/bin/add_to_config.py @@ -0,0 +1,74 @@ +# !/usr/bin/env python +# -*- coding: utf-8 -*- +# +# Filename: add_to_config.py +# Project: bin +# Author: Brian Cherinka +# Created: Monday, 18th November 2019 6:12:06 pm +# License: BSD 3-clause "New" or "Revised" License +# Copyright (c) 2019 Brian Cherinka +# Last Modified: Monday, 18th November 2019 6:57:04 pm +# Modified By: Brian Cherinka + + +from __future__ import print_function, division, absolute_import + +import os +import sys +import yaml +import argparse + + +def update_config(envvars): + ''' Update the custom config with environment variables ''' + data = {row[0]: row[1] for row in envvars} + + custom_config_fn = os.path.expanduser('~/.{0}/{0}.yml'.format('sdss_install')) + + if os.path.exists(custom_config_fn): + config = yaml.load(open(custom_config_fn), Loader=yaml.SafeLoader) + + # add the environment variables + if 'envs' in config: + config['envs'].update(data) + else: + config['envs'] = data + + # append to the file + with open(custom_config_fn, 'w+') as file: + yaml.dump(config, file) + else: + # write a new custom config + dirname = os.path.dirname(custom_config_fn) + if not os.path.exists(dirname): + os.makedirs(dirname) + with open(custom_config_fn, 'w') as file: + yaml.dump({'envs': data}, file) + + +def parse_args(): + ''' Parse the arguments ''' + + parser = argparse.ArgumentParser( + prog='update your custom sdss_install.yaml config file', usage='%(prog)s [opts]') + parser.add_argument('-v', '--verbose', action='store_true', dest='verbose', + help='Print extra information.', default=False) + parser.add_argument('-e', "--envvars", nargs='*', + type=lambda kv: kv.split("="), dest='envvars', + help='The environment variables to add to the config file; as key-value pairs') + opts = parser.parse_args() + + return opts + + +def main(args): + + # parse arguments + opts = parse_args() + + # write to config + update_config(opts.envvars) + + +if __name__ == '__main__': + main(sys.argv[1:]) diff --git a/bin/configure b/bin/configure new file mode 100755 index 0000000..d29ca0b --- /dev/null +++ b/bin/configure @@ -0,0 +1,173 @@ +#!/bin/bash +# + +# +# Runs configuration setup +# +# Prompts configuration of environment variables +# SDSS_INSTALL_PRODUCT_ROOT - a base directory where all SDSS products will be installed +# SDSS_GIT_ROOT - the directory where all SDSS github products will be installed +# SDSS_SVN_ROOT - the directory where all SDSS svn products will be installed +# SDSS_GIT_MODULES - the directory containing all SDSS git module files +# SDSS_SVN_MODULES - the directory containing all SDSS svn module files +# SDSS_INSTALL_DIR - temporary directory where current sdss_install is installed + + +steps=1 +echo 'sdss_install configuration' +echo '--------------------------' +echo 'Follow these steps to set up directory paths to your root products and modulefiles.' +echo 'Type a new absolute directory path or hit enter to accept the default.' +echo 'For new users, we recommend to accept all defaults (hit enter to all).' +echo ' ' + +# set root installation directory +defroot=$HOME'/software/sdss' +echo $steps'.) Set $SDSS_INSTALL_PRODUCT_ROOT environment variable. ' [Default: $defroot] +read myroot + +[[ -z $myroot ]] && root=$defroot || root=$myroot + + +# set git installation directory +((steps++)) +defgit=$root/github +echo $steps'.) Set $SDSS_GIT_ROOT environment variable. ' [Default: $defgit] +read mygit + +[[ -z $mygit ]] && git=$defgit || git=$mygit + + +# set svn installation directory +((steps++)) +defsvn=$root/svn +echo $steps'.) Set $SDSS_SVN_ROOT environment variable. ' [Default: $defsvn] +read mysvn + +[[ -z $mysvn ]] && svn=$defsvn || svn=$mysvn + + +# set git modulefiles directory +((steps++)) +defgitmod=$git/modulefiles +echo $steps'.) Set $SDSS_GIT_MODULES environment variable. ' [Default: $defgitmod] +read mygitmod + +[[ -z $mygitmod ]] && gitmod=$defgitmod || gitmod=$mygitmod + +# set svn modulefiles directory +((steps++)) +defsvnmod=$svn/modulefiles +echo $steps'.) Set $SDSS_SVN_MODULES environment variable. ' [Default: $defsvnmod] +read mysvnmod + +[[ -z $mysvnmod ]] && svnmod=$defsvnmod || svnmod=$mysvnmod + +# set sdss_install dir +((steps++)) +curdir=$(pwd) +defsdssinstall=${curdir/\/bin} +echo $steps'.) Set temporary $SDSS_INSTALL_DIR environment variable. ' [Default: $defsdssinstall] +read mysdssinstall + +[[ -z $mysdssinstall ]] && sdssinstall=$defsdssinstall || sdssinstall=$mysdssinstall + + +# check the defined paths +echo 'export SDSS_INSTALL_PRODUCT_ROOT='$root +echo 'export SDSS_GIT_ROOT='$git +echo 'export SDSS_SVN_ROOT='$svn +echo 'export SDSS_GIT_MODULES='$gitmod +echo 'export SDSS_SVN_MODULES='$svnmod +echo 'export SDSS_INSTALL_DIR='$sdssinstall + +echo 'Are these paths correct? (Y/N)' +read answer + +# set default to yes if they just hit enter +[[ -z $answer ]] && answer='y' + + +# set the environment variables +case "$answer" in + [yY] | [yY][eE][sS]) + export SDSS_INSTALL_PRODUCT_ROOT=${root} + export SDSS_GIT_ROOT=${git} + export SDSS_SVN_ROOT=${svn} + export SDSS_GIT_MODULES=${gitmod} + export SDSS_SVN_MODULES=${svnmod} + export SDSS_INSTALL_DIR=${sdssinstall} ;; + [nN] | [nN][oO]) + echo 'Resetting. Please rerun configure.' + return 1 ;; + *) + echo "I don't understand '$answer'" + return 1 ;; +esac + +# adding module paths to MODULESPATH +((steps++)) +echo $steps'.) Add modulefile paths to $MODULEPATH? (Y/N) [Default: Y]' +read answer +# set default to yes if they just hit enter +[[ -z $answer ]] && answer='y' + +case "$answer" in + [yY] | [yY][eE][sS]) + echo 'Adding. To make this permanent, place "module use '$gitmod'", "module use '$svnmod'" in your bashrc or tcshrc' + [ -d $gitmod ] || mkdir -p $gitmod + [ -d $svnmod ] || mkdir -p $svnmod + eval module use $gitmod + eval module use $svnmod ;; +esac + +# add environment variables into the user custom sdss_install.yaml config file +((steps++)) +echo $steps'.) Add environment variables to custom sdss_install yaml file? (Y/N) [Default: Y]' +read answer +# set default to yes if they just hit enter +[[ -z $answer ]] && answer='y' + +case "$answer" in + [yY] | [yY][eE][sS]) + python add_to_config.py -e SDSS_INSTALL_PRODUCT_ROOT=${root} SDSS_GIT_ROOT=${git} SDSS_SVN_ROOT=${svn} SDSS_GIT_MODULES=${gitmod} SDSS_SVN_MODULES=${svnmod} ;; +esac + + +# # append environment variables to bashrc file +# ((steps++)) +# echo $steps'.) Append environment variables to bashrc file?' [Default: N] +# read answer +# # set default to yes if they just hit enter +# [[ -z $answer ]] && answer='y' +# +# # add the environment variables to the bashrc +# case "$answer" in +# [yY] | [yY][eE][sS]) + +# # get the shell +# x=`sh -c 'ps -p $$ -o ppid=' | xargs ps -o comm= -p` + +# file=$HOME/.bashrc +# if [ ! -f "$file" ]; then +# echo $file does not exist +# return 1 +# fi +# echo $file +# echo '\n' >> $file +# echo '# SDSS Environment Variables' >> $file +# echo 'export SDSS_INSTALL_PRODUCT_ROOT='${root} >> $file +# echo 'export SDSS_GIT_ROOT='${git} >> $file +# echo 'export SDSS_SVN_ROOT='${svn} >> $file +# echo 'export SDSS_GIT_MODULES='${gitmod} >> $file +# echo 'export SDSS_SVN_MODULES='${svnmod} >> $file +# echo 'export SDSS_INSTALL_DIR='${sdssinstall} >> $file ;; + +# [nN] | [nN][oO]) +# return 1 ;; +# *) +# echo "I don't understand '$answer'" +# return 1 ;; +# esac + + diff --git a/bin/sdss_install b/bin/sdss_install index a68a283..8f0af8c 100755 --- a/bin/sdss_install +++ b/bin/sdss_install @@ -12,14 +12,17 @@ from sdss_install.install.modules import Modules options = Argument('sdss_install').options install = Install(options=options) -if options and options.version: print(__version__) +if options and options.version: + print(__version__) else: + # sets up product info and directory paths install.set_ready() install.set_product() install.set_directory() install.set_directory_install() install.set_directory_work() + # checkout product as a working version if not options.module_only: install.clean_directory_install() if options.github: @@ -31,10 +34,13 @@ else: install.reset_options_from_config() install.set_build_type() + + # creates the real product installation directory if not options.module_only: install.logger_build_message() install.make_directory_install() + # sets up the product module file if install.ready: install.set_modules() install.modules.set_module() @@ -44,21 +50,18 @@ else: install.modules.set_keywords() install.modules.set_directory() install.modules.build() - + # must be after install.fetch() if options.external_dependencies: install.install_external_dependencies() + # performs real product installation install.set_environ() if not options.module_only: install.build() install.build_documentation() install.build_package() - if not options.keep: install.clean() - - - if options.bootstrap: - install.checkout() + if not options.keep: + install.clean() install.finalize() - diff --git a/bin/sdss_install_bootstrap b/bin/sdss_install_bootstrap index 62f5bc7..3f71636 100755 --- a/bin/sdss_install_bootstrap +++ b/bin/sdss_install_bootstrap @@ -14,6 +14,7 @@ function usage() { echo " -r = Set or override the product root for sdss_install" echo " -t = Test mode. Do not make any changes. Implies -v." echo " -v = Verbose mode. Print lots of extra information." + echo " -s = Skip git version sub-directories " ) >&2 } # @@ -23,17 +24,30 @@ modules='' root='' test='' verbose='' -while getopts hm:r:tU:v argname; do +skipgitver='' +while getopts hm:r:tU:v:s argname; do case ${argname} in h) usage; exit 0 ;; m) modules=${OPTARG} ;; r) root=${OPTARG} ;; t) test='-t' ;; v) verbose='-v' ;; + s) skipgitver='--skip-git-verdirs' ;; *) usage; exit 1 ;; esac done shift $((OPTIND-1)) + +# +# Configure +# +source ./configure +# check that we actually have set the environment variables; if not exit the configuration +if [[ -z $SDSS_GIT_ROOT ]]; then + exit 1 +fi +root=$SDSS_INSTALL_PRODUCT_ROOT + # # Install # @@ -47,15 +61,15 @@ if [[ -n "${modules}" ]]; then fi if [[ -n "${root}" ]]; then if [[ -n "${verbose}" ]]; then echo "sdss_install_bootstrap - DEBUG - root = \"-r ${root}\" "; fi - root = "-r ${root}" + root="-r ${root}" fi if [[ -z "${MODULESHOME}" ]]; then echo "sdss_install_bootstrap - INFO - You do not appear to have Modules installed." exit 1 fi -if [[ -n "${verbose}" ]]; then echo "sdss_install_bootstrap - DEBUG - export SDSS_INSTALL_DIR=${SDSS_INSTALL_PRODUCT_ROOT}/github/sdss_install/master"; fi -export SDSS_INSTALL_DIR=${SDSS_INSTALL_PRODUCT_ROOT}/github/sdss_install/master +#if [[ -n "${verbose}" ]]; then echo "sdss_install_bootstrap - DEBUG - export SDSS_INSTALL_DIR=${SDSS_INSTALL_PRODUCT_ROOT}/github/sdss_install/master"; fi +#export SDSS_INSTALL_DIR=${SDSS_INSTALL_PRODUCT_ROOT}/github/sdss_install/master if [[ -n "${verbose}" ]]; then echo "sdss_install_bootstrap - DEBUG - export PATH=${SDSS_INSTALL_DIR}/bin:${PATH}"; fi export PATH=${SDSS_INSTALL_DIR}/bin:${PATH} @@ -68,6 +82,10 @@ else export PYTHONPATH=${SDSS_INSTALL_DIR}/python:${PYTHONPATH} fi -if [[ -n "${verbose}" ]]; then echo "sdss_install_bootstrap - INFO - {SDSS_INSTALL_DIR}/bin/sdss_install --github --bootstrap --module-only ${root} ${test} ${verbose}" ; fi -${SDSS_INSTALL_DIR}/bin/sdss_install --github --bootstrap --module-only --https ${root} ${test} ${verbose} +if [[ -n "${verbose}" ]]; then echo "sdss_install_bootstrap - INFO - {SDSS_INSTALL_DIR}/bin/sdss_install --github --bootstrap --force --https ${root} ${test} ${verbose}" ; fi +${SDSS_INSTALL_DIR}/bin/sdss_install --github --bootstrap --force --https ${root} ${test} ${verbose} ${skipgitver} +# +# Indicate installation completion +# +echo 'Installation of sdss_install is complete. You can now safely delete this sdss_install directory: '$SDSS_INSTALL_DIR diff --git a/docs/sphinx/CHANGELOG.rst b/docs/sphinx/CHANGELOG.rst index 14bed04..78e1593 100644 --- a/docs/sphinx/CHANGELOG.rst +++ b/docs/sphinx/CHANGELOG.rst @@ -1,7 +1,5 @@ .. _changelog: -Changelog -========= .. include:: ../../CHANGELOG.rst diff --git a/docs/sphinx/api.rst b/docs/sphinx/api.rst index 231a05e..523bc61 100644 --- a/docs/sphinx/api.rst +++ b/docs/sphinx/api.rst @@ -4,16 +4,36 @@ sdss_install Reference ========================= -.. _api-main: +.. _api-install: -Main ----- +Install +------- -.. automodule:: sdss_install.main +.. automodule:: sdss_install.install.Install :members: - :undoc-members: :show-inheritance: -.. _api-Repo: +.. automodule:: sdss_install.install4.Install4 + :members: + :show-inheritance: + +.. automodule:: sdss_install.install5.Install5 + :members: + :show-inheritance: + + +.. _api-modules: +Modules +------- +.. automodule:: sdss_install.install.modules + :members: + :show-inheritance: + +Module +------ + +.. automodule:: sdss_install.utils.module + :members: + :show-inheritance: diff --git a/docs/sphinx/conf.py b/docs/sphinx/conf.py index 67fec85..c554a4b 100644 --- a/docs/sphinx/conf.py +++ b/docs/sphinx/conf.py @@ -37,7 +37,11 @@ # ones. extensions = ['sphinx.ext.autodoc', 'sphinx.ext.napoleon', 'sphinx.ext.autosummary', 'sphinx.ext.todo', 'sphinx.ext.viewcode', 'sphinx.ext.mathjax', - 'sphinx.ext.intersphinx'] + 'sphinx.ext.intersphinx', 'sphinxarg.ext', 'releases'] + +# 'releases' (changelog) settings +releases_issue_uri = "https://github.com/sdss/sdss_install/issues/%s" +releases_release_uri = "https://github.com/sdss/sdss_install/tree/%s" # Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] @@ -124,6 +128,8 @@ .. |HDUList| replace:: :class:`~astropy.io.fits.HDUList` """ +releases_github_path = 'sdss/sdss_install' +releases_document_name = ['CHANGELOG'] # -- Options for HTML output ---------------------------------------------- diff --git a/docs/sphinx/index.rst b/docs/sphinx/index.rst index 599ee11..ea9e297 100644 --- a/docs/sphinx/index.rst +++ b/docs/sphinx/index.rst @@ -7,15 +7,17 @@ .. rubric:: :header_no_toc:`Welcome to sdss_install's documentation!` -This is the Sphinx documentation for the SDSS Python product sdss_install +This is the Sphinx documentation for the SDSS Python product sdss_install Introduction ------------ -A description of the package can go here. +`sdss_install` is a tool for easily installing versioned SDSS software products on Github, or SVN. * :ref:`What's new in sdss_install? ` * :ref:`Introduction to sdss_install ` +* :ref:`Installation Testing ` +* :ref:`Options for sdss_install command-line tool ` Reference --------- diff --git a/docs/sphinx/intro.rst b/docs/sphinx/intro.rst index 4d78567..31bc955 100644 --- a/docs/sphinx/intro.rst +++ b/docs/sphinx/intro.rst @@ -4,4 +4,170 @@ Introduction to sdss_install =============================== -We should write an introduction here. +`sdss_install` is a product for installing SDSS software and data repositories versioned by svn, or git as hosted on Github. + +Prerequisites +------------- + +`sdss_install` requires a modules environment manager to work, either `TCLSH modules `_ +or `LUA modules (LMOD) `_. Download and install either `TCLSH `_ +or `LMOD `_ before proceeding further. + +Installing sdss_install +----------------------- + +Installation of `sdss_install` involves checking out a temporary copy of `sdss_install` and running a configuration setup script. + +Checkout Product +^^^^^^^^^^^^^^^^ + +First checkout the git repo from the `SDSS Github organization `_ into any temporary directory. +:: + + git clone https://github.com/sdss/sdss_install.git sdss_install + +Navigate to the ``bin`` directory and run the `sdss_install_bootstrap` bash script. +:: + + cd sdss_install/bin + ./sdss_install_bootstrap + +This will prompt you for environment configuration (see below) and proceed to install the latest tagged version of +`sdss_install` into your specified software folder. To see usage help for the bootstrap command, +run ``./sdss_install_bootstrap -h``. + +Environment Configuration +^^^^^^^^^^^^^^^^^^^^^^^^^ + +During the initial installation of `sdss_install`, a series of prompts will ask you to set up some environment variables which tells +`sdss_install` where to install new products and module files. If you are a new user, or unfamiliar to modules, it is recommended +to accept all defaults (hit 'enter' on each prompt). + +**Configuration Prompts**: + +1. **Set $SDSS_INSTALL_PRODUCT_ROOT environment variable. [Default: $HOME/software/sdss]** + +This asks you to set the base directory path for all SDSS software. The default is in a new `software/sdss` folder in your home directory. + +2. **Set $SDSS_GIT_ROOT environment variable. [Default: $SDSS_INSTALL_PRODUCT_ROOT/github]** + +This asks you to set the path for all `git` installed products. The default is within the `$SDSS_INSTALL_PRODUCT_ROOT` directory. +The `$SDSS_GIT_ROOT` path supercedes the `$SDSS_INSTALL_PRODUCT_ROOT` path. + +3. **Set $SDSS_SVN_ROOT environment variable. [Default: $SDSS_INSTALL_PRODUCT_ROOT/svn]** + +This asks you to set the path for all `svn` installed products. The default is within the `$SDSS_INSTALL_PRODUCT_ROOT` directory. +The `$SDSS_SVN_ROOT` path supercedes the `$SDSS_INSTALL_PRODUCT_ROOT` path. + +4. **Set $SDSS_GIT_MODULES environment variable. [Default: $SDSS_GIT_ROOT/modulefiles]** + +This asks you to set the path for all module files of `git` installed products. The default is a directory within your +`$SDSS_GIT_ROOT`. + +5. **Set $SDSS_SVN_MODULES environment variable. [Default: $SDSS_SVN_ROOT/modulefiles]** + +This asks you to set the path for all module files of `svn` installed products. The default is a directory within your +`$SDSS_SVN_ROOT`. + +6. **Set temporary $SDSS_INSTALL_DIR environment variable. [Default: the current path to the temporary checkout]** + +This asks you to set the path to where `sdss_install` is installed, which should be the temporary `sdss_install` checkout directory +as the default. It is **recommended** to leave this as the default and hit enter. + +7. **Add modulefile paths to $MODULEPATH? (Y/N) [Default: Y]** + +This step attempts to add your specified git/svn module paths into the existing terminal shell with `module use xxxx`. To permanently add the +module paths into your system, you will need to add each `module use xxxx` command into your `bashrc` or `tcshrc` file. Once added, +`modules` will search these paths for any available loadable modules. + +8. **Add environment variables to custom sdss_install yaml file? (Y/N) [Default: Y]** + +This steps adds the newly defined environment variables into a custom configuration YAML file located at `$HOME/.sdss_install/sdss_install.yml`. +`sdss_install` will load these path definitions. This allows you to modify path locations after the installation of `sdss_install`, e.g. if +you are restructuring your filesystem but don't want to reinstall `sdss_install`. A standard example config file will look like:: + + envs: + SDSS_GIT_MODULES: /Users/bcherinka/software/sdss/github/modulefiles + SDSS_GIT_ROOT: /Users/bcherinka/software/sdss/github + SDSS_INSTALL_PRODUCT_ROOT: /Users/bcherinka/software/sdss + SDSS_SVN_MODULES: /Users/bcherinka/software/sdss/svn/modulefiles + SDSS_SVN_ROOT: /Users/bcherinka/software/sdss/svn + +To check for a successful installation, look for the following: + +- The last ``stdout`` log line should be `sdss_install - INFO - Install.py - line 1005 - Ready to load module sdss_install/x.x.x` +- Check inside the `$SDSS_GIT_ROOT` folder for a new `sdss_install` product using the latest tag, e.g. ``1.0.7``. +- Check inside the `$SDSS_GIT_MODULES` folder for a new `sdss_install` module file, named as the latest tag, e.g. ``1.0.7``. +- Run `module use $SDSS_GIT_ROOT` and check that you can load the module: `module load sdss_install` + +Once the installation of `sdss_install` is complete, and if successful, you can now safely delete your temporary checkout. + +Advanced Config +^^^^^^^^^^^^^^^ + +For advanced users, or users with existing git/svn product and module setups, you can specify separate custom +paths to match your existing setup. Here is an example of a default `svn` setup but custom directories for `git` repos and +module files:: + + envs: + SDSS_INSTALL_PRODUCT_ROOT: /Users/bcherinka/software/sdss + SDSS_GIT_ROOT: /Users/bcherinka/Work/git/sdss + SDSS_SVN_ROOT: /Users/bcherinka/software/sdss/svn + SDSS_GIT_MODULES: /Users/bcherinka/Work/git/modulefiles + SDSS_SVN_MODULES: /Users/bcherinka/software/sdss/svn/modulefiles + +`sdss_install` will now use this setup to install new SDSS products. The paths defined within the the custom YAML config +file always take precedence over similar paths defined in other locations, even when re-running the bootstrap setup of +`sdss_install` with ``./sdss_install_bootstrap``. + +Product Installation with sdss_install +-------------------------------------- + +Once installed and the module is loaded, `sdss_install` provides a command-line tool with which to install all other SDSS products. +See the :ref:`sdss_install tool` for all available command-line options. The general syntax for use is +:: + + sdss_install [product_name] [product_version] + +where **[product_name]** is any SDSS versioned product, and **[product_version]** is any version/branch/tag name, +e.g. `master`, `trunk`, `1.0.6`. To see help for the tool, type `sdss_install -h`. + +To install the `master` branch of the `sdss_access` package from its SDSS git repository from Github +:: + + # install master branch of sdss_access + sdss_install --github sdss_access master + +The default action is to install `git` products inside version-named sub-directories. The above command installs +the `sdss_access` product inside `$SDSS_GIT_ROOT/sdss_access/master`. This allows one to checkout multiple versions of the same product +at a time. To instead skip the use of version-named sub-directories, use the `skip-git-verdirs` option. +:: + + # install tag 0.3.0 of the sdssdb product + sdss_install --github --skip-git-verdirs sdssdb 0.3.0 + +The above command installs `sdssdb` at `$SDSS_GIT_ROOT/sdssdb` and checks out the tag `0.3.0` within it. + +When the `github` option is not specified, `sdss_install` installs from the SVN repository instead. To install +the `trunk` of the `idlutils` package from SVN +:: + + # install main trunk of idlutils + sdss_install sdss/idlutils trunk + +Note the **product_name** here is ``sdss/idlutils``. SVN requires the full SVN path specification to the product from the top level +`repo` or `data` directories. The `idlutils` product lives inside the top-level ``sdss`` +sub-directory at `https://svn.sdss.org/repo/sdss/idlutils`. To install the MaNGA Data Reduction Pipeline, ``mangadrp``, +which lives at, `https://svn.sdss.org/repo/manga/mangadrp`, one would specifiy +:: + + # install main trunk of the MaNGA DRP + sdss_install manga/mangadrp trunk + +You can also install `svn` products from the public SVN site, using the ``public`` keyword. +:: + + # install tag v1_1_0 for firefly product from public SVN + sdss_install --public sdss/firefly v1_1_0 + +which installs the product from `https://svn.sdss.org/public/repo/sdss/firefly/tags/v1_1_0`. diff --git a/docs/sphinx/test.rst b/docs/sphinx/test.rst new file mode 100644 index 0000000..9eb358b --- /dev/null +++ b/docs/sphinx/test.rst @@ -0,0 +1,38 @@ + +.. _testing: + +Test Installations +================== + +We aim to make ``sdss_install`` as robust as possible. ``sdss_install`` has been designed to work on Linux +and Mac OSx, with Lua and Tcsh modules environments and bash terminal shells. However ``sdss_install`` may behave +differently under specific combinations of operating systems, Python versions, Modules versions, etc. If you encounter +any problems with the installation of ``sdss_install`` itself, or its use to install SDSS software products, please file +a new `Github Issue `_. Currently, ``sdss_install`` has been +tested to work on the following systems: + +================== ================== ============================= ====================== +Operating System Module Environment Python(s) Shell +================== ================== ============================= ====================== +Mac OS X 10.14.6 Tcsh Module 4.2.4 Anaconda 2.7.15, 3.6.7, 3.7.3 bash 3.2.57(1)-release +Mac OS X 10.13.6 Lua Modules 7.8 Anaconda 3.7.2 bash 3.2.57(1)-release +GNU/Linux 3.10.0 Lua Modules 8.2.7 3.6.9 bash 4.2.46(2)-release +================== ================== ============================= ====================== + +We also use Travis-CI to run our test suite, which includes tests for installation of ``sdss_install``, as well +as a few SDSS software products versioned in both `git` and `svn`. The latest run of Travis can be found at +``_. Travis can automate product testing on a variety of +`operating systems `_, +`environment setups `_, and +`Python environments `_. More information on Travis's build +environments can be found `here `_. While Travis +constantly updates its environment images, ``sdss_install`` is currently being tested on the following setups: + +=================== ================== ======================= ====================== ================ +Operating System Module Environment Python(s) Shell Module Installer +=================== ================== ======================= ====================== ================ +Ubuntu Xenial Linux Lua Modules 5.8 2.7, 3.5, 3.6, 3.7, 3.8 bash 4.3.48(1)-release apt-get 1.2.31 +Mac OS X 10.14.6 Lua Modules 8.3.10 3.7.7 bash 3.2.57(1)-release homebrew 2.2.16 +Mac OS X 10.13.6 Lua Modules 8.3.10 3.6.5 bash 3.2.57(1)-release homebrew 2.2.16 +=================== ================== ======================= ====================== ================ + diff --git a/docs/sphinx/usage.rst b/docs/sphinx/usage.rst new file mode 100644 index 0000000..be2fa94 --- /dev/null +++ b/docs/sphinx/usage.rst @@ -0,0 +1,13 @@ + + + +.. _usage: + +sdss_install command-Line tool +------------------------------ + +.. argparse:: + :module: sdss_install.application.Argument + :func: sdss_install + :prog: sdss_install + diff --git a/python/sdss_install/__init__.py b/python/sdss_install/__init__.py index c6a97a0..d857fb7 100644 --- a/python/sdss_install/__init__.py +++ b/python/sdss_install/__init__.py @@ -7,9 +7,7 @@ import yaml import distutils.version -# Inits the logging system. Only shell logging, and exception and warning catching. -# File logging can be started by calling log.start_file_logger(name). -from .utils import log +from .utils import get_logger def merge(user, default): @@ -25,20 +23,50 @@ def merge(user, default): return user +def expand(config): + ''' Expands the environment variables inside a configuration ''' + for key, val in config.items(): + if isinstance(val, dict): + config[key] = expand(val) + elif isinstance(val, str): + config[key] = os.path.expandvars(val) + return config + + +def add_to_os(config): + ''' Add environment variables into the OS ''' + + envs = config.get('envs', None) + if not envs: + return + + for key, val in envs.items(): + os.environ[key] = os.path.expandvars(val) + + NAME = 'sdss_install' -# Loads config +# Inits the logging system. Only shell logging, and exception and warning catching. +# File logging can be started by calling log.start_file_logger(path). +log = get_logger(NAME) + + +yaml_kwds = dict() yaml_version = distutils.version.StrictVersion(yaml.__version__) +if yaml_version >= distutils.version.StrictVersion('5.1'): + yaml_kwds.update(Loader=yaml.FullLoader) + +# Loads config with open(os.path.dirname(__file__) + '/etc/{0}.yml'.format(NAME)) as ff: - if yaml_version >= distutils.version.StrictVersion('5.1'): - config = yaml.load(ff, Loader=yaml.FullLoader) - else: - config = yaml.load(ff) + config = yaml.load(ff, **yaml_kwds) # If there is a custom configuration file, updates the defaults using it. custom_config_fn = os.path.expanduser('~/.{0}/{0}.yml'.format(NAME)) if os.path.exists(custom_config_fn): - config = merge(yaml.load(open(custom_config_fn)), config) + config = merge(yaml.load(open(custom_config_fn), **yaml_kwds), config) +# expand any environment variables in the config +config = expand(config) +add_to_os(config) -__version__ = '0.2.2dev' +__version__ = '1.1.0dev' diff --git a/python/sdss_install/application/Argument.py b/python/sdss_install/application/Argument.py index dbd89e2..0e438f8 100644 --- a/python/sdss_install/application/Argument.py +++ b/python/sdss_install/application/Argument.py @@ -1,9 +1,10 @@ from __future__ import absolute_import, division, print_function, unicode_literals -# The line above will help with 2to3 support. -from sys import argv + +from argparse import ArgumentParser from os import environ, getenv from os.path import basename -from argparse import ArgumentParser +from sys import argv + class Argument: @@ -16,74 +17,82 @@ def __init__(self, name=None, args=None): self.options = None self.options._name = name if self.options else None - - + + def sdss_install(): '''Add command line arguments for bin file sdss_install''' xct = basename(argv[0]) - parser = ArgumentParser(description=__doc__,prog=xct) - parser.add_argument('-e', '--level', help='set logging level', metavar='LEVEL', choices=['debug','info','warning','error'], default='debug') + parser = ArgumentParser(description=__doc__, prog=xct) + parser.add_argument('-e', '--level', help='set logging level', metavar='LEVEL', + choices=['debug', 'info', 'warning', 'error'], default='debug') parser.add_argument('-b', '--bootstrap', action='store_true', dest='bootstrap', - help='Run in bootstrap mode to install the sdss_install product.') + help='Run in bootstrap mode to install the sdss_install product.') parser.add_argument('-G', '--github', action='store_true', dest='github', - help='Install products from GitHub, otherwise from SVN.') + help='Install products from GitHub, otherwise from SVN.') mode = parser.add_mutually_exclusive_group() mode.add_argument('-C', '--compile-c', action='store_true', dest='force_build_type', - help='Force C/C++ install mode, even if a setup.py file is detected (WARNING: this is for experts only).') + help='Force C/C++ install mode, even if a setup.py file is detected (WARNING: this is for experts only).') mode.add_argument('-E', '--evilmake', action='store_true', dest='evilmake', - help='Use evilmake to install product.') + help='Use evilmake to install product.') mode.add_argument('-T', '--make-target', action='store', dest='make_target', - help='Target to make when installing product.') + help='Target to make when installing product.') parser.add_argument('-d', '--default', action='store_true', dest='default', - help='Make this version the default version.') + help='Make this version the default version.') parser.add_argument('-n', '--no-build', action='store_true', dest='no_build', - help='Skip the automated build stage') + help='Skip the automated build stage') parser.add_argument('-s', '--skip-module', action='store_true', dest='skip_module', - help='Skip the module load product command') + help='Skip the module load product command') parser.add_argument('-L', '--module-only', action='store_true', dest='module_only', - help='Install / Reinstall only the module file') + help='Install / Reinstall only the module file') parser.add_argument('-P', '--no_python_package', action='store_true', dest='no_python_package', - help='Skip the python package stage for tagged products') + help='Skip the python package stage for tagged products') parser.add_argument('-D', '--documentation', action='store_true', dest='documentation', - help='Build any Sphinx or Doxygen documentation.') + help='Build any Sphinx or Doxygen documentation.') parser.add_argument('-F', '--force', action='store_true', dest='force', - help='Overwrite any existing installation of this product/version.') + help='Overwrite any existing installation of this product/version.') parser.add_argument('-k', '--keep', action='store_true', dest='keep', - help='Keep the exported build directory.') + help='Keep the exported build directory.') parser.add_argument('-m', '--modules-home', action='store', dest='modules_home', - metavar='DIR',help='Set or override the value of $MODULESHOME', - default=getenv('MODULESHOME')) + metavar='DIR', help='Set or override the value of $MODULESHOME', + default=getenv('MODULESHOME')) parser.add_argument('-a', '--alt-module', action='store', dest='alt_module', - metavar='ALT_MODULE',help='specify an alternate module file') + metavar='ALT_MODULE', help='specify an alternate module file') parser.add_argument('-M', '--module-dir', action='store', dest='moduledir', - metavar='DIR',help='Install module files in DIR.',default='') + metavar='DIR', help='Install module files in DIR.', default='') parser.add_argument('-r', '--root', action='store', dest='root', - metavar='DIR', help='Set or override the value of $SDSS_INSTALL_PRODUCT_ROOT', - default=getenv('SDSS_INSTALL_PRODUCT_ROOT')) - try: longpath = eval(environ['SDSS4TOOLS_LONGPATH']) - except: longpath = False + metavar='DIR', help='Set or override the value of $SDSS_INSTALL_PRODUCT_ROOT', + default=getenv('SDSS_INSTALL_PRODUCT_ROOT')) + try: + longpath = eval(environ['SDSS4TOOLS_LONGPATH']) + except (KeyError, TypeError): + longpath = False parser.add_argument('-l', '--longpath', action='store_true', dest='longpath', - help='Keep the long path hierarchy in the install directory', default=longpath) + help='Keep the long path hierarchy in the install directory', default=longpath) parser.add_argument('-t', '--test', action='store_true', dest='test', - help='Test mode. Do not actually install anything.') - parser.add_argument('-u', '--url', action='store',dest='url', - metavar='URL',help='Download software from URL.', - default='https://svn.sdss.org') - parser.add_argument('-g', '--public', action='store_true',dest='public', - help='Download software from public URL.') + help='Test mode. Do not actually install anything.') + parser.add_argument('-u', '--url', action='store', dest='url', + metavar='URL', help='Download software from URL.', + default='https://svn.sdss.org') + parser.add_argument('-g', '--public', action='store_true', dest='public', + help='Download software from public URL.') parser.add_argument('-U', '--username', action='store', dest='username', - metavar='USER',help='Specify svn --username (if necessary).') + metavar='USER', help='Specify svn --username (if necessary).') parser.add_argument('-v', '--verbose', action='store_true', dest='verbose', - help='Print extra information.') + help='Print extra information.') parser.add_argument('-V', '--version', action='store_true', dest='version', - help='Print version information.') - parser.add_argument('product',nargs='?',default='NO PACKAGE', - help='Name of product to install, starts with [repo, data, deprecated] or assumed to start with repo.') - parser.add_argument('product_version',nargs='?',default='NO VERSION', - help='Version of product to install (trunk or specified tag or branch).') + help='Print version information.') + parser.add_argument('product', nargs='?', default='NO PACKAGE', + help='Name of product to install, starts with [repo, data, deprecated] or assumed to start with repo.') + parser.add_argument('product_version', nargs='?', default='NO VERSION', + help='Version of product to install (trunk or specified tag or branch).') parser.add_argument('-x', '--external_dependencies', action='store', dest='external_dependencies', - metavar='DIR', help='Set a string of external dependency clone commands', - default=str()) + metavar='DIR', help='Set a string of external dependency clone commands', + default=str()) + parser.add_argument('-o', '--skip-git-verdirs', action='store_true', dest='skip_git_verdirs', + help='Set to skip checkout of git repos in version subdirectories') + parser.add_argument('--github-url', action='store', dest='github_url', + metavar='GITHUB_URL', help='Github url to use', + default=None) parser.add_argument('-H', '--https', action='store_true', dest='https', - help='Use GitHub https instead of ssh', default=False) + help='Use GitHub https instead of ssh', default=False) return parser diff --git a/python/sdss_install/core/exceptions.py b/python/sdss_install/core/exceptions.py index a10f255..0daf43b 100644 --- a/python/sdss_install/core/exceptions.py +++ b/python/sdss_install/core/exceptions.py @@ -11,18 +11,18 @@ from __future__ import print_function, division, absolute_import -class Sdss_installError(Exception): - """A custom core Sdss_install exception""" +class SdssInstallError(Exception): + """A custom core SdssInstall exception""" def __init__(self, message=None): message = 'There has been an error' \ if not message else message - super(Sdss_installError, self).__init__(message) + super(SdssInstallError, self).__init__(message) -class Sdss_installNotImplemented(Sdss_installError): +class SdssInstallNotImplemented(SdssInstallError): """A custom exception for not yet implemented features.""" def __init__(self, message=None): @@ -30,45 +30,45 @@ def __init__(self, message=None): message = 'This feature is not implemented yet.' \ if not message else message - super(Sdss_installNotImplemented, self).__init__(message) + super(SdssInstallNotImplemented, self).__init__(message) -class Sdss_installAPIError(Sdss_installError): +class SdssInstallAPIError(SdssInstallError): """A custom exception for API errors""" def __init__(self, message=None): if not message: - message = 'Error with Http Response from Sdss_install API' + message = 'Error with Http Response from SdssInstall API' else: - message = 'Http response error from Sdss_install API. {0}'.format(message) + message = 'Http response error from SdssInstall API. {0}'.format(message) - super(Sdss_installAPIError, self).__init__(message) + super(SdssInstallAPIError, self).__init__(message) -class Sdss_installApiAuthError(Sdss_installAPIError): +class SdssInstallApiAuthError(SdssInstallAPIError): """A custom exception for API authentication errors""" pass -class Sdss_installMissingDependency(Sdss_installError): +class SdssInstallMissingDependency(SdssInstallError): """A custom exception for missing dependencies.""" pass -class Sdss_installWarning(Warning): - """Base warning for Sdss_install.""" +class SdssInstallWarning(Warning): + """Base warning for SdssInstall.""" -class Sdss_installUserWarning(UserWarning, Sdss_installWarning): +class SdssInstallUserWarning(UserWarning, SdssInstallWarning): """The primary warning class.""" pass -class Sdss_installSkippedTestWarning(Sdss_installUserWarning): +class SdssInstallSkippedTestWarning(SdssInstallUserWarning): """A warning for when a test is skipped.""" pass -class Sdss_installDeprecationWarning(Sdss_installUserWarning): +class SdssInstallDeprecationWarning(SdssInstallUserWarning): """A warning for deprecated features.""" pass diff --git a/python/sdss_install/etc/sdss_install.yml b/python/sdss_install/etc/sdss_install.yml index 6eb68dd..8634662 100644 --- a/python/sdss_install/etc/sdss_install.yml +++ b/python/sdss_install/etc/sdss_install.yml @@ -1,9 +1,10 @@ --- -option1: - suboption1: 1 - suboption2: "some text" - suboption3: - subsuboption1: [1, 2, 3] +envs: + SDSS_INSTALL_PRODUCT_ROOT: $HOME/software/sdss + SDSS_GIT_ROOT: $SDSS_INSTALL_PRODUCT_ROOT/github + SDSS_SVN_ROOT: $SDSS_INSTALL_PRODUCT_ROOT/svn + SDSS_GIT_MODULES: $SDSS_GIT_ROOT/modulefiles + SDSS_SVN_MODULES: $SDSS_SVN_ROOT/modulefiles + SDSS_GITHUB_KEY: '' -option2: 2.0 diff --git a/python/sdss_install/install/Install.py b/python/sdss_install/install/Install.py index 259d657..4910ee1 100644 --- a/python/sdss_install/install/Install.py +++ b/python/sdss_install/install/Install.py @@ -3,26 +3,29 @@ """Install SDSS-IV and SDSS-V software. """ from __future__ import absolute_import, division, print_function, unicode_literals -# The line above will help with 2to3 support. -import glob -import logging -import subprocess + import datetime -from sys import argv, executable, path -from shutil import copyfile, copytree, rmtree +import logging +from json import load from os import chdir, environ, getcwd, getenv, makedirs, walk from os.path import basename, dirname, exists, isdir, join -from subprocess import Popen, PIPE -from argparse import ArgumentParser -from json import loads, dumps, load -try: from ConfigParser import SafeConfigParser, RawConfigParser -except ImportError: from configparser import SafeConfigParser, RawConfigParser -#from .most_recent_tag import most_recent_tag -from .modules import Modules +from shutil import copyfile, copytree, rmtree +from subprocess import PIPE, Popen +from sys import executable + from sdss_install.application import Argument from sdss_install.install4 import Install4 from sdss_install.install5 import Install5 +from .modules import Modules + + +try: + from ConfigParser import SafeConfigParser, RawConfigParser +except ImportError: + from configparser import SafeConfigParser, RawConfigParser + + class Install: ''' Main class which calls Install4 and Install5 @@ -40,8 +43,10 @@ def set_logger(self, options=None): debug = self.options.test or self.options.verbose or self.options.level == 'debug' self.logger = logging.getLogger('sdss_install') if self.logger: - if debug: self.logger.setLevel(logging.DEBUG) - else: self.logger.setLevel(logging.INFO) + if debug: + self.logger.setLevel(logging.DEBUG) + else: + self.logger.setLevel(logging.INFO) handler = logging.StreamHandler() if debug: formatter = logging.Formatter("%(name)s - " + @@ -55,14 +60,17 @@ def set_logger(self, options=None): "%(message)s") handler.setFormatter(formatter) self.logger.addHandler(handler) - else: print('ERROR: Unable to set_logger') - else: print('ERROR: Unable to set_logger. options=%r, logging=%r' - % (options,logging)) + else: + print('ERROR: Unable to set_logger') + else: + print('ERROR: Unable to set_logger. options=%r, logging=%r' + % (options, logging)) def set_options(self, options=None): '''Set self.options wrapper''' self.options = options if options else None - if not self.options: print('ERROR - Unable to set_options') + if not self.options: + print('ERROR - Unable to set_options') def initialize_data(self): '''Initialize class Install data.''' @@ -81,52 +89,56 @@ def initialize_data(self): def set_install4(self): '''Set a class Install4 instance.''' self.install4 = Install4(logger=self.logger, options=self.options) - if not self.install4: self.logger.error('Unable to set self.install4') + if not self.install4: + self.logger.error('Unable to set self.install4') def set_install5(self): '''Set a class Install5 instance.''' self.install5 = Install5(logger=self.logger, options=self.options) - if not self.install5: self.logger.error('Unable to set self.install5') + if not self.install5: + self.logger.error('Unable to set self.install5') def import_data(self): '''Sync data from class Import4 or class Import5 to class Import.''' if self.options.github: if self.install5: - self.ready = self.install5.ready - self.product = self.install5.product - self.directory = self.install5.directory + self.ready = self.install5.ready + self.product = self.install5.product + self.directory = self.install5.directory else: if self.install4: - self.ready = self.install4.ready - self.url = self.install4.url - self.product = self.install4.product - self.package = self.install4.package - self.directory = self.install4.directory - self.svncommand = self.install4.svncommand - self.exists = self.install4.exists - self.modules = self.install4.modules - self.build_type = self.install4.build_type + self.ready = self.install4.ready + self.url = self.install4.url + self.product = self.install4.product + self.package = self.install4.package + self.directory = self.install4.directory + self.svncommand = self.install4.svncommand + self.exists = self.install4.exists + self.modules = self.install4.modules + self.build_type = self.install4.build_type def export_data(self): '''Sync class Import data to class Import4 or class Import5.''' if self.options.github: if self.install5: - self.install5.ready = self.ready - self.install5.product = self.product - self.install5.directory = self.directory - else: self.logger.error('Unable to export_data to class Install5') + self.install5.ready = self.ready + self.install5.product = self.product + self.install5.directory = self.directory + else: + self.logger.error('Unable to export_data to class Install5') else: if self.install4: - self.install4.ready = self.ready - self.install4.url = self.url - self.install4.product = self.product - self.install4.package = self.package - self.install4.directory = self.directory - self.install4.svncommand = self.svncommand - self.install4.exists = self.exists - self.install4.modules = self.modules - self.install4.build_type = self.build_type - else: self.logger.error('Unable to export_data to class Install5') + self.install4.ready = self.ready + self.install4.url = self.url + self.install4.product = self.product + self.install4.package = self.package + self.install4.directory = self.directory + self.install4.svncommand = self.svncommand + self.install4.exists = self.exists + self.install4.modules = self.modules + self.install4.build_type = self.build_type + else: + self.logger.error('Unable to export_data to class Install5') def set_ready(self): '''Call set_ready() of class Install4 or class Install5.''' @@ -134,19 +146,26 @@ def set_ready(self): if self.ready: if self.options.github: self.set_install5() - if self.install5: self.install5.set_ready() - else: self.ready = False + if self.install5: + self.install5.set_ready() + else: + self.ready = False else: self.set_install4() - if self.install4: self.install4.set_ready() - else: self.ready = False - if self.ready: self.import_data() + if self.install4: + self.install4.set_ready() + else: + self.ready = False + if self.ready: + self.import_data() def set_product(self): '''Call set_product() of class Install4 or class Install5.''' if self.ready: - if self.options.github: self.install5.set_product() - else: self.install4.set_product() + if self.options.github: + self.install5.set_product() + else: + self.install4.set_product() self.import_data() def set_directory(self): @@ -156,10 +175,11 @@ def set_directory(self): ''' if self.ready: self.directory = dict() - try: self.directory['original'] = getcwd() + try: + self.directory['original'] = getcwd() except OSError as ose: self.logger.error("Check current directory: {0}" - .format(ose.strerror)) + .format(ose.strerror)) self.ready = False self.export_data() @@ -173,39 +193,53 @@ def set_directory_install(self): try: makedirs(self.options.root) self.logger.info("Creating {0}" - .format(self.options.root)) + .format(self.options.root)) except OSError as ose: self.logger.error("mkdir: " + - "cannot create directory '{0}': {1}" - .format(self.options.root,ose.strerror)) + "cannot create directory '{0}': {1}" + .format(self.options.root, ose.strerror)) self.ready = False else: self.logger.error("Please set the --root keyword or " + - "SDSS_INSTALL_PRODUCT_ROOT environmental variable " + - "to a valid directory.") + "SDSS_INSTALL_PRODUCT_ROOT environmental variable " + + "to a valid directory.") self.ready = False else: self.logger.error("Please use the --root keyword " + - "or set a SDSS_INSTALL_PRODUCT_ROOT " + - "environmental variable.") + "or set a SDSS_INSTALL_PRODUCT_ROOT " + + "environmental variable.") self.ready = False + if self.ready: if self.options.root.endswith('/'): self.options.root = dirname(self.options.root) + if self.options.root is not None: environ['SDSS_INSTALL_PRODUCT_ROOT'] = self.options.root + if self.options.longpath is not None: environ['SDSS4TOOLS_LONGPATH'] = 'True' - repo_type = 'github' if self.options.github else 'svn' - self.directory['root'] = ( - join(self.options.root, - repo_type, - self.product['root']) - if self.product['root'] - else join(self.options.root,repo_type)) - self.directory['install'] = join(self.directory['root'], - self.product['name'], - self.product['version']) + + # set the GIT or SVN product ROOT; this overrides main PRODUCT_ROOT + if self.options.github: + sub_root = environ.get("SDSS_GIT_ROOT") + else: + sub_root = environ.get("SDSS_SVN_ROOT") + + # set the root directory for the product + self.directory['root'] = (join(sub_root, self.product['root']) + if self.product['root'] else sub_root) + + # check to use version sub-directories or not + verdir = self.product['version'] + if self.options.github and self.options.skip_git_verdirs: + verdir = None + + # set the install directory for the product + self.directory['install'] = join(self.directory['root'], self.product['name']) + if verdir: + self.directory['install'] = join(self.directory['install'], verdir) + self.export_data() def set_directory_work(self): @@ -217,54 +251,57 @@ def set_directory_work(self): if self.ready: self.import_data() if self.options.module_only: - self.directory['work']=self.directory['install'] + self.directory['work'] = self.directory['install'] else: self.directory['work'] = join(self.directory['original'], - "%(name)s-%(version)s" % - self.product) + "%(name)s-%(version)s" % + self.product) if isdir(self.directory['work']): self.logger.info("Detected old working directory, " + - "%(work)s. Deleting..." % self.directory) + "%(work)s. Deleting..." % self.directory) rmtree(self.directory['work']) self.export_data() - def clean_directory_install(self,install_dir=None): + def clean_directory_install(self, install_dir=None): '''Remove existing install directory if exists and if option --force.''' if self.ready: self.import_data() install_dir = install_dir if install_dir else self.directory['install'] if isdir(install_dir) and not self.options.test: if self.options.force: - try: cwd = getcwd() + try: + cwd = getcwd() except OSError as ose: - self.logger.error("Check current directory: {0}".format(ose.strerror)) + self.logger.error( + "Check current directory: {0}".format(ose.strerror)) self.ready = False if self.ready: if cwd.startswith(install_dir): self.logger.error("Current working directory, {}, ".format(cwd) + - "is inside the install directory, {}, ".format(install_dir) + - "which will be deleted via the " + - "-F (or --force) option, so please cd to another " + - "working directory and try again!" % self.directory) + "is inside the install directory, {}, ".format(install_dir) + + "which will be deleted via the " + + "-F (or --force) option, so please cd to another " + + "working directory and try again!" % self.directory) self.ready = False else: self.logger.info("Preparing to install in " + - "{} (overwriting due to force option)".format(install_dir)) + "{} (overwriting due to force option)".format(install_dir)) rmtree(install_dir) else: self.logger.error("Install directory, %(install)s, already exists!" - % self.directory) + % self.directory) self.logger.info("Use the -F (or --force) option " + - "to overwrite.") + "to overwrite.") self.ready = False - else: self.logger.info("Preparing to install in %(install)s" - % self.directory) + else: + self.logger.info("Preparing to install in %(install)s" + % self.directory) self.export_data() - def set_sdss_github_remote_url(): - '''Wrapper for method Install5.set_sdss_github_remote_url()''' + def set_sdss_github_remote_url(self, use_public=None): + '''Set the set_sdss_github_remote_url() of class Install5''' if self.ready and self.options.github and self.install5: - self.install5.set_sdss_github_remote_url() + self.install5.set_sdss_github_remote_url(use_public=use_public) self.import_data() def set_svncommand(self): @@ -275,36 +312,40 @@ def set_svncommand(self): def set_exists(self): '''Call set_exists() of class Install4 or class Install5''' if self.ready: - if not self.options.github: self.install4.set_exists() + if not self.options.github: + self.install4.set_exists() self.import_data() def fetch(self): '''Call set_fetch() of class Install4 or class Install5''' if self.ready: - if self.options.github: self.install5.fetch() - else: self.install4.fetch() + if self.options.github: + self.install5.fetch() + else: + self.install4.fetch() self.import_data() def install_external_dependencies(self): '''Install external dependencies.''' if self.ready: if (self.options.external_dependencies and - isinstance(self.options.external_dependencies,dict) + isinstance(self.options.external_dependencies, dict) ): for key in self.options.external_dependencies: if self.ready: self.external_product = dict() external_dependency = self.options.external_dependencies[key] install_product = (external_dependency['install_product'] - if external_dependency - and 'install_product' in external_dependency - else None) + if external_dependency + and 'install_product' in external_dependency + else None) paths = (external_dependency['paths'] if external_dependency and 'paths' in external_dependency else None) if install_product: - self.install_external_product(install_product=install_product) + self.install_external_product( + install_product=install_product) else: self.logger.debug('No install_product found.') # Needs to be called after self.install_external_product() @@ -315,23 +356,25 @@ def install_external_dependencies(self): else: self.ready = False self.logger.error('Failed to install external dependencies. ' + - 'self.options.external_dependencies: {}' - .format(self.options.external_dependencies) + - 'isinstance(self.options.external_dependencies,dict): {}' - .format(isinstance(self.options.external_dependencies,dict)) - ) + 'self.options.external_dependencies: {}' + .format(self.options.external_dependencies) + + 'isinstance(self.options.external_dependencies,dict): {}' + .format(isinstance(self.options.external_dependencies, dict)) + ) - def install_external_product(self,install_product=None): + def install_external_product(self, install_product=None): '''Install external products''' if self.ready: - if install_product and isinstance(install_product,dict): + if install_product and isinstance(install_product, dict): url = (install_product['url'] if install_product and 'url' in install_product else None) if url: if 'github.com' in url: - self.install_external_github_product(github_product=install_product) + self.install_external_github_product( + github_product=install_product) elif 'svn.' in url: - self.install_external_svn_product(svn_product=install_product) + self.install_external_svn_product( + svn_product=install_product) else: self.ready = False self.logger.error('Encountered unsupported ' + @@ -339,35 +382,35 @@ def install_external_product(self,install_product=None): else: self.ready = False self.logger.error('Unable to install_external_product. ' + - 'install_product: {}, '.format(install_product) + - 'url: {}, '.format(url) - ) + 'install_product: {}, '.format(install_product) + + 'url: {}, '.format(url) + ) else: self.ready = False self.logger.error('Unable to install_external_product. ' + - 'install_product: {}, '.format(install_product) + - 'isinstance(install_product,dict): {}, ' - .format(isinstance(install_product,dict)) - ) - - def install_external_svn_product(self,svn_product=None): + 'install_product: {}, '.format(install_product) + + 'isinstance(install_product,dict): {}, ' + .format(isinstance(install_product, dict)) + ) + + def install_external_svn_product(self, svn_product=None): '''Install external dependency from SVN''' self.ready = False self.logger.error('Installing external dependencies from svn is currently ' + 'unsupported. Please make a GitHub sdss_install issue (ticket) ' + 'to request that this feature be added.') - def install_external_github_product(self,github_product=None): + def install_external_github_product(self, github_product=None): '''Install external dependency from GitHub''' if self.ready: - if github_product and isinstance(github_product,dict): + if github_product and isinstance(github_product, dict): url = (github_product['url'] if github_product and 'url' in github_product else None) github_url = dirname(url) if url else None product = basename(url) if url else None version = (github_product['version'] if github_product and 'version' in github_product else None) - github_remote_url = join(github_url,product) + github_remote_url = join(github_url, product) self.install5.validate_product_and_version(github_url=github_url, product=product, version=version) @@ -385,10 +428,10 @@ def install_external_github_product(self,github_product=None): version=version)) self.external_product['is_tag'] = ( False if self.external_product['is_branch'] else - self.install5.is_type(type='tag', - github_url=github_url, - product=product, - version=version)) + self.install5.is_type(type='tag', + github_url=github_url, + product=product, + version=version)) self.ready = self.ready and self.install5.ready if self.ready: self.set_external_product_install_dir() @@ -401,17 +444,17 @@ def install_external_github_product(self,github_product=None): else: self.ready = False self.logger.error('Unable to install_external_github_product. ' + - 'github_product: {}, '.format(github_product) + - 'isinstance(github_product,dict): {}, ' - .format(isinstance(github_product,dict)) - ) + 'github_product: {}, '.format(github_product) + + 'isinstance(github_product,dict): {}, ' + .format(isinstance(github_product, dict)) + ) def set_external_product_install_dir(self): '''Set the directory for external dependencies.''' if self.ready: if (self.external_product and 'product' in self.external_product and - 'version' in self.external_product + 'version' in self.external_product ): install_dir = join(self.directory['root'], 'external', @@ -425,47 +468,49 @@ def set_external_product_install_dir(self): self.logger.info("Creating {0}".format(install_dir)) except OSError as ose: self.logger.error("mkdir: " + - "cannot create directory '{0}': {1}" - .format(install_dir,ose.strerror)) + "cannot create directory '{0}': {1}" + .format(install_dir, ose.strerror)) self.ready = False else: self.ready = False self.logger.error('Unable to set_external_product_install_dir. ' + 'self.external_product: {}, ' - .format(self.external_product) + .format(self.external_product) ) - - def set_external_paths(self,paths=None): + + def set_external_paths(self, paths=None): '''Set external paths, like PATH, IDL_PATH, and PYTHONPATH''' if self.ready: - if paths and isinstance(paths,dict): - idl_paths = paths['idl'] if paths and 'idl' in paths else None - shell_paths = paths['shell'] if paths and 'shell' in paths else None + if paths and isinstance(paths, dict): + idl_paths = paths['idl'] if paths and 'idl' in paths else None + shell_paths = paths['shell'] if paths and 'shell' in paths else None python_paths = paths['python'] if paths and 'python' in paths else None - + install_dir = (self.external_product['install_dir'] if self.external_product else None) if idl_paths: for idl_path in idl_paths: if self.ready: - path = (join(install_dir,idl_path) - if install_dir and isinstance(idl_path,str) + path = (join(install_dir, idl_path) + if install_dir and isinstance(idl_path, str) else None) - self.set_external_path(path=path,path_type='IDL_PATH') + self.set_external_path( + path=path, path_type='IDL_PATH') elif shell_paths: for shell_path in shell_paths: if self.ready: - path = (join(install_dir,shell_path) - if install_dir and isinstance(shell_path,str) + path = (join(install_dir, shell_path) + if install_dir and isinstance(shell_path, str) else None) - self.set_external_path(path=path,path_type='PATH') + self.set_external_path(path=path, path_type='PATH') elif python_paths: for python_path in python_paths: if self.ready: - path = (join(install_dir,python_path) - if install_dir and isinstance(python_path,str) + path = (join(install_dir, python_path) + if install_dir and isinstance(python_path, str) else None) - self.set_external_path(path=path,path_type='PYTHONPATH') + self.set_external_path( + path=path, path_type='PYTHONPATH') else: self.ready = False self.logger.error('Unable to set_external_paths. ' + @@ -477,30 +522,34 @@ def set_external_paths(self,paths=None): self.logger.error('Unable to set_external_paths. ' + 'paths: {}, '.format(paths) + 'isinstance(paths,dict): {}, ' - .format(isinstance(paths,dict)) + .format(isinstance(paths, dict)) ) - - def set_external_path(self,path=None,path_type=None): + + def set_external_path(self, path=None, path_type=None): '''Prepend the given path to the given path_type.''' if self.ready: if path and path_type: - if path.endswith('/'): path = path.rstrip('/') - supported_path_types = ['PATH','IDL_PATH','PYTHONPATH'] + if path.endswith('/'): + path = path.rstrip('/') + supported_path_types = ['PATH', 'IDL_PATH', 'PYTHONPATH'] if path_type in supported_path_types: old_path = None try: - self.logger.info('Loading current {}'.format(path_type)) + self.logger.info( + 'Loading current {}'.format(path_type)) old_path = environ[path_type] if old_path and path not in old_path: new_path = path + ':' + old_path try: environ[path_type] = new_path - self.logger.info('Updated {}'.format(path_type)) + self.logger.info( + 'Updated {}'.format(path_type)) if self.options.level == 'debug': self.logger.debug("environ['{0}']: {1}" - .format(path_type,environ[path_type])) + .format(path_type, environ[path_type])) except: - self.logger.info('WARNING: Unable to update {}. Skipping.'.format(path_type)) + self.logger.info( + 'WARNING: Unable to update {}. Skipping.'.format(path_type)) except: try: environ[path_type] = path @@ -508,16 +557,17 @@ def set_external_path(self,path=None,path_type=None): 'Setting it to {}.'.format(path)) except: self.logger.info('WARNING: Unable to set or reset {}. Skipping.' - .format(path_type)) + .format(path_type)) else: - self.logger.info('WARNING: Unable to set_external_path. ' + - 'Unsupported path_type: {}. '.format(path_type) + - 'Supported path types: {}. '.format(supported_path_types) - ) + self.logger.info('WARNING: Unable to set_external_path. ' + + 'Unsupported path_type: {}. '.format(path_type) + + 'Supported path types: {}. '.format( + supported_path_types) + ) else: - self.logger.info('WARNING: Unable to set_external_path. ' + - 'path: {0}, path_type: {1}'.format(path,path_type)) + self.logger.info('WARNING: Unable to set_external_path. ' + + 'path: {0}, path_type: {1}'.format(path, path_type)) def checkout(self): '''Call Install5.checkout''' @@ -525,11 +575,6 @@ def checkout(self): self.install5.checkout() self.import_data() - def set_sdss_github_remote_url(self): - '''Set the set_sdss_github_remote_url() of class Install5''' - if self.ready and self.options.github: - self.install5.set_sdss_github_remote_url() - def reset_options_from_config(self): ''' Set absent command-line options from etc/config.ini file, @@ -541,13 +586,15 @@ def reset_options_from_config(self): # Some products may contain an optional etc/config.ini file to # determine the config self.options to build if self.ready: - config_filename = join('etc','config.ini') - config_file = join(self.directory['work'],config_filename) + config_filename = join('etc', 'config.ini') + config_file = join(self.directory['work'], config_filename) if exists(config_file): config = SafeConfigParser() - try: config.optionxform = unicode - except: config.optionxform = str - if len(config.read(config_file))==1: + try: + config.optionxform = unicode + except: + config.optionxform = str + if len(config.read(config_file)) == 1: if config.has_section('sdss4install'): self.process_install_section(config=config, section='sdss4install') @@ -555,78 +602,87 @@ def reset_options_from_config(self): self.process_install_section(config=config, section='sdss_install') if config.has_section('envs'): - missing = [env for key,env in config.items('envs') - if not getenv(env,None)] + missing = [env for key, env in config.items('envs') + if not getenv(env, None)] for env in missing: self.logger.error("Required environment variable " + - "{0} must be set prior to sdss_install" - .format(env)) - if missing: self.ready = False + "{0} must be set prior to sdss_install" + .format(env)) + if missing: + self.ready = False if config.has_section('external_dependencies'): self.process_install_section(config=config, section='external_dependencies') - def process_install_section(self,config=None,section=None): + def process_install_section(self, config=None, section=None): if config and section: if section == 'external_dependencies': if 'json_filepath' in config.options(section): json_filepath = join(self.directory['work'], - config.get(section,'json_filepath')) + config.get(section, 'json_filepath')) if exists(json_filepath): try: with open(json_filepath) as json_file: - self.options.external_dependencies = load(json_file) + self.options.external_dependencies = load( + json_file) except: - self.logger.info('WARNING: Unable to open the file {}'.format(json_filepath)) + self.logger.info( + 'WARNING: Unable to open the file {}'.format(json_filepath)) else: for option in config.options(section): - if option=='no_build' and not self.options.no_build: + if option == 'no_build' and not self.options.no_build: try: self.options.no_build = config.getboolean(section, option) if self.options.no_build: self.logger.info("Using {0} to set " + - "--no-build option" - .format(config_filename)) - except: pass - elif option=='skip_module' and not self.options.skip_module: + "--no-build option" + .format(config_filename)) + except: + pass + elif option == 'skip_module' and not self.options.skip_module: try: self.options.skip_module = config.getboolean(section, option) if self.options.skip_module: self.logger.info("Using {0} to set " + - "--skip_module option".format(config_filename)) - except: pass - elif (option=='no_python_package' and not + "--skip_module option".format(config_filename)) + except: + pass + elif (option == 'no_python_package' and not self.options.no_python_package): try: self.options.no_python_package = ( - config.getboolean(section,option)) + config.getboolean(section, option)) if self.options.no_python_package: self.logger.info("Using {0} to set " + - "--no_python_package option" - .format(config_filename)) - except: pass - elif option=='make_target' and not self.options.make_target: + "--no_python_package option" + .format(config_filename)) + except: + pass + elif option == 'make_target' and not self.options.make_target: try: - self.options.make_target = config.get(section,option) + self.options.make_target = config.get( + section, option) if self.options.make_target: self.logger.info("Using {0} to set " + - "--make_target {1} option" - .format(config_filename, - self.options.make_target)) - except: pass - elif option=='evilmake' and not self.options.evilmake: + "--make_target {1} option" + .format(config_filename, + self.options.make_target)) + except: + pass + elif option == 'evilmake' and not self.options.evilmake: try: self.options.evilmake = config.getboolean(section, option) if self.options.evilmake: self.logger.info("Using {0} to set " + - "--evilmake option".format(config_filename)) - except: pass + "--evilmake option".format(config_filename)) + except: + pass else: self.logger.error('Unable to process_install_section. ' + - 'config={0}, section={1}'.format(config,section)) + 'config={0}, section={1}'.format(config, section)) def set_build_type(self): '''Analyze the code to determine the build type''' @@ -636,24 +692,25 @@ def set_build_type(self): if self.options.no_build: self.build_message = "Proceeding without build..." else: - if (exists(join(self.directory['work'],'Makefile')) - and self.options.evilmake): + if (exists(join(self.directory['work'], 'Makefile')) + and self.options.evilmake): self.build_message = "Installing via evilmake" self.build_type.append('evilmake') - elif (exists(join(self.directory['work'],'setup.py')) + elif (exists(join(self.directory['work'], 'setup.py')) and not self.options.force_build_type): self.build_type.append('python') - if exists(join(self.directory['work'],'Makefile')): + if exists(join(self.directory['work'], 'Makefile')): self.build_type.append('c') - elif exists(join(self.directory['work'],'Makefile')): + elif exists(join(self.directory['work'], 'Makefile')): self.build_type.append('c') if not self.build_type: self.build_message = ("Proceeding without a setup.py " + - "or Makefile...") + "or Makefile...") def logger_build_message(self): '''Log the build message.''' - if self.build_message: self.logger.info(self.build_message) + if self.build_message: + self.logger.info(self.build_message) def make_directory_install(self): '''Make install directory.''' @@ -691,24 +748,25 @@ def build(self): if (self.product['is_master_or_branch'] or self.options.no_python_package or self.options.evilmake or not - self.build_type): + self.build_type): if self.options.test: self.logger.info("Skipping install in %(install)s " + - "(--test)" % self.directory) + "(--test)" % self.directory) else: self.logger.info("Installing in %(install)s" % - self.directory) - copytree(self.directory['work'],self.directory['install']) + self.directory) + copytree(self.directory['work'], self.directory['install']) chdir(self.directory['install']) if 'evilmake' in self.build_type: if not self.options.skip_module: self.modules.load(product=self.product['name'], version=self.product['version']) - command = ['evilmake','clean'] + command = ['evilmake', 'clean'] self.logger.info('Running "{0}" in {1}' - .format(' '.join(command), - self.directory['install'])) - (out,err,proc_returncode) = self.execute_command(command=command) + .format(' '.join(command), + self.directory['install'])) + (out, err, proc_returncode) = self.execute_command( + command=command) self.logger.debug(out) if proc_returncode != 0: self.logger.error("Evilmake response:") @@ -717,10 +775,10 @@ def build(self): if self.options.make_target: command += [self.options.make_target] self.logger.info('Running "{0}" in {1}' - .format(' '.join(command), - self.directory['install'])) - (out,err,proc_returncode) = self.execute_command(command=command, - argument='ignore') + .format(' '.join(command), + self.directory['install'])) + (out, err, proc_returncode) = self.execute_command(command=command, + argument='ignore') self.logger.debug(out) if proc_returncode != 0: self.logger.error("Evilmake response:") @@ -729,22 +787,23 @@ def build(self): if not self.options.skip_module: self.modules.load(product=self.product['name'], version=self.product['version']) - command = (['make','-C', 'src'] - if exists(join(self.directory['work'],'src')) + command = (['make', '-C', 'src'] + if exists(join(self.directory['work'], 'src')) else ['make']) if self.options.make_target: command += [self.options.make_target] self.logger.info('Running "{0}" in {1}' - .format(' '.join(command), - self.directory['install'])) - (out,err,proc_returncode) = self.execute_command(command=command) + .format(' '.join(command), + self.directory['install'])) + (out, err, proc_returncode) = self.execute_command( + command=command) self.logger.debug(out) if proc_returncode != 0: self.logger.error("Error during compile:") self.logger.error(err) self.ready = False if self.options.documentation: - self.logger.info('WARNING: Documentation will not be built ' + + self.logger.info('WARNING: Documentation will not be built ' + 'for trunk or branch installs!') else: self.package = True @@ -756,7 +815,8 @@ def build(self): "--prefix=%(install)s" % self.directory] self.logger.debug(' '.join(command)) if not self.options.test: - (out,err,proc_returncode) = self.execute_command(command=command) + (out, err, proc_returncode) = self.execute_command( + command=command) self.logger.debug(out) if proc_returncode != 0: self.logger.error("Error during installation:") @@ -772,16 +832,16 @@ def build(self): cf = list() for root, dirs, files in walk('etc'): for d in dirs: - md.append(join(self.directory['install'],root,d)) + md.append(join(self.directory['install'], root, d)) for name in files: if name.endswith('.module'): continue - cf.append((join(root,name), + cf.append((join(root, name), join(self.directory['install'], - root, - name))) + root, + name))) if md or cf: - etc_dir = join(self.directory['install'],'etc') + etc_dir = join(self.directory['install'], 'etc') self.logger.debug('Creating {0}'.format(etc_dir)) makedirs(etc_dir) if md: @@ -790,17 +850,17 @@ def build(self): if not self.options.test: makedirs(name) if cf: - for src,dst in cf: + for src, dst in cf: self.logger.debug('Copying {0} -> {1}' - .format(src,dst)) + .format(src, dst)) if not self.options.test: - copyfile(src,dst) + copyfile(src, dst) def build_documentation(self): '''Build the documentaion of the installed product.''' if self.ready and self.options.documentation: if 'python' in self.build_type: - if exists(join('doc','index.rst')): + if exists(join('doc', 'index.rst')): # # Assume Sphinx documentation. # @@ -817,33 +877,34 @@ def build_documentation(self): '.'.join(self.product['version'].split('.')[0:3]), 'year': datetime.date.today().year} - for sd in ('_templates','_build','_static'): - if not isdir(join('doc',sd)): + for sd in ('_templates', '_build', '_static'): + if not isdir(join('doc', sd)): try: - makedirs(join('doc',sd)) + makedirs(join('doc', sd)) except OSError as ose: self.logger.error(ose.strerror) self.ready = False - if not exists(join('doc','Makefile')): + if not exists(join('doc', 'Makefile')): copyfile(join(getenv('sdss_install_DIR'), - 'etc', - 'doc', - 'sphinx', - 'Makefile'), - join('doc','Makefile')) - if not exists(join('doc','conf.py')): + 'etc', + 'doc', + 'sphinx', + 'Makefile'), + join('doc', 'Makefile')) + if not exists(join('doc', 'conf.py')): with open(join(getenv('sdss_install_DIR'), 'etc', 'doc', 'sphinx', 'conf.py')) as conf: newconf = conf.read().format(**sphinx_keywords) - with open(join('doc','conf.py'),'w') as conf2: + with open(join('doc', 'conf.py'), 'w') as conf2: conf2.write(newconf) command = [executable, 'setup.py', 'build_sphinx'] self.logger.debug(' '.join(command)) if not self.options.test: - (out,err,proc_returncode) = self.execute_command(command=command) + (out, err, proc_returncode) = self.execute_command( + command=command) self.logger.debug(out) if proc_returncode != 0: self.logger.error( @@ -851,12 +912,12 @@ def build_documentation(self): self.logger.error(err) self.ready = False if not self.options.test: - if isdir(join('build','sphinx','html')): - copytree(join('build','sphinx','html'), - join(self.directory['install'],'doc')) + if isdir(join('build', 'sphinx', 'html')): + copytree(join('build', 'sphinx', 'html'), + join(self.directory['install'], 'doc')) else: self.logger.warning("Documentation build requested, " + - "but no documentation found.") + "but no documentation found.") else: # # This is not a Python product, assume Doxygen documentation. @@ -870,23 +931,23 @@ def build_documentation(self): 'description': ("Documentation for %(name)s built by sdss_install." % self.product)} - if not exists(join('doc','Makefile')): + if not exists(join('doc', 'Makefile')): copyfile(join(getenv('sdss_install_DIR'), 'etc', 'doc', 'doxygen', 'Makefile'), - join('doc','Makefile')) - if not exists(join('doc','Docyfile')): + join('doc', 'Makefile')) + if not exists(join('doc', 'Docyfile')): with open(join(getenv('sdss_install_DIR'), - 'etc', - 'doc','doxygen','Doxyfile')) as conf: + 'etc', + 'doc', 'doxygen', 'Doxyfile')) as conf: newconf = conf.read().format(**doxygen_keywords) - with open(join('doc','Doxyfile'),'w') as conf2: + with open(join('doc', 'Doxyfile'), 'w') as conf2: conf2.write(newconf) else: self.logger.warning("Documentation build requested, " + - "but no documentation found.") + "but no documentation found.") # # At this point either we have already completed a Python installation @@ -900,7 +961,7 @@ def build_package(self): command = ['make', 'install'] self.logger.debug(' '.join(command)) if not self.options.test: - (out,err,proc_returncode) = self.execute_command(command=command) + (out, err, proc_returncode) = self.execute_command(command=command) self.logger.debug(out) if proc_returncode != 0: self.logger.error("Error during compile:") @@ -910,8 +971,10 @@ def build_package(self): def clean(self): '''Remove the work directory tree.''' if self.ready: - try: rmtree(self.directory['work']) - except: pass + try: + rmtree(self.directory['work']) + except: + pass def finalize(self): '''Log installation final result message.''' @@ -922,34 +985,43 @@ def finalize(self): # if self.options.github and self.options.module_only: # rmtree(join(self.product['name'],self.product['version'])) finalize_ps = None - if self.options.test: finalize = "Test " + finalize - else: finalize += "!" - if basename(self.options.product)=='tree': pass - elif self.options.skip_module: finalize += " (skipped modules)" + if self.options.test: + finalize = "Test " + finalize + else: + finalize += "!" + if basename(self.options.product) == 'tree': + pass + elif self.options.skip_module: + finalize += " (skipped modules)" elif self.modules and not self.modules.built: finalize += " (failed modules)" - if self.modules.built==False: + if self.modules.built == False: finalize_ps = "Nonexistent template %r" % self.modules.file elif self.modules and self.modules.built: finalize_ps = ("Ready to load module %(name)s/%(version)s" - % self.product) + % self.product) self.logger.info(finalize) - if finalize_ps: self.logger.info(finalize_ps) + if finalize_ps: + self.logger.info(finalize_ps) def execute_command(self, command=None, argument=None): '''Execute the passed terminal command.''' - (out,err,proc_returncode) = (None,None,None) + (out, err, proc_returncode) = (None, None, None) if command: proc = Popen(command, stdout=PIPE, stderr=PIPE) if proc: - (out, err) = proc.communicate() if proc else (None,None) + (out, err) = proc.communicate() if proc else (None, None) proc_returncode = proc.returncode if proc else None if argument: - out = out.decode("utf-8",argument) if isinstance(out,bytes) else out - err = err.decode("utf-8",argument) if isinstance(err,bytes) else err + out = out.decode( + "utf-8", argument) if isinstance(out, bytes) else out + err = err.decode( + "utf-8", argument) if isinstance(err, bytes) else err else: - out = out.decode("utf-8") if isinstance(out,bytes) else out - err = err.decode("utf-8") if isinstance(err,bytes) else err + out = out.decode( + "utf-8") if isinstance(out, bytes) else out + err = err.decode( + "utf-8") if isinstance(err, bytes) else err else: self.ready = False self.logger.error('Unable to execute_command. ' + @@ -958,4 +1030,4 @@ def execute_command(self, command=None, argument=None): self.ready = False self.logger.error('Unable to execute_command. ' + 'command: {}'.format(command)) - return (out,err,proc_returncode) + return (out, err, proc_returncode) diff --git a/python/sdss_install/install/modules.py b/python/sdss_install/install/modules.py index fc3d4d1..9fb7f70 100644 --- a/python/sdss_install/install/modules.py +++ b/python/sdss_install/install/modules.py @@ -3,12 +3,14 @@ from __future__ import absolute_import, division, print_function, unicode_literals # The line above will help with 2to3 support. from sys import path, version_info -from os import environ, makedirs, sep -from os.path import basename, dirname, exists, isdir, join -from subprocess import Popen, PIPE +from os import environ, makedirs +from os.path import basename, exists, isdir, join from sdss_install.utils import Module + class Modules: + ''' Class for handling and preparing the installation of module files + ''' def __init__(self, options=None, @@ -27,7 +29,7 @@ def __init__(self, self.module = None def set_module(self): - self.module = Module(logger=self.logger,options=self.options) + self.module = Module(logger=self.logger, options=self.options) def set_ready(self): '''Set up Modules.''' @@ -39,7 +41,7 @@ def set_ready(self): if self.ready and not self.module.ready: self.ready = False self.logger.error("You do not appear to have Modules set up.") - + def set_file(self, ext='.module'): '''Set product module file path.''' if self.ready: @@ -47,7 +49,7 @@ def set_file(self, ext='.module'): filename = (self.product['name']+alt+ext if 'name' in self.product and ext else None) - self.file = (join(self.directory['work'],'etc',filename) + self.file = (join(self.directory['work'], 'etc', filename) if filename and 'work' in self.directory else None) @@ -55,36 +57,39 @@ def load_dependencies(self): '''Load dependencies.''' if self.ready: self.set_dependencies() - for (product,version) in self.dependencies: - self.load(product=product,version=version) + for (product, version) in self.dependencies: + self.load(product=product, version=version) def set_dependencies(self): '''Set the dependencies by looking for modules loaded in the modules file''' self.dependencies = list() if self.ready: if exists(self.file): - with open(self.file) as file: lines = file.readlines() + with open(self.file) as file: + lines = file.readlines() from json import dumps for product_version in [l.strip().split()[2] for l in lines if l.startswith('module load')]: - self.dependencies.append(product_version.split('/',1) + self.dependencies.append(product_version.split('/', 1) if '/' in product_version else (product_version, None)) - def load(self,product=None,version=None): + def load(self, product=None, version=None): '''Hook to module load function.''' if self.ready: if product: - product_version = join(product,version) if version else product + product_version = join( + product, version) if version else product try: - self.module.set_command('load',arguments=product_version) + self.module.set_command('load', arguments=product_version) self.module.execute_command() - self.logger.info("module load %s (dependency)" % product_version) + self.logger.info( + "module load %s (dependency)" % product_version) except: self.logger.warning("unable to module load %s (dependency)" % product_version) else: self.logger.error("module load command requires a " + - "product [version optional]") + "product [version optional]") def set_keywords(self, build_type=None): '''Set keywords to configure module.''' @@ -99,11 +104,11 @@ def set_keywords(self, build_type=None): self.keywords['needs_ld_lib'] = '# ' self.keywords['needs_idl'] = '# ' self.keywords['pyversion'] = "python{0:d}.{1:d}".format(*version_info) - if isdir(join(self.directory['work'],'bin')): + if isdir(join(self.directory['work'], 'bin')): self.keywords['needs_bin'] = '' - if isdir(join(self.directory['work'],'lib')): + if isdir(join(self.directory['work'], 'lib')): self.keywords['needs_ld_lib'] = '' - if isdir(join(self.directory['work'],'pro')): + if isdir(join(self.directory['work'], 'pro')): self.keywords['needs_idl'] = '' if 'python' in self.build_type: if (self.product['is_branch'] or @@ -134,14 +139,14 @@ def set_keywords(self, build_type=None): except KeyError: newpythonpath = lib_dir environ['PYTHONPATH'] = newpythonpath - path.insert(int(path[0] == ''),lib_dir) - elif isdir(join(self.directory['work'],'python')): + path.insert(int(path[0] == ''), lib_dir) + elif isdir(join(self.directory['work'], 'python')): self.keywords['needs_trunk_python'] = '' - if basename(self.options.product)=='sdss_install': + if basename(self.options.product) == 'sdss_install': self.keywords['sdss_install_root'] = self.options.root self.keywords['sdss_install_longpath'] = '# ' - elif basename(self.options.product)=='sdss4tools': + elif basename(self.options.product) == 'sdss4tools': self.keywords['sdss4tools_root'] = self.options.root self.keywords['sdss4tools_longpath'] = self.options.longpath @@ -154,7 +159,7 @@ def set_directory(self): if self.ready and not self.options.test: if not isdir(self.directory['modules']): self.logger.info("Creating Modules directory %(modules)s" - % self.directory) + % self.directory) try: makedirs(self.directory['modules']) except OSError as ose: @@ -167,17 +172,26 @@ def check_options(self): (if there is an etc/product.module file or for the tree product) ''' if self.ready: - if exists(self.file) or basename(self.options.product)=='tree': + # only check for modulefiles if the original template modulefile exists + if exists(self.file) or basename(self.options.product) == 'tree': if not self.options.moduledir: - repo_type = 'github' if self.options.github else 'svn' - self.options.moduledir = join(self.options.root, - repo_type, - 'modulefiles') + # check for SDSS_GIT/SVN_MODULES environment variable first + repo = 'GIT' if self.options.github else 'SVN' + mod_dir = environ.get('SDSS_{0}_MODULES'.format(repo), None) + # if none found, build the default modulefile path + if not mod_dir: + repo_type = 'github' if self.options.github else 'svn' + mod_dir = join(self.options.root, repo_type, 'modulefiles') + # set the modulefile directory + self.options.moduledir = mod_dir + + # create the modulefiles directory if it doesn't exist if not self.options.test: if not isdir(self.options.moduledir): self.logger.info("Creating Modules directory {0}" - .format(self.options.moduledir)) - try: makedirs(self.options.moduledir) + .format(self.options.moduledir)) + try: + makedirs(self.options.moduledir) except OSError as ose: self.ready = False self.logger.error(ose.strerror) @@ -191,24 +205,88 @@ def build(self): if exists(self.file): self.product['modulefile'] = join(self.directory['modules'], self.product['version']) + # read the modulefile template and substitute keywords with open(self.file) as file: mod = file.read().format(**self.keywords) if self.options.test: self.logger.debug(mod) else: self.logger.info("Adding module file %(modulefile)s" - % self.product) - with open(self.product['modulefile'],'w') as file: + % self.product) + # check if skipping version subdirectory + mod = self._check_skip_version(mod) + # write the new modulefile + with open(self.product['modulefile'], 'w') as file: file.write(mod) + # write the default .version file if self.options.default: versionfile = ["#%Module1.0\n", "set ModulesVersion \"%(version)s\"\n" % self.product] self.product['versionfile'] = ( - join(self.directory['modules'],'.version')) - with open(self.product['versionfile'],'w') as file: + join(self.directory['modules'], '.version')) + with open(self.product['versionfile'], 'w') as file: file.writelines(versionfile) self.built = True - elif basename(self.options.product)!='tree': self.built = False + elif basename(self.options.product) != 'tree': + self.built = False + + def _check_skip_version(self, data): + ''' Check to skip the version sub-directory in the module file + + Checks if the version sub-directory is being skipped or not. + If so, then it removes it from the modulefile data content. + + Parameters: + data (str): + The modulefile data content + + Returns: + The modified modulefile data content + ''' + + # do nothing if we aren't a git repo + if not self.options.github: + return data + # do nothing if we aren't skipping the version sub-directory + if not self.options.skip_git_verdirs: + return data + vstring = self._find_string('set PRODUCT_DIR', data) + if not vstring: + # return the original data if no PRODUCT_DIR can be found + return data + + # remove the $version subdirectory from the modulefile + new_vstring = vstring.replace('/$version', '') + data = data.replace(vstring, new_vstring) + return data + + + @staticmethod + def _find_string(data_string, data): + ''' Find a substring in some data + + Searches for a substring in read in modulefile data. + Searches from the substring up to the first "\n" entry. + + Parameters: + data_string (str): + A sub_string to search for + data (str): + The full string data to search in + + Returns: + The substring + ''' + # find starting index + vstart = data.find(data_string) + # if don't find anything return empty string + if vstart == -1: + return '' + # find ending index + vend = data.find('\n', vstart) + # extract the subtring + vstring = data[vstart:vend] + return vstring diff --git a/python/sdss_install/install4/Install4.py b/python/sdss_install/install4/Install4.py index 6d7e8c6..3aa24ee 100644 --- a/python/sdss_install/install4/Install4.py +++ b/python/sdss_install/install4/Install4.py @@ -3,23 +3,17 @@ """Install SDSS-IV software. """ from __future__ import absolute_import, division, print_function, unicode_literals -# The line above will help with 2to3 support. + +from os.path import basename, dirname, join +from subprocess import PIPE, Popen from sdss_install.application import Argument -import glob -import logging -import subprocess -import datetime -from sys import argv, executable, path -from shutil import copyfile, copytree, rmtree -from os import chdir, environ, getcwd, getenv, makedirs, walk -from os.path import basename, dirname, exists, isdir, join -from argparse import ArgumentParser -try: from ConfigParser import SafeConfigParser, RawConfigParser -except ImportError: from configparser import SafeConfigParser, RawConfigParser -from .most_recent_tag import most_recent_tag -from subprocess import Popen, PIPE + +try: + from ConfigParser import SafeConfigParser, RawConfigParser +except ImportError: + from configparser import SafeConfigParser, RawConfigParser class Install4: @@ -48,18 +42,20 @@ def set_options(self, options=None): '''Set command line argument options''' self.options = options if options else None if not self.options: - if self.logger: self.logger.error('Unable to set_options') - else: print('ERROR: Unable to set_options') + if self.logger: + self.logger.error('Unable to set_options') + else: + print('ERROR: Unable to set_options') def set_ready(self): '''Set self.ready after sanity check self.options.''' self.ready = self.options and self.logger if self.ready: - self.url = (join(self.options.url,'public') + self.url = (join(self.options.url, 'public') if self.options.public else self.options.url) if (self.options.product == 'NO PACKAGE' or - self.options.product_version == 'NO VERSION'): + self.options.product_version == 'NO VERSION'): self.logger.error("You must specify a product " + "and the version (after a space)!") self.ready = False @@ -69,12 +65,13 @@ def set_ready(self): if self.options.product_version.endswith('/'): self.options.product_version = ( dirname(self.options.product_version)) - svnroots = ['repo','data','deprecated'] + svnroots = ['repo', 'data', 'deprecated'] validproduct = [svnroot for svnroot in svnroots if self.options.product.startswith(svnroot)] if not validproduct: - self.options.product = join('repo',self.options.product) - else: self.url = None + self.options.product = join('repo', self.options.product) + else: + self.url = None def set_product(self): '''Set self.product dict containing product and version names etc.''' @@ -118,9 +115,9 @@ def set_exists(self): '''Check for existence of the product URL.''' if self.ready: self.logger.info("Contacting {url} ".format(url=self.url)) - command = self.svncommand + ['ls',self.product['url']] + command = self.svncommand + ['ls', self.product['url']] self.logger.debug(' '.join(command)) - (out,err,proc_returncode) = self.execute_command(command=command) + (out, err, proc_returncode) = self.execute_command(command=command) self.logger.debug(out) self.exists = proc_returncode == 0 if self.exists: @@ -140,24 +137,25 @@ def fetch(self): self.logger.info("Running %(checkout_or_export)s of %(url)s" % self.product) self.logger.debug(' '.join(command)) - (out,err,proc_returncode) = self.execute_command(command=command) + (out, err, proc_returncode) = self.execute_command(command=command) self.logger.debug(out) self.ready = not len(err) if self.ready: self.logger.info("Completed svn %(checkout_or_export)s " + "of %(url)s" % self.product) - else: self.logger.error("svn error during %(checkout_or_export)s " + - "of %(url)s: " % self.product + err) + else: + self.logger.error("svn error during %(checkout_or_export)s " + + "of %(url)s: " % self.product + err) def execute_command(self, command=None): '''Execute the passed terminal command.''' - (out,err,proc_returncode) = (None,None,None) + (out, err, proc_returncode) = (None, None, None) if command: proc = Popen(command, stdout=PIPE, stderr=PIPE) if proc: - (out, err) = proc.communicate() if proc else (None,None) - out = out.decode("utf-8") if isinstance(out,bytes) else out - err = err.decode("utf-8") if isinstance(err,bytes) else err + (out, err) = proc.communicate() if proc else (None, None) + out = out.decode("utf-8") if isinstance(out, bytes) else out + err = err.decode("utf-8") if isinstance(err, bytes) else err proc_returncode = proc.returncode if proc else None else: self.ready = False @@ -167,4 +165,4 @@ def execute_command(self, command=None): self.ready = False self.logger.error('Unable to execute_command. ' + 'command: {}'.format(command)) - return (out,err,proc_returncode) + return (out, err, proc_returncode) diff --git a/python/sdss_install/install4/get_svn_devstr.py b/python/sdss_install/install4/get_svn_devstr.py index 916eb26..bb7b73e 100644 --- a/python/sdss_install/install4/get_svn_devstr.py +++ b/python/sdss_install/install4/get_svn_devstr.py @@ -2,6 +2,8 @@ # -*- coding: utf-8 -*- from __future__ import absolute_import, division, print_function, unicode_literals # The line above will help with 2to3 support. + + def get_svn_devstr(product): """Get the svn revision number. @@ -19,15 +21,16 @@ def get_svn_devstr(product): """ from subprocess import Popen, PIPE from os import getenv - path = getenv(product.upper()+'_DIR') + path = getenv(product.upper() + '_DIR') if path is None: return '0' - proc = Popen(['svnversion','-n',path],stdout=PIPE,stderr=PIPE) + proc = Popen(['svnversion', '-n', path], stdout=PIPE, stderr=PIPE) out, err = proc.communicate() rev = out.decode() - if rev == 'Unversioned directory': rev = '0' + if rev == 'Unversioned directory': + rev = '0' elif ':' in rev: rev = rev.split(':')[1] - rev = rev.replace('M','').replace('S','').replace('P','') + rev = rev.replace('M', '').replace('S', '').replace('P', '') return rev diff --git a/python/sdss_install/install4/most_recent_tag.py b/python/sdss_install/install4/most_recent_tag.py index 1a789c5..2cdc3b7 100644 --- a/python/sdss_install/install4/most_recent_tag.py +++ b/python/sdss_install/install4/most_recent_tag.py @@ -2,7 +2,9 @@ # -*- coding: utf-8 -*- from __future__ import absolute_import, division, print_function, unicode_literals # The line above will help with 2to3 support. -def most_recent_tag(tags,username=None): + + +def most_recent_tag(tags, username=None): """Scan an SVN tags directory and return the most recent tag. Parameters @@ -34,19 +36,20 @@ def most_recent_tag(tags,username=None): from distutils.version import StrictVersion from subprocess import Popen, PIPE import re - bare_digit_re = re.compile(r'^v?([0-9]+)(a|b|)$') # match things like v3a or v4 - almost_good_re = re.compile(r'^v?([0-9]+)(\.[0-9]+)(\.[0-9]+)*(a|b|)$') # match things like v5.0.2a + bare_digit_re = re.compile(r'^v?([0-9]+)(a|b|)$') # match things like v3a or v4 + almost_good_re = re.compile(r'^v?([0-9]+)(\.[0-9]+)(\.[0-9]+)*(a|b|)$') # match things like v5.0.2a command = ['svn'] - if username is not None: command += ['--username', username] - command += ['ls',tags] - proc = Popen(command,stdout=PIPE,stderr=PIPE) + if username is not None: + command += ['--username', username] + command += ['ls', tags] + proc = Popen(command, stdout=PIPE, stderr=PIPE) out, err = proc.communicate() - out = out.decode("utf-8") if isinstance(out,bytes) else out - err = err.decode("utf-8") if isinstance(err,bytes) else err + out = out.decode("utf-8") if isinstance(out, bytes) else out + err = err.decode("utf-8") if isinstance(err, bytes) else err try: - tags = [v.rstrip('/').replace('_','.') for v in out.split('\n') if len(v) > 0] + tags = [v.rstrip('/').replace('_', '.') for v in out.split('\n') if len(v) > 0] except TypeError: - tags = [v.rstrip('/').replace('_','.') for v in out.decode('utf-8').split('\n') if len(v) > 0] + tags = [v.rstrip('/').replace('_', '.') for v in out.decode('utf-8').split('\n') if len(v) > 0] valid_tags = list() for t in tags: v = None @@ -61,20 +64,20 @@ def most_recent_tag(tags,username=None): if m is not None: g = m.groups() if len(g[1]) > 0: - v = StrictVersion(g[0]+'.0'+g[1]+'0') + v = StrictVersion(g[0] + '.0' + g[1] + '0') else: - v = StrictVersion(g[0]+'.0') + v = StrictVersion(g[0] + '.0') else: m = almost_good_re.match(t) if m is not None: g = m.groups() - tt = g[0]+g[1] + tt = g[0] + g[1] if g[2] is None: tt += '.0' else: tt += g[2] if len(g[3]) > 0: - tt += g[3]+'0' + tt += g[3] + '0' try: v = StrictVersion(tt) except ValueError: diff --git a/python/sdss_install/install4/version.py b/python/sdss_install/install4/version.py index db3cdfe..acdfbfa 100644 --- a/python/sdss_install/install4/version.py +++ b/python/sdss_install/install4/version.py @@ -2,6 +2,8 @@ # -*- coding: utf-8 -*- from __future__ import absolute_import, division, print_function, unicode_literals # The line above will help with 2to3 support. + + def version(headurl): """Returns the version of a package. @@ -26,12 +28,12 @@ def version(headurl): from . import get_svn_devstr, most_recent_tag if headurl.find('tags') > 0: # myproduct = headurl[0:headurl.find('tags')-1].split('/')[-1] - myversion = headurl[headurl.find('tags')+5:].split('/')[0] + myversion = headurl[headurl.find('tags') + 5:].split('/')[0] elif (headurl.find('trunk') > 0) or (headurl.find('branches') > 0): - url = headurl[10:len(headurl)-2] - findstr = ('branches','trunk')[int(headurl.find('trunk') > 0)] - myproduct = headurl[0:headurl.find(findstr)-1].split('/')[-1] - tagurl = url[0:url.find(findstr)]+'tags' + url = headurl[10:len(headurl) - 2] + findstr = ('branches', 'trunk')[int(headurl.find('trunk') > 0)] + myproduct = headurl[0:headurl.find(findstr) - 1].split('/')[-1] + tagurl = url[0:url.find(findstr)] + 'tags' myversion = most_recent_tag(tagurl) + '.dev' + get_svn_devstr(myproduct) else: myversion = '0.0.1.dev' diff --git a/python/sdss_install/install5/Install5.py b/python/sdss_install/install5/Install5.py index 33cb4ad..d08fc28 100644 --- a/python/sdss_install/install5/Install5.py +++ b/python/sdss_install/install5/Install5.py @@ -3,30 +3,29 @@ """Install SDSS-V software. """ from __future__ import absolute_import, division, print_function, unicode_literals -# The line above will help with 2to3 support. -from shutil import rmtree#, copyfile, copytree -from os import getcwd, environ, makedirs, chdir, remove#, getenv, walk -from os.path import isdir, join, exists, basename, dirname -from subprocess import Popen, PIPE -from inspect import stack, getmodule -from re import search, compile, match + +import re +from os import chdir, environ, makedirs +from os.path import abspath, basename, dirname, exists, isdir, join +from subprocess import PIPE, Popen + class Install5: - '''Class for sdss_install'ation of GitHub repositories.''' + '''Class for sdss_installation of GitHub repositories.''' def __init__(self, logger=None, options=None): self.set_logger(logger=logger) self.set_options(options=options) self.set_attributes() - def set_logger(self,logger=None): + def set_logger(self, logger=None): '''Set the class logger''' self.logger = logger if logger else None self.ready = bool(self.logger) if not self.ready: print('ERROR: %r> Unable to set_logger.' % self.__class__) - def set_options(self,options=None): + def set_options(self, options=None): '''Set command line argument options''' self.options = None if self.ready: @@ -43,7 +42,7 @@ def set_attributes(self): self.directory = None self.github_remote_url = None self.external_product = None - + def set_ready(self): ''' Set self.ready after sanity check self.options and @@ -51,14 +50,14 @@ def set_ready(self): ''' self.ready = self.options and self.logger if self.ready: - if (self.options.product == 'NO PACKAGE' or + if (self.options.product == 'NO PACKAGE' or self.options.product_version == 'NO VERSION' ): if self.options.bootstrap: self.options.default = True self.options.product = 'sdss_install' self.options.product_version = self.get_bootstrap_version() - if self.get_valid_version() and self.ready: + if self.get_valid_version(github_url=self.options.github_url) and self.ready: self.logger.info( "Selected sdss_install/{} for bootstrap installation." .format(self.options.product_version)) @@ -72,18 +71,20 @@ def set_ready(self): if self.options.product_version.endswith('/'): self.options.product_version = ( dirname(self.options.product_version)) - self.validate_product_and_version() + self.validate_product_and_version(github_url=self.options.github_url) else: self.ready = False - self.logger.error('Invalid product. {} '.format(self.options.product) ) + self.logger.error( + 'Invalid product. {} '.format(self.options.product)) - def validate_product_and_version(self,github_url=None,product=None,version=None): + def validate_product_and_version(self, github_url=None, product=None, version=None): '''Validate the product and product version.''' if self.ready: - if self.get_valid_product(github_url=github_url,product=product,version=version): - self.get_valid_version(github_url=github_url,product=product,version=version) + if self.get_valid_product(github_url=github_url, product=product, version=version): + self.get_valid_version( + github_url=github_url, product=product, version=version) - def get_valid_product(self,github_url=None,product=None,version=None): + def get_valid_product(self, github_url=None, product=None, version=None): '''Validate the product''' valid_product = None if self.ready: @@ -96,13 +97,13 @@ def get_valid_product(self,github_url=None,product=None,version=None): version=version) if self.ready: if valid_product: - self.logger.debug('Valid product: {} '.format(product) ) + self.logger.debug('Valid product: {} '.format(product)) else: self.ready = False - self.logger.error('Invalid product: {} '.format(product) ) + self.logger.error('Invalid product: {} '.format(product)) return valid_product - def get_valid_version(self,github_url=None,product=None,version=None): + def get_valid_version(self, github_url=None, product=None, version=None): '''Validate the product version''' valid_version = None if self.ready: @@ -113,11 +114,11 @@ def get_valid_version(self,github_url=None,product=None,version=None): github_url=github_url, product=product, version=version) - is_tag = False if is_branch else self.is_type(type='tag', - github_url=github_url, - product=product, - version=version) - valid_version = bool(is_master or is_branch or is_tag) + is_tag = False if is_branch else self.is_type(type='tag', + github_url=github_url, + product=product, + version=version) + valid_version = bool(is_master or is_branch or is_tag) if self.ready: if valid_version: self.logger.debug('Valid version: {}'.format(version)) @@ -131,49 +132,70 @@ def get_bootstrap_version(self): '''Return latest sdss_install tag, if present, otherwise 'master'.''' version = 'master' if self.ready: - try: product_root = environ['SDSS_INSTALL_PRODUCT_ROOT'] - except: - product_root = None - self.logger.error('Environmental variable not found: SDSS_INSTALL_PRODUCT_ROOT. ' - 'Please set before running sdss_install_bootstrap.') - sdss_install_dir = (join(product_root,'github','sdss_install') - if product_root else None) - sdss_install_master_dir = (join(sdss_install_dir,'master') - if sdss_install_dir else None) + # set the product root + product_root = environ.get('SDSS_GIT_ROOT', None) + if not product_root: + root = environ.get("SDSS_INSTALL_PRODUCT_ROOT", None) + if not root: + self.logger.error('SDSS_INSTALL_PRODUCT_ROOT environment variable not found. ' + 'Please rerun sdss_install_bootstrap and set it during configuration.') + product_root = join(root, 'github') if root else None + if not exists(product_root): + self.logger.info('Creating product_root {0}'.format(product_root)) + makedirs(product_root) + + # set the sdss_install directory path + sdss_install_dir = environ.get("SDSS_INSTALL_DIR", None) + if not sdss_install_dir: + if product_root: + sdss_install_dir = join(product_root, 'sdss_install') + else: + sdss_install_dir = abspath(join(dirname(__file__), '../../../')) + + # check for version sub-directory + if version in sdss_install_dir: + sdss_install_dir = dirname(sdss_install_dir) + sdss_install_master_dir = join(sdss_install_dir, version) + else: + sdss_install_master_dir = sdss_install_dir + + # run checkout commands to switch to tag if sdss_install_master_dir and isdir(sdss_install_master_dir): - self.logger.debug('Changing directory to: {}'.format(sdss_install_master_dir)) + self.logger.debug('Changing directory to: {}'.format( + sdss_install_master_dir)) chdir(sdss_install_master_dir) + # get most recent tag information - command = ['git','describe','--tags'] + command = ['git', 'describe', '--tags'] self.logger.debug('Running command: %s' % ' '.join(command)) - (out,err,proc_returncode) = self.execute_command(command=command) + (out, err, proc_returncode) = self.execute_command(command=command) + if proc_returncode == 0: # set version to most recent tag regex = '^(\d+\.)(\d+\.)(\d+)' - matches = self.get_matches(regex=regex,string=out) if out else list() + matches = self.get_matches( + regex=regex, string=out) if out else list() match = matches[0] if matches else str() if match: version = match.strip() - # rename directory name master to version name + + # exit if skipping version subdirectories + if self.options.skip_git_verdirs: + return version + + # changing to directory above self.logger.debug('Changing directory to: {}'.format(sdss_install_dir)) chdir(sdss_install_dir) - sdss_install_version_dir = (join(sdss_install_dir,version) - if sdss_install_dir else None) - command = ['mv',sdss_install_master_dir,sdss_install_version_dir] - self.logger.debug('Running command: %s' % ' '.join(command)) - (out,err,proc_returncode) = self.execute_command(command=command) - if not proc_returncode == 0: - self.ready = False - self.logger.error('Error encountered while running command: {}. ' - .format(' '.join(command)) + - 'err: {}.'.format(err)) - else: version = 'master' + else: + version = 'master' + + # changing directory to the product root self.logger.debug('Changing directory to: {}'.format(product_root)) chdir(product_root) else: self.ready = False self.logger.error('Error encountered while running command: {}. ' - .format(' '.join(command)) + + .format(' '.join(command)) + 'err: {}.'.format(err)) else: self.ready = False @@ -183,17 +205,20 @@ def get_bootstrap_version(self): 'sdss_install to $SDSS_INSTALL_PRODUCT_ROOT/github/master') return version - def get_matches(self,regex=None,string=None): + def get_matches(self, regex=None, string=None): matches = None if self.ready: if regex and string: - string = string.decode("utf-8") if isinstance(string,bytes) else string - pattern = compile(regex) + string = string.decode( + "utf-8") if isinstance(string, bytes) else string + pattern = re.compile(regex) iterator = pattern.finditer(string) matches = list() for match in iterator: - text = string[match.start() : match.end()] if match else None - if text: matches.append(text) + text = string[match.start(): match.end() + ] if match else None + if text: + matches.append(text) else: self.ready = False self.logger.error('Unable to check_match. ' + @@ -202,7 +227,6 @@ def get_matches(self,regex=None,string=None): ) return matches - def set_product(self): '''Set self.product dict containing product and version names etc.''' # @@ -212,56 +236,61 @@ def set_product(self): # if self.ready: self.product = dict() - self.product['root'] = None # No GitHub directory structure to preserve, as in SVN - self.product['name'] = self.options.product - self.product['version'] = self.options.product_version - self.product['is_master'] = (self.options.product_version == 'master') + # No GitHub directory structure to preserve, as in SVN + self.product['root'] = None + self.product['name'] = self.options.product + self.product['version'] = self.options.product_version + self.product['is_master'] = ( + self.options.product_version == 'master') self.product['is_branch'] = (True if self.product['is_master'] - else self.is_type(type='branch')) - self.product['is_tag'] = (False if self.product['is_branch'] - else self.is_type(type='tag')) + else self.is_type(type='branch', github_url=self.options.github_url)) + self.product['is_tag'] = (False if self.product['is_branch'] + else self.is_type(type='tag', github_url=self.options.github_url)) self.product['is_master_or_branch'] = (self.product['is_master'] or self.product['is_branch']) self.product['checkout_or_export'] = ('checkout' if self.product['is_master_or_branch'] else 'export') - def is_type(self,type=None,github_url=None,product=None,version=None): + def is_type(self, type=None, github_url=None, product=None, version=None): '''Check if the product_version is a valid Github branch.''' if self.ready: if type: product = product if product else self.options.product version = version if version else self.options.product_version - github_url = (github_url if github_url - else 'https://github.com/sdss' if self.options.https - else 'git@github.com:sdss') - url = join(github_url,product + '.git') - options = {'repository' : '--heads', # for validating product - 'branch' : '--heads', # for validating product_version - 'tag' : '--tags', # for validating product_version + github_url = ( + github_url if github_url else 'https://github.com/sdss' + if self.options.https else 'git@github.com:sdss') + url = join(github_url, product + '.git') + options = {'repository': '--heads', # for validating product + 'branch': '--heads', # for validating product_version + 'tag': '--tags', # for validating product_version } if type in options: product_version = (version if type != 'repository' - else 'master') # every product has master branch + else 'master') # every product has master branch command = ['git', 'ls-remote', options[type], url, product_version] - self.logger.debug('Running command: %s' % ' '.join(command)) - (out,err,proc_returncode) = self.execute_command(command=command) + self.logger.debug('Running command: %s' % + ' '.join(command)) + (out, err, proc_returncode) = self.execute_command( + command=command) if proc_returncode != 0: regex = '(?i)Permission denied \(publickey\)' - matches = self.get_matches(regex=regex,string=err) if err else list() + matches = self.get_matches( + regex=regex, string=err) if err else list() match = matches[0] if matches else str() s = ('While running the command\n{0}\nthe following error occurred:\n{1}\n' - .format(' '.join(command),err)) + .format(' '.join(command), err)) if match: s += ('Please see the following URL for more informaiton: \n' + 'https://help.github.com' + '/en/articles/error-permission-denied-publickey' - ) + ) self.ready = False self.logger.error(s) else: @@ -276,16 +305,14 @@ def is_type(self,type=None,github_url=None,product=None,version=None): 'type: {}'.format(type)) return bool(out) - def set_sdss_github_remote_url(self): + def set_sdss_github_remote_url(self, use_public=None): '''Set the SDSS GitHub HTTPS remote URL''' self.github_remote_url = None if self.ready: product = self.options.product if self.options else None - if product: - self.github_remote_url = ('https://github.com/sdss/{!s}.git'.format(product) - if self.options.https else - 'git@github.com:sdss/{!s}.git'.format(product) - ) + url = 'https://github.com/sdss' if use_public or self.options.https \ + else 'git@github.com:sdss' + self.github_remote_url = join(url, '{!s}.git'.format(product)) if product else None def fetch(self): ''' @@ -294,7 +321,7 @@ def fetch(self): ''' self.clone() self.checkout() - + def clone(self): '''Clone the GitHub repository for the product.''' if self.ready: @@ -305,98 +332,103 @@ def clone(self): clone_dir = (self.external_product['install_dir'] if self.external_product else self.directory['work']) - command = ['git','clone',github_remote_url,clone_dir] + command = ['git', 'clone', github_remote_url, clone_dir] self.logger.debug('Running command: %s' % ' '.join(command)) - (out,err,proc_returncode) = self.execute_command(command=command) + (out, err, proc_returncode) = self.execute_command(command=command) # NOTE: err is non-empty even when git clone is successful. if proc_returncode == 0: self.logger.info("Completed GitHub clone of repository {}" - .format(basename(github_remote_url) - .replace('.git',str()))) + .format(basename(github_remote_url) + .replace('.git', str()))) else: self.ready = False self.logger.error('Error encountered while running command: {}. ' - .format(' '.join(command)) + + .format(' '.join(command)) + 'err: {}.'.format(err)) def checkout(self): '''Checkout branch or tag and, if tag, remove git remote origin.''' - if self.ready: - version = None - install_dir = None - if self.external_product: - install_dir = self.external_product['install_dir'] - if self.external_product['is_master']: - self.logger.debug('Skipping checkout for {} branch' - .format(self.external_product['version'])) - else: - if self.external_product['is_branch']: - version = self.external_product['version'] - s = 'Completed checkout of branch {}'.format(version) - remove = False - elif self.external_product['is_tag']: - version = 'tags/' + self.external_product['version'] - s = ('Completed checkout of tag {} '.format(version) + - 'and removal of git remote origin') - remove = True - else: - version = None + if not self.ready: + return + + version = None + install_dir = None + if self.external_product: + install_dir = self.external_product['install_dir'] + if self.external_product['is_master']: + self.logger.debug('Skipping checkout for {} branch' + .format(self.external_product['version'])) else: - install_dir = self.directory['work'] - if self.product['is_master']: - self.logger.debug('Skipping checkout for {} branch' - .format(self.product['version'])) + if self.external_product['is_branch']: + version = self.external_product['version'] + s = 'Completed checkout of branch {}'.format(version) + remove = False + elif self.external_product['is_tag']: + version = 'tags/' + self.external_product['version'] + s = ('Completed checkout of tag {} '.format(version) + + 'and removal of git remote origin') + remove = True else: - if self.product['is_branch']: - version = self.product['version'] - s = 'Completed checkout of branch {}'.format(version) - remove = False - elif self.product['is_tag']: - version = 'tags/' + self.product['version'] - s = ('Completed checkout of tag {} '.format(version) + - 'and removal of git remote origin') - remove = True - else: - version = None - if version and install_dir: - chdir(install_dir) - command = ['git','checkout',version] - self.logger.debug('Running command: %s' % ' '.join(command)) - (out,err,proc_returncode) = self.execute_command(command=command) - # NOTE: err is non-empty even when git checkout is successful. - if proc_returncode == 0: - chdir(self.directory['original']) - if remove: self.export() - if self.ready: self.logger.info(s) + version = None + else: + install_dir = self.directory['work'] + if self.product['is_master']: + self.logger.debug('Skipping checkout for {} branch' + .format(self.product['version'])) + else: + if self.product['is_branch']: + version = self.product['version'] + s = 'Completed checkout of branch {}'.format(version) + remove = False + elif self.product['is_tag']: + version = 'tags/' + self.product['version'] + s = ('Completed checkout of tag {} '.format(version) + + 'and removal of git remote origin') + remove = True else: - self.ready = False - self.logger.error('Error encountered while running command: {}. ' - .format(' '.join(command)) + - 'err: {}.'.format(err)) - else: pass # version and install_dir can be None when is_master + version = None + if version and install_dir: + chdir(install_dir) + command = ['git', 'checkout', version] + self.logger.debug('Running command: %s' % ' '.join(command)) + (out, err, proc_returncode) = self.execute_command(command=command) + # NOTE: err is non-empty even when git checkout is successful. + if proc_returncode == 0: + chdir(self.directory['original']) + if remove: + self.export() + if self.ready: + self.logger.info(s) + else: + self.ready = False + self.logger.error('Error encountered while running command: {}. ' + .format(' '.join(command)) + + 'err: {}.'.format(err)) + else: + pass # version and install_dir can be None when is_master def export(self): '''Remove git remote origin.''' if self.ready: chdir(self.directory['work']) - command = ['git','remote','rm','origin'] + command = ['git', 'remote', 'rm', 'origin'] self.logger.debug('Running command: %s' % ' '.join(command)) - (out,err,proc_returncode) = self.execute_command(command=command) + (out, err, proc_returncode) = self.execute_command(command=command) if not proc_returncode == 0: self.ready = False self.logger.error('Error encountered while running command: {}. ' - .format(' '.join(command)) + + .format(' '.join(command)) + 'err: {}.'.format(err)) def execute_command(self, command=None): '''Execute the passed terminal command.''' - (out,err,proc_returncode) = (None,None,None) + (out, err, proc_returncode) = (None, None, None) if command: proc = Popen(command, stdout=PIPE, stderr=PIPE) if proc: - (out, err) = proc.communicate() if proc else (None,None) - out = out.decode("utf-8") if isinstance(out,bytes) else out - err = err.decode("utf-8") if isinstance(err,bytes) else err + (out, err) = proc.communicate() if proc else (None, None) + out = out.decode("utf-8") if isinstance(out, bytes) else out + err = err.decode("utf-8") if isinstance(err, bytes) else err proc_returncode = proc.returncode if proc else None else: self.ready = False @@ -406,5 +438,4 @@ def execute_command(self, command=None): self.ready = False self.logger.error('Unable to execute_command. ' + 'command: {}'.format(command)) - return (out,err,proc_returncode) - + return (out, err, proc_returncode) diff --git a/python/sdss_install/main.py b/python/sdss_install/main.py deleted file mode 100644 index f3e66e8..0000000 --- a/python/sdss_install/main.py +++ /dev/null @@ -1,88 +0,0 @@ -# encoding: utf-8 -# -# @Author: José Sánchez-Gallego -# @Date: Oct 12, 2017 -# @Filename: main.py -# @License: BSD 3-Clause -# @Copyright: José Sánchez-Gallego - - -from __future__ import division -from __future__ import print_function -from __future__ import absolute_import -from __future__ import unicode_literals - -import operator - - -__all__ = ('math', 'MyClass') - - -def math(arg1, arg2, arith_operator='+'): - """Performs an arithmetic operation. - - This function accepts to numbers and performs an arithmetic operation - with them. The arithmetic operation can be passed as a string. By default, - the addition operator is assumed. - - Parameters: - arg1,arg2 (float): - The numbers that we will sub/subtract/multiply/divide. - arith_operator ({'+', '-', '*', '/'}): - A string indicating the arithmetic operation to perform. - - Returns: - result (float): - The result of the arithmetic operation. - - Example: - >>> math(2, 2, arith_operator='*') - >>> 4 - - """ - - str_to_operator = {'+': operator.add, - '-': operator.sub, - '*': operator.mul, - '/': operator.truediv} - - return str_to_operator[arith_operator](arg1, arg2) - - -class MyClass(object): - """A description of the class. - - The top docstring in a class describes the class in general, and the - parameters to be passed to the class ``__init__``. - - Parameters: - arg1 (float): - The first argument. - arg2 (int): - The second argument. - kwarg1 (str): - A keyword argument. - - Attributes: - name (str): A description of what names gives acces to. - - """ - - def __init__(self, arg1, arg2, kwarg1='a'): - - self.name = arg1 - - def do_something(self): - """A description of what this method does.""" - - pass - - def do_something_else(self, param): - """A description of what this method does. - - If the class only has one or two arguments, you can describe them - inline. ``param`` is the parameter that we use to do something else. - - """ - - pass diff --git a/python/sdss_install/tests/conftest.py b/python/sdss_install/tests/conftest.py index 71b51e1..0003165 100644 --- a/python/sdss_install/tests/conftest.py +++ b/python/sdss_install/tests/conftest.py @@ -17,3 +17,229 @@ underlying directories. See https://docs.pytest.org/en/2.7.3/plugins.html for more information. """ + +import pytest +import sys +import os +import subprocess +from sdss_install.application import Argument +from sdss_install.install import Install + + +# PYTEST MODIFIERS +# ----------------- +def pytest_addoption(parser): + """Add new options""" + # run public svn tests only + parser.addoption('--public-svn-only', action='store_true', + default=False, help='Run public svn tests.') + + +def pytest_runtest_setup(item): + """Skip private svn tests.""" + if 'privatesvn' in item.keywords and item.config.getoption('--public-svn-only'): + pytest.skip('Not a public svn test') + +# +# Fixtures + + +@pytest.fixture(scope='function', autouse=True) +def monkey_setup(monkeypatch, tmpdir): + ''' Fixture that automatically sets up a temporary install directory ''' + tmproot = tmpdir.mkdir("software").mkdir("sdss") + tmpwork = tmproot.mkdir("work") + tmpgit = tmproot.mkdir("github") + tmpsvn = tmproot.mkdir("svn") + tmpgitmod = tmpgit.mkdir("modulefiles") + tmpsvnmod = tmpsvn.mkdir("modulefiles") + # change to working directory + os.chdir(str(tmpwork)) + + sdss_install_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), '../../../')) + #shutil.copytree(sdss_install_dir, tmpgit / 'sdss_install/master') + #sdss_install_dir = tmpgit / 'sdss_install/master' + + monkeypatch.setenv("SDSS_INSTALL_PRODUCT_ROOT", str(tmproot)) + monkeypatch.setenv("SDSS_GIT_ROOT", str(tmpgit)) + monkeypatch.setenv("SDSS_SVN_ROOT", str(tmpsvn)) + monkeypatch.setenv("SDSS_GIT_MODULES", str(tmpgitmod)) + monkeypatch.setenv("SDSS_SVN_MODULES", str(tmpsvnmod)) + monkeypatch.setenv("SDSS_INSTALL_DIR", str(sdss_install_dir)) + monkeypatch.setenv("SDSS4TOOLS_LONGPATH", 'True') + monkeypatch.setenv("WORKDIR", tmpwork) + + +@pytest.fixture(scope='function') +def setup_sdss_install(monkeypatch, request): + skipver = hasattr(request, 'param') + + def git(*args): + return subprocess.check_call(['git'] + list(args)) + + #tmpgit = os.environ.get("SDSS_GIT_ROOT") + #tmpwork = os.path.join(os.path.dirname(os.getenv("SDSS_GIT_ROOT")), 'work') + tmpwork = os.environ.get("WORKDIR") + install_dir = os.path.join(tmpwork, 'sdss_install') + if skipver: + os.chdir(str(tmpwork)) + git("clone", "https://github.com/sdss/sdss_install") + else: + install_dir = os.path.join(tmpwork, 'sdss_install') + os.makedirs(install_dir) + os.chdir(install_dir) + git("clone", "https://github.com/sdss/sdss_install", "master") + sdss_install_dir = os.path.join(install_dir, "master") if not skipver else install_dir + monkeypatch.setenv("SDSS_INSTALL_DIR", str(sdss_install_dir)) + + +@pytest.fixture(scope='function') +def monkey_diffdir(monkeypatch, tmpdir): + ''' Fixture to monkeypatch different git and svn root directories ''' + tmproot = tmpdir / "software" / "sdss" + tmpwork = (tmpdir / 'software').mkdir('Work') + tmpgit = tmpwork.mkdir('git') + tmpsvn = (tmpdir / 'software').mkdir('svn') + tmpgitmod = tmpwork.mkdir("modulefiles") + tmpsvnmod = tmpsvn.mkdir("modulefiles") + + monkeypatch.setenv("SDSS_INSTALL_PRODUCT_ROOT", str(tmproot)) + monkeypatch.setenv("SDSS_GIT_ROOT", str(tmpgit)) + monkeypatch.setenv("SDSS_SVN_ROOT", str(tmpsvn)) + monkeypatch.setenv("SDSS_GIT_MODULES", str(tmpgitmod)) + monkeypatch.setenv("SDSS_SVN_MODULES", str(tmpsvnmod)) + monkeypatch.setenv("WORKDIR", tmpwork) + + +def core_install(params): + ''' sets up the core sdss_install object + + Parameters: + params (list): + A list of command-line options for sdss_install to use + Returns: + a core Install instance + ''' + sys.argv[1:] = params + options = Argument('sdss_install').options + options.github_url = 'https://github.com/sdss' + install = Install(options=options) + return install + + +@pytest.fixture() +def install(request): + ''' Fixture to generate a parameterized sdss_install instance ''' + params = request.param + core = core_install(params) + yield core + core = None + + +@pytest.fixture() +def setup(install): + ''' Fixture to generate an Install up to setup ''' + install.set_ready() + install.set_product() + install.set_directory() + install.set_directory_install() + install.set_directory_work() + yield install + install = None + + +@pytest.fixture() +def work(setup): + ''' Fixture to generate an Install up to work-product checkout ''' + options = setup.options + if not options.module_only: + setup.clean_directory_install() + if options.github: + setup.set_sdss_github_remote_url(use_public=True) + else: + setup.set_svncommand() + setup.set_exists() + setup.fetch() + yield setup + setup = None + + +def _run_module(install): + ''' run the module setup ''' + install.reset_options_from_config() + install.set_build_type() + + if install.ready: + install.set_modules() + install.modules.set_module() + install.modules.set_ready() + install.modules.set_file() + install.modules.load_dependencies() + install.modules.set_keywords() + install.modules.set_directory() + return install + + +@pytest.fixture() +def module_setup(work): + ''' Fixture to generate an Install up to modulefile setup but without build ''' + work = _run_module(work) + yield work + work = None + + +@pytest.fixture() +def module(work): + ''' Fixture to generate an Install up to modulefile generation ''' + work = _run_module(work) + work.modules.build() + yield work + work = None + + +@pytest.fixture() +def external(module): + ''' Fixture to generate and Install up to external dependency installation ''' + options = module.options + # must be after module.fetch() + if options.external_dependencies: + module.install_external_dependencies() + yield module + module = None + + +@pytest.fixture() +def build(external): + ''' Fixture to generate an Install for full install process ''' + options = external.options + + external.set_environ() + if not options.module_only: + external.build() + external.build_documentation() + external.build_package() + if not options.keep: + external.clean() + yield external + external = None + + +@pytest.fixture() +def finalize(build): + ''' Fixture to finalize an installation ''' + build.finalize() + + yield build + build = None + + +@pytest.fixture() +def bootstrap(finalize): + ''' Fixture for running bootstrap step for sdss_install + + As of PR #48, this now just returns the finalize fixture + since bootstrap now just uses sdss_install to install + sdss_install + ''' + yield finalize + finalize = None diff --git a/python/sdss_install/tests/test_main.py b/python/sdss_install/tests/test_main.py deleted file mode 100644 index 3203881..0000000 --- a/python/sdss_install/tests/test_main.py +++ /dev/null @@ -1,23 +0,0 @@ -# encoding: utf-8 -# -# main.py - - -from __future__ import division -from __future__ import print_function -from __future__ import absolute_import -from __future__ import unicode_literals - -from pytest import mark - -from sdss_install.main import math - - -class TestMath(object): - """Tests for the ``math`` function in main.py.""" - - @mark.parametrize(('arg1', 'arg2', 'operator', 'result'), - [(1, 2, '+', 3), (2, 2, '-', 0), (3, 5, '*', 15), (10, 2, '/', 5)]) - def test_math(self, arg1, arg2, operator, result): - - assert math(arg1, arg2, arith_operator=operator) == result diff --git a/python/sdss_install/tests/test_module.py b/python/sdss_install/tests/test_module.py new file mode 100644 index 0000000..7a4f7b9 --- /dev/null +++ b/python/sdss_install/tests/test_module.py @@ -0,0 +1,37 @@ +# !/usr/bin/env python +# -*- coding: utf-8 -*- +# +# Filename: test_module.py +# Project: tests +# Author: Brian Cherinka +# Created: Tuesday, 25th February 2020 9:28:37 am +# License: BSD 3-clause "New" or "Revised" License +# Copyright (c) 2020 Brian Cherinka +# Last Modified: Friday, 28th February 2020 2:00:03 pm +# Modified By: Brian Cherinka + + +from __future__ import print_function, division, absolute_import + +import pytest +import os +from sdss_install.utils.module import Module + + +@pytest.fixture() +def module(install): + module = Module(logger=install.logger, options=install.options) + yield module + module = None + + +@pytest.mark.parametrize('install', [('-v', '-t', '--github')], ids=['options'], indirect=True) +def test_module_ready(module): + assert module.ready is True + + +@pytest.mark.skipif(os.getenv("TRAVIS") is not None, reason='skipping if on travis') +@pytest.mark.parametrize('install', [('-v', '-t', '--github')], ids=['options'], indirect=True) +def test_module_list(module): + loaded_modules = module.list_modules() + assert loaded_modules diff --git a/python/sdss_install/tests/test_sdss_install.py b/python/sdss_install/tests/test_sdss_install.py new file mode 100644 index 0000000..5d2790b --- /dev/null +++ b/python/sdss_install/tests/test_sdss_install.py @@ -0,0 +1,432 @@ +# !/usr/bin/env python +# -*- coding: utf-8 -*- +# +# Filename: test_sdss_install.py +# Project: tests +# Author: Brian Cherinka +# Created: Tuesday, 19th November 2019 11:02:59 am +# License: BSD 3-clause "New" or "Revised" License +# Copyright (c) 2019 Brian Cherinka +# Last Modified: Monday, 24th February 2020 6:08:28 pm +# Modified By: Brian Cherinka + + +from __future__ import print_function, division, absolute_import +import pytest +import logging +import subprocess +import os +import sys +from sdss_install.install.modules import Modules +from sdss_install.utils.module import Module + +# +# 68.92 seconds to run w/ "pytest" using conftest.py +# +# by class +# TestOptions (0.13s) +# TestGitSetup (0.6s) +# TestSvnSetup (0.06s) +# TestDiffDirs (21.07s) +# TestNoVerDirs (9.7s) +# TestInstall (17.85s) +# TestModules (9.6s) +# TestBuild (1.11s) +# TestBoostrap (9.38s) + + +class TestOptions(object): + + @pytest.mark.parametrize('install', [('-v', '-t', '--github')], ids=['options'], indirect=True) + def test_correct_options(self, install): + ''' test that core options can be set ''' + assert install.options.verbose is True + assert install.options.test is True + assert install.options.github is True + assert install.ready is False + assert install.options.root == os.environ.get('SDSS_INSTALL_PRODUCT_ROOT') + assert 'pytest' in install.options.root + assert install.options.github_url == 'https://github.com/sdss' + + +def _assert_product(install, repo, name, version, tag=False, branch=False): + ''' assert a set of product stuff ''' + root = os.path.dirname(install.options.product) if repo == 'svn' else None + assert install.product['root'] == root + assert install.product['name'] == name + assert install.product['version'] == version + is_master = version in ['master', 'trunk'] + assert install.product['is_master'] is is_master + assert install.product['checkout_or_export'] == 'checkout' if not tag else 'export' + assert install.product['is_branch'] is branch or is_master + if repo == 'git': + assert install.product['is_tag'] is tag + + +def _assert_dirs(install, repo, name, version, diff=None, noverdir=None): + ''' assert a set of directory stuff ''' + assert install.product['name'] == name + assert install.directory['original'] == os.getcwd() + # handle the product root directory + envvar = os.environ.get('SDSS_{0}_ROOT'.format(repo.upper())) + root = os.path.join(envvar, install.product['root']) if install.product['root'] else envvar + assert install.directory['root'] == root + # handle whether version is attached or not for install directory + skipver = noverdir and repo == 'git' + installdir = os.path.join(root, name, version) if not skipver else os.path.join(root, name) + assert install.directory['install'] == installdir + # handle work directory + curdir = os.path.join(os.getcwd(), '{0}-{1}'.format(name, version)) + assert install.directory['work'] == curdir + # assert that root directory not part of product root or not + if diff: + assert not install.directory['root'].startswith(install.options.root) + else: + assert install.directory['root'].startswith(install.options.root) + # assert if no version sub-directory or not + if skipver: + assert version not in install.directory['install'] + else: + assert version in install.directory['install'] + + +@pytest.mark.parametrize('install', [('-t', '--github', 'sdssdb', 'master')], ids=['sdssdb'], indirect=True) +class TestGitSetup(object): + ''' tests the initial setup when installing a git product ''' + + def test_ready(self, install): + install.set_ready() + assert isinstance(install.ready, logging.Logger) + assert hasattr(install, 'install5') + + def test_product(self, install): + install.set_ready() + install.set_product() + _assert_product(install, 'git', 'sdssdb', 'master') + + def test_directories(self, setup): + _assert_dirs(setup, 'git', 'sdssdb', 'master') + + +@pytest.mark.parametrize('install', [('-t', 'sdss/transfer', 'trunk')], ids=['transfer'], indirect=True) +@pytest.mark.privatesvn +class TestSvnSetup(object): + ''' tests the initial setup when installing an svn product ''' + + def test_ready(self, install): + install.set_ready() + assert isinstance(install.ready, logging.Logger) + assert hasattr(install, 'install4') + + def test_product(self, install): + install.set_ready() + install.set_product() + _assert_product(install, 'svn', 'transfer', 'trunk') + + def test_directories(self, setup): + _assert_dirs(setup, 'svn', 'transfer', 'trunk') + + +class TestDiffDirs(object): + ''' tests when GIT_ROOT and SVN_ROOT are different than SDSS_INSTALL_PRODUCT_ROOT ''' + + @pytest.mark.parametrize('install', [('-t', '--github', 'sdssdb', 'master')], ids=['sdssdb'], indirect=True) + def test_git_root(self, monkey_diffdir, setup): + root = setup.options.root + assert root not in setup.directory['root'] + _assert_dirs(setup, 'git', 'sdssdb', 'master', diff=True) + + + @pytest.mark.parametrize('install', [('-t', 'transfer', 'trunk')], ids=['transfer'], indirect=True) + @pytest.mark.privatesvn + def test_svn_root(self, monkey_diffdir, setup): + root = setup.options.root + assert root not in setup.directory['root'] + _assert_dirs(setup, 'svn', 'transfer', 'trunk', diff=True) + + @pytest.mark.parametrize('install', [('--github', 'sdssdb', 'master')], ids=['sdssdb'], indirect=True) + def test_git_install(self, monkey_diffdir, work): + work.set_build_type() + work.build() + assert os.listdir(work.directory['install']) + assert os.environ.get("SDSS_GIT_ROOT") in work.directory['install'] + + @pytest.mark.parametrize('install', [('sdss/transfer', 'trunk')], ids=['sdssdb'], indirect=True) + @pytest.mark.privatesvn + def test_svn_install(self, monkey_diffdir, work): + work.set_build_type() + work.build() + assert os.listdir(work.directory['install']) + assert os.environ.get("SDSS_SVN_ROOT") in work.directory['install'] + + @pytest.mark.parametrize('install', [('--github', 'sdssdb', 'master')], ids=['sdssdb'], indirect=True) + def test_git_mod(self, monkey_diffdir, module): + ''' test that git modules get placed in custom directory ''' + _assert_mod_setup(module, 'git', 'sdssdb', 'master', work=True) + _assert_mod_build(module, 'git', 'sdssdb', 'master') + + @pytest.mark.parametrize('install', [('sdss/template', 'trunk')], ids=['template'], indirect=True) + @pytest.mark.privatesvn + def test_svn_mod(self, monkey_diffdir, module): + ''' test that svn modules get placed in custom directory ''' + # note transfer product has old etc/module.in; use template product instead for etc/template.module + _assert_mod_setup(module, 'svn', 'template', 'trunk', work=True) + _assert_mod_build(module, 'svn', 'template', 'trunk') + + +class TestNoVerDirs(object): + ''' Tests for not setting a version sub-directory for git installs ''' + + @pytest.mark.parametrize('install', [('-t', '--skip-git-verdirs', '--github', 'sdssdb', 'master')], ids=['sdssdb'], indirect=True) + def test_no_version_dir(self, setup): + assert setup.options.skip_git_verdirs is True + _assert_dirs(setup, 'git', 'sdssdb', 'master', + noverdir=setup.options.skip_git_verdirs) + + @pytest.mark.parametrize('install', [('-t', '--skip-git-verdirs', 'sdss/transfer', 'trunk')], ids=['transfer'], indirect=True) + @pytest.mark.privatesvn + def test_svn_no_version_dir(self, setup): + ''' ensure version is still set for svn installs ''' + assert setup.options.skip_git_verdirs is True + _assert_dirs(setup, 'svn', 'transfer', 'trunk', + noverdir=setup.options.skip_git_verdirs) + + @pytest.mark.parametrize('install', [('--skip-git-verdirs', '--github', 'sdssdb', 'master')], ids=['sdssdb'], indirect=True) + def test_nover_install(self, work): + ''' test for correct checkout of git repo not in version sub-directory''' + work.set_build_type() + work.build() + idirs = os.listdir(work.directory['install']) + assert os.listdir(work.directory['work']) + assert idirs + assert 'master' not in idirs + assert 'python' in idirs + assert os.path.isdir(os.path.join(work.directory['install'], 'python/sdssdb')) + + @pytest.mark.parametrize('install', [('--skip-git-verdirs', '--github', 'sdssdb', 'master')], ids=['sdssdb'], indirect=True) + def test_nover_modulefile(self, module): + _assert_mod_setup(module, 'git', 'sdssdb', 'master', work=True) + _assert_mod_build(module, 'git', 'sdssdb', 'master') + _assert_mod_version(module, 'sdssdb', 'master', skipver=True) + + @pytest.mark.parametrize('install', [('--skip-git-verdirs', 'sdss/template', 'trunk')], ids=['template'], indirect=True) + @pytest.mark.privatesvn + def test_svn_nover_modulefile(self, module): + _assert_mod_setup(module, 'svn', 'template', 'trunk', work=True) + _assert_mod_build(module, 'svn', 'template', 'trunk') + _assert_mod_version(module, 'template', 'trunk') + + +class TestInstall(object): + ''' Test various installation steps for Git install ''' + + def _assert_work(self, install, repo, name): + if repo == 'svn': + assert 'svn' in install.svncommand + assert install.product['name'] == name + assert os.listdir(install.directory['work']) + + def _assert_build(self, install): + assert install.build_type is None + install.set_build_type() + assert 'python' in install.build_type + + def _assert_install(self, install): + install.make_directory_install() + assert not os.path.exists(install.directory['install']) + install.build() + assert os.listdir(install.directory['install']) + + @pytest.mark.parametrize('install', [('--github', 'sdssdb', 'master')], ids=['sdssdb'], indirect=True) + def test_git_install(self, work): + # assert work dir stuff + self._assert_work(work, 'git', 'sdssdb') + + # assert the proper build type + self._assert_build(work) + + # assert the install directory gets populated + self._assert_install(work) + + @pytest.mark.parametrize('install', [('sdss/transfer', 'trunk')], ids=['transfer'], indirect=True) + @pytest.mark.privatesvn + def test_svn_install(self, work): + # assert work dir stuff + self._assert_work(work, 'svn', 'transfer') + + # assert the proper build type + self._assert_build(work) + + # assert the install directory gets populated + self._assert_install(work) + + @pytest.mark.parametrize('install', [('--public', 'sdss/sdss4tools', 'trunk')], ids=['sdss4tools'], indirect=True) + def test_public_svn(self, work): + # assert work dir stuff + self._assert_work(work, 'svn', 'sdss4tools') + + # assert the proper build type + self._assert_build(work) + + # assert the install directory gets populated + self._assert_install(work) + + assert 'public' in work.product['url'] + + +def _assert_mod_setup(install, repo, name, version, work=False): + ''' assert some stuff regarding modulefile setup ''' + assert isinstance(install.modules, Modules) + assert isinstance(install.modules.module, Module) + assert isinstance(install.modules.ready, Module) + modname = '{0}/etc/{1}.module'.format(version, name) + assert modname in install.modules.file + assert install.modules.keywords['name'] == name + assert install.modules.keywords['version'] == version + moddir = os.environ.get('SDSS_{0}_MODULES'.format(repo.upper())) + assert install.modules.directory['modules'] == name if not work else os.path.join( + moddir, name) + + +def _assert_mod_build(install, repo, name, version): + ''' assert some stuff regarding post built module file ''' + assert install.modules.built is True + moddir = os.environ.get('SDSS_{0}_MODULES'.format(repo.upper())) + modfile = os.path.join(moddir, name, version) + assert install.modules.product['modulefile'] == modfile + assert os.path.exists(modfile) + + +class TestModules(object): + + @pytest.mark.parametrize('install', [('--module-only', '--github', 'sdssdb', 'master')], ids=['sdssdb'], indirect=True) + def test_git_setup(self, module_setup): + _assert_mod_setup(module_setup, 'git', 'sdssdb', 'master') + + @pytest.mark.parametrize('install', [('--module-only', 'sdss/transfer', 'trunk')], ids=['transfer'], indirect=True) + @pytest.mark.privatesvn + def test_svn_setup(self, module_setup): + _assert_mod_setup(module_setup, 'svn', 'transfer', 'trunk') + + @pytest.mark.parametrize('install', [('--module-only', '--github', 'sdssdb', 'master')], ids=['sdssdb'], indirect=True) + def test_build_fail_no_work(self, module): + ''' test that the build fails when the product is not already checked out ''' + _assert_mod_setup(module, 'git', 'sdssdb', 'master') + assert module.modules.built is False + assert 'modulefile' not in module.modules.product + + @pytest.mark.parametrize('install', [('--github', 'sdssdb', 'master')], ids=['sdssdb'], indirect=True) + def test_git_build(self, module): + ''' test that the build works when the product is already checked out ''' + _assert_mod_setup(module, 'git', 'sdssdb', 'master', work=True) + _assert_mod_build(module, 'git', 'sdssdb', 'master') + _assert_mod_version(module, 'sdssdb', 'master') + + @pytest.mark.parametrize('install', [('sdss/template', 'trunk')], ids=['template'], indirect=True) + @pytest.mark.privatesvn + def test_svn_build(self, module): + ''' test that the build works when the product is already checked out ''' + # note transfer product has old etc/module.in; use template product instead for etc/template.module + _assert_mod_setup(module, 'svn', 'template', 'trunk', work=True) + _assert_mod_build(module, 'svn', 'template', 'trunk') + _assert_mod_version(module, 'template', 'trunk') + + +def _find_string(data_string, data): + ''' Find a substring in some data + + Parameters: + data_string (str): + A sub_string to search for + data (str): + The full string data to search in + ''' + vstart = data.find(data_string) + vend = data.find('\n', vstart) + vstring = data[vstart:vend] + return vstring + + +def _assert_mod_version(install, name, version, skipver=None): + ''' asserts version inside output module file ''' + + modfile = install.modules.product['modulefile'] + # open file + with open(modfile, 'r') as file: + data = file.read() + + # find strings + name_string = _find_string('set product', data) + version_string = _find_string('set version', data) + product_string = _find_string('set PRODUCT_DIR', data) + + assert name in name_string + assert version in version_string + + if skipver: + assert '$version' not in product_string + else: + assert '$version' in product_string + + +class TestBuild(object): + + @pytest.mark.parametrize('install', [('-D', '--github', 'sdssdb', 'master')], ids=['sdssdb'], indirect=True) + def test_git_build(self, build): + assert os.listdir(build.directory['install']) + + +def git(*args): + ''' run a git command ''' + if sys.version_info.major == 3 and sys.version_info.minor >= 7: + kwargs = {'capture_output': True} + else: + kwargs = {'stdout': subprocess.PIPE} + return subprocess.run(['git'] + list(args), **kwargs) + + +def get_tag(): + ''' get the latest git tag ''' + tmp = git('describe', '--tags') + tag = tmp.stdout.strip().decode('utf-8').split('-')[0] + return tag + + +class TestBootstrap(object): + ''' Test for bootstrap install of sdss_install ''' + tag = get_tag() + + @pytest.mark.parametrize('install', [('--github', '--bootstrap')], ids=['sdss_install'], indirect=True) + def test_setup(self, setup_sdss_install, setup): + assert setup.options.product == 'sdss_install' + assert setup.product['name'] == 'sdss_install' + assert setup.product['version'] == self.tag + assert setup.product['is_tag'] is True + assert setup.directory['install'] == os.path.join(os.getenv("SDSS_GIT_ROOT"), 'sdss_install', self.tag) + assert setup.directory['work'] == os.path.join(os.getenv("SDSS_GIT_ROOT"), 'sdss_install-{0}'.format(self.tag)) + + @pytest.mark.parametrize('install', [('--github', '--bootstrap')], ids=['sdss_install'], indirect=True) + def test_module(self, setup_sdss_install, module): + assert module.modules.built is True + assert os.path.isfile(module.modules.product['modulefile']) + assert os.environ.get("SDSS_GIT_MODULES") in module.modules.product['modulefile'] + + # test the modulefile + _assert_mod_setup(module, 'git', 'sdss_install', self.tag, work=True) + _assert_mod_build(module, 'git', 'sdss_install', self.tag) + _assert_mod_version(module, 'sdss_install', self.tag) + + @pytest.mark.parametrize('install', [('--github', '--bootstrap')], ids=['sdss_install'], indirect=True) + def test_bootstrap(self, setup_sdss_install, bootstrap): + assert os.listdir(bootstrap.directory['install']) + os.chdir(bootstrap.directory['install']) + status = git('status') + assert 'HEAD detached at {0}'.format(self.tag) in str(status.stdout) + + @pytest.mark.parametrize('setup_sdss_install', [('--skip-git-verdirs',)], ids=['novers'], indirect=True) + @pytest.mark.parametrize('install', [('--github', '--bootstrap', '--skip-git-verdirs')], ids=['sdss_install'], indirect=True) + def test_diffdirs_novers(self, monkey_diffdir, setup_sdss_install, bootstrap): + assert os.listdir(bootstrap.directory['install']) + os.chdir(bootstrap.directory['install']) + status = git('status') + assert 'HEAD detached at {0}'.format(self.tag) in str(status.stdout) diff --git a/python/sdss_install/utils/__init__.py b/python/sdss_install/utils/__init__.py index 9e61afa..b1e5470 100644 --- a/python/sdss_install/utils/__init__.py +++ b/python/sdss_install/utils/__init__.py @@ -1,3 +1,3 @@ -from .logger import log +from .logger import * from .module import Module diff --git a/python/sdss_install/utils/logger.py b/python/sdss_install/utils/logger.py index 963c836..1a4b88f 100644 --- a/python/sdss_install/utils/logger.py +++ b/python/sdss_install/utils/logger.py @@ -6,10 +6,7 @@ # @License: BSD 3-Clause # @Copyright: José Sánchez-Gallego -# Adapted from astropy's logging system. - -from __future__ import absolute_import, division, print_function, unicode_literals - +import copy import datetime import logging import os @@ -27,55 +24,46 @@ from .color_print import color_text -# Adds custom log level for print and twisted messages -PRINT = 15 -logging.addLevelName(PRINT, 'PRINT') - - -def print_log_level(self, message, *args, **kws): - self._log(PRINT, message, args, **kws) - +__all__ = ['get_logger'] -logging.Logger._print = print_log_level +def get_exception_formatted(tp, value, tb): + """Adds colours to tracebacks.""" -def print_exception_formatted(type, value, tb): - """A custom hook for printing tracebacks with colours.""" - - tbtext = ''.join(traceback.format_exception(type, value, tb)) + tbtext = ''.join(traceback.format_exception(tp, value, tb)) lexer = get_lexer_by_name('pytb', stripall=True) formatter = TerminalFormatter() - sys.stderr.write(highlight(tbtext, lexer, formatter)) + return highlight(tbtext, lexer, formatter) def colored_formatter(record): """Prints log messages with colours.""" - colours = {'info': ('blue', 'normal'), - 'debug': ('magenta', 'normal'), - 'warning': ('yellow', 'normal'), - 'print': ('green', 'normal'), - 'error': ('red', 'bold')} + colours = {'info': 'blue', + 'debug': 'magenta', + 'warning': 'yellow', + 'critical': 'red', + 'error': 'red'} levelname = record.levelname.lower() - if levelname == 'error': - return - if levelname.lower() in colours: - levelname_color = colours[levelname][0] + levelname_color = colours[levelname] header = color_text('[{}]: '.format(levelname.upper()), levelname_color) message = record.getMessage() if levelname == 'warning': - warning_category_groups = re.match(r'^\w*?(.+?Warning) (.*)', message) + warning_category_groups = re.match( + r'^.*?\s*?(\w*?Warning): (.*)', message) if warning_category_groups is not None: warning_category, warning_text = warning_category_groups.groups() - warning_category_colour = color_text('({})'.format(warning_category), 'cyan') - message = '{} {}'.format(color_text(warning_text, ''), warning_category_colour) + warning_category_colour = color_text( + '({})'.format(warning_category), 'cyan') + message = '{} {}'.format(color_text( + warning_text, ''), warning_category_colour) sys.__stdout__.write('{}{}\n'.format(header, message)) sys.__stdout__.flush() @@ -83,164 +71,104 @@ def colored_formatter(record): return -class MyFormatter(logging.Formatter): - - base_fmt = '%(asctime)s - %(levelname)s - %(message)s [%(funcName)s @ %(filename)s]' +class SDSSFormatter(logging.Formatter): + """Custom `Formatter `.""" + base_fmt = '%(asctime)s - %(levelname)s - %(message)s' ansi_escape = re.compile(r'\x1b[^m]*m') - def __init__(self, fmt='%(levelname)s - %(message)s [%(funcName)s @ %(filename)s]'): - logging.Formatter.__init__(self, fmt, datefmt='%Y-%m-%d %H:%M:%S') + def __init__(self, fmt=base_fmt): + logging.Formatter.__init__(self, fmt) def format(self, record): - # Save the original format configured by the user - # when the logger formatter was instantiated - format_orig = self._fmt - - # Replace the original format with one customized by logging level - if record.levelno == logging.DEBUG: - self._fmt = MyFormatter.base_fmt - - elif record.levelno == logging.getLevelName('PRINT'): - self._fmt = MyFormatter.base_fmt - - elif record.levelno == logging.INFO: - self._fmt = MyFormatter.base_fmt - - elif record.levelno == logging.ERROR: - self._fmt = MyFormatter.base_fmt - - elif record.levelno == logging.WARNING: - self._fmt = MyFormatter.base_fmt - - record.msg = self.ansi_escape.sub('', record.msg) - - # Call the original formatter class to do the grunt work - result = logging.Formatter.format(self, record) - - # Restore the original format configured by the user - self._fmt = format_orig - - return result - - -Logger = logging.getLoggerClass() -fmt = MyFormatter() - - -class LoggerStdout(object): - """A pipe for stdout to a logger.""" - - def __init__(self, level): - self.level = level - - def write(self, message): - - if message != '\n': - self.level(message) + # Copy the record so that any modifications we make do not + # affect how the record is displayed in other handlers. + record_cp = copy.copy(record) + + record_cp.msg = self.ansi_escape.sub('', record_cp.msg) + + # The format of a warnings redirected with warnings.captureWarnings + # has the format : : message\n . + # We reorganise that into a cleaner message. For some reason in this + # case the message is in record.args instead of in record.msg. + if record_cp.levelno == logging.WARNING and len(record_cp.args) > 0: + match = re.match( + r'^(.*?):\s*?(\w*?Warning): (.*)', record_cp.args[0]) + if match: + message = '{1} - {2} [{0}]'.format(*match.groups()) + record_cp.args = tuple([message] + list(record_cp.args[1:])) + + return logging.Formatter.format(self, record_cp) + + +class SDSSLogger(logging.Logger): + """Custom logging system. + Parameters + ---------- + name : str + The name of the logger. + log_level : int + The initial logging level for the console handler. + capture_warnings : bool + Whether to capture warnings and redirect them to the log. + """ - def flush(self): - pass + def __init__(self, name): + super(SDSSLogger, self).__init__(name) -class MyLogger(Logger): - """This class is used to set up the logging system. + def init(self, log_level=logging.INFO, capture_warnings=True): - The main functionality added by this class over the built-in - logging.Logger class is the ability to keep track of the origin of the - messages, the ability to enable logging of warnings.warn calls and - exceptions, and the addition of colorized output and context managers to - easily capture messages to a file or list. + # Set levels + self.setLevel(logging.DEBUG) - It is addapted from the astropy logging system. + # Sets the console handler + self.sh = logging.StreamHandler() + self.sh.emit = colored_formatter + self.addHandler(self.sh) + self.sh.setLevel(log_level) - """ + # Placeholders for the file handler. + self.fh = None + self.log_filename = None - INFO = 15 + # Catches exceptions + sys.excepthook = self._catch_exceptions - # The default actor to log to. It is set by the set_actor() method. - _actor = None + self.warnings_logger = None - def save_log(self, path): - shutil.copyfile(self.log_filename, os.path.expanduser(path)) + if capture_warnings: + self.capture_warnings() def _catch_exceptions(self, exctype, value, tb): """Catches all exceptions and logs them.""" - # Now we log it. - self.error('Uncaught exception', exc_info=(exctype, value, tb)) - - # First, we print to stdout with some colouring. - print_exception_formatted(exctype, value, tb) - - def warning(self, msg, category=None, use_filters=True): - """Custom ``logging.warning``. - - Behaves like the default ``logging.warning`` but accepts ``category`` - and ``use_filters`` as arguments. ``category`` is the type of warning - we are issuing (defaults to `UserWarning`). If ``use_filters=True``, - checks whether there are global filters set for the message or the - warning category and behaves accordingly. + self.error(get_exception_formatted(exctype, value, tb)) + def capture_warnings(self): + """Capture warnings. + When `logging.captureWarnings` is `True`, all the warnings are + redirected to a logger called ``py.warnings``. We add our handlers + to the warning logger. """ - if category is None: - category = UserWarning - - full_msg = '{0} {1}'.format(category.__name__, msg) - - n_issued = 0 - if category in self.warning_registry: - if msg in self.warning_registry[category]: - n_issued = self.warning_registry[category] - - if use_filters: - - category_filter = None - regex_filter = None - for warnings_filter in warnings.filters: - if issubclass(category, warnings_filter[2]): - category_filter = warnings_filter[0] - regex_filter = warnings_filter[1] - - if (category_filter == 'ignore') or (category_filter == 'once' and n_issued >= 1): - if regex_filter is None or regex_filter.search(msg) is not None: - return - - if category_filter == 'error': - raise ValueError(full_msg) + logging.captureWarnings(True) - super(MyLogger, self).warning(full_msg) - - if msg in self.warning_registry[category]: - self.warning_registry[category][msg] += 1 - else: - self.warning_registry[category][msg] = 1 - - def _set_defaults(self, log_level=logging.INFO, redirect_stdout=False): - """Reset logger to its initial state.""" - - # Remove all previous handlers - for handler in self.handlers[:]: - self.removeHandler(handler) - - # Set up the stdout handler - self.fh = None - self.sh = logging.StreamHandler() - self.sh.emit = colored_formatter - self.addHandler(self.sh) + self.warnings_logger = logging.getLogger('py.warnings') - self.sh.setLevel(log_level) + # Only enable the sh handler if none is attached to the warnings + # logger yet. Prevents duplicated prints of the warnings. + for handler in self.warnings_logger.handlers: + if isinstance(handler, logging.StreamHandler): + return - # Redirects all stdout to the logger - if redirect_stdout: - sys.stdout = LoggerStdout(self._print) + self.warnings_logger.addHandler(self.sh) - # Catches exceptions - sys.excepthook = self._catch_exceptions + def save_log(self, path): + shutil.copyfile(self.log_filename, os.path.expanduser(path)) - def start_file_logger(self, path, log_file_level=logging.DEBUG): + def start_file_logger(self, path, log_level=logging.DEBUG): """Start file logging.""" log_file_path = os.path.expanduser(path) @@ -249,23 +177,31 @@ def start_file_logger(self, path, log_file_level=logging.DEBUG): try: if not os.path.exists(logdir): - os.mkdir(logdir) + os.makedirs(logdir) if os.path.exists(log_file_path): - strtime = datetime.datetime.utcnow().strftime( - '%Y-%m-%d_%H:%M:%S') + strtime = datetime.datetime.utcnow().strftime('%Y-%m-%d_%H:%M:%S') shutil.move(log_file_path, log_file_path + '.' + strtime) self.fh = TimedRotatingFileHandler( str(log_file_path), when='midnight', utc=True) + self.fh.suffix = '%Y-%m-%d_%H:%M:%S' + except (IOError, OSError) as ee: - self.warning('log file {0!r} could not be opened for writing: {1}'.format( - log_file_path, ee), RuntimeWarning) + + warnings.warn('log file {0!r} could not be opened for ' + 'writing: {1}'.format(log_file_path, ee), + RuntimeWarning) + else: - self.fh.setFormatter(fmt) + + self.fh.setFormatter(SDSSFormatter()) self.addHandler(self.fh) - self.fh.setLevel(log_file_level) + self.fh.setLevel(log_level) + + if self.warnings_logger: + self.warnings_logger.addHandler(self.fh) self.log_filename = log_file_path @@ -278,6 +214,16 @@ def set_level(self, level): self.fh.setLevel(level) -logging.setLoggerClass(MyLogger) -log = logging.getLogger(__name__) -log._set_defaults() # Inits sh handler +def get_logger(name, **kwargs): + """Gets a new logger.""" + + orig_logger = logging.getLoggerClass() + + logging.setLoggerClass(SDSSLogger) + + log = logging.getLogger(name) + log.init(**kwargs) + + logging.setLoggerClass(orig_logger) + + return log diff --git a/python/sdss_install/utils/module.py b/python/sdss_install/utils/module.py index f469168..c5d1f0e 100644 --- a/python/sdss_install/utils/module.py +++ b/python/sdss_install/utils/module.py @@ -8,11 +8,10 @@ from __future__ import absolute_import, division, print_function, unicode_literals -import string from os import environ from subprocess import Popen, PIPE from os.path import join, exists -from re import search, compile +import re class Module: @@ -20,29 +19,29 @@ class Module: Replaces system module's python shell, which has poor pipe handling.''' - def __init__(self,logger=None,options=None): + def __init__(self, logger=None, options=None): self.set_logger(logger=logger) self.set_options(options=options) self.set_modules() self.set_version() self.set_version_major_minor_patch() - def set_logger(self,logger=None): + def set_logger(self, logger=None): '''Set the class logger''' self.logger = logger if logger else None self.ready = bool(self.logger) if not self.ready: print('ERROR: %r> Unable to set_logger.' % self.__class__) - def set_options(self,options=None): + def set_options(self, options=None): '''Set command line argument options''' self.options = None if self.ready: self.options = options if options else None if not self.options: self.ready = False - self.logger.error('Unable to set_options' + - 'self.options: {}'.format(self.options)) + self.logger.warning('Unable to set_options' + + 'self.options: {}'.format(self.options)) def set_modules(self): self.set_modules_home() @@ -55,11 +54,13 @@ def set_modules_home(self): self.modules_home = dict() if self.ready: modules_home = None - if self.options.modules_home: + if self.options and self.options.modules_home: modules_home = self.options.modules_home else: - try: modules_home = environ['MODULESHOME'] - except: modules_home = dict() + try: + modules_home = environ['MODULESHOME'] + except: + modules_home = dict() if modules_home: self.modules_home = {'dir': modules_home} self.modules_home['lmod'] = join(modules_home, "libexec", "lmod") @@ -83,10 +84,12 @@ def set_tclsh(self): '''Set tclsh exec directory.''' self.tclsh = None if self.ready and self.modules_lang['tcl']: - try: self.tclsh = environ['TCLSH'] + try: + self.tclsh = environ['TCLSH'] except: - for self.tclsh in ['/usr/bin/tclsh','/bin/tclsh', None]: - if self.tclsh and exists(self.tclsh): break + for self.tclsh in ['/usr/bin/tclsh', '/bin/tclsh', None]: + if self.tclsh and exists(self.tclsh): + break if not self.tclsh: self.ready = False self.logger.error('Unable to set_tclsh. ' + @@ -108,7 +111,7 @@ def set_command(self, command=None, arguments=None): self.command = ([self.modules_home['lmod'], 'bash', command] if self.modules_lang['lua'] else [self.tclsh, self.modules_home['tcl'], 'python', command] - if self.modules_lang['tcl'] else None ) + if self.modules_lang['tcl'] else None) if self.command and arguments: self.command.append(join(arguments)) else: @@ -120,7 +123,9 @@ def execute_command(self): if self.command: proc = Popen(self.command, stdout=PIPE, stderr=PIPE) if proc: - (self.stdout, self.stderr) = proc.communicate() if proc else (None,None) + (out, err) = proc.communicate() if proc else (None, None) + self.stdout = out.decode("utf-8") if isinstance(out, bytes) else out + self.stderr = err.decode("utf-8") if isinstance(err, bytes) else err self.returncode = proc.returncode if proc else None else: self.ready = False @@ -153,11 +158,12 @@ def set_version_major_minor_patch(self): ) version_string = str(self.version) if self.version else str() text = str() - p1 = compile('\d+\.\d+\.\d+|\d+\.\d+') + p1 = re.compile('\d+\.\d+\.\d+|\d+\.\d+') iterator = p1.finditer(version_string) for match in iterator: - text = version_string[match.start() : match.end()] if match else str() - if text: break + text = version_string[match.start(): match.end()] if match else str() + if text: + break split = text.split('.') if text else list() self.major = split[0].strip() if split and len(split) >= 1 else str() self.minor = split[1].strip() if split and len(split) >= 2 else str() @@ -169,6 +175,19 @@ def set_version_major_minor_patch(self): 'self.verson_lang: {}.'.format(self.verson_lang) ) + def list_modules(self): + ''' List the currently loaded modules ''' + + self.set_command(command='list') + self.execute_command() + + if self.returncode != 0: + self.logger.info('module list failed.') + return + + modules = re.findall('[a-z_]+/[a-z_.0-9]+', self.stderr) + return modules + diff --git a/requirements.txt b/requirements.txt index 5421ce1..e48ac86 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,5 +3,3 @@ # astropy pyyaml pygments -sdss-tree>=2.15.2 -sdss-access>=0.2.3 diff --git a/requirements_doc.txt b/requirements_doc.txt index 3075ae7..06e8991 100644 --- a/requirements_doc.txt +++ b/requirements_doc.txt @@ -2,3 +2,6 @@ Sphinx>=1.6.5 sphinx_bootstrap_theme>=0.4.12 +sphinx-argparse>=0.2.5 +releases>=1.6.3 +semantic-version==2.6.0 \ No newline at end of file diff --git a/setup.py b/setup.py index 8369c1f..3fa303c 100644 --- a/setup.py +++ b/setup.py @@ -19,7 +19,7 @@ # The NAME variable should be of the format "sdss-sdss_install". # Please check your NAME adheres to that format. NAME = 'sdss_install' -VERSION = '0.2.2dev' +VERSION = '1.1.0dev' RELEASE = 'dev' in VERSION