From dd361bd664d583a4ed85ffe983bffa3aaecc8f44 Mon Sep 17 00:00:00 2001 From: Eric Charles Date: Thu, 19 Mar 2020 14:39:39 -0700 Subject: [PATCH] changes for python 3 (#322) * changes for python 3 * Added six to setup.py * fixes for python3 * fixes for python3 * fixes for python3 * fixes for python3 * fixes for python3 * changes for python 3 * update condainstall.sh * switch to using special env for fermipy-test-build * Added -y option in condainstall.sh * fix typo in condainstall.sh * fix typo in condainstall.sh * more fixes to condainstall.sh * udpated requirements in setup.py * moved to using install_requires for conditional deps in setup.py * moved to using install_requires for conditional deps in setup.py * udpated requirements in setup.py * more setup stuff * more setup stuff * more setup stuff * test for travis * more travis testing * test for travis * travis testing * more travis tesing * more travis testing * more travis testing * more travis testing * more travis testing * remove unused coordsys_to_frame import that was broken in gammapy 0.16 * fixes for python 3 string handling * cleanup travis.yaml * cleanup fake_travis.yaml * Added notes to condainstall.sh and updated install.rst in docs --- .travis.yml | 14 ++--- condainstall.sh | 41 +++++++++----- condasetup.sh | 9 ---- docs/install.rst | 72 +++++++++++-------------- fake_travis.sh | 47 +++++++--------- fermipy/jobs/job_archive.py | 8 +-- fermipy/jobs/tests/test_file_archive.py | 13 +++-- fermipy/jobs/tests/test_job_archive.py | 19 ++++--- fermipy/utils.py | 6 ++- fermipy/wcs_utils.py | 6 ++- setup.py | 27 ++++++---- travistests.sh | 2 +- 12 files changed, 140 insertions(+), 124 deletions(-) diff --git a/.travis.yml b/.travis.yml index 7aba7d20..9235e690 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,11 +16,13 @@ env: - FERMI_DIR=$HOME/ScienceTools/x86_64-unknown-linux-gnu-libc2.17 - SLAC_ST_BUILD=false - PIP_DEPS='coverage pytest-cov' - - CONDA2='conda install -y -c conda-forge healpy' - INSTALL_CMD='python setup.py install' - CONDA_CHANNELS=conda-forge - - CONDA_DEPS='gammapy numpy astropy scipy matplotlib pytest pyyaml' - + - CONDA_DEPS='gammapy numpy astropy scipy matplotlib pyyaml' + - CONDA2_DEPS='subprocess32 pytest' + - FERMI_CONDA_CHANNELS='-c conda-forge/label/cf201901 -c fermi' + - FERMIPY_CONDA_ENV='fermipy' + matrix: include: @@ -30,7 +32,7 @@ matrix: env: NAME=main PYTHON_VERSION=2.7 CONDA_DOWNLOAD=Miniconda2-latest-Linux-x86_64.sh - ST_INSTALL='conda install -y -c conda-forge/label/cf201901 -c fermi fermitools' + ST_INSTALL="conda install -y --name $FERMIPY_CONDA_ENV $FERMI_CONDA_CHANNELS -c $CONDA_CHANNELS fermitools" DOCKER_INSTALL='' # The old build: @@ -39,7 +41,7 @@ matrix: env: NAME=old PYTHON_VERSION=2.7 CONDA_DOWNLOAD=Miniconda2-latest-Linux-x86_64.sh - ST_INSTALL='conda install -y -c conda-forge/label/cf201901 -c fermi fermitools=1.0.2' + ST_INSTALL="conda install -y --name $FERMIPY_CONDA_ENV $FERMI_CONDA_CHANNELS -c $CONDA_CHANNELS fermitools=1.0.2" DOCKER_INSTALL='' # The ancient build, using a tarball @@ -60,7 +62,7 @@ matrix: PYTHON_VERSION=3.6 ST_INSTALL='' DOCKER_INSTALL='' - PIP_DEPS='coverage pytest-cov coveralls sphinx sphinx_rtd_theme' + PIP_DEPS='coverage pytest-cov coveralls' CONDA_DOWNLOAD=Miniconda3-latest-Linux-x86_64.sh CONDA_DEPS='gammapy numpy astropy scipy matplotlib pytest pyyaml sphinx sphinx_rtd_theme' diff --git a/condainstall.sh b/condainstall.sh index 10a89399..529e4f05 100644 --- a/condainstall.sh +++ b/condainstall.sh @@ -1,9 +1,11 @@ #!/usr/bin/env bash +# Default python version if [[ -z $PYTHON_VERSION ]]; then PYTHON_VERSION=2.7 fi +# Default conda download if [[ -z $CONDA_DOWNLOAD ]]; then if [[ `uname` == "Darwin" ]]; then echo "Detected Mac OS X..." @@ -14,18 +16,23 @@ if [[ -z $CONDA_DOWNLOAD ]]; then fi fi +# Default conda deps if [[ -z $CONDA_DEPS ]]; then - CONDA_DEPS='scipy matplotlib pyyaml ipython numpy astropy' -fi - -if [[ -z $CONDA2 ]]; then - CONDA2='conda install -y healpy subprocess32 fermipy jupyter' + CONDA_DEPS='scipy matplotlib pyyaml numpy astropyy gammapy healpy' fi +# Default conda path if [[ -z $CONDA_PATH ]]; then CONDA_PATH=$HOME/miniconda fi + +# Default conda env +if [[ -z $FERMIPY_CONDA_ENV ]]; then + FERMIPY_CONDA_ENV=fermipy +fi + + if [ ! -d "$CONDA_PATH/bin" ]; then echo Creating a new conda installation under $CONDA_PATH curl -o miniconda.sh -L http://repo.continuum.io/miniconda/$CONDA_DOWNLOAD @@ -39,28 +46,36 @@ else echo "Using existing conda installation under $CONDA_PATH" fi -export PATH="$CONDA_PATH/bin:$PATH" +# Make sure we have conda setup +. $CONDA_PATH/etc/profile.d/conda.sh + +# First we update conda and make an env +conda update -q conda -y +conda create --name $FERMIPY_CONDA_ENV $FERMI_CONDA_CHANNELS -y python=$PYTHON_VERSION +conda activate $FERMIPY_CONDA_ENV +# Install the science tools, if requested if [[ -n $ST_INSTALL ]]; then $ST_INSTALL fi -conda update -q conda -y -conda config --add channels conda-forge -conda config --add channels conda-forge/label/cf201901 -conda info -a -conda install -y python=$PYTHON_VERSION pip pytest $CONDA_DEPS +# Ok, now we install fermipy dependencies +conda install -n $FERMIPY_CONDA_ENV -y -c conda-forge $CONDA_DEPS -if [[ -n $CONDA2 ]]; then - $CONDA2 +# Install extra conda deps if needed +if [[ -n $CONDA2_DEPS ]]; then + conda install -n $FERMIPY_CONDA_ENV -y -c conda-forge $CONDA2_DEPS fi +# Install stuff from pip (for travis coverage testing) if [[ -n $PIP_DEPS ]]; then python -m pip install $PIP_DEPS fi source condasetup.sh +# We don't set this up by default. Let the user specify how to do it. +# This is just here from the travis testing if [[ -n $INSTALL_CMD ]]; then $INSTALL_CMD fi diff --git a/condasetup.sh b/condasetup.sh index 4c60545f..057c86d7 100644 --- a/condasetup.sh +++ b/condasetup.sh @@ -15,14 +15,5 @@ if [[ $SLAC_ST_BUILD == true ]]; then elif [[ -e $FERMI_DIR/fermi-init.sh ]]; then source $FERMI_DIR/fermi-init.sh export PYTHONPATH=$FERMI_DIR/lib/python:$FERMI_DIR/lib -else - export CONDA_PREFIX=$CONDA_PATH - source $CONDA_PREFIX/etc/conda/activate.d/activate_fermitools.sh -fi - -export PATH=$CONDA_PATH/bin:$PATH - -if [[ -n $CONDA_ENV_PATH ]]; then - export PATH=$CONDA_ENV_PATH/bin:$PATH fi diff --git a/docs/install.rst b/docs/install.rst index 06f7a647..1aaab50e 100644 --- a/docs/install.rst +++ b/docs/install.rst @@ -5,21 +5,19 @@ Installation .. note:: - Fermipy is only compatible with Science Tools v10r0p5 or later. If - you are using an earlier version, you will need to download and + From version 0.19.0 fermipy is only compatible with + fermitools version 1.2.23 or later. If you are using an earlier + version, you will need to download and install the latest version from the `FSSC - `_. Note - that it is recommended to use the *non-ROOT* binary distributions - of the Science Tools. + `_. These instructions assume that you already have a local installation of the Fermi Science Tools (STs). For more information about installing and setting up the STs see :ref:`stinstall`. If you are running at SLAC you can follow the `Running at SLAC`_ instructions. -For Unix/Linux users we currently recommend following the -:ref:`condainstall` instructions. For OSX users we recommend -following the :ref:`pipinstall` instructions. The -:ref:`dockerinstall` instructions can be used to install the STs on +For we currently recommend following the :ref:`condainstall` +instructions. However the :ref:`pipinstall` instructions should +also work. The :ref:`dockerinstall` instructions can be used to install the STs on OSX and Linux machines that are new enough to support Docker. To install the development version of Fermipy follow the :ref:`devinstall` instructions. @@ -29,25 +27,23 @@ install the development version of Fermipy follow the Installing the Fermi Science Tools ---------------------------------- -The Fermi STs are a prerequisite for fermipy. To install the STs we -recommend using one of the non-ROOT binary distributions available -from the `FSSC -`_. The -following example illustrates how to install the binary distribution -on a Linux machine running Ubuntu Trusty: +The fermitools are a prerequisite for fermipy. The +following example illustrates how the fermitools in an existing +anaconda installation. .. code-block:: bash - $ curl -OL http://fermi.gsfc.nasa.gov/ssc/data/analysis/software/tar/ScienceTools-v10r0p5-fssc-20150518-x86_64-unknown-linux-gnu-libc2.19-10-without-rootA.tar.gz - $ tar xzf ScienceTools-v10r0p5-fssc-20150518-x86_64-unknown-linux-gnu-libc2.19-10-without-rootA.tar.gz - $ export FERMI_DIR=ScienceTools-v10r0p5-fssc-20150518-x86_64-unknown-linux-gnu-libc2.19-10-without-rootA/x86_64-unknown-linux-gnu-libc2.19-10 - $ source $FERMI_DIR/fermi-init.sh + $ conda create --name fermi -y python=$PYTHON_VERSION + $ conda activate fermi + $ conda install -y --name fermi -c conda-forge/label/cf201901 -c + fermi -c conda-forge fermitools" -More information about installing the STs as well as the complete list -of the available binary distributions is available on the `FSSC +More information about installing the fermitools is available on the `FSSC software page -`_. - +`_. More +information about setting up an anaconda installation is included in +the :ref:`condainstall` instructions below. + .. _pipinstall: Installing with pip @@ -55,21 +51,21 @@ Installing with pip These instructions cover installation with the ``pip`` package management tool. This will install fermipy and its dependencies into -the python distribution that comes with the Fermi Science Tools. -First verify that you're running the python from the Science Tools +the python distribution that comes with the fermitools. +First verify that you're running the python from the fermitools .. code-block:: bash $ which python -If this doesn't point to the python in your Science Tools install +If this doesn't point to the python in your fermitools install (i.e. it returns /usr/bin/python or /usr/local/bin/python) then the -Science Tools are not properly setup. +fermitools are not properly setup. Before starting the installation process, you will need to determine whether you have setuptools and pip installed in your local python environment. You may need to install these packages if you are -running with the binary version of the Fermi Science Tools distributed +running with the binary version of the fermitools distributed by the FSSC. The following command will install both packages in your local environment: @@ -119,26 +115,20 @@ Finally, check that fermipy imports: Installing with Anaconda Python ------------------------------- -.. note:: - - The following instructions have only been verified to work with - binary Linux distributions of the Fermi STs. If you are using OSX - or you have installed the STs from source you should follow the - :ref:`pipinstall` thread above. - These instructions cover how to use fermipy with a new or existing anaconda python installation. These instructions assume that you have -already downloaded and installed the Fermi STs from the FSSC and you -have set the ``FERMI_DIR`` environment variable to point to the location -of this installation. +already downloaded and installed the fermitools from the FSSC in +and conda environment nammed 'fermi' and you have set the +``FERMI_DIR`` +environment variable to point to the location of this installation. If you already have an existing anaconda python installation then fermipy can be installed from the conda-forge channel as follows: .. code-block:: bash - $ conda config --append channels conda-forge - $ conda install fermipy + $ conda activate fermi + $ conda install -n fermi -c conda-forge fermipy If you do not have an anaconda installation, the ``condainstall.sh`` script can be used to create a minimal anaconda installation from @@ -173,7 +163,7 @@ switch to this environment before running the script: .. code-block:: bash - $ source activate fermi-env + $ conda activate fermi-env $ source condasetup.sh .. _dockerinstall: diff --git a/fake_travis.sh b/fake_travis.sh index ff7bd982..914ff092 100644 --- a/fake_travis.sh +++ b/fake_travis.sh @@ -1,35 +1,31 @@ export SLAC_ST_BUILD=false -export PIP_DEPS='coverage pytest-cov' -export CONDA2='conda install -y -c conda-forge healpy' +export PIP_DEPS='coverage pytest-cov coveralls' export INSTALL_CMD='python setup.py install' export CONDA_CHANNELS="conda-forge" -export CONDA_DEPS='gammapy numpy astropy scipy matplotlib pytest pyyaml' -#export CONDA_DEPS='pytest pyyaml' -export CONDA_PATH='/u/ek/echarles/dmcat/software/build_test/miniconda' +export FERMI_CONDA_CHANNELS="-c conda-forge/label/cf201901 -c fermi" +export CONDA_DEPS='gammapy numpy astropy scipy matplotlib pyyaml' +export CONDA2_DEPS='subprocess32 pytest' +export DOCKER_INSTALL='' -NAME='main' +NAME='docs' +export FERMIPY_CONDA_ENV="fermipy-test-$NAME" case $NAME in main) export PYTHON_VERSION="2.7" - export CONDA_DOWNLOAD="Miniconda2-latest-Linux-x86_64.sh" - export ST_INSTALL="conda install -y -c conda-forge -c conda-forge/label/cf201901 -c fermi fermitools" + export CONDA_PATH="/Users/echarles/anaconda2" + export ST_INSTALL="conda install -y --name $FERMIPY_CONDA_ENV $FERMI_CONDA_CHANNELS -c $CONDA_CHANNELS -c fermi fermitools" ;; old) export PYTHON_VERSION="2.7" - export CONDA_DOWNLOAD="Miniconda2-latest-Linux-x86_64.sh" - export ST_INSTALL="conda install -y -c conda-forge/label/cf201901 -c fermi fermitools=1.0.2" - ;; - veryold) - export PYTHON_VERSION="2.7" - export CONDA_DOWNLOAD="Miniconda2-latest-Linux-x86_64.sh" - export ST_INSTALL="conda install -y -c conda-forge/label/cf201901 -c fermi fermitools=1.0.0" + export CONDA_PATH="/Users/echarles/anaconda2" + export ST_INSTALL="conda install -y --name $FERMIPY_CONDA_ENV $FERMI_CONDA_CHANNELS -c $CONDA_CHANNELS -c fermi fermitools=1.0.2" ;; ancient) export PYTHON_VERSION="2.7" - export CONDA_DOWNLOAD="Miniconda2-latest-Linux-x86_64.sh" + export CONDA_PATH="/Users/echarles/anaconda2" export ST_PATH='/u/ek/echarles/dmcat/software/build_test' export ST_INSTALL="bash stinstall.sh $ST_PATH" export FERMI_DIR='/u/ek/echarles/dmcat/software/build_test/ScienceTools/x86_64-unknown-linux-gnu-libc2.17' @@ -37,36 +33,33 @@ case $NAME in ;; docs) export PYTHON_VERSION="3.6" + export CONDA_PATH="/Users/echarles/anaconda3" export ST_INSTALL='' - export DOCKER_INSTALL='' - export PIP_DEPS='coverage pytest-cov coveralls sphinx sphinx_rtd_theme' - export CONDA_DOWNLOAD='Miniconda3-latest-Linux-x86_64.sh' export CONDA_DEPS='gammapy numpy astropy scipy matplotlib pytest pyyaml sphinx sphinx_rtd_theme' ;; - py36_st-no_dep-yes) + py36_st-no_dep) export PYTHON_VERSION="3.6" + export CONDA_PATH="/Users/echarles/anaconda3" export ST_INSTALL='' - export DOCKER_INSTALL='' - export CONDA_DOWNLOAD='Miniconda3-latest-Linux-x86_64.sh' ;; - py2_st-no_dep-yes) + py2_st-no_dep) export PYTHON_VERSION="2.7" + export CONDA_PATH="/Users/echarles/anaconda2" export ST_INSTALL='' - export DOCKER_INSTALL='' - export CONDA_DOWNLOAD='Miniconda2-latest-Linux-x86_64.sh' ;; slac*) export PYTHON_VERSION="2.7" - export CONDA_DOWNLOAD="Miniconda-latest-Linux-x86_64.sh" export ST_INSTALL="" export SLAC_ST_BUILD=true export INST_DIR='/u/ek/echarles/dmcat/software/git-releases/FT_01-00-01_orig' ;; + none) + exit + ;; esac echo Running fake_travis for build $NAME -\rm -rf $CONDA_PATH source condainstall.sh bash travistests.sh diff --git a/fermipy/jobs/job_archive.py b/fermipy/jobs/job_archive.py index 164786b7..94efbba0 100644 --- a/fermipy/jobs/job_archive.py +++ b/fermipy/jobs/job_archive.py @@ -418,10 +418,10 @@ def append_to_tables(self, table, table_ids): intfile_refs[1] = f_ptr table.add_row(dict(dbkey=self.dbkey, - jobname=self.jobname, - jobkey=self.jobkey, - appname=self.appname, - logfile=self.logfile, + jobname=str(self.jobname), + jobkey=str(self.jobkey), + appname=str(self.appname), + logfile=str(self.logfile), job_config=str(self.job_config), timestamp=self.timestamp, infile_refs=infile_refs, diff --git a/fermipy/jobs/tests/test_file_archive.py b/fermipy/jobs/tests/test_file_archive.py index a1412442..58c749fd 100644 --- a/fermipy/jobs/tests/test_file_archive.py +++ b/fermipy/jobs/tests/test_file_archive.py @@ -5,6 +5,13 @@ from fermipy.jobs.file_archive import FileStatus, FileHandle, FileArchive +def assert_str_eq(str1, str2): + try: + test_str = str2.decode() + except AttributeError: + test_str = str2 + assert test_str == str1 + def test_file_handle(): """ Test that we can build a `FileHandle` """ @@ -19,9 +26,9 @@ def test_file_handle(): file_dict2 = FileHandle.make_dict(table) file_handle2 = file_dict2[file_handle.key] - assert file_handle.path == file_handle2.path - assert file_handle.key == file_handle2.key - assert file_handle.creator == file_handle2.creator + assert_str_eq(file_handle.path, file_handle2.path) + assert_str_eq(file_handle.key, file_handle2.key) + assert_str_eq(file_handle.creator, file_handle2.creator) assert file_handle.timestamp == file_handle2.timestamp assert file_handle.status == file_handle2.status diff --git a/fermipy/jobs/tests/test_job_archive.py b/fermipy/jobs/tests/test_job_archive.py index a086f5ab..f680ea20 100644 --- a/fermipy/jobs/tests/test_job_archive.py +++ b/fermipy/jobs/tests/test_job_archive.py @@ -8,6 +8,13 @@ from fermipy.jobs.chain import Link +def assert_str_eq(str1, str2): + try: + test_str = str2.decode() + except AttributeError: + test_str = str2 + assert test_str == str1 + def test_job_details(): """ Test that we can build a `JobDetails` object """ @@ -27,10 +34,10 @@ def test_job_details(): job_dict2 = JobDetails.make_dict(table) job2 = job_dict2[job.dbkey] - assert job.jobname == job2.jobname - assert job.dbkey == job2.dbkey - assert job.logfile == job2.logfile - assert job.status == job2.status + assert_str_eq(job.jobname, job2.jobname) + assert_str_eq(job.dbkey, job2.dbkey) + assert_str_eq(job.logfile, job2.logfile) + assert_str_eq(job.status, job2.status) def test_job_archive(): @@ -80,10 +87,10 @@ class DummyLink(Link): outfile2='output2_2.fits') link.update_args(config_1) - job = job_archive.register_job_from_link(link, 'dummy1', logfile='dummy1.log') + job = job_archive.register_job_from_link(link, str('dummy1'), logfile=str('dummy1.log')) link.update_args(config_2) - job2 = job_archive.register_job_from_link(link, 'dummy2', logfile='dummy2.log') + job2 = job_archive.register_job_from_link(link, str('dummy2'), logfile=str('dummy2.log')) assert job assert job2 diff --git a/fermipy/utils.py b/fermipy/utils.py index 8e87365a..b281a68a 100644 --- a/fermipy/utils.py +++ b/fermipy/utils.py @@ -16,7 +16,10 @@ from scipy.ndimage.measurements import label import scipy.special as special from numpy.core import defchararray -from astropy.extern import six +try: + from astropy.extern import six +except ImportError: + import six def init_matplotlib_backend(backend=None): @@ -47,7 +50,6 @@ def unicode_representer(dumper, uni): yaml.add_representer(six.text_type, unicode_representer) - def load_yaml(infile, **kwargs): return yaml.load(open(infile), **kwargs) diff --git a/fermipy/wcs_utils.py b/fermipy/wcs_utils.py index 51f34367..91a2c3e8 100644 --- a/fermipy/wcs_utils.py +++ b/fermipy/wcs_utils.py @@ -6,8 +6,10 @@ from astropy.io import fits from astropy import units as u from astropy.coordinates import SkyCoord -from astropy.extern import six -from gammapy.maps.geom import coordsys_to_frame +try: + from astropy.extern import six +except ImportError: + import six class WCSProj(object): diff --git a/setup.py b/setup.py index 4bf65536..c76f1524 100644 --- a/setup.py +++ b/setup.py @@ -102,17 +102,24 @@ 'fermipy-plot-sed-summary-bands = fermipy.jobs.target_plotting:PlotSEDSummaryBands.main', 'fermipy-plot-sed-summary-bands-sg = fermipy.jobs.target_plotting:PlotSEDSummaryBands_SG.main', ]}, - #install_requires=[], install_requires=[ - 'numpy >= 1.10.1', - 'astropy >= 2.0.7', - 'matplotlib >= 1.5.0', - 'scipy >= 0.14', - 'pyyaml', - 'healpy', - 'gammapy >= 0.8', - ], -# extras_require={ + #"numpy >= 1.17; python_version>='3.0'", + #"numpy; python_version=='2.7'", + "numpy", + "pyyaml", + "scipy", + "astropy", + #"astropy >= 2.0.7, < 4.0", + #"matplotlib >= 1.5.0; python_version>='3.0'", + #"matplotlib; python_version=='2.7'", + "matplotlib", + "healpy", + #"astropy_healpix", + #"astropy_healpix < 0.5; python_version>='3.0'", + #"gammapy == 0.10; python_version=='2.7'", + #"cython", + #"gammapy" + ], # ':python_version=="2.7"': [ # 'numpy >= 1.6.1', # 'astropy >= 2.0.7', diff --git a/travistests.sh b/travistests.sh index 844ee36f..2d2cf453 100644 --- a/travistests.sh +++ b/travistests.sh @@ -9,7 +9,7 @@ if [[ $TRAVIS_OS_NAME == 'linux' ]]; then export QT_API=pyqt; else export DISPLAY=:99.0; - /usr/bin/Xvfb :99 -screen 0 1280x1024x24 & + #/usr/bin/Xvfb :99 -screen 0 1280x1024x24 & fi python -m pytest -vv --cov=fermipy --cov-config=fermipy/tests/coveragerc --durations=30