changes for python 3 (#322)

* changes for python 3 * Added six to setup.py * fixes for python3 * fixes for python3 * fixes for python3 * fixes for python3 * fixes for python3 * changes for python 3 * update condainstall.sh * switch to using special env for fermipy-test-build * Added -y option in condainstall.sh * fix typo in condainstall.sh * fix typo in condainstall.sh * more fixes to condainstall.sh * udpated requirements in setup.py * moved to using install_requires for conditional deps in setup.py * moved to using install_requires for conditional deps in setup.py * udpated requirements in setup.py * more setup stuff * more setup stuff * more setup stuff * test for travis * more travis testing * test for travis * travis testing * more travis tesing * more travis testing * more travis testing * more travis testing * more travis testing * remove unused coordsys_to_frame import that was broken in gammapy 0.16 * fixes for python 3 string handling * cleanup travis.yaml * cleanup fake_travis.yaml * Added notes to condainstall.sh and updated install.rst in docs
fermiPy · Mar 19, 2020 · dd361bd · dd361bd
1 parent b54348d
commit dd361bd
Show file tree

Hide file tree

Showing 12 changed files with 140 additions and 124 deletions.
diff --git a/.travis.yml b/.travis.yml
@@ -16,11 +16,13 @@ env:
       - FERMI_DIR=$HOME/ScienceTools/x86_64-unknown-linux-gnu-libc2.17
       - SLAC_ST_BUILD=false
       - PIP_DEPS='coverage pytest-cov'
-      - CONDA2='conda install -y -c conda-forge healpy'
       - INSTALL_CMD='python setup.py install'
       - CONDA_CHANNELS=conda-forge
-      - CONDA_DEPS='gammapy numpy astropy scipy matplotlib pytest pyyaml'
-
+      - CONDA_DEPS='gammapy numpy astropy scipy matplotlib pyyaml'
+      - CONDA2_DEPS='subprocess32 pytest'
+      - FERMI_CONDA_CHANNELS='-c conda-forge/label/cf201901 -c fermi'
+      - FERMIPY_CONDA_ENV='fermipy'
+
 matrix:
     include:
 
@@ -30,7 +32,7 @@ matrix:
           env: NAME=main
                PYTHON_VERSION=2.7
                CONDA_DOWNLOAD=Miniconda2-latest-Linux-x86_64.sh
-               ST_INSTALL='conda install -y -c conda-forge/label/cf201901 -c fermi fermitools'
+               ST_INSTALL="conda install -y --name $FERMIPY_CONDA_ENV $FERMI_CONDA_CHANNELS -c $CONDA_CHANNELS fermitools"
                DOCKER_INSTALL=''
 
         # The old build:
@@ -39,7 +41,7 @@ matrix:
           env: NAME=old
                PYTHON_VERSION=2.7
                CONDA_DOWNLOAD=Miniconda2-latest-Linux-x86_64.sh
-               ST_INSTALL='conda install -y -c conda-forge/label/cf201901 -c fermi fermitools=1.0.2'
+               ST_INSTALL="conda install -y --name $FERMIPY_CONDA_ENV $FERMI_CONDA_CHANNELS -c $CONDA_CHANNELS fermitools=1.0.2"
                DOCKER_INSTALL=''
 
         # The ancient build, using a tarball
@@ -60,7 +62,7 @@ matrix:
                PYTHON_VERSION=3.6
                ST_INSTALL=''
                DOCKER_INSTALL=''
-               PIP_DEPS='coverage pytest-cov coveralls sphinx sphinx_rtd_theme'
+               PIP_DEPS='coverage pytest-cov coveralls'
                CONDA_DOWNLOAD=Miniconda3-latest-Linux-x86_64.sh
                CONDA_DEPS='gammapy numpy astropy scipy matplotlib pytest pyyaml sphinx sphinx_rtd_theme'
 

diff --git a/condainstall.sh b/condainstall.sh
@@ -1,9 +1,11 @@
 #!/usr/bin/env bash
 
+# Default python version
 if [[ -z $PYTHON_VERSION ]]; then
     PYTHON_VERSION=2.7
 fi
 
+# Default conda download
 if [[ -z $CONDA_DOWNLOAD ]]; then
     if [[ `uname` == "Darwin" ]]; then
 	echo "Detected Mac OS X..."
@@ -14,18 +16,23 @@ if [[ -z $CONDA_DOWNLOAD ]]; then
     fi
 fi
 
+# Default conda deps
 if [[ -z $CONDA_DEPS ]]; then
-    CONDA_DEPS='scipy matplotlib pyyaml ipython numpy astropy'
-fi
-
-if [[ -z $CONDA2 ]]; then
-    CONDA2='conda install -y healpy subprocess32 fermipy jupyter'
+    CONDA_DEPS='scipy matplotlib pyyaml numpy astropyy gammapy healpy'
 fi
 
+# Default conda path
 if [[ -z $CONDA_PATH ]]; then
     CONDA_PATH=$HOME/miniconda
 fi
 
+
+# Default conda env
+if [[ -z $FERMIPY_CONDA_ENV ]]; then
+    FERMIPY_CONDA_ENV=fermipy
+fi
+
+
 if [ ! -d "$CONDA_PATH/bin" ]; then
     echo Creating a new conda installation under $CONDA_PATH
     curl -o miniconda.sh -L http://repo.continuum.io/miniconda/$CONDA_DOWNLOAD
@@ -39,28 +46,36 @@ else
     echo "Using existing conda installation under $CONDA_PATH"
 fi
 
-export PATH="$CONDA_PATH/bin:$PATH"
+# Make sure we have conda setup
+. $CONDA_PATH/etc/profile.d/conda.sh
+
+# First we update conda and make an env
+conda update -q conda -y
+conda create --name $FERMIPY_CONDA_ENV $FERMI_CONDA_CHANNELS -y python=$PYTHON_VERSION
+conda activate $FERMIPY_CONDA_ENV
 
+# Install the science tools, if requested
 if [[ -n $ST_INSTALL ]]; then
     $ST_INSTALL
 fi
 
-conda update -q conda -y
-conda config --add channels conda-forge
-conda config --add channels conda-forge/label/cf201901
-conda info -a
-conda install -y python=$PYTHON_VERSION pip pytest $CONDA_DEPS
+# Ok, now we install fermipy dependencies
+conda install -n $FERMIPY_CONDA_ENV -y -c conda-forge $CONDA_DEPS
 
-if [[ -n $CONDA2 ]]; then
-    $CONDA2
+# Install extra conda deps if needed
+if [[ -n $CONDA2_DEPS ]]; then
+    conda install -n $FERMIPY_CONDA_ENV -y -c conda-forge $CONDA2_DEPS
 fi
 
+# Install stuff from pip (for travis coverage testing)
 if [[ -n $PIP_DEPS ]]; then
     python -m pip install $PIP_DEPS
 fi
 
 source condasetup.sh
 
+# We don't set this up by default.  Let the user specify how to do it.
+# This is just here from the travis testing
 if [[ -n $INSTALL_CMD ]]; then
     $INSTALL_CMD
 fi

diff --git a/condasetup.sh b/condasetup.sh
@@ -15,14 +15,5 @@ if [[ $SLAC_ST_BUILD == true ]]; then
 elif [[ -e $FERMI_DIR/fermi-init.sh ]]; then    
     source $FERMI_DIR/fermi-init.sh
     export PYTHONPATH=$FERMI_DIR/lib/python:$FERMI_DIR/lib
-else    
-    export CONDA_PREFIX=$CONDA_PATH
-    source $CONDA_PREFIX/etc/conda/activate.d/activate_fermitools.sh
-fi
-
-export PATH=$CONDA_PATH/bin:$PATH
-
-if [[ -n $CONDA_ENV_PATH ]]; then
-   export PATH=$CONDA_ENV_PATH/bin:$PATH
 fi
 
diff --git a/docs/install.rst b/docs/install.rst
@@ -5,21 +5,19 @@ Installation
 
 .. note:: 
 
-   Fermipy is only compatible with Science Tools v10r0p5 or later.  If
-   you are using an earlier version, you will need to download and
+   From version 0.19.0 fermipy is only compatible with
+   fermitools version 1.2.23 or later.  If you are using an earlier
+   version, you will need to download and
    install the latest version from the `FSSC
-   <http://fermi.gsfc.nasa.gov/ssc/data/analysis/software/>`_.  Note
-   that it is recommended to use the *non-ROOT* binary distributions
-   of the Science Tools.
+   <http://fermi.gsfc.nasa.gov/ssc/data/analysis/software/>`_.  
 
 These instructions assume that you already have a local installation
 of the Fermi Science Tools (STs).  For more information about
 installing and setting up the STs see :ref:`stinstall`.  If you are
 running at SLAC you can follow the `Running at SLAC`_ instructions.
-For Unix/Linux users we currently recommend following the
-:ref:`condainstall` instructions.  For OSX users we recommend
-following the :ref:`pipinstall` instructions.  The
-:ref:`dockerinstall` instructions can be used to install the STs on
+For we currently recommend following the :ref:`condainstall`
+instructions.  However the :ref:`pipinstall` instructions should
+also work.   The :ref:`dockerinstall` instructions can be used to install the STs on
 OSX and Linux machines that are new enough to support Docker.  To
 install the development version of Fermipy follow the
 :ref:`devinstall` instructions.
@@ -29,47 +27,45 @@ install the development version of Fermipy follow the
 Installing the Fermi Science Tools
 ----------------------------------
 
-The Fermi STs are a prerequisite for fermipy.  To install the STs we
-recommend using one of the non-ROOT binary distributions available
-from the `FSSC
-<http://fermi.gsfc.nasa.gov/ssc/data/analysis/software/>`_.  The
-following example illustrates how to install the binary distribution
-on a Linux machine running Ubuntu Trusty:
+The fermitools are a prerequisite for fermipy.  The
+following example illustrates how the fermitools in an existing
+anaconda installation.
 
 .. code-block:: bash
 
-   $ curl -OL http://fermi.gsfc.nasa.gov/ssc/data/analysis/software/tar/ScienceTools-v10r0p5-fssc-20150518-x86_64-unknown-linux-gnu-libc2.19-10-without-rootA.tar.gz
-   $ tar xzf ScienceTools-v10r0p5-fssc-20150518-x86_64-unknown-linux-gnu-libc2.19-10-without-rootA.tar.gz
-   $ export FERMI_DIR=ScienceTools-v10r0p5-fssc-20150518-x86_64-unknown-linux-gnu-libc2.19-10-without-rootA/x86_64-unknown-linux-gnu-libc2.19-10
-   $ source $FERMI_DIR/fermi-init.sh
+   $ conda create --name fermi -y python=$PYTHON_VERSION
+   $ conda activate fermi
+   $ conda install -y --name fermi -c conda-forge/label/cf201901 -c
+   fermi -c conda-forge fermitools"
 
-More information about installing the STs as well as the complete list
-of the available binary distributions is available on the `FSSC
+More information about installing the fermitools is available on the `FSSC
 software page
-<http://fermi.gsfc.nasa.gov/ssc/data/analysis/software/>`_.
-
+<http://fermi.gsfc.nasa.gov/ssc/data/analysis/software/>`_.   More
+information about setting up an anaconda installation is included in
+the :ref:`condainstall` instructions below.
+	
 .. _pipinstall:
 
 Installing with pip
 -------------------
 
 These instructions cover installation with the ``pip`` package
 management tool.  This will install fermipy and its dependencies into
-the python distribution that comes with the Fermi Science Tools.
-First verify that you're running the python from the Science Tools
+the python distribution that comes with the fermitools.
+First verify that you're running the python from the fermitools
 
 .. code-block:: bash
 
    $ which python
 
-If this doesn't point to the python in your Science Tools install
+If this doesn't point to the python in your fermitools install
 (i.e. it returns /usr/bin/python or /usr/local/bin/python) then the
-Science Tools are not properly setup.
+fermitools are not properly setup.
 
 Before starting the installation process, you will need to determine
 whether you have setuptools and pip installed in your local python
 environment.  You may need to install these packages if you are
-running with the binary version of the Fermi Science Tools distributed
+running with the binary version of the fermitools distributed
 by the FSSC.  The following command will install both packages in your
 local environment:
 
@@ -119,26 +115,20 @@ Finally, check that fermipy imports:
 Installing with Anaconda Python
 -------------------------------
 
-.. note:: 
-
-   The following instructions have only been verified to work with
-   binary Linux distributions of the Fermi STs.  If you are using OSX
-   or you have installed the STs from source you should follow the
-   :ref:`pipinstall` thread above.
-
 These instructions cover how to use fermipy with a new or existing
 anaconda python installation.  These instructions assume that you have
-already downloaded and installed the Fermi STs from the FSSC and you
-have set the ``FERMI_DIR`` environment variable to point to the location
-of this installation.
+already downloaded and installed the fermitools from the FSSC in
+and conda environment nammed 'fermi' and you have set the
+``FERMI_DIR``
+environment variable to point to the location of this installation.
 
 If you already have an existing anaconda python installation then fermipy
 can be installed from the conda-forge channel as follows:
 
 .. code-block:: bash
 
-   $ conda config --append channels conda-forge
-   $ conda install fermipy
+   $ conda activate fermi
+   $ conda install -n fermi -c conda-forge fermipy
    
 If you do not have an anaconda installation, the ``condainstall.sh``
 script can be used to create a minimal anaconda installation from
@@ -173,7 +163,7 @@ switch to this environment before running the script:
    
 .. code-block:: bash
 
-   $ source activate fermi-env
+   $ conda activate fermi-env
    $ source condasetup.sh
 
 .. _dockerinstall:

diff --git a/fake_travis.sh b/fake_travis.sh
@@ -1,72 +1,65 @@
 
 export SLAC_ST_BUILD=false
-export PIP_DEPS='coverage pytest-cov'
-export CONDA2='conda install -y -c conda-forge healpy'
+export PIP_DEPS='coverage pytest-cov coveralls'
 export INSTALL_CMD='python setup.py install'
 export CONDA_CHANNELS="conda-forge"
-export CONDA_DEPS='gammapy numpy astropy scipy matplotlib pytest pyyaml'
-#export CONDA_DEPS='pytest pyyaml'
-export CONDA_PATH='/u/ek/echarles/dmcat/software/build_test/miniconda'
+export FERMI_CONDA_CHANNELS="-c conda-forge/label/cf201901 -c fermi"
+export CONDA_DEPS='gammapy numpy astropy scipy matplotlib pyyaml'
+export CONDA2_DEPS='subprocess32 pytest'
+export DOCKER_INSTALL=''
 
-NAME='main'
+NAME='docs'
+export FERMIPY_CONDA_ENV="fermipy-test-$NAME"
 
 
 case $NAME in
     main)
 	export PYTHON_VERSION="2.7"
-	export CONDA_DOWNLOAD="Miniconda2-latest-Linux-x86_64.sh"
-	export ST_INSTALL="conda install -y -c conda-forge -c conda-forge/label/cf201901 -c fermi fermitools"
+	export CONDA_PATH="/Users/echarles/anaconda2"
+	export ST_INSTALL="conda install -y --name $FERMIPY_CONDA_ENV $FERMI_CONDA_CHANNELS -c $CONDA_CHANNELS -c fermi fermitools"
 	;;
     old)
 	export PYTHON_VERSION="2.7"
-	export CONDA_DOWNLOAD="Miniconda2-latest-Linux-x86_64.sh"
-	export ST_INSTALL="conda install -y -c conda-forge/label/cf201901 -c fermi fermitools=1.0.2"
-	;;
-    veryold)
-	export PYTHON_VERSION="2.7"
-	export CONDA_DOWNLOAD="Miniconda2-latest-Linux-x86_64.sh"
-	export ST_INSTALL="conda install -y -c conda-forge/label/cf201901 -c fermi fermitools=1.0.0"
+	export CONDA_PATH="/Users/echarles/anaconda2"
+	export ST_INSTALL="conda install -y --name $FERMIPY_CONDA_ENV $FERMI_CONDA_CHANNELS -c $CONDA_CHANNELS -c fermi fermitools=1.0.2"
 	;;
     ancient)
 	export PYTHON_VERSION="2.7"
-	export CONDA_DOWNLOAD="Miniconda2-latest-Linux-x86_64.sh"
+	export CONDA_PATH="/Users/echarles/anaconda2"
 	export ST_PATH='/u/ek/echarles/dmcat/software/build_test'
 	export ST_INSTALL="bash stinstall.sh $ST_PATH"
 	export FERMI_DIR='/u/ek/echarles/dmcat/software/build_test/ScienceTools/x86_64-unknown-linux-gnu-libc2.17'
 	export CONDA_DEPS='gammapy numpy astropy scipy matplotlib pytest pyyaml'
 	;;
     docs)
 	export PYTHON_VERSION="3.6"
+	export CONDA_PATH="/Users/echarles/anaconda3"
         export ST_INSTALL=''
-        export DOCKER_INSTALL=''
-        export PIP_DEPS='coverage pytest-cov coveralls sphinx sphinx_rtd_theme'
-	export CONDA_DOWNLOAD='Miniconda3-latest-Linux-x86_64.sh'
         export CONDA_DEPS='gammapy numpy astropy scipy matplotlib pytest pyyaml sphinx sphinx_rtd_theme'
 	;;
-    py36_st-no_dep-yes)
+    py36_st-no_dep)
 	export PYTHON_VERSION="3.6"
+	export CONDA_PATH="/Users/echarles/anaconda3"
         export ST_INSTALL=''
-        export DOCKER_INSTALL=''
-	export CONDA_DOWNLOAD='Miniconda3-latest-Linux-x86_64.sh'
 	;;
-    py2_st-no_dep-yes)
+    py2_st-no_dep)
 	export PYTHON_VERSION="2.7"
+	export CONDA_PATH="/Users/echarles/anaconda2"
         export ST_INSTALL=''
-        export DOCKER_INSTALL=''
-	export CONDA_DOWNLOAD='Miniconda2-latest-Linux-x86_64.sh'
 	;;
     slac*)
 	export PYTHON_VERSION="2.7"
-	export CONDA_DOWNLOAD="Miniconda-latest-Linux-x86_64.sh"
 	export ST_INSTALL=""
 	export SLAC_ST_BUILD=true
 	export INST_DIR='/u/ek/echarles/dmcat/software/git-releases/FT_01-00-01_orig'
 	;;
+    none)
+	exit
+        ;;
 esac
 
 echo Running fake_travis for build $NAME
 
-\rm -rf $CONDA_PATH
 source condainstall.sh 
 
 bash travistests.sh

diff --git a/fermipy/jobs/job_archive.py b/fermipy/jobs/job_archive.py
@@ -418,10 +418,10 @@ def append_to_tables(self, table, table_ids):
         intfile_refs[1] = f_ptr
 
         table.add_row(dict(dbkey=self.dbkey,
-                           jobname=self.jobname,
-                           jobkey=self.jobkey,
-                           appname=self.appname,
-                           logfile=self.logfile,
+                           jobname=str(self.jobname),
+                           jobkey=str(self.jobkey),
+                           appname=str(self.appname),
+                           logfile=str(self.logfile),
                            job_config=str(self.job_config),
                            timestamp=self.timestamp,
                            infile_refs=infile_refs,