Merge pull request #374 from broadinstitute/dp-osx

OSX compatibility

.travis.yml

@@ -1,5 +1,21 @@
 language: python
-sudo: false
+
+matrix:
+    include:
+        - os: linux
+          sudo: false
+          python: 2.7
+        - os: linux
+          sudo: false
+          python: 3.4
+        - os: linux
+          sudo: false
+          python: 3.5
+#        - os: osx
+#          language: generic
+#          env:
+#              - TRAVIS_OSX_PYTHON_VERSION=py35
+#              - TRAVIS_PYTHON_VERSION=3.5.1
 
 cache:
  directories:
@@ -17,26 +33,16 @@ env:
   - PYTHONIOENCODING=UTF8
   - secure: l9tLtFKGNhaRdRN2N7Fiks63VatVCOtDUG7FI/pi7JNJu/EriTwDRlncoVCRCJZKOdxG8OrwC1BLX6CNqpVjJISEPGV/djsf2wCV9vi6oa+OsvMymsJAjOYkLezwRLVZp/0l/sGumPGz+q+XIM8VnkOZezIvZjGaaAtBpRTHdmA=
 
-python:
-  - 2.7
-  - 3.4
-  - 3.5
-
 git:
   depth: 3
 
 before_install:
   - travis/before_install.sh
 
 install:
-  - travis/install-conda.sh
-  - export PATH="$PATH:$HOME/miniconda/bin"
-  - hash -r
+  - source travis/install-conda.sh
   - travis/install-pip.sh
-  - travis/install-tools.sh
-
-#before_script:
-#  - travis/flake8.sh # alternative to landscape.io
+  - source travis/install-tools.sh
 
 script:
   - travis/tests-unit.sh

test/integration/test_diamond.py

@@ -11,6 +11,7 @@
 from Bio import SeqIO
 import metagenomics
 from test.integration import snake
+import tools
 import tools.diamond
 import tools.picard
 import util.file
@@ -129,6 +130,8 @@ def diamond_db(request, tmpdir_factory, diamond, db_type):
     return db
 
 
+@pytest.mark.skipif(tools.is_osx(),
+                    reason="Diamond osx binary does not yet exist on bioconda")
 @pytest.mark.skipif(sys.version_info < (3,2),
                     reason="Python version is too old for snakemake.")
 def test_pipes(tmpdir, diamond_db, taxonomy_db, input_bam):
@@ -153,6 +156,8 @@ def test_pipes(tmpdir, diamond_db, taxonomy_db, input_bam):
     # assert os.path.getsize(krona_out) > 0
 
 
+@pytest.mark.skipif(tools.is_osx(),
+                    reason="Diamond osx binary does not yet exist on bioconda")
 def test_diamond(diamond_db, taxonomy_db, input_bam):
     out_report = util.file.mkstempfname('.report')
     out_lca = util.file.mkstempfname('.lca.tsv')

test/integration/test_intrahost.py

@@ -8,12 +8,15 @@
 import shutil
 import tempfile
 import argparse
+import unittest
 
 # module-specific
 import intrahost
 import util.file
 import test
+import tools
 
+@unittest.skipIf(tools.is_osx(), "vphaser2 osx binary from bioconda has issues")
 class TestPerSample(test.TestCaseWithTmp):
     ''' This tests step 1 of the iSNV calling process
         (intrahost.vphaser_one_sample), which runs V-Phaser2 on

test/integration/test_kraken.py

@@ -9,6 +9,7 @@
 import pytest
 import metagenomics
 import util.file
+import tools
 import tools.kraken
 import tools.krona
 import tools.picard
@@ -105,6 +106,8 @@ def krona_db(request, tmpdir_factory, krona, db_type):
     return db
 
 
+@pytest.mark.skipif(tools.is_osx(),
+                    reason="kraken osx binary does not yet exist on bioconda")
 def test_kraken_tool(tmpdir, kraken, kraken_db, input_bam):
     outdir = tempfile.mkdtemp('-kraken')
     out = join(outdir, 'zaire_ebola.kraken')
@@ -117,6 +120,8 @@ def test_kraken_tool(tmpdir, kraken, kraken_db, input_bam):
     assert os.path.getsize(out_filtered) > 0
 
 
+@pytest.mark.skipif(tools.is_osx(),
+                    reason="kraken osx binary does not yet exist on bioconda")
 def test_kraken(kraken_db, input_bam):
     out_report = util.file.mkstempfname('.report')
     out_reads = util.file.mkstempfname('.reads.gz')
@@ -129,6 +134,8 @@ def test_kraken(kraken_db, input_bam):
     assert os.path.getsize(out_reads) > 0
 
 
+@pytest.mark.skipif(tools.is_osx(),
+                    reason="kraken osx binary does not yet exist on bioconda")
 @pytest.mark.skipif(sys.version_info < (3,2),
                     reason="Python version is too old for snakemake.")
 def test_pipes(tmpdir, kraken_db, krona_db, input_bam):
@@ -155,6 +162,8 @@ def test_pipes(tmpdir, kraken_db, krona_db, input_bam):
     # assert os.path.getsize(krona_out) > 0
 
 
+@pytest.mark.skipif(tools.is_osx(),
+                    reason="kraken osx binary does not yet exist on bioconda")
 @pytest.mark.skipif(True, reason="krona create db takes too much disk io")
 def test_kraken_krona(tmpdir, kraken_db, krona_db, input_bam):
     out_report = util.file.mkstempfname('.report')

test/unit/test_intrahost.py

@@ -151,6 +151,8 @@ def test_single_strand_bias_hard_filter(self):
         self.assertAlmostEqual(float(output[0][6]), expected[6], places=4)
         self.assertEqual(output[0][7:], expected[7:])
 
+
+@unittest.skipIf(tools.is_osx(), "vphaser2 osx binary from bioconda has issues")
 class TestPerSample(test.TestCaseWithTmp):
     ''' This tests step 1 of the iSNV calling process
         (intrahost.vphaser_one_sample), which runs V-Phaser2 on

test/unit/test_tools_kraken.py

@@ -5,6 +5,7 @@
 import unittest
 import mock
 import subprocess
+import unittest
 import util.file
 import util.misc
 import tools.kraken
@@ -89,6 +90,7 @@ def test_classify_num_threads(self):
             self.assertEqual(actual, str(expected), "failure for requested %s, expected %s, actual %s" % (requested, expected, actual))
 
 
+@unittest.skipIf(tools.is_osx(), "kraken osx binary does not yet exist on bioconda")
 class TestToolKrakenExecute(TestCaseWithTmp):
 
     def setUp(self):

test/unit/test_tools_vphaser2.py

@@ -2,14 +2,16 @@
 
 __author__ = "irwin@broadinstitute.org"
 
-import unittest
 import os
 import pickle
+import unittest
 import util.file
+import tools
 from intrahost import vphaser_main
 from test import TestCaseWithTmp
 
 
+@unittest.skipIf(tools.is_osx(), "vphaser2 osx binary from bioconda has issues")
 class TestVPhaser2(TestCaseWithTmp):
 
     def test_vphaser2(self):

travis/install-conda.sh

@@ -4,22 +4,45 @@ set -e
 # the miniconda directory may exist if it has been restored from cache
 if [ -d "$MINICONDA_DIR" ] && [ -e "$MINICONDA_DIR/bin/conda" ]; then
     echo "Miniconda install already present from cache: $MINICONDA_DIR"
+    if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then
+        # on OSX we need to rely on the conda Python rather than the Travis-supplied system Python
+        # so conda has a higher precedence
+        export PATH="$MINICONDA_DIR/bin:$PATH"
+    else
+        export PATH="$PATH:$MINICONDA_DIR/bin"
+    fi
+    hash -r
 else # if it does not exist, we need to install miniconda
     rm -rf "$MINICONDA_DIR" # remove the directory in case we have an empty cached directory
 
     if [[ "$TRAVIS_PYTHON_VERSION" == 2* ]]; then
-        wget https://repo.continuum.io/miniconda/Miniconda-latest-Linux-x86_64.sh -O miniconda.sh;
-    else
-        wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh;
+        if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then
+            wget https://repo.continuum.io/miniconda/Miniconda-latest-MacOSX-x86_64.sh -O miniconda.sh;
+        else
+            wget https://repo.continuum.io/miniconda/Miniconda-latest-Linux-x86_64.sh -O miniconda.sh;
+        fi
+     else
+        if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then
+            wget https://repo.continuum.io/miniconda/Miniconda3-latest-MacOSX-x86_64.sh -O miniconda.sh;
+        else
+            wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh;
+        fi
     fi
 
     bash miniconda.sh -b -p "$MINICONDA_DIR"
     chown -R "$USER" "$MINICONDA_DIR"
-    export PATH="$PATH:$MINICONDA_DIR/bin"
+    if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then
+        # on OSX we need to rely on the conda Python rather than the Travis-supplied system Python
+        # so conda has a higher precedence
+        export PATH="$MINICONDA_DIR/bin:$PATH"
+    else
+        export PATH="$PATH:$MINICONDA_DIR/bin"
+    fi
     hash -r
     conda config --set always_yes yes --set changeps1 no
     conda config --add channels bioconda
     conda config --add channels r
     conda update -q conda
-    conda info -a # for debugging
-fi
+fi
+
+conda info -a # for debugging

travis/install-tools.sh

@@ -1,9 +1,9 @@
 #!/bin/bash
 set -e
 
-if [ ! -d $GATK_PATH -o ! -d $NOVOALIGN_PATH ]; then
+if [ ! -d $GATK_PATH ]; then
   if [ -z "$BUNDLE_SECRET" ]; then
-    echo "ERROR: GATK and/or Novoalign is missing, but secret key is not set for auto-download."
+    echo "ERROR: GATK is missing, but secret key is not set for auto-download."
     exit 1
 
   else
@@ -16,8 +16,17 @@ if [ ! -d $GATK_PATH -o ! -d $NOVOALIGN_PATH ]; then
   fi
 fi
 
+if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then
+    # encrypted bundle contains linux binary for Novoalign, remove that here
+    unset NOVOALIGN_PATH
+    # some conda packages dont exist on OSX
+    cat requirements-conda.txt | grep -v diamond | grep -v kraken > $HOME/requirements-conda.txt
+else
+    # for linux, just use requirements-conda as-is
+    cp requirements-conda.txt $HOME
+fi
+
 echo "Installing and validating bioinformatic tools"
 export CONDA_ENVS_PATH=tools/conda-cache:tools/conda-tools/default
-conda create -y -m -c bioconda -p tools/conda-tools/default --file requirements-conda.txt
-#py.test -n 1 -v --durations=50 test/unit/test_tools.py
+conda create -y -m -c bioconda -p tools/conda-tools/default --file $HOME/requirements-conda.txt
 ./install_tools.py