Merge pull request #116 from joachimwolff/develop

Documentation, galaxy wrappers and bug fixes

.travis.yml

@@ -1,46 +1,59 @@
-language: c
+language: python
 sudo: false
 services:
   - postgresql
-env:
-  - TRAVIS_PYTHON_VERSION=2.7
+addons:
+  apt:
+    packages:
+    - samtools
+python:
+  - 2.7
+
 os:
   - linux
-  #- osx
-
-# Setup anaconda
-before_install:
-  - export HIC_TEST_DATA_DIR="`pwd`/hicexplorer/test/test_data/"
-  - echo $HIC_TEST_DATA_DIR
-  - if [[ "$TRAVIS_OS_NAME" == "linux" && "$TRAVIS_PYTHON_VERSION" == "2.7" ]]; then pip install virtualenv --user ; fi
-  - if [[ "$TRAVIS_OS_NAME" == "linux" && "$TRAVIS_PYTHON_VERSION" == "2.7" ]]; then virtualenv planemo_test_env; source planemo_test_env/bin/activate; pip install six; pip install planemo ; deactivate ; fi
-  - if [[ "$TRAVIS_OS_NAME" == "linux" && "$TRAVIS_PYTHON_VERSION" == "2.7" ]]; then curl https://repo.continuum.io/miniconda/Miniconda-latest-Linux-x86_64.sh -o miniconda.sh ; fi
-  #- if [[ "$TRAVIS_OS_NAME" == "linux" && "$TRAVIS_PYTHON_VERSION" == "3.5" ]]; then curl https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -o miniconda.sh ; fi
-  - if [[ "$TRAVIS_OS_NAME" == "osx" && "$TRAVIS_PYTHON_VERSION" == "2.7" ]]; then curl https://repo.continuum.io/miniconda/Miniconda-latest-MacOSX-x86_64.sh -o miniconda.sh ; fi
-  #- if [[ "$TRAVIS_OS_NAME" == "osx" && "$TRAVIS_PYTHON_VERSION" == "3.5" ]]; then curl https://repo.continuum.io/miniconda/Miniconda3-latest-MacOSX-x86_64.sh -o miniconda.sh ; fi
-  - bash miniconda.sh -b -p $HOME/miniconda
-  - export PATH="$HOME/miniconda/bin:$PATH"
-  - hash -r
-  - conda config --set always_yes yes --set changeps1 no
-  - conda update -q conda
 
-  # Useful for debugging any issues with conda
-  - conda info -a
+jobs:
+  include:
+    - stage: lint
+      env:
+        - LINT=1
+      before_install:
+        - curl https://repo.continuum.io/miniconda/Miniconda-latest-Linux-x86_64.sh -o miniconda.sh
+        - bash miniconda.sh -b -p $HOME/miniconda
+        - PATH_WITHOUT_CONDA="$PATH"
+        - export PATH="$HOME/miniconda/bin:$PATH"
+        - hash -r
+        - conda config --set always_yes yes --set changeps1 no
+        - conda update -q conda
+      install:
+        - conda install --yes python=2.7 flake8
+      script:
+        - if [[ "$LINT" == "1" ]]; then flake8 . --exclude=.venv,.build,planemo_test_env,build --ignore=E501,F403,E402,F999,F405,E712 ; fi
+    - stage: test
+      # Setup anaconda
+      before_install:
+        - export HIC_TEST_DATA_DIR="`pwd`/hicexplorer/test/test_data/"
+        - echo $HIC_TEST_DATA_DIR
+        - curl https://repo.continuum.io/miniconda/Miniconda-latest-Linux-x86_64.sh -o miniconda.sh
+        # - if [[ "$TRAVIS_OS_NAME" == "linux" && "$TRAVIS_PYTHON_VERSION" == "3.5" ]]; then curl https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -o miniconda.sh ; fi
+        # - if [[ "$TRAVIS_OS_NAME" == "linux" && "$TRAVIS_PYTHON_VERSION" == "3.6" ]]; then curl https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -o miniconda.sh ; fi
+        - bash miniconda.sh -b -p $HOME/miniconda
+        - PATH_WITHOUT_CONDA="$PATH"
+        - export PATH="$HOME/miniconda/bin:$PATH"
+        - hash -r
+        - conda config --set always_yes yes --set changeps1 no
+        - conda update -q conda
+        # Useful for debugging any issues with conda
+        - conda info -a
 
-# Install packages
-install:
-  - conda install --yes python=$TRAVIS_PYTHON_VERSION numpy scipy matplotlib=2.0.0 nose flake8 pytables biopython pandas
-  - conda install --yes -c bioconda pysam intervaltree
-  - pip install pyBigWig
-  - python setup.py install
+      # Install packages
+      install:
+        - conda create --name hicexplorer_pytest --yes python=$TRAVIS_PYTHON_VERSION pytest
+        - source activate hicexplorer_pytest
+        - conda install --yes -c defaults -c conda-forge -c bioconda --file requirements.txt
+        - conda install --yes -c defaults -c conda-forge -c bioconda configparser
+        - python setup.py install
 
-# command to run tests
-script:
-  - flake8 . --exclude=.venv,.build,planemo_test_env,build --ignore=E501,F403,E402,F999,F405,E712
-  - export owd=`pwd`
-  - cd ~/ && nosetests --with-doctest -sv hicexplorer
-  - cd ${owd}
-  - echo ${owd}
-  #- if [[ "$TRAVIS_OS_NAME" == "linux" && "$TRAVIS_PYTHON_VERSION" == "2.7" ]]; then /home/travis/build/maxplanck-ie/HiCExplorer/foo/bin/planemo lint galaxy/wrapper/ ; fi
-  - if [[ "$TRAVIS_OS_NAME" == "linux" && "$TRAVIS_PYTHON_VERSION" == "2.7" ]]; then bash .planemo.sh ; fi
-  # - if [[ "$TRAVIS_OS_NAME" == "linux" && "$TRAVIS_PYTHON_VERSION" == "2.7" ]]; then bash /home/travis/build/maxplanck-ie/HiCExplorer/.planemo.sh ; fi
+      # command to run tests
+      script:
+        - py.test hicexplorer --doctest-modules

README.rst

@@ -16,8 +16,8 @@ Set of programs to process, analyze and visualize Hi-C data
 -----------------------------------------------------------
 
 Sequencing techniques that probe the 3D organization of the genome generate large amounts of data whose processing,
-analysis and visualization is challenging. Here, we present Hi-C Explorer, a set of tools for the analysis and
-visualization of chromosome conformation data. Hi-C explorer facilitates the creation of contact matrices, correction
+analysis and visualization is challenging. Here, we present HiCExplorer, a set of tools for the analysis and
+visualization of chromosome conformation data. HiCExplorer facilitates the creation of contact matrices, correction
 of contacts, TAD detection, merging, reordering or chromosomes, conversion from different formats and detection of
 long-range contacts. Moreover, it allows the visualization of multiple contact matrices along with other types of
 data like genes, compartments, ChIP-seq coverage tracks (and in general any type of genomic scores) and long range contacts.

docs/conf.py

@@ -65,8 +65,8 @@
 
 # General information about the project.
 project = u'HiCExplorer'
-copyright = u'2015, Fidel Ramírez'
-author = u'Fidel Ramírez'
+copyright = u'2017, Fidel Ramírez, Bjoern Gruening, Vivek Bhardwaj, Joachim Wolff'
+author = u'Fidel Ramírez, Bjoern Gruening, Vivek Bhardwaj, Joachim Wolff'
 
 # The version info for the project you're documenting, acts as replacement for
 # |version| and |release|, also used in various other places throughout the

docs/content/HiCExport.rst

@@ -4,9 +4,9 @@ Importing and Exporting HiCExplorer data
 Exporting HiCExplorer output to Bioconductor
 --------------------------------------------
 
-It's possible to export HiC Matrices produced by HiCExplorer to
+It's possible to export Hi-C Matrices produced by HiCExplorer to
 `bioconductor <http://bioconductor.org/>`__ in R, which allows us to use
-existing bioconductor infrastructure for differential HiC analysis. The
+existing bioconductor infrastructure for differential Hi-C analysis. The
 tool **hicExport** allows us to write Hi-C matrices in a format that can
 eaisily be imported in bioconductor as **GInteractions** object. Below
 is an example.

docs/content/News.rst

@@ -36,15 +36,15 @@ Our #biorXiv preprint on DNA sequences behind Fly genome architecture is online!
 
 Read the article here : `<http://biorxiv.org/content/early/2017/03/08/115063>`_
 
-In this article, we introduce HiCExplorer : Our easy to use tool for HiC data analysis, also available in `Galaxy <https://galaxyproject.org/>`_.
+In this article, we introduce HiCExplorer : Our easy to use tool for Hi-C data analysis, also available in `Galaxy <https://galaxyproject.org/>`_.
 
-We also introduce `HiCBrowser <https://github.com/maxplanck-ie/HiCBrowser>`_ : A standalone software to visualize HiC along with other genomic datasets.
+We also introduce `HiCBrowser <https://github.com/maxplanck-ie/HiCBrowser>`_ : A standalone software to visualize Hi-C along with other genomic datasets.
 
 Based on HiCExplorer and HiCBrowser, we built a useful resource for anyone to browse and download the chromosome
 conformation datasets in Human, Mouse and Flies. It's called `the chorogenome navigator <http://chorogenome.ie-freiburg.mpg.de/>`_
 
 Along with these resources, we present an analysis of DNA sequences behind 3D genome of Flies. Using high-resolution
-HiC analysis, we find a set of DNA motifs that characterize TAD boundaries in Flies and show the importance of these motifs in genome organization.
+Hi-C analysis, we find a set of DNA motifs that characterize TAD boundaries in Flies and show the importance of these motifs in genome organization.
 
 We hope that these resources and analysis would be useful for the community and welcome any feedback.
 

docs/content/example_usage.rst

@@ -73,6 +73,7 @@ containing the restriction sites, this file can be created with the tool :ref:`f
 :ref:`findRestSite`
 that is part of HiCExplorer.
 
+
 .. code-block:: bash
 
    # build matrix from independently mated read pairs
@@ -81,8 +82,10 @@ that is part of HiCExplorer.
    $ hicBuildMatrix --samFiles mate_R1.bam mate_R2.bam \
                     --binSize 10000 \
                     --restrictionSequence GATC \
+                    --threads 4
+                    --inputBufferSize 100000
                     --outBam hic.bam \
-                    -o hic_matrix.npz
+                    -o hic_matrix.h5
                     --QCfolder ./hicQC
 
 
@@ -109,7 +112,7 @@ diagnostic plot as follows:
 
 .. code-block:: bash
 
-   $ hicCorrectMatrix diagnostic_plot -m hic_matrix.npz -o hic_corrected.npz
+   $ hicCorrectMatrix diagnostic_plot -m hic_matrix.h5 -o hic_corrected.h5
 
 
 The plot should look like this:
@@ -132,7 +135,7 @@ Once the thresholds have been decided, the matrix can be corrected
 .. code-block:: bash
 
    # correct Hi-C matrix
-   $ hicCorrectMatrix -m hic_matrix.npz --filterThreshold -1.5 5 -o hic_corrected.npz
+   $ hicCorrectMatrix -m hic_matrix.h5 --filterThreshold -1.5 5 -o hic_corrected.h5
 
 
 Visualization of results
@@ -148,7 +151,7 @@ plot the counts using the `--log1p` option.
 
 .. code-block:: bash
 
-   $ hicPlotMatrix -m hic_corrected.npz -o hic_plot.png --region 1:20000000-80000000 --log1p
+   $ hicPlotMatrix -m hic_corrected.h5 -o hic_plot.png --region 1:20000000-80000000 --log1p
 
 
 
@@ -170,7 +173,7 @@ boundaries.
 
 .. code-block:: bash
 
-   $ hicFindTADs -m hic_corrected.npz --outPrefix hic_corrected --numberOfProcessors 16
+   $ hicFindTADs -m hic_corrected.h5 --outPrefix hic_corrected --numberOfProcessors 16
 
 
 This code will produce several files: 1. The TAD-separation score file, 2. the z-score matrix, 3. a bed file

docs/content/installation.rst

@@ -8,14 +8,18 @@ Requirements
 -------------
 
 * Python 2.7
-* numpy >= 1.10.4
-* scipy >= 0.17.1
-* matplotlib >= 1.5.3
-* pysam >= 0.8.3
+* numpy >= 1.12.1
+* scipy >= 0.19.0
+* matplotlib >= 2.0.0
+* pysam >= 0.11.2
 * intervaltree >= 2.1.0
-* biopython >= 1.65
-* tables >= 3.2.2
-* pyBigWig >=0.2.8
+* biopython >= 1.68
+* pytables >= 3.3.0
+* pandas >= 0.20.2
+* pybigwig >= 0.3.4
+
+Command line installation using ``conda``
+-----------------------------------------
 
 The fastet way to obtain **Python 2.7 together with numpy and scipy** is
 via the `Anaconda Scientific Python
@@ -69,6 +73,8 @@ a specific folder using the ``--prefix`` option)
 	$ python setup.py install --prefix /User/Tools/hicexplorer
 
 
+
+
 Galaxy installation
 --------------------
 

docs/content/list-of-tools.rst

@@ -30,7 +30,7 @@ HiCExplorer tools
    +--------------------------------+------------------+-----------------------------------+---------------------------------------------+-----------------------------------------------------------------------------------+
    |:ref:`hicPlotDistVsCounts`      | visualization    | hicMatrix object                  | log log plot of Hi-C contacts per distance  | Quality control                                                                   |
    +--------------------------------+------------------+-----------------------------------+---------------------------------------------+-----------------------------------------------------------------------------------+
-   |:ref:`hicExport`                | data integration | multiple hiC file formats         | HiC matices/outputs in several formats      | Export matrix to different formats                                                |
+   |:ref:`hicExport`                | data integration | multiple Hi-C file formats         | Hi-C matices/outputs in several formats      | Export matrix to different formats                                                |
    +--------------------------------+------------------+-----------------------------------+---------------------------------------------+-----------------------------------------------------------------------------------+
    |:ref:`hicInfo`                  | information      | one or more hicMatrix objects     | Screen info                                 | Prints information about  matrices, like size, maximum, minimux, bin size, etc.   |
    +--------------------------------+------------------+-----------------------------------+---------------------------------------------+-----------------------------------------------------------------------------------+