Merge branch 'tree_186' of https://github.com/widdowquinn/pyani into …

…tree_186
widdowquinn · Jun 17, 2022 · 7255824 · 7255824
2 parents 60182b1 + bc3eae0
commit 7255824
Show file tree

Hide file tree

Showing 64 changed files with 58,810 additions and 173 deletions.
diff --git a/.all-contributorsrc b/.all-contributorsrc
@@ -124,6 +124,15 @@
       "contributions": [
         "bug"
       ]
+    },
+    {
+      "login": "zakisaati",
+      "name": "Zaki Saati Santamaría",
+      "avatar_url": "https://avatars.githubusercontent.com/u/50806485?v=4",
+      "profile": "https://www.researchgate.net/profile/Zaki-Saati-Santamaria",
+      "contributions": [
+        "bug"
+      ]
     }
   ],
   "contributorsPerLine": 7,

diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -11,6 +11,7 @@ workflows:
   version: 2.1
   test:
     jobs:
+      - test-3_10
       - test-3_9
       - test-3_8
       - test-3_7
@@ -24,6 +25,7 @@ workflows:
               only:
                 - master
     jobs:
+      - test-3_10
       - test-3_9
       - test-3_8
       - test-3_7
@@ -32,7 +34,7 @@ workflows:
 jobs:
   test-3_8: &test-template
     docker:
-      - image: circleci/python:3.8.9
+      - image: cimg/python:3.8.9
 
     working_directory: ~/repo
 
@@ -41,8 +43,7 @@ jobs:
 
       - restore_cache:
           keys:
-            - pyani-dependencies-pip-{{ .Branch }}-{{ checksum "requirements.txt" }}-{{ checksum "requirements-dev.txt" }}-{{ checksum "requirements-pip.txt" }}
-            - pyani-dependencies-pip-
+            - pyani-dependencies-pip-{{ .Environment.CIRCLE_JOB }}-{{ .Branch }}-{{ checksum "requirements.txt" }}-{{ checksum "requirements-dev.txt" }}-{{ checksum "requirements-pip.txt" }}-{{ checksum "requirements-thirdparty.txt" }}-{{ checksum "requirements-fastani.txt" }}-{{ checksum "requirements-pyqt-conda.txt" }}-{{ checksum "requirements-pyqt-pip.txt" }}
 
       - run:
           name: update apt-get
@@ -60,53 +61,77 @@ jobs:
             tar -zxf legacy_blast.tar.gz
             echo 'export PATH=$PWD/blast-2.2.26/bin:$PATH' >> $BASH_ENV
             source $BASH_ENV
+      - run:
+          name: install fastANI
+          command: |
+            wget https://github.com/ParBLiSS/FastANI/releases/download/v1.33/fastANI-Linux64-v1.33.zip
+            unzip fastANI-Linux64-v1.33.zip
+            echo 'export PATH=$PWD:$PATH' >> $BASH_ENV
+            source $BASH_ENV
+            fastANI -h
+
 
       - run:
           name: install dependencies
           command: |
-            python3 -m venv venv
-            . venv/bin/activate
             pip install --upgrade pip setuptools virtualenv
             pip install -r requirements.txt
             pip install -r requirements-dev.txt
             pip install -r requirements-pip.txt
+            pip install -r requirements-pyqt-pip.txt
 
       - save_cache:
           paths:
-            - ./venv
-          key: pyani-dependencies-pip-{{ .Branch }}-{{ checksum "requirements.txt" }}-{{ checksum "requirements-dev.txt" }}-{{ checksum "requirements-pip.txt" }}
+            - /home/circleci/.pyenv
+          key: pyani-dependencies-pip-{{ .Environment.CIRCLE_JOB }}-{{ .Branch }}-{{ checksum "requirements.txt" }}-{{ checksum "requirements-dev.txt" }}-{{ checksum "requirements-pip.txt" }}-{{ checksum "requirements-thirdparty.txt" }}-{{ checksum "requirements-fastani.txt" }}-{{ checksum "requirements-pyqt-conda.txt" }}-{{ checksum "requirements-pyqt-pip.txt" }}
 
       - run:
           name: install pyani
           command: |
-            . venv/bin/activate
             pip install -e .  # Install the repo as a package
 
+      # Slow tests are separated out to avoid hitting context deadlines on CircleCI
       - run:
-          name: run tests
+          name: run fast tests
           command: |
-            . venv/bin/activate
-            pytest -v --cov=pyani --cov-report xml:.coverage.xml
+            python -m pytest -v -m "not slow" --cov=pyani --cov-report xml:.coverage_fast.xml
+
+      # Slow tests are separated out to avoid hitting context deadlines on CircleCI
+      - run:
+          name: run slow tests
+          no_output_timeout: 30m
+          command: |
+            python -m pytest -v -m "slow" --cov=pyani --cov-report xml:.coverage_slow.xml
 
       - store_artifacts:
           path: tests/test_output
           destination: tests/test_output
       - store_artifacts:
-          path: .coverage.xml
-          destination: .coverage.xml
+          path: .coverage_fast.xml
+          destination: .coverage_fast.xml
+      - store_artifacts:
+          path: .coverage_slow.xml
+          destination: .coverage_slow.xml
 
+      # CodeCov merges separate report uploads: https://docs.codecov.com/docs/merging-reports
+      - codecov/upload:
+          file: '.coverage_fast.xml'
       - codecov/upload:
-          file: .coverage.xml
+          file: '.coverage_slow.xml'
 
   test-3_6:
     <<: *test-template
     docker:
-      - image: circleci/python:3.6.9
+      - image: cimg/python:3.6.9
   test-3_7:
     <<: *test-template
     docker:
-      - image: circleci/python:3.7.10
+      - image: cimg/python:3.7.10
   test-3_9:
     <<: *test-template
     docker:
-      - image: circleci/python:3.9.5
+      - image: cimg/python:3.9.5
+  test-3_10:
+    <<: *test-template
+    docker:
+      - image: cimg/python:3.10.1
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -1,20 +1,23 @@
 repos:
 -   repo: https://github.com/psf/black
-    rev: 21.5b1
+    rev: 22.3.0
     hooks:
     - id: black
       language_version: python3
 -   repo: https://github.com/asottile/blacken-docs
-    rev: v1.10.0
+    rev: v1.12.1
     hooks:
     -   id: blacken-docs
         additional_dependencies: [black]
 -   repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v1.2.3
+    rev: v4.2.0
     hooks:
-    - id: flake8
     - id: check-case-conflict
     - id: check-docstring-first
     - id: check-json
     - id: check-merge-conflict
 #    - id: check-yaml
+-   repo: https://gitlab.com/pycqa/flake8
+    rev: 3.9.2
+    hooks:
+    - id: flake8
diff --git a/Makefile b/Makefile
@@ -3,18 +3,30 @@
 # This file is part of the pyani package distribution
 # (https://github.com/widdowquinn/pyani)
 
-# Set up all development dependencies in the current conda environment
-setup_env:
+# Install conda dependencies
+setup_conda:
 	@conda install --file requirements-dev.txt --yes
 	@conda install --file requirements.txt --yes
 	@conda install --file requirements-thirdparty.txt --yes
+	@conda install --file requirements-fastani.txt --yes
+	@conda install --file requirements-pyqt-conda.txt --yes
+
+# Install pip dependencies
+setup_pip:
 	@pip install -r requirements-pip.txt
+
+# Install dependencies, but not pre-commit
+setup_dependencies: setup_conda setup_pip
+	@pip install -U -e .
+
+# Set up all development dependencies and pre-commit in the current conda environment
+setup_env: setup_conda setup_pip
 	@pre-commit install
 	@pip install -U -e .
 
 # Run all tests and display coverage report in a browser
 test:
-	@pytest --cov-report=html --cov=pyani -v tests/ && open htmlcov/index.html
+	@python -m pytest --cov-report=html --cov=pyani -v tests/ && open htmlcov/index.html
 
 # Build and display documentation
 docs: clean_docs
@@ -39,22 +51,22 @@ clean_walkthrough:
 
 # Run walkthrough
 walkthrough: clean_walkthrough
-	pyani download --email my.email@my.domain -t 203804 C_blochmannia
+	pyani download --email my.email@my.domain -t 203804 -o C_blochmannia
 	pyani createdb -f
 	pyani anim -i C_blochmannia -o C_blochmannia_ANIm \
         --name "C. blochmannia run 1" \
         --labels C_blochmannia/labels.txt --classes C_blochmannia/classes.txt
-	pyani report --runs C_blochmannia_ANIm/ --formats html,excel,stdout
-	pyani report --run_results 1 --formats html,excel,stdout C_blochmannia_ANIm/
-	pyani report --run_matrices 1 --formats html,excel,stdout C_blochmannia_ANIm/
-	pyani plot --formats png,pdf --method seaborn C_blochmannia_ANIm 1
+	pyani report --runs -o C_blochmannia_ANIm/ --formats html excel stdout
+	pyani report --run_results 1 --formats html excel stdout -o C_blochmannia_ANIm/
+	pyani report --run_matrices 1 --formats html excel stdout -o C_blochmannia_ANIm/
+	pyani plot --formats png pdf --method seaborn -o C_blochmannia_ANIm --run_id 1
 	# pyani anib C_blochmannia C_blochmannia_ANIb \
     #     --name "C. blochmannia run 2" \
     #     --labels C_blochmannia/labels.txt --classes C_blochmannia/classes.txt
 	# pyani report --runs C_blochmannia_ANIb/ --formats html,excel,stdout
 	# pyani report --run_results 2 --formats html,excel,stdout C_blochmannia_ANIb/
 	# pyani report --run_matrices 2 --formats html,excel,stdout C_blochmannia_ANIb/
-	# pyani plot --formats png,pdf --method seaborn C_blochmannia_ANIb 2	
+	# pyani plot --formats png,pdf --method seaborn C_blochmannia_ANIb 2
 
 uml:
 	pyreverse -o pdf -p pyani pyani
diff --git a/README.md b/README.md
@@ -35,6 +35,7 @@ and we are grateful to all who have contributed to this software:
     <td align="center"><a href="https://github.com/sammywinchester19"><img src="https://avatars.githubusercontent.com/u/67588791?v=4?s=100" width="100px;" alt=""/><br /><sub><b>sammywinchester19</b></sub></a><br /><a href="https://github.com/widdowquinn/pyani/issues?q=author%3Asammywinchester19" title="Bug reports">🐛</a></td>
     <td align="center"><a href="https://github.com/TSL-RamKrishna"><img src="https://avatars.githubusercontent.com/u/20773891?v=4?s=100" width="100px;" alt=""/><br /><sub><b>Ram Krishna Shrestha</b></sub></a><br /><a href="https://github.com/widdowquinn/pyani/commits?author=TSL-RamKrishna" title="Tests">⚠️</a> <a href="https://github.com/widdowquinn/pyani/commits?author=TSL-RamKrishna" title="Code">💻</a> <a href="#ideas-TSL-RamKrishna" title="Ideas, Planning, & Feedback">🤔</a></td>
     <td align="center"><a href="https://github.com/dparks1134"><img src="https://avatars.githubusercontent.com/u/3688336?v=4?s=100" width="100px;" alt=""/><br /><sub><b>Donovan Parks</b></sub></a><br /><a href="https://github.com/widdowquinn/pyani/issues?q=author%3Adparks1134" title="Bug reports">🐛</a></td>
+    <td align="center"><a href="https://www.researchgate.net/profile/Zaki-Saati-Santamaria"><img src="https://avatars.githubusercontent.com/u/50806485?v=4?s=100" width="100px;" alt=""/><br /><sub><b>Zaki Saati Santamaría</b></sub></a><br /><a href="https://github.com/widdowquinn/pyani/issues?q=author%3Azakisaati" title="Bug reports">🐛</a></td>
   </tr>
 </table>
 
@@ -64,7 +65,7 @@ DOI: [10.1039/C5AY02550H](https://doi.org/10.1039/C5AY02550H)
 [![pyani sourcerank](https://img.shields.io/librariesio/sourcerank/pypi/pyani.svg?logo=koding&logoColor=white)](https://libraries.io/pypi/pyani)
 
 <!-- ALL-CONTRIBUTORS-BADGE:START - Do not remove or modify this section -->
-[![All Contributors](https://img.shields.io/badge/all_contributors-11-orange.svg?style=flat-square)](#contributors-)
+[![All Contributors](https://img.shields.io/badge/all_contributors-12-orange.svg?style=flat-square)](#contributors-)
 <!-- ALL-CONTRIBUTORS-BADGE:END -->
 
 [![pyani PyPi version](https://img.shields.io/pypi/v/pyani "PyPI version")](https://pypi.python.org/pypi/pyani)
@@ -171,26 +172,68 @@ pip3 install pyani
 
 ### Third-party tools
 
-Three alignment packages are required, to use all of `pyani`'s methods: `mummer`, `BLAST+`, and legacy `BLAST`. (*Note that it is not necessary to install all three tools to use `pyani`. You need only install the tools that are required for the method you wish to use.*)
+Four alignment packages are required, to use all of `pyani`'s methods: `mummer`, `BLAST+`, legacy `BLAST`, and `fastANI`. (*Note that it is not necessary to install all four tools to use `pyani`. You need only install the tools that are required for the method you wish to use.*)
 
 The simplest route to obtaining these tools is to use `conda`/`bioconda`:
 
 ```bash
-conda install mummer blast legacy-blast -y
+conda install mummer blast legacy-blast fastani -y
 ```
 
 But they can also be installed by following instructions from the tools' own websites.
 
 - **BLAST+** (for `anib`) [ftp://ftp.ncbi.nlm.nih.gov/blast/executables/blast+/LATEST/](ftp://ftp.ncbi.nlm.nih.gov/blast/executables/blast+/LATEST/)
 - **legacy BLAST** (for `aniblastall`) [ftp://ftp.ncbi.nlm.nih.gov/blast/executables/release/LATEST/](ftp://ftp.ncbi.nlm.nih.gov/blast/executables/release/LATEST/)
 - **MUMmer** (for `anim`) [http://mummer.sourceforge.net/](ftp://ftp.ncbi.nlm.nih.gov/blast/executables/release/LATEST/)
+- **fastANI** (for `fastani`) [https://github.com/ParBLiSS/FastANI](https://github.com/ParBLiSS/FastANI)
 
 ### NOTE: Installing legacy BLAST
 
 **The legacy BLAST executable available from NCBI will not run on macOS Big Sur.**
 
 If you wish to use `pyani blastall` or the `ANIblastall` method with the legacy `pyani` interface, you will require a locally-installed copy of this package. This is one of the packages linked in the `requirements-thirdparty.txt` file.
 
+### Note: Installing fastANI
+
+**`fastANI` installation (via `conda`) will fail for Python 3.9 on macOS.**
+
+If you are using Python ≥3.9 to run `pyani` on macOS, `fastANI` will fail to install due to errors in the `conda` recipe (for build `h0a26cfa_0`). Using an earlier version of Python may be the simplest solution. However, you can also install `fastANI` yourself—via `conda`—using a slightly altered version of the command produced from the `conda` recipe, as shown below.
+
+*N.B. We have placed the `fastANI` requirement in its own `requirements-fastani.txt` file so that this will not also cause other installations to fail.*
+
+##### Bypassing `conda`
+
+```bash
+c++ -O3 -DNDEBUG -std=c++11 -Isrc \
+-I ${ENV_DIR}/lib/include -mmacosx-version-min=10.7 \
+-stdlib=libc++ -Xpreprocessor -fopenmp -lomp \
+-DUSE_BOOST src/cgi/core_genome_identity.cpp -o fastANI \
+${ENV_DIR}/lib/lib/libboost_math_c99.a \
+-lstdc++ -lz -lm
+```
+
+Two alterations need to be made to the file name specified in the fifth line here:
+
+1. One of the `/lib`s should be removed.
+1. The file extension should be changed from `.a`, to `.dylib`.
+
+The resultant command ought to look something like this (with ${ENV_DIR} being wherever you're installing `fastANI`):
+
+```bash
+c++ -O3 -DNDEBUG -std=c++11 -Isrc \
+-I ${ENV_DIR}/lib/include -mmacosx-version-min=10.7 \
+-stdlib=libc++ -Xpreprocessor -fopenmp -lomp \
+-DUSE_BOOST src/cgi/core_genome_identity.cpp -o fastANI \
+${ENV_DIR}/lib/libboost_math_c99.dylib \
+-lstdc++ -lz -lm
+```
+
+For a more technical overview of the issue, and other solutions that may be shared there, please see https://github.com/widdowquinn/pyani/issues/377.
+
+##### Troubleshooting
+
+This solution is how one of our developers managed to solve the problem. We hope it works for you, but can not guarantee this. Unfortunately, if this does not work for you, we will also not be able to troubleshoot the issue. In this instance, please file an issue at https://github.com/bioconda/bioconda-recipes.
+
 -----
 
 ## Documentation (v0.3)

diff --git a/docs/about.rst b/docs/about.rst
@@ -9,6 +9,7 @@ About ``pyani``
 - ANIb (average nucleotide identity using ``BLAST+``)
 - ANIblastall (average nucleotide identity using ``legacy BLAST``)
 - ANIm (average nucleotide identity using ``MUMmer``)
+- fastANI (average nucleotide identity using ``fastANI``)
 - TETRA (4-mer sequence profiles)
 
 -------------------