📝🎨⬆️🔥 update build files and respective docs

.github/workflows/ci_autometa.yml → .github/workflows/pytest_codecov.yml

@@ -15,12 +15,12 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        os: [ubuntu-latest, macOS-latest]
-        python-version: [3.7, 3.8, 3.9]
+        os: [ubuntu-latest]
+        python-version: [3.7]
     env:
       OS: ${{ matrix.os }}
       PYTHON: ${{ matrix.python-version }}
-    name: Python ${{ matrix.python-version }} - ${{ matrix.os }} pytest
+    name: pytest & codecov
     steps:
       - uses: actions/checkout@v2
       - name: Setup conda
@@ -30,12 +30,9 @@ jobs:
           python-version: ${{ matrix.python-version }}
           conda-channels: anaconda, conda-forge, bioconda
       - run: conda --version
-      - run: conda install --file=requirements.txt
-      - run: python -m pip install trimap
-      - run: conda install pytest codecov pytest-cov pytest-variables gdown
-      - name: Generate coverage report
-        run: |
-          python -m pip install --requirement=tests/requirements.txt
+      - run: |
+          conda env update -n $CONDA_DEFAULT_ENV --file=tests/environment.yml
+          conda list
           gdown https://drive.google.com/uc\?\id=1bSlPldaq3C6Cf9Y5Rm7iwtUDcjxAaeEk -O tests/data/test_data.json
           python -m pytest --cov-report=xml --cov=autometa tests/
       - name: Upload coverage to Codecov

Dockerfile

@@ -20,14 +20,13 @@ LABEL maintainer="jason.kwan@wisc.edu"
 # along with Autometa. If not, see <http://www.gnu.org/licenses/>.
 
 RUN apt-get update --allow-releaseinfo-change \
-    && apt-get install -y procps \
+    && apt-get install -y procps make \
     && apt-get clean \
     && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
 
-COPY requirements.txt ./
-RUN conda install -c bioconda -c conda-forge --file=requirements.txt \
-    && conda clean --all -y \
-    && python -m pip install trimap
+COPY autometa-env.yml ./
+RUN conda env update -n base --file=autometa-env.yml \
+    && conda clean --all -y
 
 
 COPY . .

Makefile

@@ -34,11 +34,11 @@ black:
 	black --exclude autometa/validation autometa
 
 ## Set up python interpreter environment
-create_environment: requirements.txt
+create_environment: autometa-env.yml
 ifeq (True,$(HAS_CONDA))
 		@echo ">>> Detected conda, creating conda environment."
 ifeq (3,$(findstring 3,$(PYTHON_INTERPRETER)))
-	conda create -c conda-forge -c bioconda --name $(PROJECT_NAME) python=3 --file=requirements.txt
+	conda env create --file=autometa-env.yml
 else
 	@echo "It looks like you are not using python 3. Autometa is only compatible with python 3. Please upgrade."
 endif
@@ -57,7 +57,7 @@ endif
 
 ## Install autometa from source
 install: setup.py
-	python3 setup.py install
+	python3 -m pip install . --ignore-installed --no-deps -vv
 
 ## Install dependencies for test environment
 test_environment: tests/requirements.txt

autometa-env.yml

@@ -0,0 +1,30 @@
+name: autometa
+channels:
+  - conda-forge
+  - bioconda
+  - defaults
+dependencies:
+  - attrs # test-data requirement
+  - bedtools
+  - biopython
+  - bowtie2
+  - diamond>=2.0
+  - gdown
+  - hdbscan
+  - hmmer
+  - numba>=0.47
+  - numpy>=1.13
+  - pandas>=1.1
+  - parallel
+  - pip
+  - prodigal # NOTE: 2.5 and 2.6 output format is different for sequence headers
+  - python-annoy>=1.11 # required for trimap installation.
+  - requests
+  - rsync
+  - samtools>=1.11
+  - scikit-bio
+  - scikit-learn==0.24 # prevent error from joblib in multiprocessing distance calculations
+  - tqdm
+  - trimap
+  - tsne
+  - umap-learn>=0.5 # https://github.com/lmcinnes/umap/releases/tag/0.5.0 : umap, densmap

docs/source/bash-workflow.rst

@@ -21,7 +21,7 @@ After installing conda, running the following command will create a minimal Cond
 
 .. code-block:: bash
 
-    conda create -c conda-forge -c bioconda --name autometa --file=https://raw.githubusercontent.com/KwanLab/Autometa/main/requirements.txt
+    conda env create --file=https://raw.githubusercontent.com/KwanLab/Autometa/main/autometa-env.yml
 
 If you receive the message...
 
@@ -34,7 +34,7 @@ the environment then add the :code:`--force` flag to the end of the command.
 
 .. code-block:: bash
 
-    conda create -c conda-forge -c bioconda --name autometa --file=https://raw.githubusercontent.com/KwanLab/Autometa/main/requirements.txt --force
+    conda env create --file=https://raw.githubusercontent.com/KwanLab/Autometa/main/autometa-env.yml --force
 
 Once Conda has finished creating the environment be sure to activate it:
 

docs/source/installation.rst

@@ -25,40 +25,39 @@ Download and install miniconda_. Now run the following commands:
 
     # Navigate to the directory where you would like to clone Autometa
     cd $HOME
+
     # Clone the Autometa repository
     git clone https://github.com/KwanLab/Autometa.git
+
     # Navigate into the cloned repository
     cd Autometa
-    # List all make commands
-    make
+
     # create autometa conda environment
     make create_environment
+
     # activate autometa conda environment
     conda activate autometa
-    # install trimap for kmer embedding
-    python -m pip install trimap
+
     # install autometa source code in autometa environment
     make install
 
+.. note::
+
+    You can see a list of all available make commands by running ``make`` without any other arguments.
+
 Install from source (full commands)
 ===================================
 
 Download and install miniconda_. Now run the following commands:
 
 .. code-block:: bash
 
-    # Navigate to the directory where you need to clone Autometa
-    cd $HOME
-    # Clone the Autometa repository
-    git clone https://github.com/KwanLab/Autometa.git
-    # Navigate into the cloned repository
-    cd Autometa
-    # Construct the autometa environment from requirements.txt
-    conda create -n autometa -c conda-forge -c bioconda --file=requirements.txt
+    # Construct the autometa environment from autometa-env.yml
+    conda env create --file=https://raw.githubusercontent.com/KwanLab/Autometa/main/autometa-env.yml
+
     # Activate environment
     conda activate autometa
-    # install trimap for kmer embedding
-    pip install trimap
+
     # Install the autometa code base from source
     python setup.py install
 
@@ -74,13 +73,19 @@ You can build a docker image for your clone of the Autometa repository.
 
     # Navigate to the directory where you need to clone Autometa
     cd $HOME
+
     # Clone the Autometa repository
     git clone https://github.com/KwanLab/Autometa.git
+
     # Navigate into the cloned repository
     cd Autometa
+
     # This will tag the image as jasonkwan/autometa:<your current branch>
     make image
 
+    # (or the full command from within the Autometa repo)
+    docker build . -t jasonkwan/autometa:`git branch --show-current`
+
 Testing Autometa
 ================
 
@@ -92,10 +97,13 @@ To run the tests, however, you'll first need to install the following packages a
 
     # Activate your autometa conda environment
     conda activate autometa
+
     # List all make options
     make
+
     # Install dependencies for test environment
     make test_environment
+
     # Download test_data.json for unit testing to tests/data/
     make unit_test_data_download
 
@@ -105,8 +113,10 @@ You can now run different unit tests using the following commands:
 
     # Run all unit tests
     make unit_test
+
     # Run unit tests marked with entrypoint
     make unit_test_entrypoints
+
     # Run unit tests marked with WIP
     make unit_test_wip
 

docs/source/nextflow-workflow.rst

@@ -156,7 +156,7 @@ Conda environment named "autometa-nf", and install Nextflow and nf-core tools.
 
 .. code-block:: bash
 
-    conda env create --file=https://raw.githubusercontent.com/KwanLab/Autometa/main/environment.yml
+    conda env create --file=https://raw.githubusercontent.com/KwanLab/Autometa/main/nextflow-env.yml
 
 If you receive the message...
 
@@ -169,7 +169,7 @@ the environment then add the :code:`--force` flag to the end of the command.
 
 .. code-block:: bash
 
-    conda env create --file=https://raw.githubusercontent.com/KwanLab/Autometa/main/environment.yml --force
+    conda env create --file=https://raw.githubusercontent.com/KwanLab/Autometa/main/nextflow-env.yml --force
 
 Once Conda has finished creating the environment be sure to activate it:
 

tests/environment.yml

@@ -0,0 +1,39 @@
+name: autometa
+channels:
+  - conda-forge
+  - bioconda
+  - defaults
+dependencies:
+  - attrs # test-data requirement
+  - bedtools
+  - biopython
+  - black
+  - bowtie2
+  - diamond>=2.0
+  - gdown
+  - hdbscan
+  - hmmer
+  - numba>=0.47
+  - numpy>=1.13
+  - pandas>=1.1
+  - parallel
+  - pip
+  - pre_commit
+  - prodigal # NOTE: 2.5 and 2.6 output format is different for sequence headers
+  - pytest
+  - pytest-cov
+  - pytest-html
+  - pytest-repeat
+  - pytest-variables
+  - python-annoy>=1.11 # required for trimap installation.
+  - requests
+  - rsync
+  - samtools>=1.11
+  - scikit-bio
+  - scikit-learn==0.24 # prevent error from joblib in multiprocessing distance calculations
+  - sphinx
+  - sphinx_rtd_theme
+  - tqdm
+  - trimap
+  - tsne
+  - umap-learn>=0.5 # https://github.com/lmcinnes/umap/releases/tag/0.5.0 : umap, densmap