Merge pull request #228 from nf-core/dev

PR for Release 1.4

.travis.yml

@@ -9,6 +9,9 @@ python:
 before_install:
   # PRs to master are only ok if coming from dev branch
   - '[ $TRAVIS_PULL_REQUEST = "false" ] || [ $TRAVIS_BRANCH != "master" ] || ([ $TRAVIS_PULL_REQUEST_SLUG = $TRAVIS_REPO_SLUG ] && [ $TRAVIS_PULL_REQUEST_BRANCH = "dev" ])'
+  # Check that the changelog has been updated if this is a PR
+  - git config remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*" && git fetch
+  - '[ $TRAVIS_PULL_REQUEST = "false" ] || [ $(git diff --name-only HEAD origin/${TRAVIS_BRANCH} | grep CHANGELOG.md | wc -l) -eq 1 ]'
 install:
   # Install Nextflow
   - mkdir /tmp/nextflow
@@ -41,5 +44,5 @@ deploy:
     python: '3.6'
 
 # Sync pipelines with possible nf-core template changes
-after_deploy: 
+after_deploy:
   - ./bin/sync

CHANGELOG.md

@@ -1,5 +1,28 @@
 # nf-core/tools: Changelog
 
+## [v1.4](https://github.com/nf-core/tools/releases/tag/1.4) - 2018-12-12 Tantalum Butterfly
+
+#### Template pipeline
+* Institutional custom config profiles moved to github `nf-core/configs`
+    * These will now be maintained centrally as opposed to being shipped with the pipelines in `conf/`
+    * Load `base.config` by default for all profiles
+    * Removed profiles named `standard` and `none`
+    * Added parameter `--igenomesIgnore` so `igenomes.config` is not loaded if parameter clashes are observed
+    * Added parameter `--custom_config_version` for custom config version control. Can use this parameter to provide commit id for reproducibility. Defaults to `master`
+    * Deleted custom configs from template in `conf/` directory i.e. `uzh.config`, `binac.config` and `cfc.config`
+* `multiqc_config` and `output_md` are now put into channels instead of using the files directly (see issue [#222](https://github.com/nf-core/tools/issues/222))
+* Added `local.md` to cookiecutter template in `docs/configuration/`. This was referenced in `README.md` but not present.
+* Major overhaul of docs to add/remove parameters, unify linking of files and added description for providing custom configs where necessary
+* Travis: Pull the `dev` tagged docker image for testing
+* Removed UPPMAX-specific documentation from the template.
+
+#### Tools helper code
+* Make Travis CI tests fail on pull requests if the `CHANGELOG.md` file hasn't been updated
+* Minor bugfixing in Python code (eg. removing unused import statements)
+* Made the web requests caching work on multi-user installations
+* Handle exception if nextflow isn't installed
+* Linting: Update for Travis: Pull the `dev` tagged docker image for testing
+
 ## [v1.3](https://github.com/nf-core/tools/releases/tag/1.3) - 2018-11-21
 * `nf-core create` command line interface updated
     * Interactive prompts for required arguments if not given

README.md

@@ -3,6 +3,7 @@
 # [nf-core/tools](https://github.com/nf-core/tools)
 [![Build Status](https://travis-ci.org/nf-core/tools.svg?branch=master)](https://travis-ci.org/nf-core/tools)
 [![codecov](https://codecov.io/gh/nf-core/tools/branch/master/graph/badge.svg)](https://codecov.io/gh/nf-core/tools)
+[![install with bioconda](https://img.shields.io/badge/install%20with-bioconda-brightgreen.svg?style=flat-square)](http://bioconda.github.io/recipes/nf-core/README.html)
 
 A python package with helper tools for the nf-core community.
 
@@ -298,8 +299,8 @@ INFO: Updating version in nextflow.config
  + container = 'nfcore/mypipeline:1.0'
 
 INFO: Updating version in .travis.yml
- - docker tag nfcore/mypipeline nfcore/mypipeline:latest
- + docker tag nfcore/mypipeline nfcore/mypipeline:1.0
+ - docker tag nfcore/mypipeline:dev nfcore/mypipeline:latest
+ + docker tag nfcore/mypipeline:dev nfcore/mypipeline:1.0
 
 INFO: Updating version in Singularity
  - VERSION 1.0dev

bin/sync

@@ -1,15 +1,10 @@
 #!/usr/bin/env python
 
-from cookiecutter.main import cookiecutter
-import git
 import json
 import os
 import requests
 from requests.auth import HTTPBasicAuth
-import shutil
 import sys
-import subprocess
-import tempfile
 import syncutils.template
 
 # Set the default nf-core pipeline template branch
@@ -53,7 +48,7 @@ def main():
     pipelines = res.json().get('remote_workflows')
     if not pipelines:
         print("Pipeline information was empty!")
-    
+
     # TODO: Remove this line, once we go for production
     pipelines = [
         {
@@ -82,21 +77,20 @@ def main():
         else:
             print("Created pull-request for pipeline \'{pipeline}\' successfully."
                 .format(pipeline=pipeline["name"]))
-    
+
     for pipeline, exception in sync_errors:
         print("Sync for pipeline {name} failed.".format(name=pipeline))
         print(exception)
 
     for pipeline, return_code, content in pr_errors:
-        print("Pull-request for pipeline \'{pipeline}\' failed," 
+        print("Pull-request for pipeline \'{pipeline}\' failed,"
                 " got return code {return_code}."
                 .format(pipeline=pipeline, return_code=return_code))
         print(content)
-    
+
     if pr_errors or sync_errors: sys.exit(1)
-    
+
     sys.exit(0)
-            
+
 if __name__ == "__main__":
     main()
-

bin/syncutils/template.py

@@ -3,8 +3,6 @@
 import git
 import os
 import shutil
-import sys
-from cookiecutter.main import cookiecutter
 
 import nf_core.create
 
@@ -29,7 +27,7 @@ def __init__(self, pipeline, branch=TEMPLATE_BRANCH, repo_url=""):
         self.templatedir = tempfile.mkdtemp()
         self.repo = git.Repo.clone_from(self.repo_url, self.tmpdir)
         assert self.repo
-    
+
     def sync(self):
         """Execute the template update.
         """
@@ -49,7 +47,7 @@ def context_from_nextflow(self, nf_project_dir):
 
         # Fetch the config variables from the Nextflow pipeline
         config = utils.fetch_wf_config(wf_path=nf_project_dir)
-        
+
         # Checkout again to configured template branch
         self.repo.git.checkout("origin/{branch}".format(branch=self.branch),
             b="{branch}".format(branch=self.branch))

bin/syncutils/utils.py

@@ -10,6 +10,9 @@ def fetch_wf_config(wf_path):
     try:
         with open(os.devnull, 'w') as devnull:
             nfconfig_raw = subprocess.check_output(['nextflow', 'config', '-flat', wf_path], stderr=devnull)
+    except OSError as e:
+        if e.errno == os.errno.ENOENT:
+            raise AssertionError("It looks like Nextflow is not installed. It is required for most nf-core functions.")
     except subprocess.CalledProcessError as e:
         raise AssertionError("`nextflow config` returned non-zero error code: %s,\n   %s", e.returncode, e.output)
     else:
@@ -20,10 +23,10 @@ def fetch_wf_config(wf_path):
     return config
 
 def create_context(config):
-    """Consumes a flat Nextflow config file and will create 
+    """Consumes a flat Nextflow config file and will create
     a context dictionary with information for the nf-core template creation.
 
-    Returns: A dictionary with: 
+    Returns: A dictionary with:
         {
             'pipeline_name': '<parsed_name>'
             'pipeline_short_description': '<parsed_description>'
@@ -32,7 +35,7 @@ def create_context(config):
     """
     context = {}
     context["pipeline_name"] = config.get("manifest.name") if config.get("manifest.name") else get_name_from_url(config.get("manifest.homePage"))
-    context["pipeline_short_description"] = config.get("manifest.description") 
+    context["pipeline_short_description"] = config.get("manifest.description")
     context["version"] = config.get("manifest.version") if config.get("manifest.version") else config.get("params.version")
     context["author"] = config.get("manifest.author") if config.get("manifest.author") else "No author provided"
     return context

docs/lint_errors.md

@@ -107,7 +107,7 @@ nf-core pipelines must have CI testing with Travis or Circle CI.
 This test fails if the following happens:
 
 * `.travis.yml` does not contain the string `nf-core lint ${TRAVIS_BUILD_DIR}` under `script`
-* `.travis.yml` does not contain the string `docker pull <container>` under `before_install`
+* `.travis.yml` does not contain the string `docker pull <container>:dev` under `before_install`
     * Where `<container>` is fetched from `params.container` in the `nextflow.config` file, without the docker tag _(if we have the tag the tests fail when making a release)_
 * `.travis.yml` does not test the Nextflow version specified in the pipeline as `manifest.nextflowVersion`
     * This is expected in the `env` section of the config, eg:

nf_core/bump_version.py

@@ -39,8 +39,8 @@ def bump_pipeline_version(lint_obj, new_version):
     update_file_version("nextflow.config", lint_obj, nfconfig_pattern, nfconfig_newstr)
 
     # Update travis image tag
-    nfconfig_pattern = r"docker tag nfcore/{name} nfcore/{name}:(?:{tag}|latest)".format(name=lint_obj.pipeline_name.lower(), tag=current_version.replace('.',r'\.'))
-    nfconfig_newstr = "docker tag nfcore/{name} nfcore/{name}:{tag}".format(name=lint_obj.pipeline_name.lower(), tag=docker_tag)
+    nfconfig_pattern = r"docker tag nfcore/{name}:dev nfcore/{name}:(?:{tag}|latest)".format(name=lint_obj.pipeline_name.lower(), tag=current_version.replace('.',r'\.'))
+    nfconfig_newstr = "docker tag nfcore/{name}:dev nfcore/{name}:{tag}".format(name=lint_obj.pipeline_name.lower(), tag=docker_tag)
     update_file_version(".travis.yml", lint_obj, nfconfig_pattern, nfconfig_newstr)
 
     # Update Singularity version name

nf_core/create.py

@@ -9,7 +9,6 @@
 import git
 import logging
 import os
-import re
 import shutil
 import sys
 import tempfile

nf_core/download.py

@@ -8,7 +8,6 @@
 import logging
 import hashlib
 import os
-import re
 import requests
 import requests_cache
 import subprocess
@@ -77,8 +76,8 @@ def download_workflow(self):
                         self.pull_singularity_image(container)
 
     def fetch_workflow_details(self, wfs):
-        """ Fetch details of nf-core workflow to download 
-        
+        """ Fetch details of nf-core workflow to download
+
         params:
         - wfs   A nf_core.list.Workflows object
         """

nf_core/lint.py

@@ -5,31 +5,20 @@
 the nf-core community guidelines.
 """
 
-import datetime
 import logging
 import io
 import os
 import re
 import shlex
-import subprocess
-import tempfile
 
 import click
 import requests
-import requests_cache
 import yaml
 
 import nf_core.utils
 
 # Set up local caching for requests to speed up remote queries
-cachedir = os.path.join(tempfile.gettempdir(), 'nfcore_cache')
-if not os.path.exists(cachedir):
-    os.mkdir(cachedir)
-requests_cache.install_cache(
-    os.path.join(cachedir, 'nfcore_cache'),
-    expire_after=datetime.timedelta(hours=1),
-    backend='sqlite',
-)
+nf_core.utils.setup_requests_cachedir()
 
 # Don't pick up debug logs from the requests package
 logging.getLogger("requests").setLevel(logging.WARNING)
@@ -400,7 +389,7 @@ def check_ci_config(self):
                 # Check that we're pulling the right docker image
                 if self.config.get('params.container', ''):
                     docker_notag = re.sub(r':(?:[\.\d]+|latest)$', '', self.config.get('params.container', '').strip('"\''))
-                    docker_pull_cmd = 'docker pull {}'.format(docker_notag)
+                    docker_pull_cmd = 'docker pull {}:dev'.format(docker_notag)
                     try:
                         assert(docker_pull_cmd in ciconf.get('before_install', []))
                     except AssertionError:
@@ -409,7 +398,7 @@ def check_ci_config(self):
                         self.passed.append((5, "CI is pulling the correct docker image: {}".format(docker_pull_cmd)))
 
                     # Check that we tag the docker image properly
-                    docker_tag_cmd = 'docker tag {} {}'.format(docker_notag, self.config.get('params.container', '').strip('"\''))
+                    docker_tag_cmd = 'docker tag {}:dev {}'.format(docker_notag, self.config.get('params.container', '').strip('"\''))
                     try:
                         assert(docker_tag_cmd in ciconf.get('before_install'))
                     except AssertionError:

nf_core/list.py

@@ -11,22 +11,15 @@
 import re
 import subprocess
 import sys
-import tempfile
 
 import git
 import requests
-import requests_cache
 import tabulate
 
+import nf_core.utils
+
 # Set up local caching for requests to speed up remote queries
-cachedir = os.path.join(tempfile.gettempdir(), 'nfcore_cache')
-if not os.path.exists(cachedir):
-    os.mkdir(cachedir)
-requests_cache.install_cache(
-    os.path.join(cachedir, 'nfcore_cache'),
-    expire_after=datetime.timedelta(hours=1),
-    backend='sqlite',
-)
+nf_core.utils.setup_requests_cachedir()
 
 def list_workflows(sort='release', json=False, keywords=[]):
     """ Main function to list all nf-core workflows """
@@ -81,6 +74,9 @@ def get_local_nf_workflows(self):
             try:
                 with open(os.devnull, 'w') as devnull:
                     nflist_raw = subprocess.check_output(['nextflow', 'list'], stderr=devnull)
+            except OSError as e:
+                if e.errno == os.errno.ENOENT:
+                    raise AssertionError("It looks like Nextflow is not installed. It is required for most nf-core functions.")
             except subprocess.CalledProcessError as e:
                 raise AssertionError("`nextflow list` returned non-zero error code: %s,\n   %s", e.returncode, e.output)
             else:
@@ -244,6 +240,9 @@ def get_local_nf_workflow_details(self):
                 try:
                     with open(os.devnull, 'w') as devnull:
                         nfinfo_raw = subprocess.check_output(['nextflow', 'info', '-d', self.full_name], stderr=devnull)
+                except OSError as e:
+                    if e.errno == os.errno.ENOENT:
+                        raise AssertionError("It looks like Nextflow is not installed. It is required for most nf-core functions.")
                 except subprocess.CalledProcessError as e:
                     raise AssertionError("`nextflow list` returned non-zero error code: %s,\n   %s", e.returncode, e.output)
                 else:

nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.travis.yml

@@ -11,17 +11,17 @@ before_install:
   # PRs to master are only ok if coming from dev branch
   - '[ $TRAVIS_PULL_REQUEST = "false" ] || [ $TRAVIS_BRANCH != "master" ] || ([ $TRAVIS_PULL_REQUEST_SLUG = $TRAVIS_REPO_SLUG ] && [ $TRAVIS_PULL_REQUEST_BRANCH = "dev" ])'
   # Pull the docker image first so the test doesn't wait for this
-  - docker pull {{ cookiecutter.name_docker }}
+  - docker pull {{ cookiecutter.name_docker }}:dev
   # Fake the tag locally so that the pipeline runs properly
-  - docker tag {{ cookiecutter.name_docker }} {{ cookiecutter.name_docker }}:latest
+  - docker tag {{ cookiecutter.name_docker }}:dev {{ cookiecutter.name_docker }}:latest
 
 install:
   # Install Nextflow
   - mkdir /tmp/nextflow && cd /tmp/nextflow
   - wget -qO- get.nextflow.io | bash
   - sudo ln -s /tmp/nextflow/nextflow /usr/local/bin/nextflow
   # Install nf-core/tools
-  - pip install --upgrade pip 
+  - pip install --upgrade pip
   - pip install nf-core
   # Reset
   - mkdir ${TRAVIS_BUILD_DIR}/tests && cd ${TRAVIS_BUILD_DIR}/tests

nf_core/pipeline-template/{{cookiecutter.name_noslash}}/README.md

@@ -20,6 +20,7 @@ The {{ cookiecutter.name }} pipeline comes with documentation about the pipeline
 2. Pipeline configuration
     * [Local installation](docs/configuration/local.md)
     * [Adding your own system](docs/configuration/adding_your_own.md)
+    * [Reference genomes](docs/configuration/reference_genomes.md)  
 3. [Running the pipeline](docs/usage.md)
 4. [Output and how to interpret the results](docs/output.md)
 5. [Troubleshooting](docs/troubleshooting.md)