AiiDA conda environment (#2081)

* add an `environment.yml` file for installing AiiDA's dependencies using conda
* add a pre-commit hook that updates this file based on `setup_requirements.py`
* add a Travis test that tests the conda install and prohibits adding new dependencies if they are not available on conda (either the `defaults` or the `conda-forge` channel`)
* update reentry to 1.2.2, which is now available on `conda-forge`

Note: Using 'defaults' channel (anaconda channel is currently breaking installs)
conda/conda#7872

.pre-commit-config.yaml

@@ -58,12 +58,23 @@
   hooks:
   - id: pyproject
     name: Validating pyproject.toml
-    entry: python ./utils/validate_pyproject.py
+    entry: python ./utils/validate_pyproject.py version
     language: system
     files: >-
       (?x)^(
         setup_requirements.py|
-        utils/validate_pyproject|
+        utils/validate_pyproject.py|
+      )$
+    pass_filenames: false
+  - id: conda
+    name: Validating environment.yml
+    entry: python ./utils/validate_pyproject.py conda
+    language: system
+    files: >-
+      (?x)^(
+        setup_requirements.py|
+        utils/validate_pyproject.py|
+        environment.yml|
       )$
     pass_filenames: false
 

.travis-data/install_conda.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+# See https://conda.io/docs/user-guide/tasks/use-conda-with-travis-ci.html#the-travis-yml-file
+if [[ "$TRAVIS_PYTHON_VERSION" == "2.7" ]]; then
+  wget https://repo.continuum.io/miniconda/Miniconda2-latest-Linux-x86_64.sh -O miniconda.sh;
+else
+  wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh;
+fi
+bash miniconda.sh -b -p $HOME/miniconda
+export PATH="$HOME/miniconda/bin:$PATH"
+hash -r
+conda config --set always_yes yes --set changeps1 no
+conda update -q conda
+# Useful for debugging any issues with conda
+conda info -a

.travis-data/setup_profiles.sh

@@ -1,7 +1,6 @@
 set -ev
 
-if [[ "$TEST_TYPE" != "pre-commit" ]]
-    # no setup at all required for pre-commit to run
+if [[ "$TEST_TYPE" == "tests" || "$TEST_TYPE" == "docs" ]]
 then
     # Here I create the actual DB for submission
     psql -c "CREATE DATABASE $TEST_AIIDA_BACKEND;" -U postgres

.travis-data/test_script.sh

@@ -3,6 +3,8 @@
 # Be verbose, and stop with error as soon there's one
 set -ev
 
+DATA_DIR=${TRAVIS_BUILD_DIR}/.travis-data
+
 case "$TEST_TYPE" in
     docs)
         # Compile the docs (HTML format); 
@@ -12,7 +14,6 @@ case "$TEST_TYPE" in
         SPHINXOPTS="-nW" make -C docs
         ;;
     tests)
-        DATA_DIR=${TRAVIS_BUILD_DIR}/.travis-data
 
         # make sure we have the correct pg_ctl in our path for pgtest, to prevent issue #1722
         # this must match the version request in travis.yml
@@ -37,4 +38,12 @@ case "$TEST_TYPE" in
     pre-commit)
         pre-commit run --all-files || ( git status --short ; git diff ; exit 1 )
         ;;
+    conda)
+        # Note: Not added to install in order not to slow down other tests
+        source ${DATA_DIR}/install_conda.sh
+
+        # Replace dep1 dep2 ... with your dependencies
+        conda env create -f environment.yml -n test-environment python=$TRAVIS_PYTHON_VERSION
+        source activate test-environment
+        ;;
 esac

.travis.yml

@@ -49,6 +49,7 @@ env:
     - TEST_AIIDA_BACKEND=django TEST_TYPE="docs"
     - TEST_AIIDA_BACKEND=django TEST_TYPE="tests"
     - TEST_AIIDA_BACKEND=sqlalchemy TEST_TYPE="tests"
+    - TEST_TYPE="conda"
 
 
 before_script:

docs/requirements_for_rtd.txt

@@ -55,7 +55,7 @@ python-memcached==1.59
 python-mimeparse==1.6.0
 pytz==2018.4
 qe-tools==1.1.0
-reentry==1.2.1
+reentry==1.2.2
 seekpath==1.8.1
 singledispatch >= 3.4.0.3
 six==1.11.0

environment.yml

@@ -0,0 +1,47 @@
+# Usage: conda env create -f environment.yml
+---
+channels:
+- defaults
+- conda-forge
+- etetoolkit
+dependencies:
+- reentry==1.2.2
+- python-dateutil==2.7.2
+- python-mimeparse==1.6.0
+- django==1.7.11
+- django-extensions==1.5.0
+- tzlocal==1.5.1
+- pytz==2018.4
+- PyYAML==3.12
+- six==1.11.0
+- future==0.16.0
+- pathlib2==2.3.0
+- celery==3.1.25
+- billiard==3.3.0.23
+- amqp==1.4.9
+- anyjson==0.3.3
+- plumpy==0.7.12
+- portalocker==1.1.0
+- psutil==5.4.5
+- meld3==1.0.2
+- numpy==1.14.3
+- SQLAlchemy==1.0.19
+- SQLAlchemy-Utils==0.33.0
+- alembic==0.9.9
+- ujson==1.35
+- enum34==1.1.6
+- voluptuous==0.11.1
+- aldjemy==0.8.0
+- passlib==1.7.1
+- validate_email==1.3
+- click==6.7
+- click-plugins==1.0.3
+- click-spinner==0.1.7
+- tabulate==0.8.2
+- ete3==3.1.1
+- uritools==2.1.0
+- psycopg2==2.7.4
+- paramiko==2.4.2
+- ecdsa==0.13
+- ipython<6.0
+name: aiida

pyproject.toml

@@ -1,3 +1,3 @@
 [build-system]
 # Minimum requirements for the build system to execute.
-requires = ["setuptools", "wheel", "reentry==1.2.1"]
+requires = ["setuptools", "wheel", "reentry==1.2.2"]

setup_requirements.py

@@ -9,7 +9,7 @@
 ###########################################################################
 
 install_requires = [
-    'reentry==1.2.1',
+    'reentry==1.2.2',
     'python-dateutil==2.7.2',
     'python-mimeparse==1.6.0',
     'django==1.7.11',  # Upgrade to Django 1.9 does prevent AiiDA functioning

utils/validate_pyproject.py

@@ -13,19 +13,24 @@
 import sys
 import toml
 
-@click.command()
+dir_path = os.path.dirname(os.path.realpath(__file__))
+sys.path.append(os.path.join(dir_path, os.pardir))
+import setup_requirements
+
+
+@click.group()
+def cli():
+    pass
+
+@click.command('version')
 def validate_pyproject():
     """
     Ensure that the version of reentry in setup_requirements.py and pyproject.toml are identical
     """
     filename_pyproject = 'pyproject.toml'
     filename_requirements = 'setup_requirements.py'
 
-    dir_path = os.path.dirname(os.path.realpath(__file__))
     toml_file = os.path.join(dir_path, os.pardir, filename_pyproject)
-    sys.path.append(os.path.join(dir_path, os.pardir))
-
-    import setup_requirements
 
     reentry_requirement = None
 
@@ -63,5 +68,39 @@ def validate_pyproject():
         sys.exit(1)
 
 
+@click.command('conda')
+def update_environment_yml():
+    """
+    Updates environment.yml file for conda.
+    """
+    from setup_requirements import install_requires
+    import yaml
+
+    # fix incompatibilities between conda and pypi
+    replacements = {
+        'psycopg2-binary' : 'psycopg2',
+        'validate-email' : 'validate_email',
+    }
+    sep = '%'  # use something forbidden in conda package names
+    pkg_string = sep.join(install_requires)
+    for (pypi_pkg_name, conda_pkg_name) in iter(replacements.items()):
+        pkg_string = pkg_string.replace(pypi_pkg_name, conda_pkg_name)
+    install_requires = pkg_string.split(sep)
+    environment = dict(
+        name = 'aiida',
+        channels = ['defaults', 'conda-forge', 'etetoolkit'],
+        dependencies = install_requires,
+    )
+
+    environment_filename = 'environment.yml'
+    file_path = os.path.join(dir_path, os.pardir, environment_filename)
+    with open(file_path, 'w') as env_file:
+        env_file.write('# Usage: conda env create -f environment.yml\n')
+        yaml.dump(environment, env_file, explicit_start=True, 
+                  default_flow_style=False)
+
+cli.add_command(validate_pyproject)
+cli.add_command(update_environment_yml)
+
 if __name__ == '__main__':
-    validate_pyproject()  # pylint: disable=no-value-for-parameter
+    cli()  # pylint: disable=no-value-for-parameter