config.yml

version: 2.1

variables:
  install_conda: &install_conda
    run:
      name: Install miniconda3
      command: |
        sudo apt-get update && sudo apt-get install -y build-essential libz-dev libcurl3-dev libarchive-dev gcc && \
        wget --quiet https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O ~/miniconda.sh && \
        /usr/bin/bash ~/miniconda.sh -b && \
        rm ~/miniconda.sh && \
        ~/miniconda3/bin/conda clean -tipsy && \
        sudo ln -s ~/miniconda3/etc/profile.d/conda.sh /etc/profile.d/conda.sh && \
        echo ". ~/miniconda3/etc/profile.d/conda.sh" >> $BASH_ENV && \
        echo "conda activate base" >> $BASH_ENV
  update_conda: &update_conda
    run:
      name: Update conda
      command: conda update -n base conda
  install_mamba_solver: &install_mamba_solver
    run:
      name: Install mamba solver (experimental)
      command: conda install -n base conda-libmamba-solver
  install_sys_deps: &install_sys_deps
    run:
      name: install build-essential
      command: apt-get update --allow-releaseinfo-change && apt-get install -y build-essential libz-dev libcurl3-dev libarchive-dev gcc
  install_apptainer: &install_apptainer
    run:
      name: Install Apptainer from source
      command: |
        echo 'export CI_JOB_PY_YAML="$CI_JOB_PY_YAML"' >> $BASH_ENV
        sudo apt-get -q update && sudo apt-get install -y build-essential squashfs-tools libseccomp-dev cryptsetup libhdf5-serial-dev pkg-config
        wget https://github.com/apptainer/apptainer/releases/download/v1.0.1/apptainer_1.0.1_amd64.deb 
        sudo apt install ~/repo/apptainer_1.0.1_amd64.deb
        apptainer --version
        rm ~/repo/apptainer_1.0.1_amd64.deb
  install_kipoi_plugins: &install_kipoi_plugins
    run:
      name: Install kipoi plugins
      command: |
        pip install kipoi_interpret
        pip install kipoiseq
        pip install pyfaidx
        conda install --yes -c conda-forge fastparquet zarr numcodecs
  export_shortcuts: &export_shortcuts
    run:
      name: Export shortcuts
      command: |
        # for no good reason the wrong version of pytest is choosen by default...
        echo 'export PYTEST=$(conda info --base)/envs/kipoi-env/bin/pytest' >> $BASH_ENV
  install_from_dev_requirements_py36: &install_from_dev_requirements_py36
    run:
      name: Install from dev-requirements-py36.yml
      command: |
        echo 'export CI_JOB_PY_YAML="$CI_JOB_PY_YAML"' >> $BASH_ENV
        conda env create --name kipoi-env -f dev-requirements-py36.yml --experimental-solver=libmamba
        source activate kipoi-env
        pip install -e .
        git lfs install
        # ln -s /opt/conda/bin/python /usr/bin/python
  install_from_dev_requirements_py37: &install_from_dev_requirements_py37
    run:
      name: Install from dev-requirements-py37.yml
      command: |  
        echo 'export CI_JOB_PY_YAML="$CI_JOB_PY_YAML"' >> $BASH_ENV
        conda env create --name kipoi-env -f dev-requirements-py37.yml --experimental-solver=libmamba
        source activate kipoi-env
        pip install -e .
        git lfs install
        # ln -s /opt/conda/bin/python /usr/bin/python
  install_from_dev_requirements: &install_from_dev_requirements
    run:
      name: Install from dev-requirements.yml
      command: |
        echo 'export CI_JOB_PY_YAML="$CI_JOB_PY_YAML"' >> $BASH_ENV
        conda create --name kipoi-env python=<< parameters.python-version >>

        source activate kipoi-env
        conda env update --name kipoi-env --file dev-requirements.yml --experimental-solver=libmamba 
        pip install -e .
        git lfs install    
      no_output_timeout: 30m
  install_cyvcf2: &install_cyvcf2
    run:
      name: Install cyvcf2 (conda package is not yet available for 3.10)
      command: |
        source activate kipoi-env
        git clone --recursive https://github.com/brentp/cyvcf2
        cd cyvcf2/htslib
        autoheader
        autoconf
        ./configure --enable-libcurl
        make
        cd ..
        pip install -r requirements.txt
        CYTHONIZE=1 pip install -e .
  install_pybigwig: &install_pybigwig
    run:
      name: Install pybigwig (conda package is not yet available for 3.10)
      command: |
        source activate kipoi-env
        git clone https://github.com/deeptools/pyBigWig.git
        cd pyBigWig
        pip install .    
  install_kipoi: &install_kipoi
    run:
      name: Install Kipoi
      command: pip install '.[develop]'
  install_requests: &install_requests
    run:
      name: Install requirements for requests correctly
      command: pip install requests
  kipoi_ls1: &kipoi_ls
    run:
      name: List kipoi packages
      command: kipoi ls
  kipoi_ls2: &activate_and_kipoi_ls
    run:
      name: List kipoi packages
      command: |  
        source activate kipoi-env
        kipoi ls
  activate_and_run_tests_py367: &activate_and_run_tests_py367
    run:
      name: Run tests
      command: |
        source activate kipoi-env
        mkdir test-reports
        $PYTEST --cov=kipoi/ tests/ -p no:sugar --no-cov-on-fail --junitxml=test-reports/junit.xml
      no_output_timeout: 30m
  activate_and_run_tests: &activate_and_run_tests
    run:
      name: Run tests
      command: |
        source activate kipoi-env
        mkdir test-reports
        $PYTEST --cov=kipoi/ tests/ -k "not legacy" -p no:sugar --no-cov-on-fail --junitxml=test-reports/junit.xml
      no_output_timeout: 30m
  # Special case for running without concise 
  # run_coveralls: &run_coveralls
  #   run:
  #     name: run coveralls
  #     command: coveralls || true
  activate_and_run_coveralls: &activate_and_run_coveralls
    run:
      name: activate and run coveralls
      command: |
        source activate kipoi-env
        coveralls
  store_test_results: &store_test_results
    store_test_results:
      path: test-reports
  store_test_artifacts: &store_test_artifacts
    store_artifacts:
      path: test-reports
      destination: test-reports
  # deployment
  build_pypi_package: &build_pypi_package
    run:
      name: build python packages
      command: |
        source activate kipoi-env
        python setup.py sdist bdist_wheel
  conda_install_twine: &conda_install_twine
    run:
      name: Install twine
      command: conda install twine
  pypi_create_pypirc: &pypi_create_pypirc
    run:
      name: init .pypirc
      command: |
        echo -e "[distutils]" >> ~/.pypirc
        echo -e "index-servers = " >> ~/.pypirc
        echo -e "    pypi" >> ~/.pypirc
        echo -e "    testpypi" >> ~/.pypirc
        echo -e "" >> ~/.pypirc
        echo -e "[testpypi]" >> ~/.pypirc
        echo -e "repository: https://test.pypi.org/legacy/" >> ~/.pypirc
        echo -e "username: $TEST_PYPI_USERNAME" >> ~/.pypirc
        echo -e "password: $TEST_PYPI_PASSWORD" >> ~/.pypirc
        echo -e "[pypi]" >> ~/.pypirc
        echo -e "username: $PYPI_USERNAME" >> ~/.pypirc
        echo -e "password: $PYPI_PASSWORD" >> ~/.pypirc

jobs:
  test:
    parameters:
      python-version:
        type: string
    machine: # executor type
      image: ubuntu-2004:current  
    resource_class: medium
    working_directory: ~/repo  
    steps:
      - checkout
      - *install_apptainer
      - *install_conda
      - *update_conda
      - *install_mamba_solver
      - *install_from_dev_requirements
      - *install_cyvcf2
      - *install_pybigwig
      - *export_shortcuts
      - *activate_and_kipoi_ls
      - *activate_and_run_tests
      # - coveralls/upload
      # - *activate_and_run_coveralls
      - *store_test_results
      - *store_test_artifacts
  test-36:
    machine: # executor type
      image: ubuntu-2004:current
    resource_class: medium
    working_directory: ~/repo
    steps:
      - checkout
      - *install_apptainer
      - *install_conda
      - *update_conda
      - *install_mamba_solver
      # - *install_sys_deps # Installed in install_conda step
      - *install_from_dev_requirements_py36
      - *export_shortcuts
      - *activate_and_kipoi_ls
      - *activate_and_run_tests_py367
      # - coveralls/upload
      # - *activate_and_run_coveralls
      - *store_test_results
      - *store_test_artifacts
  test-37:
    machine: # executor type
      image: ubuntu-2004:current
    resource_class: medium
    working_directory: ~/repo
    steps:
      - checkout
      - *install_apptainer
      - *install_conda
      - *update_conda
      - *install_mamba_solver
      # - *install_sys_deps # Installed in install_conda step
      - *install_from_dev_requirements_py37
      - *export_shortcuts
      - *activate_and_kipoi_ls
      - *activate_and_run_tests_py367
      - *activate_and_run_coveralls
      - *store_test_results
      - *store_test_artifacts
  build-deploy-docs:
    docker:
      # - image: continuumio/miniconda3:4.5.12
      - image: continuumio/anaconda3:latest
      # - image: circleci/python:3.5
    working_directory: ~/repo
    steps:
      - add_ssh_keys:
          fingerprints:
            - 60:0f:05:31:12:3d:bc:8b:df:9d:08:da:71:a5:43:b6
      - checkout
      - run:
          name: Install Kipoi in a conda environment
          command: |
            conda create -n py39 -y python=3.9
            source activate py39
            pip install -e .
      - run:
          name: Install build deps
          command: |
            source activate py39
            pip install pandas mistune jinja2 nbconvert tornado mkdocs git+https://github.com/kipoi/pydoc-markdown@master
      - run:
          name: Build docs
          command: |
            source activate py39
            cd docs/
            ./render_ipynb.bash
            pydocmd build
      - run:
          name: Deploy docs
          command: .circleci/deploy_docs.bash

  test-deploy-pypi:
    parameters:
      python-version:
        type: string
        default: "3.9"
    machine: # executor type
      image: ubuntu-2004:current
    resource_class: medium
    working_directory: ~/repo
    steps:
      - checkout
      - *install_conda
      # - *install_mamba
      - *update_conda 
      - *install_mamba_solver
      # - *install_sys_deps # Installed in install_conda step
      - *install_from_dev_requirements
      - *export_shortcuts
      - *activate_and_kipoi_ls
      - *build_pypi_package
      - *conda_install_twine
      - *pypi_create_pypirc
      - run:
          name: check if commit is tagged
          command: git describe --exact-match --tags HEAD && echo "commit is tagged; continuing" || circleci step halt
      - run:
          name: upload to TestPyPI
          command: twine upload --repository testpypi dist/* --verbose
      - run:
          name: pip install from TestPyPI
          command: |
            source activate kipoi-env
            python -m pip install --user --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple kipoi --verbose
            python -c "import kipoi; print(kipoi.__version__)"

  productive-deploy-pypi:
    parameters:
      python-version:
        type: string
        default: "3.9"
    machine: # executor type
      image: ubuntu-2004:current
    resource_class: medium
    working_directory: ~/repo
    steps:
      - checkout
      - *install_conda
      # - *install_mamba
      - *update_conda
      - *install_mamba_solver
      # - *install_sys_deps # Installed in install_conda step
      - *install_from_dev_requirements
      - *export_shortcuts
      - *activate_and_kipoi_ls
      - *build_pypi_package
      - *conda_install_twine
      - *pypi_create_pypirc
      - run:
          name: check if commit is tagged
          command: git describe --exact-match --tags HEAD && echo "commit is tagged; continuing" || circleci step halt
      - run:
          name: upload to PyPI
          command: twine upload --repository pypi dist/* --verbose
      - run:
          name: pip install from PyPI
          command: |
            source activate kipoi-env
            python -m pip install --user kipoi --verbose
            python -c "import kipoi; print(kipoi.__version__)"
          
      - run:
          name: Update all containers with new version of kipoi
          command: |
            curl -X POST https://api.github.com/repos/kipoi/kipoi-containers/dispatches \
            -H 'Accept: application/vnd.github.v3+json' \
            -u $SYNC_TOKEN \
            --data '{"event_type": "updateall"}' 

workflows:
  version: 2.1

  # TODO - consider running the deploy only on
  # tag commits: https://circleci.com/docs/2.0/workflows/#git-tag-job-execution
  test-n-deploy:
    jobs:
      - test:
          matrix:
            parameters:
              python-version: ["3.8", "3.9", "3.10"]
      - test-36
      - test-37
      - build-deploy-docs:
          requires:
            - test-36
            - test-37
            - test
          filters:
            branches:
              only:
                - master
                - test_deployment_script
      - test-deploy-pypi:
          requires:
            - test-36
            - test-37
            - test
          filters:
#            tags:
#              only: /^v.*/
            branches:
              only:
                - test_deployment_pypi
                - test_deployment_script
      - productive-deploy-pypi:
          requires:
            - test-36
            - test-37
            - test
          filters:
#            # Trigger only on a tagged commit starting with v.
#            # https://circleci.com/docs/2.0/workflows/#executing-workflows-for-a-git-tag
#            tags:
#              only: /^v.*/
            branches:
              only:
                - master
  kipoi-nightly-test:
    triggers:
      - schedule:
          cron: "0 0 * * *"
          filters:
            branches:
              only:
                - master
    jobs:
      - test:
          matrix:
            parameters:
              python-version: ["3.8", "3.9", "3.10"]
      - test-36
      - test-37