Merge pull request #71 from kipoi/cleanup

Cleanup

.github/workflows/release-workflow.yml

@@ -16,29 +16,48 @@ jobs:
         "attentivechrome", "bpnet-oskn", "seqvec", "deepflybrain", "aparent-site_probabilities", 
         "aparent-veff"]
     runs-on: ubuntu-latest
+    env:
+      SINGULARITY_PULL_FOLDER: "/home/runner/singularity/"
+      ZENODO_ACCESS_TOKEN: ${{ secrets.ZENODOACCESSTOKEN }}
     # Steps represent a sequence of tasks that will be executed as part of the job
     steps:
     - uses: actions/checkout@v2
     - name: Set up Python 3.9
       uses: actions/setup-python@v2
       with:
         python-version: 3.9
+    - name: Install conda environment py39
+      uses: conda-incubator/setup-miniconda@v2
+      with:
+        activate-environment: py39
+        environment-file: environment.ubuntu.singularity.yml
+        python-version: 3.9
+        auto-activate-base: false
     - name: Install dependencies
+      shell: bash -l {0}
       run: |
+        mkdir -p /home/runner/singularity/
+        sudo apt-get update && sudo apt-get install -y libhdf5-serial-dev pkg-config
         python -m pip install --upgrade pip
-        pip install pytest
-        if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+        pip install -e .
     - name: Build and test ${{ matrix.image }}
+      shell: bash -l {0}
       run: |
         docker build -f dockerfiles/Dockerfile.kipoi-base-env -t kipoi/kipoi-docker:kipoi-base-env . 
         docker build -f dockerfiles/Dockerfile.${{ matrix.image }} -t kipoi/kipoi-docker:${{ matrix.image }} . 
         pytest -s test-containers/test_containers_from_command_line.py --image=kipoi/kipoi-docker:${{ matrix.image }} 
     - name: Push ${{ matrix.image }}
+      shell: bash -l {0}
       if: ${{ success() }}
       run: |
         docker login --username ${{ secrets.DOCKERUSERNAME }} --password ${{ secrets.DOCKERPASSWORD }}
         docker push kipoi/kipoi-docker:${{ matrix.image }} 
         docker system prune -a -f
+    - name: Update the singularity version of ${{ matrix.image }}
+      shell: bash -l {0}
+      if: ${{ success() }}
+      run: |
+        update_all_singularity kipoi/kipoi-docker:${{ matrix.image }} 
 
   buildandtestsharedpy3keras2:
     # The type of runner that the job will run on
@@ -55,11 +74,6 @@ jobs:
       uses: actions/setup-python@v2
       with:
         python-version: 3.9
-    - name: Install dependencies
-      run: |
-        python -m pip install --upgrade pip
-        pip install pytest
-        if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
     - name: Build and test sharedpy3keras2 with ${{ matrix.modelgroup }}
       run: |
         docker build -f dockerfiles/Dockerfile.kipoi-base-env -t kipoi/kipoi-docker:kipoi-base-env . 
@@ -69,28 +83,46 @@ jobs:
 
   buildandpushkipoibaseenvandsharedpy3keras2:
     needs: [buildandtestsharedpy3keras2, buildtestandpush]
-      # The type of runner that the job will run on
     runs-on: ubuntu-latest
-      # Steps represent a sequence of tasks that will be executed as part of the job
+    env:
+      SINGULARITY_PULL_FOLDER: "/home/runner/singularity/"
+      ZENODO_ACCESS_TOKEN: ${{ secrets.ZENODOACCESSTOKEN }}
     steps:
     - uses: actions/checkout@v2
     - name: Set up Python 3.9
       uses: actions/setup-python@v2
       with:
         python-version: 3.9
+    - name: Install conda environment py39
+      uses: conda-incubator/setup-miniconda@v2
+      with:
+        activate-environment: py39
+        environment-file: environment.ubuntu.singularity.yml
+        python-version: 3.9
+        auto-activate-base: false
     - name: Install dependencies
+      shell: bash -l {0}
       run: |
+        mkdir -p /home/runner/singularity/
+        sudo apt-get update && sudo apt-get install -y libhdf5-serial-dev pkg-config
         python -m pip install --upgrade pip
-        pip install pytest
-        if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+        pip install -e .
     - name: Build sharedpy3keras2 and kipoi-base-env
+      shell: bash -l {0}
       run: |
         docker build -f dockerfiles/Dockerfile.kipoi-base-env -t kipoi/kipoi-docker:kipoi-base-env . 
         docker build -f dockerfiles/Dockerfile.sharedpy3keras2 -t kipoi/kipoi-docker:sharedpy3keras2 . 
     - name: Push kipoi-base-env and sharedpy3keras2
+      shell: bash -l {0}
       if: ${{ success() }}
       run: |
         docker login --username ${{ secrets.DOCKERUSERNAME }} --password ${{ secrets.DOCKERPASSWORD }}
         docker push kipoi/kipoi-docker:kipoi-base-env
         docker push kipoi/kipoi-docker:sharedpy3keras2
         docker system prune -a -f
+    - name: Update the singularity version of ${{ matrix.image }}
+      shell: bash -l {0}
+      if: ${{ success() }}
+      run: |
+        update_all_singularity kipoi/kipoi-docker:kipoi-base-env
+        update_all_singularity kipoi/kipoi-docker:sharedpy3keras2

.github/workflows/sync-with-model-repo.yml

@@ -36,13 +36,14 @@ jobs:
         python-version: 3.9
         auto-activate-base: false
     - name: Install pip dependencies
+      shell: bash -l {0}
       run: |
         sudo apt-get update && sudo apt-get install -y libhdf5-serial-dev pkg-config
         python -m pip install --upgrade pip
-        pip install pytest
-        pip install -r requirements.txt
+        pip install -e .
     - name: Sync with kipoi model repo
+      shell: bash -l {0}
       run: |
-        python -m  modelupdater.updateoradd
+        python kipoi_containers/updateoradd.py
     - name: Create Pull Request
       uses: peter-evans/create-pull-request@v3.8.2

.github/workflows/test-images.yml

@@ -1,8 +1,7 @@
 # This workflow will install Python dependencies, run tests and lint with a single version of Python
 # For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
 
-name: Test each model group with one representative model per group with compatible
-  docker images
+name: CI
 
 on:
   push:
@@ -26,8 +25,8 @@ jobs:
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
-        pip install flake8 pytest
-        if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+        pip install flake8 
+        pip install -e .
     - name: Lint with flake8
       run: |
         # stop the build if there are Python syntax errors or undefined names
@@ -48,24 +47,35 @@ jobs:
           "SeqVec/embedding", "Xpresso/human_GM12878", "epidermal_basset/encode-roadmap.basset.clf.testfold-0",
           "DeepFlyBrain", "APARENT/site_probabilities", "APARENT/veff"]
     runs-on: ubuntu-latest
+    env:
+      SINGULARITY_PULL_FOLDER: "/home/runner/singularity/"
     steps:
     - uses: actions/checkout@v2
     - name: Set up Python 3.9
       uses: actions/setup-python@v2
       with:
         python-version: 3.9
+    - name: Install 
+      uses: conda-incubator/setup-miniconda@v2
+      with:
+        activate-environment: py39
+        environment-file: environment.ubuntu.singularity.yml
+        python-version: 3.9
+        auto-activate-base: false
     - name: Install dependencies
+      shell: bash -l {0}
       run: |
+        mkdir -p /home/runner/singularity/
         sudo apt-get update && sudo apt-get install -y libhdf5-serial-dev pkg-config
         python -m pip install --upgrade pip
-        pip install pytest
-        if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+        pip install -e .
     - name: Test with pytest
+      shell: bash -l {0}
       run: |
         pytest -s test-containers/test_models_from_command_line.py --model=${{ matrix.model }}
         docker system prune -a -f
 
-  testupdateoradd:
+  testdocker:
 
     runs-on: ubuntu-latest
     steps:
@@ -78,12 +88,11 @@ jobs:
       run: |
         sudo apt-get update && sudo apt-get install -y libhdf5-serial-dev pkg-config
         python -m pip install --upgrade pip
-        pip install pytest
-        if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+        pip install -e .
     - name: Test updateoradd
       run: |-
         docker pull kipoi/kipoi-docker:mmsplice
-        python -m pytest -s test-modelupdater/test_updateoradd.py
+        pytest -s test-docker/test_docker_handler.py
 
 
   testsingularity:
@@ -97,19 +106,13 @@ jobs:
       uses: actions/setup-python@v2
       with:
         python-version: 3.9
-    - name: Install conda dependencies
-      uses: conda-incubator/setup-miniconda@v2
-      with:
-        activate-environment: py39
-        environment-file: environment.ubuntu.singularity.yml
-        python-version: 3.9
-        auto-activate-base: false
     - name: Install pip dependencies
       run: |
         sudo apt-get update && sudo apt-get install -y libhdf5-serial-dev pkg-config
         python -m pip install --upgrade pip
-        pip install pytest
-        pip install -r requirements.txt
+        pip install -e .
     - name: Test singularity
       run: |
-        python -m pytest -s test-singularity-modification/test_zenodo.py
+        pytest -s test-singularity
+        pytest -s test-containers/test_update_all_singularity_images.py
+

.pre-commit-config.yaml

@@ -5,7 +5,7 @@ repos:
         - id: black
           language_version: python3
     -   repo: https://gitlab.com/pycqa/flake8
-        rev: 3.7.9
+        rev: 4.0.0
         hooks:
         - id: flake8