test_deploy.yml

name: Test, Integration Tests and Deploy

on:
  push:
    branches:
      - "**"
    tags:
      - "v*.*.*"
env:
  DEPENDENCY_CACHE_PREFIX: "v2"
  NETWORK_CACHE_PREFIX: "v1"
jobs:
  set-matrix:
    runs-on: ubuntu-latest
    outputs:
      matrix: ${{ steps.set-matrix.outputs.matrix }}
    steps:
      - name: set matrix
        id: set-matrix
        env:
          REF: ${{ github.ref }}
          DEFAULT_BRANCH: ${{ github.event.repository.default_branch }}
        run: |
          if [ "$REF" == "refs/heads/$DEFAULT_BRANCH" ] || [[ "$REF" =~ ^refs/tags/.* ]]
          then
              echo "::set-output name=matrix::{\"python-version\": [\"3.8\", \"3.9\"]}"
          else
              echo "::set-output name=matrix::{\"python-version\": [\"3.7\"]}"
          fi

  cleanup-runs:
    runs-on: ubuntu-latest
    steps:
      - uses: rokroskar/workflow-run-cleanup-action@v0.3.0
        env:
          GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
    if: "!startsWith(github.ref, 'refs/tags/') && !endsWith(github.ref, github.event.repository.default_branch)"

  year-week:
    runs-on: ubuntu-latest
    outputs:
      date: ${{ steps.year-week.outputs.date }}
    steps:
      - id: year-week
        run: echo "::set-output name=date::$(date '+%Y-%V')"

  style-check:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v2
      - run: git fetch --depth=1 origin +refs/tags/*:refs/tags/*
      - name: Set up Python 3.8
        uses: actions/setup-python@v1
        with:
          python-version: "3.8"
      - uses: actions/cache@master
        id: dependency-cache
        with:
          path: |
            ${{ env.pythonLocation }}
            renku/templates
          key: ${{env.DEPENDENCY_CACHE_PREFIX}}-${{ runner.os }}-${{ env.pythonLocation }}-${{ hashFiles('pyproject.toml') }}-${{hashFiles('poetry.lock')}}-${{hashFiles('Makefile')}}
      - name: Install dependencies
        if: steps.dependency-cache.outputs.cache-hit != 'true' || 'refs/heads/master' == github.ref || 'refs/heads/develop' == github.ref || startsWith(github.ref, 'refs/tags/')
        run: |
          python -m pip install --upgrade pip
          python -m pip install coveralls poetry wheel twine
          make download-templates
          python -m pip install .[all]
      - name: Install renku into cache
        if: steps.dependency-cache.outputs.cache-hit == 'true'
        run: python -m pip install --no-deps .
      - name: Set Git config
        run: |
          git config --global --add user.name "Renku Bot"
          git config --global --add user.email "renku@datascience.ch"
      - name: Test style
        env:
          POETRY_VIRTUALENVS_CREATE: false
        run: ./run-tests.sh -s

  docs-linux:
    runs-on: ubuntu-latest
    needs: [set-matrix]
    strategy:
      max-parallel: 3
      matrix: ${{fromJson(needs.set-matrix.outputs.matrix)}}
    steps:
      - uses: actions/checkout@v2
      - run: git fetch --depth=1 origin +refs/tags/*:refs/tags/*
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v1
        with:
          python-version: ${{ matrix.python-version }}
      - uses: actions/cache@master
        id: dependency-cache
        with:
          path: |
            ${{ env.pythonLocation }}
            renku/templates
          key: ${{env.DEPENDENCY_CACHE_PREFIX}}-${{ runner.os }}-${{ env.pythonLocation }}-${{ hashFiles('pyproject.toml') }}-${{hashFiles('poetry.lock')}}-${{hashFiles('Makefile')}}
      - name: Install dependencies
        if: steps.dependency-cache.outputs.cache-hit != 'true' || 'refs/heads/master' == github.ref || 'refs/heads/develop' == github.ref || startsWith(github.ref, 'refs/tags/')
        run: |
          python -m pip install --upgrade pip
          python -m pip install coveralls poetry wheel twine
          make download-templates
          python -m pip install .[all]
      - name: Install renku into cache
        if: steps.dependency-cache.outputs.cache-hit == 'true'
        run: python -m pip install --no-deps .
      - name: Set Git config
        run: |
          git config --global --add user.name "Renku Bot"
          git config --global --add user.email "renku@datascience.ch"
      - name: Test docs
        env:
          POETRY_VIRTUALENVS_CREATE: false
          SIDEBAR: 1
        run: ./run-tests.sh -d

  mypy:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v2
      - run: git fetch --depth=1 origin +refs/tags/*:refs/tags/*
      - name: Set up Python 3.8
        uses: actions/setup-python@v1
        with:
          python-version: "3.8"
      - uses: actions/cache@master
        id: dependency-cache
        with:
          path: |
            ${{ env.pythonLocation }}
            renku/templates
          key: ${{env.DEPENDENCY_CACHE_PREFIX}}-${{ runner.os }}-${{ env.pythonLocation }}-${{ hashFiles('pyproject.toml') }}-${{hashFiles('poetry.lock')}}-${{hashFiles('Makefile')}}
      - name: Install dependencies
        if: steps.dependency-cache.outputs.cache-hit != 'true' || 'refs/heads/master' == github.ref || 'refs/heads/develop' == github.ref || startsWith(github.ref, 'refs/tags/')
        run: |
          python -m pip install --upgrade pip
          python -m pip install coveralls poetry wheel twine
          make download-templates
          python -m pip install .[all]
      - name: Install renku into cache
        if: steps.dependency-cache.outputs.cache-hit == 'true'
        run: python -m pip install --no-deps .
      - name: Set Git config
        run: |
          git config --global --add user.name "Renku Bot"
          git config --global --add user.email "renku@datascience.ch"
      - name: Test mypy
        env:
          POETRY_VIRTUALENVS_CREATE: false
        run: mypy

  cheatsheet:
    runs-on: ubuntu-latest
    if: "'refs/heads/master' != github.ref && 'refs/heads/develop' != github.ref && !startsWith(github.ref, 'refs/tags/')"
    steps:
      - uses: actions/checkout@v2
      - run: git fetch --depth=1 origin +refs/tags/*:refs/tags/*
      - name: Set up Python 3.7
        uses: actions/setup-python@v1
        with:
          python-version: 3.7
      - name: Install xetex
        run: |
          sudo add-apt-repository "deb http://archive.ubuntu.com/ubuntu jammy main restricted multiverse universe"
          sudo apt-get update -y
          sudo apt-get install -y -t jammy texlive-xetex

      - uses: actions/cache@master
        id: cache
        with:
          path: |
            ${{ env.pythonLocation }}
            renku/templates
          key: ${{env.CACHE_PREFIX}}-${{ runner.os }}-${{ env.pythonLocation }}-${{ hashFiles('pyproject.toml') }}-${{hashFiles('poetry.lock')}}-${{hashFiles('build.py')}}
      - name: Install dependencies
        if: steps.cache.outputs.cache-hit != 'true'
        run: |
          python -m pip install --upgrade pip
          python -m pip install coveralls poetry wheel twine
          python -m pip install .[all]
      - name: Install renku into cache
        if: steps.cache.outputs.cache-hit == 'true'
        run: python -m pip install --no-deps .
      - name: Set Git config
        run: |
          git config --global --add user.name "Renku Bot"
          git config --global --add user.email "renku@datascience.ch"
      - name: Build cheatsheet
        run: |
          sphinx-build -b latex -c docs/cheatsheet/ docs docs/latex/ -W
          cd docs/latex/
          xelatex cheatsheet
          cat <(md5sum cheatsheet.tex) <(md5sum sdsc_cheatsheet.cls) > ../cheatsheet_hash_new
          cd ../..
      - name: Publish cheatsheet if changed
        id: publish
        if:  ${{ hashFiles('docs/cheatsheet_hash') != hashFiles('docs/cheatsheet_hash_new') }}
        run: |
          cat docs/cheatsheet_hash
          cat docs/cheatsheet_hash_new
          cd docs
          rm cheatsheet_hash
          mv cheatsheet_hash_new cheatsheet_hash
          cp latex/cheatsheet.pdf _static/cheatsheet/
          git add cheatsheet_hash _static/cheatsheet/cheatsheet.pdf
          git commit -m "chore: Update cheatsheet"
      - name: Push changes
        uses: ad-m/github-push-action@master
        if: steps.publish.outcome != 'Skipped' && success()
        with:
          github_token: ${{ secrets.GITHUB_TOKEN }}
          branch: ${{ github.ref }}

  test-distro:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v2
        with:
          fetch-depth: 0
      - run: git fetch --depth=1 origin +refs/tags/*:refs/tags/*
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v1
        with:
          python-version: "3.7"
      - uses: actions/cache@master
        id: dependency-cache
        with:
          path: |
            ${{ env.pythonLocation }}
            renku/templates
          key: ${{env.DEPENDENCY_CACHE_PREFIX}}-${{ runner.os }}-${{ env.pythonLocation }}-${{ hashFiles('pyproject.toml') }}-${{hashFiles('poetry.lock')}}-${{hashFiles('Makefile')}}
      - name: Install dependencies
        if: steps.dependency-cache.outputs.cache-hit != 'true' || 'refs/heads/master' == github.ref || 'refs/heads/develop' == github.ref || startsWith(github.ref, 'refs/tags/')
        run: |
          python -m pip install --upgrade pip
          python -m pip install coveralls poetry wheel twine
          make download-templates
          pip install .[all]
      - name: Install renku into cache
        if: steps.dependency-cache.outputs.cache-hit == 'true'
        run: python -m pip install --no-deps .
      - name: Build Package
        env:
          POETRY_VIRTUALENVS_CREATE: false
        run: poetry build
      - name: test with twine
        run: twine check dist/*

  test-linux-api:
    runs-on: ubuntu-latest
    needs: [set-matrix]
    strategy:
      max-parallel: 3
      matrix: ${{fromJson(needs.set-matrix.outputs.matrix)}}
    steps:
      - uses: actions/checkout@v2
        with:
          fetch-depth: 0
      - run: git fetch --depth=1 origin +refs/tags/*:refs/tags/*
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v1
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install system packages
        run: |
          sudo apt-get update -y
          sudo apt-get install -y libyaml-0-2 libyaml-dev
      - uses: actions/cache@master
        id: dependency-cache
        with:
          path: |
            ${{ env.pythonLocation }}
            renku/templates
          key: ${{env.DEPENDENCY_CACHE_PREFIX}}-${{ runner.os }}-${{ env.pythonLocation }}-${{ hashFiles('pyproject.toml') }}-${{hashFiles('poetry.lock')}}-${{hashFiles('Makefile')}}
      - name: Install dependencies
        if: steps.dependency-cache.outputs.cache-hit != 'true' || 'refs/heads/master' == github.ref || 'refs/heads/develop' == github.ref || startsWith(github.ref, 'refs/tags/')
        run: |
          python -m pip install --upgrade pip
          python -m pip install coveralls poetry wheel twine
          make download-templates
          python -m pip install .[all]
      - name: Install renku into cache
        if: steps.dependency-cache.outputs.cache-hit == 'true'
        run: python -m pip install --no-deps .
      - name: Set Git config
        run: |
          git config --global --add user.name "Renku Bot"
          git config --global --add user.email "renku@datascience.ch"
      - name: Test with pytest
        env:
          POETRY_VIRTUALENVS_CREATE: false
          LANG: en_US.UTF-8
          LC_ALL: en_US.UTF-8
        run: pytest -v -m "not integration and not publish and not serial" -n auto tests/api
      - name: Coveralls
        env:
          COVERALLS_REPO_TOKEN: ${{ secrets.COVERALLS_REPO_TOKEN }}
          GITHUB_TOKEN: $COVERALLS_REPO_TOKEN
          RENKU_REQUESTS_TIMEOUT_SECONDS: 120
          COVERALLS_PARALLEL: true
        run: coveralls --service=github-actions
        continue-on-error: true

  test-linux-cli:
    runs-on: ubuntu-latest
    needs: [set-matrix]
    strategy:
      max-parallel: 3
      matrix: ${{fromJson(needs.set-matrix.outputs.matrix)}}
    steps:
      - uses: actions/checkout@v2
        with:
          fetch-depth: 0
      - run: git fetch --depth=1 origin +refs/tags/*:refs/tags/*
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v1
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install system packages
        run: |
          sudo apt-get update -y
          sudo apt-get install -y libyaml-0-2 libyaml-dev
      - uses: actions/cache@master
        id: dependency-cache
        with:
          path: |
            ${{ env.pythonLocation }}
            renku/templates
          key: ${{env.DEPENDENCY_CACHE_PREFIX}}-${{ runner.os }}-${{ env.pythonLocation }}-${{ hashFiles('pyproject.toml') }}-${{hashFiles('poetry.lock')}}-${{hashFiles('Makefile')}}
      - name: Install dependencies
        if: steps.dependency-cache.outputs.cache-hit != 'true' || 'refs/heads/master' == github.ref || 'refs/heads/develop' == github.ref || startsWith(github.ref, 'refs/tags/')
        run: |
          python -m pip install --upgrade pip
          python -m pip install coveralls poetry wheel twine
          make download-templates
          python -m pip install .[all]
      - name: Install renku into cache
        if: steps.dependency-cache.outputs.cache-hit == 'true'
        run: python -m pip install --no-deps .
      - name: Set Git config
        run: |
          git config --global --add user.name "Renku Bot"
          git config --global --add user.email "renku@datascience.ch"
      - name: Test with pytest
        env:
          POETRY_VIRTUALENVS_CREATE: false
          LANG: en_US.UTF-8
          LC_ALL: en_US.UTF-8
        run: pytest -v -m "not integration and not publish and not serial" -n auto  tests/cli
      - name: Test with pytest(serial)
        env:
          POETRY_VIRTUALENVS_CREATE: false
          LANG: en_US.UTF-8
          LC_ALL: en_US.UTF-8
        run: pytest -v --cov-append -m "not integration and not publish and serial"  tests/cli
      - name: Coveralls
        env:
          COVERALLS_REPO_TOKEN: ${{ secrets.COVERALLS_REPO_TOKEN }}
          GITHUB_TOKEN: $COVERALLS_REPO_TOKEN
          RENKU_REQUESTS_TIMEOUT_SECONDS: 120
          COVERALLS_PARALLEL: true
        run: coveralls --service=github-actions
        continue-on-error: true

  test-linux-core:
    runs-on: ubuntu-latest
    needs: [set-matrix]
    strategy:
      max-parallel: 3
      matrix: ${{fromJson(needs.set-matrix.outputs.matrix)}}
    steps:
      - uses: actions/checkout@v2
        with:
          fetch-depth: 0
      - run: git fetch --depth=1 origin +refs/tags/*:refs/tags/*
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v1
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install system packages
        run: |
          sudo apt-get update -y
          sudo apt-get install -y libyaml-0-2 libyaml-dev
      - uses: actions/cache@master
        id: dependency-cache
        with:
          path: |
            ${{ env.pythonLocation }}
            renku/templates
          key: ${{env.DEPENDENCY_CACHE_PREFIX}}-${{ runner.os }}-${{ env.pythonLocation }}-${{ hashFiles('pyproject.toml') }}-${{hashFiles('poetry.lock')}}-${{hashFiles('Makefile')}}
      - name: Install dependencies
        if: steps.dependency-cache.outputs.cache-hit != 'true' || 'refs/heads/master' == github.ref || 'refs/heads/develop' == github.ref || startsWith(github.ref, 'refs/tags/')
        run: |
          python -m pip install --upgrade pip
          python -m pip install coveralls poetry wheel twine
          make download-templates
          python -m pip install .[all]
      - name: Install renku into cache
        if: steps.dependency-cache.outputs.cache-hit == 'true'
        run: python -m pip install --no-deps .
      - name: Set Git config
        run: |
          git config --global --add user.name "Renku Bot"
          git config --global --add user.email "renku@datascience.ch"
      - name: Test with pytest
        env:
          POETRY_VIRTUALENVS_CREATE: false
          LANG: en_US.UTF-8
          LC_ALL: en_US.UTF-8
        run: pytest -v -m "not integration and not publish and not serial" -n auto tests/core
      - name: Test with pytest(serial)
        env:
          POETRY_VIRTUALENVS_CREATE: false
          LANG: en_US.UTF-8
          LC_ALL: en_US.UTF-8
        run: pytest -v --cov-append -m "not integration and not publish and serial" tests/core
      - name: Coveralls
        env:
          COVERALLS_REPO_TOKEN: ${{ secrets.COVERALLS_REPO_TOKEN }}
          GITHUB_TOKEN: $COVERALLS_REPO_TOKEN
          RENKU_REQUESTS_TIMEOUT_SECONDS: 120
          COVERALLS_PARALLEL: true
        run: coveralls --service=github-actions
        continue-on-error: true

  test-linux-service:
    runs-on: ubuntu-latest
    needs: [set-matrix]
    strategy:
      max-parallel: 3
      matrix: ${{fromJson(needs.set-matrix.outputs.matrix)}}
    steps:
      - uses: actions/checkout@v2
        with:
          fetch-depth: 0
      - run: git fetch --depth=1 origin +refs/tags/*:refs/tags/*
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v1
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install system packages
        run: |
          sudo apt-get update -y
          sudo apt-get install -y libyaml-0-2 libyaml-dev
      - uses: actions/cache@master
        id: dependency-cache
        with:
          path: |
            ${{ env.pythonLocation }}
            renku/templates
          key: ${{env.DEPENDENCY_CACHE_PREFIX}}-${{ runner.os }}-${{ env.pythonLocation }}-${{ hashFiles('pyproject.toml') }}-${{hashFiles('poetry.lock')}}-${{hashFiles('Makefile')}}
      - name: Install dependencies
        if: steps.dependency-cache.outputs.cache-hit != 'true' || 'refs/heads/master' == github.ref || 'refs/heads/develop' == github.ref || startsWith(github.ref, 'refs/tags/')
        run: |
          python -m pip install --upgrade pip
          python -m pip install coveralls poetry wheel twine
          make download-templates
          python -m pip install .[all]
      - name: Install renku into cache
        if: steps.dependency-cache.outputs.cache-hit == 'true'
        run: python -m pip install --no-deps .
      - name: Set Git config
        run: |
          git config --global --add user.name "Renku Bot"
          git config --global --add user.email "renku@datascience.ch"
      - name: Test with pytest
        env:
          POETRY_VIRTUALENVS_CREATE: false
          LANG: en_US.UTF-8
          LC_ALL: en_US.UTF-8
        run: pytest -v -m "not integration and not publish" -n auto tests/service
      - name: Coveralls
        env:
          COVERALLS_REPO_TOKEN: ${{ secrets.COVERALLS_REPO_TOKEN }}
          GITHUB_TOKEN: $COVERALLS_REPO_TOKEN
          RENKU_REQUESTS_TIMEOUT_SECONDS: 120
          COVERALLS_PARALLEL: true
        run: coveralls --service=github-actions
        continue-on-error: true

  test-macos-api:
    runs-on: macos-latest
    needs: [set-matrix]
    strategy:
      max-parallel: 3
      matrix: ${{fromJson(needs.set-matrix.outputs.matrix)}}
    steps:
      - uses: actions/checkout@v2
        with:
          fetch-depth: 0
      - run: git fetch --depth=1 origin +refs/tags/*:refs/tags/*
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v1
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install dependencies
        run: |
          brew update || true
          brew unlink git-lfs || true
          curl -L https://raw.githubusercontent.com/Homebrew/homebrew-core/43842898fd3ff43273466052722f5ba2789196cb/Formula/git-lfs.rb > git-lfs.rb && brew install git-lfs.rb && rm git-lfs.rb
          brew install shellcheck node || brew link --overwrite node
          python -m pip install --upgrade pip
          python -m pip install wheel
          make download-templates
          python -m pip install .[all]
          git config --global --add user.name "Renku Bot"
          git config --global --add user.email "renku@datascience.ch"
      - name: Test with pytest
        env:
          POETRY_VIRTUALENVS_CREATE: false
          LANG: en_US.UTF-8
          LC_ALL: en_US.UTF-8
          RENKU_REQUESTS_TIMEOUT_SECONDS: 120
        run: pytest -v -m "not integration and not publish and not serial" tests/api

  test-macos-cli:
    runs-on: macos-latest
    needs: [set-matrix]
    strategy:
      max-parallel: 3
      matrix: ${{fromJson(needs.set-matrix.outputs.matrix)}}
    steps:
      - uses: actions/checkout@v2
        with:
          fetch-depth: 0
      - run: git fetch --depth=1 origin +refs/tags/*:refs/tags/*
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v1
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install dependencies
        run: |
          brew update || true
          brew unlink git-lfs || true
          curl -L https://raw.githubusercontent.com/Homebrew/homebrew-core/43842898fd3ff43273466052722f5ba2789196cb/Formula/git-lfs.rb > git-lfs.rb && brew install git-lfs.rb && rm git-lfs.rb
          brew install shellcheck node || brew link --overwrite node
          python -m pip install --upgrade pip
          python -m pip install wheel
          make download-templates
          python -m pip install .[all]
          git config --global --add user.name "Renku Bot"
          git config --global --add user.email "renku@datascience.ch"
      - name: Test with pytest
        env:
          POETRY_VIRTUALENVS_CREATE: false
          LANG: en_US.UTF-8
          LC_ALL: en_US.UTF-8
          RENKU_REQUESTS_TIMEOUT_SECONDS: 120
        run: pytest -v -m "not integration and not publish and not serial" tests/cli
      - name: Test with pytest(serial)
        env:
          POETRY_VIRTUALENVS_CREATE: false
          LANG: en_US.UTF-8
          LC_ALL: en_US.UTF-8
          RENKU_REQUESTS_TIMEOUT_SECONDS: 120
        run: pytest -v -m "not integration and not publish and serial" tests/cli

  test-macos-core:
    runs-on: macos-latest
    needs: [set-matrix]
    strategy:
      max-parallel: 3
      matrix: ${{fromJson(needs.set-matrix.outputs.matrix)}}
    steps:
      - uses: actions/checkout@v2
        with:
          fetch-depth: 0
      - run: git fetch --depth=1 origin +refs/tags/*:refs/tags/*
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v1
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install dependencies
        run: |
          brew update || true
          brew unlink git-lfs || true
          curl -L https://raw.githubusercontent.com/Homebrew/homebrew-core/43842898fd3ff43273466052722f5ba2789196cb/Formula/git-lfs.rb > git-lfs.rb && brew install git-lfs.rb && rm git-lfs.rb
          brew install shellcheck node || brew link --overwrite node
          python -m pip install --upgrade pip
          python -m pip install wheel
          make download-templates
          python -m pip install .[all]
          git config --global --add user.name "Renku Bot"
          git config --global --add user.email "renku@datascience.ch"
      - name: Test with pytest
        env:
          POETRY_VIRTUALENVS_CREATE: false
          LANG: en_US.UTF-8
          LC_ALL: en_US.UTF-8
          RENKU_REQUESTS_TIMEOUT_SECONDS: 120
        run: pytest -v -m "not integration and not publish and not serial" tests/core
      - name: Test with pytest(serial)
        env:
          LANG: en_US.UTF-8
          LC_ALL: en_US.UTF-8
          RENKU_REQUESTS_TIMEOUT_SECONDS: 120
        run: pytest -v -m "not integration and not publish and serial" tests/core

  test-macos-service:
    runs-on: macos-latest
    needs: [set-matrix]
    strategy:
      max-parallel: 3
      matrix: ${{fromJson(needs.set-matrix.outputs.matrix)}}
    steps:
      - uses: actions/checkout@v2
        with:
          fetch-depth: 0
      - run: git fetch --depth=1 origin +refs/tags/*:refs/tags/*
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v1
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install dependencies
        run: |
          brew update || true
          brew unlink git-lfs || true
          curl -L https://raw.githubusercontent.com/Homebrew/homebrew-core/43842898fd3ff43273466052722f5ba2789196cb/Formula/git-lfs.rb > git-lfs.rb && brew install git-lfs.rb && rm git-lfs.rb
          brew install shellcheck node || brew link --overwrite node
          python -m pip install --upgrade pip
          python -m pip install wheel
          make download-templates
          python -m pip install .[all]
          git config --global --add user.name "Renku Bot"
          git config --global --add user.email "renku@datascience.ch"
      - name: Test with pytest
        env:
          POETRY_VIRTUALENVS_CREATE: false
          LANG: en_US.UTF-8
          LC_ALL: en_US.UTF-8
          RENKU_REQUESTS_TIMEOUT_SECONDS: 120
        run: pytest -v -m "not integration and not publish" tests/service

  test-linux-integration:
    runs-on: ubuntu-latest
    needs: [set-matrix, year-week]
    strategy:
      max-parallel: 3
      matrix: ${{fromJson(needs.set-matrix.outputs.matrix)}}
    steps:
      - uses: actions/checkout@v2
        with:
          fetch-depth: 0
      - run: git fetch --depth=1 origin +refs/tags/*:refs/tags/*
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v1
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install system packages
        run: |
          sudo apt-get update -y
          sudo apt-get install -y libyaml-0-2 libyaml-dev
      - uses: actions/cache@master
        id: dependency-cache
        with:
          path: |
            ${{ env.pythonLocation }}
            renku/templates
          key: ${{env.DEPENDENCY_CACHE_PREFIX}}-${{ runner.os }}-${{ env.pythonLocation }}-${{ hashFiles('pyproject.toml') }}-${{hashFiles('poetry.lock')}}-${{hashFiles('Makefile')}}
      - name: Install dependencies
        if: steps.dependency-cache.outputs.cache-hit != 'true' || 'refs/heads/master' == github.ref || 'refs/heads/develop' == github.ref || startsWith(github.ref, 'refs/tags/')
        run: |
          python -m pip install --upgrade pip
          python -m pip install coveralls poetry wheel twine
          make download-templates
          python -m pip install .[all]
      - name: Install renku into cache
        if: steps.dependency-cache.outputs.cache-hit == 'true'
        run: python -m pip install --no-deps .
      - name: Set Git config
        run: |
          git config --global --add user.name "Renku Bot"
          git config --global --add user.email "renku@datascience.ch"
      - uses: actions/cache@master
        id: network-cache
        with:
          path: cassettes
          key: ${{ env.NETWORK_CACHE_PREFIX }}-${{ needs.year-week.outputs.date }}-${{ hashFiles('tests') }}-${{ github.job }}
      - name: Test with pytest
        env:
          POETRY_VIRTUALENVS_CREATE: false
          LANG: en_US.UTF-8
          LC_ALL: en_US.UTF-8
          DATAVERSE_ACCESS_TOKEN: ${{ secrets.DATAVERSE_ACCESS_TOKEN }}
          IT_OAUTH_GIT_TOKEN: ${{ secrets.IT_OAUTH_GIT_TOKEN }}
          ZENODO_ACCESS_TOKEN: ${{ secrets.ZENODO_ACCESS_TOKEN }}
          OLOS_ACCESS_TOKEN: ${{ secrets.OLOS_ACCESS_TOKEN }}
          RENKU_REQUESTS_TIMEOUT_SECONDS: 120
        run: pytest -m "integration and not service and not serial" -v --timeout=600 -n auto
      - name: Test with pytest (serial)
        env:
          POETRY_VIRTUALENVS_CREATE: false
          LANG: en_US.UTF-8
          LC_ALL: en_US.UTF-8
          DATAVERSE_ACCESS_TOKEN: ${{ secrets.DATAVERSE_ACCESS_TOKEN }}
          IT_OAUTH_GIT_TOKEN: ${{ secrets.IT_OAUTH_GIT_TOKEN }}
          ZENODO_ACCESS_TOKEN: ${{ secrets.ZENODO_ACCESS_TOKEN }}
          OLOS_ACCESS_TOKEN: ${{ secrets.OLOS_ACCESS_TOKEN }}
          RENKU_REQUESTS_TIMEOUT_SECONDS: 120
        run: pytest --cov-append -m "integration and not service and serial" -v --timeout=600
      - name: Coveralls
        env:
          COVERALLS_REPO_TOKEN: ${{ secrets.COVERALLS_REPO_TOKEN }}
          GITHUB_TOKEN: $COVERALLS_REPO_TOKEN
          RENKU_REQUESTS_TIMEOUT_SECONDS: 120
          COVERALLS_PARALLEL: true
        run: coveralls --service=github-actions
        continue-on-error: true

  test-linux-integration-service:
    runs-on: ubuntu-latest
    needs: [set-matrix, year-week]
    strategy:
      max-parallel: 3
      matrix: ${{fromJson(needs.set-matrix.outputs.matrix)}}
    steps:
      - uses: actions/checkout@v2
        with:
          fetch-depth: 0
      - run: git fetch --depth=1 origin +refs/tags/*:refs/tags/*
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v1
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install system packages
        run: |
          sudo apt-get update -y
          sudo apt-get install -y libyaml-0-2 libyaml-dev
      - uses: actions/cache@master
        id: dependency-cache
        with:
          path: |
            ${{ env.pythonLocation }}
            renku/templates
          key: ${{env.DEPENDENCY_CACHE_PREFIX}}-${{ runner.os }}-${{ env.pythonLocation }}-${{ hashFiles('pyproject.toml') }}-${{hashFiles('poetry.lock')}}-${{hashFiles('Makefile')}}
      - name: Install dependencies
        if: steps.dependency-cache.outputs.cache-hit != 'true' || 'refs/heads/master' == github.ref || 'refs/heads/develop' == github.ref || startsWith(github.ref, 'refs/tags/')
        run: |
          python -m pip install --upgrade pip
          python -m pip install coveralls poetry wheel twine
          make download-templates
          python -m pip install .[all]
      - name: Install renku into cache
        if: steps.dependency-cache.outputs.cache-hit == 'true'
        run: python -m pip install --no-deps .
      - name: Set Git config
        run: |
          git config --global --add user.name "Renku Bot"
          git config --global --add user.email "renku@datascience.ch"
      - uses: actions/cache@master
        id: network-cache
        with:
          path: cassettes
          key: ${{ env.NETWORK_CACHE_PREFIX }}-${{ needs.year-week.outputs.date }}-${{ hashFiles('tests') }}-${{ github.job }}
      - name: Test with pytest
        env:
          POETRY_VIRTUALENVS_CREATE: false
          LANG: en_US.UTF-8
          LC_ALL: en_US.UTF-8
          DATAVERSE_ACCESS_TOKEN: ${{ secrets.DATAVERSE_ACCESS_TOKEN }}
          IT_OAUTH_GIT_TOKEN: ${{ secrets.IT_OAUTH_GIT_TOKEN }}
          ZENODO_ACCESS_TOKEN: ${{ secrets.ZENODO_ACCESS_TOKEN }}
          OLOS_ACCESS_TOKEN: ${{ secrets.OLOS_ACCESS_TOKEN }}
          RENKU_REQUESTS_TIMEOUT_SECONDS: 120
        run: pytest -m "integration and service and not serial" -k test_mirgate_wrong_template_failure
      - name: Test with pytest (serial)
        env:
          POETRY_VIRTUALENVS_CREATE: false
          LANG: en_US.UTF-8
          LC_ALL: en_US.UTF-8
          DATAVERSE_ACCESS_TOKEN: ${{ secrets.DATAVERSE_ACCESS_TOKEN }}
          IT_OAUTH_GIT_TOKEN: ${{ secrets.IT_OAUTH_GIT_TOKEN }}
          ZENODO_ACCESS_TOKEN: ${{ secrets.ZENODO_ACCESS_TOKEN }}
          OLOS_ACCESS_TOKEN: ${{ secrets.OLOS_ACCESS_TOKEN }}
          RENKU_REQUESTS_TIMEOUT_SECONDS: 120
        run: pytest --cov-append -m "integration and service and serial" -v --timeout=600
      - name: Coveralls
        env:
          COVERALLS_REPO_TOKEN: ${{ secrets.COVERALLS_REPO_TOKEN }}
          GITHUB_TOKEN: $COVERALLS_REPO_TOKEN
          RENKU_REQUESTS_TIMEOUT_SECONDS: 120
          COVERALLS_PARALLEL: true
        run: coveralls --service=github-actions
        continue-on-error: true

  test-macos-integration:
    runs-on: macos-latest
    needs: [set-matrix]
    strategy:
      max-parallel: 1
      matrix: ${{fromJson(needs.set-matrix.outputs.matrix)}}
    if: "startsWith(github.ref, 'refs/tags/') || endsWith(github.ref, github.event.repository.default_branch)"
    steps:
      - uses: actions/checkout@v2
        with:
          fetch-depth: 0
      - run: git fetch --depth=1 origin +refs/tags/*:refs/tags/*
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v1
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install dependencies
        run: |
          brew update || true
          brew unlink git-lfs || true
          curl -L https://raw.githubusercontent.com/Homebrew/homebrew-core/43842898fd3ff43273466052722f5ba2789196cb/Formula/git-lfs.rb > git-lfs.rb && brew install git-lfs.rb && rm git-lfs.rb
          brew install shellcheck node || brew link --overwrite node
          python -m pip install --upgrade pip
          python -m pip install wheel
          make download-templates
          python -m pip install .[all]
          git config --global --add user.name "Renku Bot"
          git config --global --add user.email "renku@datascience.ch"
      - name: Test with pytest
        env:
          POETRY_VIRTUALENVS_CREATE: false
          LANG: en_US.UTF-8
          LC_ALL: en_US.UTF-8
          DATAVERSE_ACCESS_TOKEN: ${{ secrets.DATAVERSE_ACCESS_TOKEN }}
          IT_OAUTH_GIT_TOKEN: ${{ secrets.IT_OAUTH_GIT_TOKEN }}
          ZENODO_ACCESS_TOKEN: ${{ secrets.ZENODO_ACCESS_TOKEN }}
          OLOS_ACCESS_TOKEN: ${{ secrets.OLOS_ACCESS_TOKEN }}
          RENKU_REQUESTS_TIMEOUT_SECONDS: 120
        run: pytest -m "integration and not serial" -v
      - name: Test with pytest(serial)
        env:
          POETRY_VIRTUALENVS_CREATE: false
          LANG: en_US.UTF-8
          LC_ALL: en_US.UTF-8
          DATAVERSE_ACCESS_TOKEN: ${{ secrets.DATAVERSE_ACCESS_TOKEN }}
          IT_OAUTH_GIT_TOKEN: ${{ secrets.IT_OAUTH_GIT_TOKEN }}
          ZENODO_ACCESS_TOKEN: ${{ secrets.ZENODO_ACCESS_TOKEN }}
          OLOS_ACCESS_TOKEN: ${{ secrets.OLOS_ACCESS_TOKEN }}
          RENKU_REQUESTS_TIMEOUT_SECONDS: 120
        run: pytest -m "integration and serial" -v

  publish-pypi:
    runs-on: ubuntu-latest
    if: "startsWith(github.ref, 'refs/tags/')"
    steps:
      - uses: actions/checkout@v2
        with:
          fetch-depth: 0
      - run: git fetch --depth=1 origin +refs/tags/*:refs/tags/*
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v1
        with:
          python-version: "3.7"
      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
          python -m pip install poetry poetry-dynamic-versioning poetry-lock-package twine
          python -m pip install .[all]
          git config --global --add user.name "Renku Bot"
          git config --global --add user.email "renku@datascience.ch"
      - name: Tag if necessary
        if: "!startsWith(github.ref, 'refs/tags/')"
        run: |
          renku --version
          export GIT_TAG=$(renku --version)
          git tag $GIT_TAG
      - name: Build and publish to pypi
        env:
          POETRY_VIRTUALENVS_CREATE: false
          PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }}
        run: make publish-package
      - name: Build and publish lock package
        env:
          POETRY_VIRTUALENVS_CREATE: false
          PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }}
        run: make publish-lock-package

  update-develop-branch:
    runs-on: ubuntu-latest
    if: "startsWith(github.ref, 'refs/tags/')"
    steps:
      - uses: actions/checkout@v2
        with:
          fetch-depth: 0
          ref: 'develop'
      - name: Set Git config
        run: |
          git config --global --add user.name "Renku Bot"
          git config --global --add user.email "renku@datascience.ch"
      - name: merge changes from master
        run: |
          git merge origin/master --message "Merge branch 'master' into develop"
          git push

  build-images:
    runs-on: ubuntu-latest
    needs:
      [
        docs-linux,
        test-linux-cli,
        test-linux-core,
        test-linux-service,
        test-macos-cli,
        test-macos-core,
        test-macos-service,
      ]
    steps:
      - uses: actions/checkout@v2
      - name: build images
        run: make docker-tag
      - name: Docker Login
        if: "startsWith(github.ref, 'refs/tags/') || endsWith(github.ref, github.event.repository.default_branch)"
        uses: Azure/docker-login@v1
        with:
          username: ${{ secrets.RENKU_DOCKER_USERNAME }}
          password: ${{ secrets.RENKU_DOCKER_PASSWORD }}
      - name: push images
        if: "startsWith(github.ref, 'refs/tags/') ||  endsWith(github.ref, github.event.repository.default_branch)"
        run: |
          make docker-push

  publish-chart:
    runs-on: ubuntu-latest
    needs:
      [
        docs-linux,
        test-linux-cli,
        test-linux-core,
        test-linux-service,
        test-macos-cli,
        test-macos-core,
        test-macos-service,
      ]
    if: startsWith(github.ref, 'refs/tags/')
    steps:
      - uses: actions/checkout@v2
        with:
          fetch-depth: 0
      - name: set up environment variables
        run: |
          echo "GIT_USER=Renku Bot" >> $GITHUB_ENV
          echo "GIT_EMAIL=renku@datascience.ch" >> $GITHUB_ENV
      - name: Push chart and images
        uses: SwissDataScienceCenter/renku-actions/publish-chart@v0.5.1
        env:
          CHART_NAME: renku-core
          GITHUB_TOKEN: ${{ secrets.RENKUBOT_GITHUB_TOKEN }}
          DOCKER_USERNAME: ${{ secrets.RENKU_DOCKER_USERNAME }}
          DOCKER_PASSWORD: ${{ secrets.RENKU_DOCKER_PASSWORD }}
      - name: Wait for chart to be available
        run: sleep 120
      - name: Update component version
        uses: SwissDataScienceCenter/renku-actions/update-component-version@v0.5.1
        env:
          CHART_NAME: renku-core
          GITHUB_TOKEN: ${{ secrets.RENKUBOT_GITHUB_TOKEN }}

  coveralls-final:
    name: Aggregate coveralls data
    needs:
      [
        test-linux-cli,
        test-linux-core,
        test-linux-service,
        test-linux-api,
        test-linux-integration,
        test-linux-integration-service,
      ]
    runs-on: ubuntu-latest
    permissions:
      contents: read
    steps:
      - name: Coveralls Finished
        uses: coverallsapp/github-action@master
        with:
          github-token: ${{ secrets.github_token }}
          parallel-finished: true