test_deploy.yml

name: Test, Integration Tests and Deploy

on:
  push:
    branches:
      - "**"
    tags:
      - "v*.*.*"
env:
  DEPENDENCY_CACHE_PREFIX: "v2"
  NETWORK_CACHE_PREFIX: "v1"
  TEMPLATES_VERSION: "0.4.1" # To create a new cache key when only templates are updated
jobs:
  set-matrix:
    runs-on: ubuntu-latest
    outputs:
      matrix: ${{ steps.set-matrix.outputs.matrix }}
    steps:
      - name: set matrix
        id: set-matrix
        env:
          REF: ${{ github.ref }}
          DEFAULT_BRANCH: ${{ github.event.repository.default_branch }}
        run: |
          if [ "$REF" == "refs/heads/$DEFAULT_BRANCH" ] || [[ "$REF" =~ ^refs/tags/.* ]]
          then
              echo "::set-output name=matrix::{\"python-version\": [\"3.7\", \"3.10\"]}"
          else
              echo "::set-output name=matrix::{\"python-version\": [\"3.9\"]}"
          fi

  cleanup-runs:
    runs-on: ubuntu-latest
    steps:
      - uses: rokroskar/workflow-run-cleanup-action@v0.3.3
        env:
          GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
    if: "!startsWith(github.ref, 'refs/tags/') && !endsWith(github.ref, github.event.repository.default_branch)"

  year-week:
    runs-on: ubuntu-latest
    outputs:
      date: ${{ steps.year-week.outputs.date }}
    steps:
      - id: year-week
        run: echo "::set-output name=date::$(date '+%Y-%V')"

  style-check:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3.2.0
        with:
          fetch-depth: 0
      - run: git fetch --depth=1 origin +refs/tags/*:refs/tags/*
      - name: Set up Python 3.9
        uses: actions/setup-python@v4
        with:
          python-version: "3.9"
      - uses: actions/cache@master
        id: dependency-cache
        with:
          path: |
            ${{ env.pythonLocation }}
            renku/templates
          key: ${{ env.DEPENDENCY_CACHE_PREFIX }}-${{ runner.os }}-${{ env.pythonLocation }}-${{ hashFiles('pyproject.toml') }}-${{ hashFiles('poetry.lock') }}-${{ hashFiles('Makefile') }}-${{ env.TEMPLATES_VERSION }}
      - name: Install dependencies
        if: steps.dependency-cache.outputs.cache-hit != 'true' || 'refs/heads/master' == github.ref || 'refs/heads/develop' == github.ref || startsWith(github.ref, 'refs/tags/')
        env:
          POETRY_VIRTUALENVS_CREATE: false
        run: |
          python -m pip install --upgrade pip
          python -m pip install coveralls poetry poetry-dynamic-versioning[plugin] wheel twine
          poetry self add "poetry-dynamic-versioning[plugin]"
          make download-templates
          poetry install -E service --with dev,tests,docs
          python -c "import pydocstyle;print(pydocstyle.__file__)"
          echo "${{ env.pythonLocation }}"
      - name: Install renku into cache
        if: steps.dependency-cache.outputs.cache-hit == 'true'
        env:
          POETRY_VIRTUALENVS_CREATE: false
        run: |
          poetry install -E service --with dev,tests,docs
      - name: Set Git config
        run: |
          git config --global --add user.name "Renku Bot"
          git config --global --add user.email "renku@datascience.ch"
      - name: Test style
        env:
          POETRY_VIRTUALENVS_CREATE: false
        run: ./run-tests.sh -s
      - name: Upload poetry lock file
        if: always()
        uses: actions/upload-artifact@v3
        with:
          name: poetry-lock
          path: poetry.lock
          retention-days: 5

  docs-linux:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3.2.0
        with:
          fetch-depth: 0
      - run: git fetch --depth=1 origin +refs/tags/*:refs/tags/*
      - name: Set up Python
        uses: actions/setup-python@v4
        with:
          python-version: "3.9"
      - uses: actions/cache@master
        id: dependency-cache
        with:
          path: |
            ${{ env.pythonLocation }}
            renku/templates
          key: ${{ env.DEPENDENCY_CACHE_PREFIX }}-${{ runner.os }}-${{ env.pythonLocation }}-${{ hashFiles('pyproject.toml') }}-${{ hashFiles('poetry.lock') }}-${{ hashFiles('Makefile') }}-${{ env.TEMPLATES_VERSION }}
      - name: Install dependencies
        if: steps.dependency-cache.outputs.cache-hit != 'true' || 'refs/heads/master' == github.ref || 'refs/heads/develop' == github.ref || startsWith(github.ref, 'refs/tags/')
        env:
          POETRY_VIRTUALENVS_CREATE: false
        run: |
          python -m pip install --upgrade pip
          python -m pip install coveralls poetry wheel twine
          poetry self add "poetry-dynamic-versioning[plugin]"
          make download-templates
          poetry install -E service --with dev,tests,docs
      - name: Install renku into cache
        if: steps.dependency-cache.outputs.cache-hit == 'true'
        env:
          POETRY_VIRTUALENVS_CREATE: false
        run: |
          poetry self add "poetry-dynamic-versioning[plugin]"
          poetry install -E service --with dev,tests,docs
      - name: Set Git config
        run: |
          git config --global --add user.name "Renku Bot"
          git config --global --add user.email "renku@datascience.ch"
      - name: Test docs
        env:
          POETRY_VIRTUALENVS_CREATE: false
          SIDEBAR: 1
        run: ./run-tests.sh -d

  mypy:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3.2.0
        with:
          fetch-depth: 0
      - run: git fetch --depth=1 origin +refs/tags/*:refs/tags/*
      - name: Set up Python 3.9
        uses: actions/setup-python@v4
        with:
          python-version: "3.9"
      - uses: actions/cache@master
        id: dependency-cache
        with:
          path: |
            ${{ env.pythonLocation }}
            renku/templates
          key: ${{ env.DEPENDENCY_CACHE_PREFIX }}-${{ runner.os }}-${{ env.pythonLocation }}-${{ hashFiles('pyproject.toml') }}-${{ hashFiles('poetry.lock') }}-${{ hashFiles('Makefile') }}-${{ env.TEMPLATES_VERSION }}
      - name: Install dependencies
        if: steps.dependency-cache.outputs.cache-hit != 'true' || 'refs/heads/master' == github.ref || 'refs/heads/develop' == github.ref || startsWith(github.ref, 'refs/tags/')
        env:
          POETRY_VIRTUALENVS_CREATE: false
        run: |
          python -m pip install --upgrade pip
          python -m pip install coveralls poetry wheel twine
          poetry self add "poetry-dynamic-versioning[plugin]"
          make download-templates
          poetry install -E service --with dev,tests,docs
      - name: Install renku into cache
        if: steps.dependency-cache.outputs.cache-hit == 'true'
        env:
          POETRY_VIRTUALENVS_CREATE: false
        run: |
          poetry install -E service --with dev,tests,docs
      - name: Set Git config
        run: |
          git config --global --add user.name "Renku Bot"
          git config --global --add user.email "renku@datascience.ch"
      - name: Test mypy
        env:
          POETRY_VIRTUALENVS_CREATE: false
        run: mypy renku

  test-distro:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3.2.0
        with:
          fetch-depth: 0
      - run: git fetch --depth=1 origin +refs/tags/*:refs/tags/*
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v4
        with:
          python-version: "3.9"
      - uses: actions/cache@master
        id: dependency-cache
        with:
          path: |
            ${{ env.pythonLocation }}
            renku/templates
          key: ${{ env.DEPENDENCY_CACHE_PREFIX }}-${{ runner.os }}-${{ env.pythonLocation }}-${{ hashFiles('pyproject.toml') }}-${{ hashFiles('poetry.lock') }}-${{ hashFiles('Makefile') }}-${{ env.TEMPLATES_VERSION }}
      - name: Install dependencies
        if: steps.dependency-cache.outputs.cache-hit != 'true' || 'refs/heads/master' == github.ref || 'refs/heads/develop' == github.ref || startsWith(github.ref, 'refs/tags/')
        run: |
          python -m pip install --upgrade pip
          python -m pip install coveralls poetry wheel twine
          poetry self add "poetry-dynamic-versioning[plugin]"
          make download-templates
          poetry install -E service --with dev,tests,docs
      - name: Install renku into cache
        if: steps.dependency-cache.outputs.cache-hit == 'true'
        env:
          POETRY_VIRTUALENVS_CREATE: false
        run: |
          poetry install -E service --with dev,tests,docs
      - name: Build Package
        env:
          POETRY_VIRTUALENVS_CREATE: false
        run: poetry build
      - name: test with twine
        run: twine check dist/*

  test-linux-api:
    runs-on: ubuntu-latest
    needs: [set-matrix]
    strategy:
      max-parallel: 3
      matrix: ${{fromJson(needs.set-matrix.outputs.matrix)}}
    steps:
      - uses: actions/checkout@v3.2.0
        with:
          fetch-depth: 0
      - run: git fetch --depth=1 origin +refs/tags/*:refs/tags/*
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install system packages
        run: |
          sudo apt-get update -y
          sudo apt-get install -y libyaml-0-2 libyaml-dev
      - uses: actions/cache@master
        id: dependency-cache
        with:
          path: |
            ${{ env.pythonLocation }}
            renku/templates
          key: ${{ env.DEPENDENCY_CACHE_PREFIX }}-${{ runner.os }}-${{ env.pythonLocation }}-${{ hashFiles('pyproject.toml') }}-${{ hashFiles('poetry.lock') }}-${{ hashFiles('Makefile') }}-${{ env.TEMPLATES_VERSION }}
      - name: Install dependencies
        if: steps.dependency-cache.outputs.cache-hit != 'true' || 'refs/heads/master' == github.ref || 'refs/heads/develop' == github.ref || startsWith(github.ref, 'refs/tags/')
        env:
          POETRY_VIRTUALENVS_CREATE: false
        run: |
          python -m pip install --upgrade pip
          python -m pip install coveralls poetry wheel twine
          poetry self add "poetry-dynamic-versioning[plugin]"
          make download-templates
          poetry install -E service --with dev,tests,docs
      - name: Install renku into cache
        if: steps.dependency-cache.outputs.cache-hit == 'true'
        env:
          POETRY_VIRTUALENVS_CREATE: false
        run: |
          poetry install -E service --with dev,tests,docs
      - name: Set Git config
        run: |
          git config --global --add user.name "Renku Bot"
          git config --global --add user.email "renku@datascience.ch"
      - name: Set coveralls path mapping
        run: |
          echo "

          [tool.coverage.paths]
          source = [
              \"renku/\",
              \"$(python -c 'import sysconfig; print(sysconfig.get_paths()["purelib"])')/renku/\"
          ]" >> pyproject.toml
      - name: Test with pytest
        env:
          POETRY_VIRTUALENVS_CREATE: false
          LANG: en_US.UTF-8
          LC_ALL: en_US.UTF-8
        run: pytest -v -m "not integration and not publish and not serial" -n auto tests/api
      - name: Coveralls
        env:
          COVERALLS_REPO_TOKEN: ${{ secrets.COVERALLS_REPO_TOKEN }}
          GITHUB_TOKEN: $COVERALLS_REPO_TOKEN
          RENKU_REQUESTS_TIMEOUT_SECONDS: 120
          COVERALLS_PARALLEL: true
        if: always()
        run: coveralls --service=github-actions
        continue-on-error: true

  test-linux-cli:
    runs-on: ubuntu-latest
    needs: [set-matrix]
    strategy:
      max-parallel: 3
      matrix: ${{fromJson(needs.set-matrix.outputs.matrix)}}
    steps:
      - uses: actions/checkout@v3.2.0
        with:
          fetch-depth: 0
      - run: git fetch --depth=1 origin +refs/tags/*:refs/tags/*
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install system packages
        run: |
          sudo apt-get update -y
          sudo apt-get install -y libyaml-0-2 libyaml-dev
      - uses: actions/cache@master
        id: dependency-cache
        with:
          path: |
            ${{ env.pythonLocation }}
            renku/templates
          key: ${{ env.DEPENDENCY_CACHE_PREFIX }}-${{ runner.os }}-${{ env.pythonLocation }}-${{ hashFiles('pyproject.toml') }}-${{ hashFiles('poetry.lock') }}-${{ hashFiles('Makefile') }}-${{ env.TEMPLATES_VERSION }}
      - name: Install dependencies
        if: steps.dependency-cache.outputs.cache-hit != 'true' || 'refs/heads/master' == github.ref || 'refs/heads/develop' == github.ref || startsWith(github.ref, 'refs/tags/')
        env:
          POETRY_VIRTUALENVS_CREATE: false
        run: |
          python -m pip install --upgrade pip
          python -m pip install coveralls poetry wheel twine
          poetry self add "poetry-dynamic-versioning[plugin]"
          make download-templates
          poetry install -E service --with dev,tests,docs
      - name: Install renku into cache
        if: steps.dependency-cache.outputs.cache-hit == 'true'
        env:
          POETRY_VIRTUALENVS_CREATE: false
        run: |
          poetry install -E service --with dev,tests,docs
      - name: Set Git config
        run: |
          git config --global --add user.name "Renku Bot"
          git config --global --add user.email "renku@datascience.ch"
      - name: Set coveralls path mapping
        run: |
          echo "

          [tool.coverage.paths]
          source = [
              \"renku/\",
              \"$(python -c 'import sysconfig; print(sysconfig.get_paths()["purelib"])')/renku/\"
          ]" >> pyproject.toml
      - name: Test with pytest
        env:
          POETRY_VIRTUALENVS_CREATE: false
          LANG: en_US.UTF-8
          LC_ALL: en_US.UTF-8
        run: pytest -v -m "not integration and not publish and not serial" -n auto  tests/cli
      - name: Test with pytest(serial)
        env:
          POETRY_VIRTUALENVS_CREATE: false
          LANG: en_US.UTF-8
          LC_ALL: en_US.UTF-8
        run: pytest -v --cov-append -m "not integration and not publish and serial"  tests/cli
      - name: Coveralls
        env:
          COVERALLS_REPO_TOKEN: ${{ secrets.COVERALLS_REPO_TOKEN }}
          GITHUB_TOKEN: $COVERALLS_REPO_TOKEN
          RENKU_REQUESTS_TIMEOUT_SECONDS: 120
          COVERALLS_PARALLEL: true
        if: always()
        run: coveralls --service=github-actions
        continue-on-error: true

  test-linux-core:
    runs-on: ubuntu-latest
    needs: [set-matrix]
    strategy:
      max-parallel: 3
      matrix: ${{fromJson(needs.set-matrix.outputs.matrix)}}
    steps:
      - uses: actions/checkout@v3.2.0
        with:
          fetch-depth: 0
      - run: git fetch --depth=1 origin +refs/tags/*:refs/tags/*
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install system packages
        run: |
          sudo apt-get update -y
          sudo apt-get install -y libyaml-0-2 libyaml-dev
      - uses: actions/cache@master
        id: dependency-cache
        with:
          path: |
            ${{ env.pythonLocation }}
            renku/templates
          key: ${{ env.DEPENDENCY_CACHE_PREFIX }}-${{ runner.os }}-${{ env.pythonLocation }}-${{ hashFiles('pyproject.toml') }}-${{ hashFiles('poetry.lock') }}-${{ hashFiles('Makefile') }}-${{ env.TEMPLATES_VERSION }}
      - name: Install dependencies
        if: steps.dependency-cache.outputs.cache-hit != 'true' || 'refs/heads/master' == github.ref || 'refs/heads/develop' == github.ref || startsWith(github.ref, 'refs/tags/')
        env:
          POETRY_VIRTUALENVS_CREATE: false
        run: |
          python -m pip install --upgrade pip
          python -m pip install coveralls poetry wheel twine
          poetry self add "poetry-dynamic-versioning[plugin]"
          make download-templates
          poetry install -E service --with dev,tests,docs
      - name: Install renku into cache
        if: steps.dependency-cache.outputs.cache-hit == 'true'
        env:
          POETRY_VIRTUALENVS_CREATE: false
        run: |
          poetry install -E service --with dev,tests,docs
      - name: Set Git config
        run: |
          git config --global --add user.name "Renku Bot"
          git config --global --add user.email "renku@datascience.ch"
      - name: Set coveralls path mapping
        run: |
          echo "

          [tool.coverage.paths]
          source = [
              \"renku/\",
              \"$(python -c 'import sysconfig; print(sysconfig.get_paths()["purelib"])')/renku/\"
          ]" >> pyproject.toml
      - name: Test with pytest
        env:
          POETRY_VIRTUALENVS_CREATE: false
          LANG: en_US.UTF-8
          LC_ALL: en_US.UTF-8
        run: pytest -v -m "not integration and not publish and not serial" -n auto tests/core
      - name: Test with pytest(serial)
        env:
          POETRY_VIRTUALENVS_CREATE: false
          LANG: en_US.UTF-8
          LC_ALL: en_US.UTF-8
        run: pytest -v --cov-append -m "not integration and not publish and serial" tests/core
      - name: Coveralls
        env:
          COVERALLS_REPO_TOKEN: ${{ secrets.COVERALLS_REPO_TOKEN }}
          GITHUB_TOKEN: $COVERALLS_REPO_TOKEN
          RENKU_REQUESTS_TIMEOUT_SECONDS: 120
          COVERALLS_PARALLEL: true
        if: always()
        run: coveralls --service=github-actions
        continue-on-error: true

  test-linux-service:
    runs-on: ubuntu-latest
    needs: [set-matrix]
    strategy:
      max-parallel: 3
      matrix: ${{fromJson(needs.set-matrix.outputs.matrix)}}
    steps:
      - uses: actions/checkout@v3.2.0
        with:
          fetch-depth: 0
      - run: git fetch --depth=1 origin +refs/tags/*:refs/tags/*
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install system packages
        run: |
          sudo apt-get update -y
          sudo apt-get install -y libyaml-0-2 libyaml-dev
      - uses: actions/cache@master
        id: dependency-cache
        with:
          path: |
            ${{ env.pythonLocation }}
            renku/templates
          key: ${{ env.DEPENDENCY_CACHE_PREFIX }}-${{ runner.os }}-${{ env.pythonLocation }}-${{ hashFiles('pyproject.toml') }}-${{ hashFiles('poetry.lock') }}-${{ hashFiles('Makefile') }}-${{ env.TEMPLATES_VERSION }}
      - name: Install dependencies
        if: steps.dependency-cache.outputs.cache-hit != 'true' || 'refs/heads/master' == github.ref || 'refs/heads/develop' == github.ref || startsWith(github.ref, 'refs/tags/')
        env:
          POETRY_VIRTUALENVS_CREATE: false
        run: |
          python -m pip install --upgrade pip
          python -m pip install coveralls poetry wheel twine
          poetry self add "poetry-dynamic-versioning[plugin]"
          make download-templates
          poetry install -E service --with dev,tests,docs
      - name: Install renku into cache
        if: steps.dependency-cache.outputs.cache-hit == 'true'
        env:
          POETRY_VIRTUALENVS_CREATE: false
        run: |
          poetry install -E service --with dev,tests,docs
      - name: Set Git config
        run: |
          git config --global --add user.name "Renku Bot"
          git config --global --add user.email "renku@datascience.ch"
      - name: Set coveralls path mapping
        run: |
          echo "

          [tool.coverage.paths]
          source = [
              \"renku/\",
              \"$(python -c 'import sysconfig; print(sysconfig.get_paths()["purelib"])')/renku/\"
          ]" >> pyproject.toml
      - name: Test with pytest
        env:
          POETRY_VIRTUALENVS_CREATE: false
          LANG: en_US.UTF-8
          LC_ALL: en_US.UTF-8
        run: pytest -v -m "not integration and not publish" -n auto tests/service
      - name: Coveralls
        env:
          COVERALLS_REPO_TOKEN: ${{ secrets.COVERALLS_REPO_TOKEN }}
          GITHUB_TOKEN: $COVERALLS_REPO_TOKEN
          RENKU_REQUESTS_TIMEOUT_SECONDS: 120
          COVERALLS_PARALLEL: true
        if: always()
        run: coveralls --service=github-actions
        continue-on-error: true

  test-macos-api:
    runs-on: macos-latest
    needs: [set-matrix]
    strategy:
      max-parallel: 3
      matrix: ${{fromJson(needs.set-matrix.outputs.matrix)}}
    steps:
      - uses: actions/checkout@v3.2.0
        with:
          fetch-depth: 0
      - run: git fetch --depth=1 origin +refs/tags/*:refs/tags/*
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install dependencies
        env:
          POETRY_VIRTUALENVS_CREATE: false
        run: |
          brew update || true
          brew unlink git-lfs || true
          curl -L https://raw.githubusercontent.com/Homebrew/homebrew-core/43842898fd3ff43273466052722f5ba2789196cb/Formula/git-lfs.rb > git-lfs.rb && brew install git-lfs.rb && rm git-lfs.rb
          brew install shellcheck node || brew link --overwrite node
          python -m pip install --upgrade pip
          python -m pip install wheel poetry
          poetry self add "poetry-dynamic-versioning[plugin]"
          make download-templates
          poetry install -E service --with dev,tests,docs
          git config --global --add user.name "Renku Bot"
          git config --global --add user.email "renku@datascience.ch"
      - name: Test with pytest
        env:
          POETRY_VIRTUALENVS_CREATE: false
          LANG: en_US.UTF-8
          LC_ALL: en_US.UTF-8
          RENKU_REQUESTS_TIMEOUT_SECONDS: 120
        run: pytest -v -m "not integration and not publish and not serial" tests/api

  test-macos-cli:
    runs-on: macos-latest
    needs: [set-matrix]
    strategy:
      max-parallel: 3
      matrix: ${{fromJson(needs.set-matrix.outputs.matrix)}}
    steps:
      - uses: actions/checkout@v3.2.0
        with:
          fetch-depth: 0
      - run: git fetch --depth=1 origin +refs/tags/*:refs/tags/*
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install dependencies
        env:
          POETRY_VIRTUALENVS_CREATE: false
        run: |
          brew update || true
          brew unlink git-lfs || true
          curl -L https://raw.githubusercontent.com/Homebrew/homebrew-core/43842898fd3ff43273466052722f5ba2789196cb/Formula/git-lfs.rb > git-lfs.rb && brew install git-lfs.rb && rm git-lfs.rb
          brew install shellcheck node || brew link --overwrite node
          python -m pip install --upgrade pip
          python -m pip install wheel poetry
          poetry self add "poetry-dynamic-versioning[plugin]"
          make download-templates
          poetry install -E service --with dev,tests,docs
          git config --global --add user.name "Renku Bot"
          git config --global --add user.email "renku@datascience.ch"
      - name: Test with pytest
        env:
          POETRY_VIRTUALENVS_CREATE: false
          LANG: en_US.UTF-8
          LC_ALL: en_US.UTF-8
          RENKU_REQUESTS_TIMEOUT_SECONDS: 120
        run: pytest -v -m "not integration and not publish and not serial" tests/cli
      - name: Test with pytest(serial)
        env:
          POETRY_VIRTUALENVS_CREATE: false
          LANG: en_US.UTF-8
          LC_ALL: en_US.UTF-8
          RENKU_REQUESTS_TIMEOUT_SECONDS: 120
        run: pytest -v -m "not integration and not publish and serial" tests/cli

  test-macos-core:
    runs-on: macos-latest
    needs: [set-matrix]
    strategy:
      max-parallel: 3
      matrix: ${{fromJson(needs.set-matrix.outputs.matrix)}}
    steps:
      - uses: actions/checkout@v3.2.0
        with:
          fetch-depth: 0
      - run: git fetch --depth=1 origin +refs/tags/*:refs/tags/*
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install dependencies
        env:
          POETRY_VIRTUALENVS_CREATE: false
        run: |
          brew update || true
          brew unlink git-lfs || true
          curl -L https://raw.githubusercontent.com/Homebrew/homebrew-core/43842898fd3ff43273466052722f5ba2789196cb/Formula/git-lfs.rb > git-lfs.rb && brew install git-lfs.rb && rm git-lfs.rb
          brew install shellcheck node || brew link --overwrite node
          python -m pip install --upgrade pip
          python -m pip install wheel poetry
          poetry self add "poetry-dynamic-versioning[plugin]"
          make download-templates
          poetry install -E service --with dev,tests,docs
          git config --global --add user.name "Renku Bot"
          git config --global --add user.email "renku@datascience.ch"
      - name: Test with pytest
        env:
          POETRY_VIRTUALENVS_CREATE: false
          LANG: en_US.UTF-8
          LC_ALL: en_US.UTF-8
          RENKU_REQUESTS_TIMEOUT_SECONDS: 120
        run: pytest -v -m "not integration and not publish and not serial" tests/core
      - name: Test with pytest(serial)
        env:
          LANG: en_US.UTF-8
          LC_ALL: en_US.UTF-8
          RENKU_REQUESTS_TIMEOUT_SECONDS: 120
        run: pytest -v -m "not integration and not publish and serial" tests/core

  test-macos-service:
    runs-on: macos-latest
    needs: [set-matrix]
    strategy:
      max-parallel: 3
      matrix: ${{fromJson(needs.set-matrix.outputs.matrix)}}
    steps:
      - uses: actions/checkout@v3.2.0
        with:
          fetch-depth: 0
      - run: git fetch --depth=1 origin +refs/tags/*:refs/tags/*
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install dependencies
        env:
          POETRY_VIRTUALENVS_CREATE: false
        run: |
          brew update || true
          brew unlink git-lfs || true
          curl -L https://raw.githubusercontent.com/Homebrew/homebrew-core/43842898fd3ff43273466052722f5ba2789196cb/Formula/git-lfs.rb > git-lfs.rb && brew install git-lfs.rb && rm git-lfs.rb
          brew install shellcheck node || brew link --overwrite node
          python -m pip install --upgrade pip
          python -m pip install wheel poetry
          poetry self add "poetry-dynamic-versioning[plugin]"
          make download-templates
          poetry install -E service --with dev,tests,docs
          git config --global --add user.name "Renku Bot"
          git config --global --add user.email "renku@datascience.ch"
      - name: Test with pytest
        env:
          POETRY_VIRTUALENVS_CREATE: false
          LANG: en_US.UTF-8
          LC_ALL: en_US.UTF-8
          RENKU_REQUESTS_TIMEOUT_SECONDS: 120
        run: pytest -v -m "not integration and not publish" tests/service

  test-linux-integration:
    runs-on: ubuntu-latest
    needs: [set-matrix, year-week]
    strategy:
      max-parallel: 3
      matrix: ${{fromJson(needs.set-matrix.outputs.matrix)}}
    steps:
      - uses: actions/checkout@v3.2.0
        with:
          fetch-depth: 0
      - run: git fetch --depth=1 origin +refs/tags/*:refs/tags/*
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install system packages
        run: |
          sudo apt-get update -y
          sudo apt-get install -y libyaml-0-2 libyaml-dev rclone fuse
      - uses: actions/cache@master
        id: dependency-cache
        with:
          path: |
            ${{ env.pythonLocation }}
            renku/templates
          key: ${{ env.DEPENDENCY_CACHE_PREFIX }}-${{ runner.os }}-${{ env.pythonLocation }}-${{ hashFiles('pyproject.toml') }}-${{ hashFiles('poetry.lock') }}-${{ hashFiles('Makefile') }}-${{ env.TEMPLATES_VERSION }}
      - name: Install dependencies
        if: steps.dependency-cache.outputs.cache-hit != 'true' || 'refs/heads/master' == github.ref || 'refs/heads/develop' == github.ref || startsWith(github.ref, 'refs/tags/')
        env:
          POETRY_VIRTUALENVS_CREATE: false
        run: |
          python -m pip install --upgrade pip
          python -m pip install coveralls poetry wheel twine
          poetry self add "poetry-dynamic-versioning[plugin]"
          make download-templates
          poetry install -E service --with dev,tests,docs
      - name: Install renku into cache
        if: steps.dependency-cache.outputs.cache-hit == 'true'
        env:
          POETRY_VIRTUALENVS_CREATE: false
        run: |
          poetry install -E service --with dev,tests,docs
      - name: Set Git config
        run: |
          git config --global --add user.name "Renku Bot"
          git config --global --add user.email "renku@datascience.ch"
      - uses: actions/cache@master
        id: network-cache
        with:
          path: cassettes
          key: ${{ env.NETWORK_CACHE_PREFIX }}-${{ needs.year-week.outputs.date }}-${{ hashFiles('tests') }}-${{ github.job }}
      - name: Set coveralls path mapping
        run: |
          echo "

          [tool.coverage.paths]
          source = [
              \"renku/\",
              \"$(python -c 'import sysconfig; print(sysconfig.get_paths()["purelib"])')/renku/\"
          ]" >> pyproject.toml
      - name: Test with pytest
        env:
          POETRY_VIRTUALENVS_CREATE: false
          LANG: en_US.UTF-8
          LC_ALL: en_US.UTF-8
          DATAVERSE_ACCESS_TOKEN: ${{ secrets.DATAVERSE_ACCESS_TOKEN }}
          IT_OAUTH_GIT_TOKEN: ${{ secrets.IT_OAUTH_GIT_TOKEN }}
          ZENODO_ACCESS_TOKEN: ${{ secrets.ZENODO_ACCESS_TOKEN }}
          OLOS_ACCESS_TOKEN: ${{ secrets.OLOS_ACCESS_TOKEN }}
          RENKU_REQUESTS_TIMEOUT_SECONDS: 120
          CLOUD_STORAGE_AZURE_KEY: ${{ secrets.CLOUD_STORAGE_AZURE_KEY }}
          CLOUD_STORAGE_S3_ACCESS_KEY_ID: ${{ secrets.CLOUD_STORAGE_S3_ACCESS_KEY_ID }}
          CLOUD_STORAGE_S3_SECRET_ACCESS_KEY: ${{ secrets.CLOUD_STORAGE_S3_SECRET_ACCESS_KEY }}
        run: pytest -m "integration and not service and not serial" -v --timeout=600 -n auto
      - name: Start Redis
        uses: supercharge/redis-github-action@1.4.0
      - name: Test with pytest (serial)
        env:
          POETRY_VIRTUALENVS_CREATE: false
          LANG: en_US.UTF-8
          LC_ALL: en_US.UTF-8
          DATAVERSE_ACCESS_TOKEN: ${{ secrets.DATAVERSE_ACCESS_TOKEN }}
          IT_OAUTH_GIT_TOKEN: ${{ secrets.IT_OAUTH_GIT_TOKEN }}
          ZENODO_ACCESS_TOKEN: ${{ secrets.ZENODO_ACCESS_TOKEN }}
          OLOS_ACCESS_TOKEN: ${{ secrets.OLOS_ACCESS_TOKEN }}
          RENKU_REQUESTS_TIMEOUT_SECONDS: 120
        run: pytest --cov-append -m "integration and not service and serial" -v --timeout=600
      - name: Coveralls
        env:
          COVERALLS_REPO_TOKEN: ${{ secrets.COVERALLS_REPO_TOKEN }}
          GITHUB_TOKEN: $COVERALLS_REPO_TOKEN
          RENKU_REQUESTS_TIMEOUT_SECONDS: 120
          COVERALLS_PARALLEL: true
        if: always()
        run: coveralls --service=github-actions
        continue-on-error: true

  test-linux-integration-service:
    runs-on: ubuntu-latest
    needs: [set-matrix, year-week]
    strategy:
      max-parallel: 3
      matrix: ${{fromJson(needs.set-matrix.outputs.matrix)}}
    steps:
      - uses: actions/checkout@v3.2.0
        with:
          fetch-depth: 0
      - run: git fetch --depth=1 origin +refs/tags/*:refs/tags/*
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install system packages
        run: |
          sudo apt-get update -y
          sudo apt-get install -y libyaml-0-2 libyaml-dev rclone fuse
      - uses: actions/cache@master
        id: dependency-cache
        with:
          path: |
            ${{ env.pythonLocation }}
            renku/templates
          key: ${{ env.DEPENDENCY_CACHE_PREFIX }}-${{ runner.os }}-${{ env.pythonLocation }}-${{ hashFiles('pyproject.toml') }}-${{ hashFiles('poetry.lock') }}-${{ hashFiles('Makefile') }}-${{ env.TEMPLATES_VERSION }}
      - name: Install dependencies
        if: steps.dependency-cache.outputs.cache-hit != 'true' || 'refs/heads/master' == github.ref || 'refs/heads/develop' == github.ref || startsWith(github.ref, 'refs/tags/')
        env:
          POETRY_VIRTUALENVS_CREATE: false
        run: |
          python -m pip install --upgrade pip
          python -m pip install coveralls poetry wheel twine
          poetry self add "poetry-dynamic-versioning[plugin]"
          make download-templates
          poetry install -E service --with dev,tests,docs
      - name: Install renku into cache
        if: steps.dependency-cache.outputs.cache-hit == 'true'
        env:
          POETRY_VIRTUALENVS_CREATE: false
        run: |
          poetry install -E service --with dev,tests,docs
      - name: Set Git config
        run: |
          git config --global --add user.name "Renku Bot"
          git config --global --add user.email "renku@datascience.ch"
      - uses: actions/cache@master
        id: network-cache
        with:
          path: cassettes
          key: ${{ env.NETWORK_CACHE_PREFIX }}-${{ needs.year-week.outputs.date }}-${{ hashFiles('tests') }}-${{ github.job }}
      - name: Set coveralls path mapping
        run: |
          echo "

          [tool.coverage.paths]
          source = [
              \"renku/\",
              \"$(python -c 'import sysconfig; print(sysconfig.get_paths()["purelib"])')/renku/\"
          ]" >> pyproject.toml
      - name: Test with pytest
        env:
          POETRY_VIRTUALENVS_CREATE: false
          LANG: en_US.UTF-8
          LC_ALL: en_US.UTF-8
          DATAVERSE_ACCESS_TOKEN: ${{ secrets.DATAVERSE_ACCESS_TOKEN }}
          IT_OAUTH_GIT_TOKEN: ${{ secrets.IT_OAUTH_GIT_TOKEN }}
          ZENODO_ACCESS_TOKEN: ${{ secrets.ZENODO_ACCESS_TOKEN }}
          OLOS_ACCESS_TOKEN: ${{ secrets.OLOS_ACCESS_TOKEN }}
          RENKU_REQUESTS_TIMEOUT_SECONDS: 120
        run: pytest -m "integration and service and not serial" -v --timeout=600 -n auto
      - name: Test with pytest (serial)
        env:
          POETRY_VIRTUALENVS_CREATE: false
          LANG: en_US.UTF-8
          LC_ALL: en_US.UTF-8
          DATAVERSE_ACCESS_TOKEN: ${{ secrets.DATAVERSE_ACCESS_TOKEN }}
          IT_OAUTH_GIT_TOKEN: ${{ secrets.IT_OAUTH_GIT_TOKEN }}
          ZENODO_ACCESS_TOKEN: ${{ secrets.ZENODO_ACCESS_TOKEN }}
          OLOS_ACCESS_TOKEN: ${{ secrets.OLOS_ACCESS_TOKEN }}
          RENKU_REQUESTS_TIMEOUT_SECONDS: 120
        run: pytest --cov-append -m "integration and service and serial" -v --timeout=600
      - name: Coveralls
        env:
          COVERALLS_REPO_TOKEN: ${{ secrets.COVERALLS_REPO_TOKEN }}
          GITHUB_TOKEN: $COVERALLS_REPO_TOKEN
          RENKU_REQUESTS_TIMEOUT_SECONDS: 120
          COVERALLS_PARALLEL: true
        if: always()
        run: coveralls --service=github-actions
        continue-on-error: true

  test-macos-integration:
    runs-on: macos-latest
    needs: [set-matrix]
    strategy:
      max-parallel: 1
      matrix: ${{fromJson(needs.set-matrix.outputs.matrix)}}
    if: "startsWith(github.ref, 'refs/tags/') || endsWith(github.ref, github.event.repository.default_branch)"
    steps:
      - uses: actions/checkout@v3.2.0
        with:
          fetch-depth: 0
      - run: git fetch --depth=1 origin +refs/tags/*:refs/tags/*
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install dependencies
        env:
          POETRY_VIRTUALENVS_CREATE: false
        run: |
          brew update || true
          brew unlink git-lfs || true
          curl -L https://raw.githubusercontent.com/Homebrew/homebrew-core/43842898fd3ff43273466052722f5ba2789196cb/Formula/git-lfs.rb > git-lfs.rb && brew install git-lfs.rb && rm git-lfs.rb
          brew install shellcheck node || brew link --overwrite node
          python -m pip install --upgrade pip
          python -m pip install wheel poetry
          poetry self add "poetry-dynamic-versioning[plugin]"
          make download-templates
          poetry install -E service --with dev,tests,docs
          git config --global --add user.name "Renku Bot"
          git config --global --add user.email "renku@datascience.ch"
          brew install rclone osxfuse
      - name: Test with pytest
        env:
          POETRY_VIRTUALENVS_CREATE: false
          LANG: en_US.UTF-8
          LC_ALL: en_US.UTF-8
          DATAVERSE_ACCESS_TOKEN: ${{ secrets.DATAVERSE_ACCESS_TOKEN }}
          IT_OAUTH_GIT_TOKEN: ${{ secrets.IT_OAUTH_GIT_TOKEN }}
          ZENODO_ACCESS_TOKEN: ${{ secrets.ZENODO_ACCESS_TOKEN }}
          OLOS_ACCESS_TOKEN: ${{ secrets.OLOS_ACCESS_TOKEN }}
          RENKU_REQUESTS_TIMEOUT_SECONDS: 120
          CLOUD_STORAGE_AZURE_KEY: ${{ secrets.CLOUD_STORAGE_AZURE_KEY }}
          CLOUD_STORAGE_S3_ACCESS_KEY_ID: ${{ secrets.CLOUD_STORAGE_S3_ACCESS_KEY_ID }}
          CLOUD_STORAGE_S3_SECRET_ACCESS_KEY: ${{ secrets.CLOUD_STORAGE_S3_SECRET_ACCESS_KEY }}
        run: pytest -m "integration and not serial" -v
      - name: Start Redis
        uses: supercharge/redis-github-action@1.4.0
      - name: Test with pytest(serial)
        env:
          POETRY_VIRTUALENVS_CREATE: false
          LANG: en_US.UTF-8
          LC_ALL: en_US.UTF-8
          DATAVERSE_ACCESS_TOKEN: ${{ secrets.DATAVERSE_ACCESS_TOKEN }}
          IT_OAUTH_GIT_TOKEN: ${{ secrets.IT_OAUTH_GIT_TOKEN }}
          ZENODO_ACCESS_TOKEN: ${{ secrets.ZENODO_ACCESS_TOKEN }}
          OLOS_ACCESS_TOKEN: ${{ secrets.OLOS_ACCESS_TOKEN }}
          RENKU_REQUESTS_TIMEOUT_SECONDS: 120
        run: pytest -m "integration and serial" -v

  publish-pypi:
    runs-on: ubuntu-latest
    if: "startsWith(github.ref, 'refs/tags/')"
    steps:
      - uses: actions/checkout@v3.2.0
        with:
          fetch-depth: 0
      - run: git fetch --depth=1 origin +refs/tags/*:refs/tags/*
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v4
        with:
          python-version: "3.9"
      - name: Install dependencies
        env:
          POETRY_VIRTUALENVS_CREATE: false
        run: |
          python -m pip install --upgrade pip
          python -m pip install poetry poetry-lock-package twine
          poetry self add "poetry-dynamic-versioning[plugin]"
          poetry install -E service --with dev,tests,docs
          git config --global --add user.name "Renku Bot"
          git config --global --add user.email "renku@datascience.ch"
      - name: Tag if necessary
        if: "!startsWith(github.ref, 'refs/tags/')"
        run: |
          renku --version
          export GIT_TAG=$(renku --version)
          git tag $GIT_TAG
      - name: Build and publish to pypi
        env:
          POETRY_VIRTUALENVS_CREATE: false
          PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }}
        run: make publish-package
      - name: Build and publish lock package
        env:
          POETRY_VIRTUALENVS_CREATE: false
          PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }}
        run: make publish-lock-package

  update-develop-branch:
    runs-on: ubuntu-latest
    if: "startsWith(github.ref, 'refs/tags/')"
    steps:
      - uses: actions/checkout@v3.2.0
        with:
          fetch-depth: 0
          ref: "develop"
      - name: Set Git config
        run: |
          git config --global --add user.name "Renku Bot"
          git config --global --add user.email "renku@datascience.ch"
      - name: merge changes from master
        env:
          GITHUB_TOKEN: ${{ secrets.RENKUBOT_GITHUB_TOKEN }}
        run: |
          git merge origin/master --message "Merge branch 'master' into develop"
          git push

  build-images:
    runs-on: ubuntu-latest
    needs:
      [
        docs-linux,
        test-linux-cli,
        test-linux-core,
        test-linux-service,
        test-macos-cli,
        test-macos-core,
        test-macos-service,
      ]
    steps:
      - uses: actions/checkout@v3.2.0
        with:
          fetch-depth: 0
      - name: build images
        run: make docker-tag
      - name: Docker Login
        if: "startsWith(github.ref, 'refs/tags/') || endsWith(github.ref, github.event.repository.default_branch)"
        uses: Azure/docker-login@v1
        with:
          username: ${{ secrets.RENKU_DOCKER_USERNAME }}
          password: ${{ secrets.RENKU_DOCKER_PASSWORD }}
      - name: push images
        if: "startsWith(github.ref, 'refs/tags/') ||  endsWith(github.ref, github.event.repository.default_branch)"
        run: |
          make docker-push

  publish-chart:
    runs-on: ubuntu-latest
    needs:
      [
        docs-linux,
        test-linux-cli,
        test-linux-core,
        test-linux-service,
        test-macos-cli,
        test-macos-core,
        test-macos-service,
      ]
    if: startsWith(github.ref, 'refs/tags/')
    steps:
      - uses: actions/checkout@v3.2.0
        with:
          fetch-depth: 0
      - name: set up environment variables
        run: |
          echo "GIT_USER=Renku Bot" >> $GITHUB_ENV
          echo "GIT_EMAIL=renku@datascience.ch" >> $GITHUB_ENV
      - name: Push chart and images
        uses: SwissDataScienceCenter/renku-actions/publish-chart@v1.4.1
        env:
          CHART_NAME: renku-core
          GITHUB_TOKEN: ${{ secrets.RENKUBOT_GITHUB_TOKEN }}
          DOCKER_USERNAME: ${{ secrets.RENKU_DOCKER_USERNAME }}
          DOCKER_PASSWORD: ${{ secrets.RENKU_DOCKER_PASSWORD }}
      - name: Wait for chart to be available
        run: sleep 120
      - name: Update component version
        uses: SwissDataScienceCenter/renku-actions/update-component-version@v1.4.1
        env:
          CHART_NAME: renku-core
          GITHUB_TOKEN: ${{ secrets.RENKUBOT_GITHUB_TOKEN }}

  coveralls-final:
    name: Aggregate coveralls data
    needs:
      [
        test-linux-cli,
        test-linux-core,
        test-linux-service,
        test-linux-api,
        test-linux-integration,
        test-linux-integration-service,
      ]
    if: always()
    runs-on: ubuntu-latest
    permissions:
      contents: read
    steps:
      - name: Coveralls Finished
        uses: coverallsapp/github-action@master
        with:
          github-token: ${{ secrets.github_token }}
          parallel-finished: true
          base-path: renku/