run-tests.yml

name: 🧪 QGIS tests

on:
  push:
    branches:
      - master
      - release-**
      - queued_ltr_backports
    paths:
    - 'src/**'
    - 'external/**'
    - 'python/**'
    - 'tests/**'
    - 'CMakeLists.txt'
    - '.github/workflows/run-tests.yml'
    - '.docker/**'
    - '.ci/**'
  pull_request:
    branches:
      - master
      - release-**
    # paths cannot be filtered on this workflow on pull request as it is a required one in the branch protection
    # feature request and hacks: https://github.community/t/feature-request-conditional-required-checks/16761

jobs:
  build:
    env:
      DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
      DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
      GH_WORKSPACE: ${{ github.workspace }} # used in docker compose
      RUN_FLAKY_TESTS: ${{ contains( github.event.pull_request.labels.*.name, 'run flaky tests') }}

    runs-on: ubuntu-latest

    strategy:
      matrix:
        # tests run on 20.04 (Qt 5.12), compile test on 20.10 (Qt 5.14) and 21.04 (Qt 5.15)
        include:
          - distro-version: '20.04'
            qt-version: 5
            run-tests: true
            docker-tag-suffix: ''
            with-qt6: OFF
            with-qt5: ON
            with-3d: ON
            with-quick: ON
            patch-qt-3d: true
            LD_PRELOAD: /lib/x86_64-linux-gnu/libSegFault.so

          - distro-version: '21.04'
            qt-version: 5
            run-tests: false
            docker-tag-suffix: _21.04
            with-qt6: OFF
            with-qt5: ON
            with-3d: OFF
            with-quick: ON
            patch-qt-3d: false
            LD_PRELOAD: /lib/x86_64-linux-gnu/libSegFault.so

          - distro-version: 'rawhide'
            qt-version: 6
            run-tests: false
            docker-tag-suffix: ''
            with-qt6: ON
            with-qt5: OFF
            with-3d: OFF
            with-quick: OFF
            patch-qt-3d: false
            LD_PRELOAD: ''


      fail-fast: false

    outputs:
      compile_outcome: ${{ steps.compile.outcome }}
      tests_failing: ${{ steps.tests.outputs.TESTS_FAILING }}
      cdash_url: ${{ steps.tests.outputs.CDASH_URL }}
      runners_outcome: ${{ steps.runners.outcome }}

    steps:
      - name: Checkout
        uses: actions/checkout@v2

      - name: Set vars
        env:
          GITHUB_EVENT_NAME: ${{ github.event_name }}
          GITHUB_REF: ${{ github.ref }}
          GITHUB_PR_NUMBER: ${{github.event.number}}
        run: |
          # Be aware that these instructions are duplicated in run-tests job
          DOCKER_TAG=$(echo $( [[ ${GITHUB_EVENT_NAME} =~ ^pull_request$ ]] && echo ${GITHUB_BASE_REF} || echo ${GITHUB_REF##*/} ) | sed 's/^master$/latest/')${{ matrix.docker-tag-suffix }}
          CTEST_BUILD_NAME=$( [[ ${GITHUB_EVENT_NAME} =~ ^pull_request$ ]] && echo "PR${GITHUB_PR_NUMBER}" || echo ${GITHUB_REF##*/} )"_${GITHUB_SHA}"
          echo "DOCKER_TAG=${DOCKER_TAG}" >> $GITHUB_ENV
          echo "CTEST_BUILD_NAME=${CTEST_BUILD_NAME}" >> $GITHUB_ENV

      - name: Print vars
        run: |
          echo DOCKER_TAG: ${DOCKER_TAG}
          echo CTEST_BUILD_NAME: ${CTEST_BUILD_NAME}

      - name: Build Docker Container with Build Environment
        id: docker-build
        uses: whoan/docker-build-with-cache-action@v5
        with:
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}
          registry: docker.pkg.github.com
          image_name: qgis3-qt6-build-deps
          dockerfile: .docker/qgis3-qt${{ matrix.qt-version }}-build-deps.dockerfile
          build_extra_args: "--build-arg=DISTRO_VERSION=${{ matrix.distro-version }}"
          push_git_tag: true
          push_image_and_stages: on:push
          pull_image_and_stages: ${{ github.event_name != 'workflow_dispatch' }}

      - name: Tag image
        run: |
          docker tag ${DOCKER_IMAGE} qgis3-build-deps
        env:
          DOCKER_IMAGE: ${{ steps.docker-build.outputs.FULL_IMAGE_NAME }}

      - name: Push Qt5 deps image to Docker hub
        if: ${{ github.repository_owner == 'qgis' && github.event_name != 'pull_request' && matrix.qt-version == '5' }}
        run: |
          docker tag qgis3-build-deps qgis/qgis3-build-deps:${DOCKER_TAG}
          docker login -u="${DOCKER_USERNAME}" -p="${DOCKER_PASSWORD}"
          docker push "qgis/qgis3-build-deps:${DOCKER_TAG}"

      - name: Prepare build cache for pull request
        uses: pat-s/always-upload-cache@v2.1.5
        if: github.event_name == 'pull_request'
        with:
          path: /home/runner/QGIS/.ccache
          key: build-ccache-${{ matrix.distro-version }}-qt${{ matrix.qt-version }}-${{ github.actor }}-${{ github.head_ref }}-${{ github.sha }}
          # The head_ref or source branch of the pull request in a workflow run.
          # The base_ref or target branch of the pull request in a workflow run.
          restore-keys: |
            build-ccache-${{ matrix.distro-version }}-qt${{ matrix.qt-version }}-${{ github.actor }}-${{ github.head_ref }}-
            build-ccache-${{ matrix.distro-version }}-qt${{ matrix.qt-version }}-refs/heads/${{ github.base_ref }}-
            build-ccache-${{ matrix.distro-version }}-qt${{ matrix.qt-version }}-refs/heads/master-

      - name: Prepare build cache for branch/tag
        # use a fork of actions/cache@v2 to upload cache even when the build or test failed
        uses: pat-s/always-upload-cache@v2.1.5
        if: github.event_name != 'pull_request'
        with:
          path: /home/runner/QGIS/.ccache
          # The branch or tag ref that triggered the workflow run. For branches this in the format refs/heads/<branch_name>, and for tags it is refs/tags/<tag_name>
          key: build-ccache-${{ matrix.distro-version }}-qt${{ matrix.qt-version }}-${{ github.ref }}-${{ github.sha }}
          restore-keys: |
            build-ccache-${{ matrix.distro-version }}-qt${{ matrix.qt-version }}-${{ github.ref }}-
            build-ccache-${{ matrix.distro-version }}-qt${{ matrix.qt-version }}-refs/heads/master-

      - name: Compile QGIS
        id: compile
        run: |
          docker run -t --name qgis_container \
                     -v $(pwd):/root/QGIS \
                     -v /home/runner/QGIS/.ccache:/root/.ccache \
                     --env-file .docker/docker-variables.env \
                     --env PUSH_TO_CDASH=true \
                     --env WITH_QT5=${{ matrix.with-qt5 }} \
                     --env WITH_QT6=${{ matrix.with-qt6 }} \
                     --env WITH_QUICK=${{ matrix.with-quick }} \
                     --env WITH_3D=${{ matrix.with-3d }} \
                     --env PATCH_QT_3D=${{ matrix.patch-qt-3d }} \
                     --env LD_PRELOAD=${{ matrix.LD_PRELOAD }} \
                     qgis3-build-deps \
                     /root/QGIS/.docker/docker-qgis-build.sh

      - name: Push artifact
        id: push_artifact
        if: ${{ matrix.run-tests }}
        run: tar --exclude='*.o' -cvzf build.tgz build

      - uses: actions/upload-artifact@v2.2.4
        if: ${{ matrix.run-tests }}
        with:
          name: build.tgz
          path: ./build.tgz
          retention-days: 1

#      - name: Test QGIS runners
#        id: runners
#        if: ${{ matrix.distro-version == env.DEFAULT_DISTRO_VERSION }}
#        run: |
#          docker run -d --name qgis-testing-environment \
#                     -v $(pwd):/root/QGIS \
#                     -v $(pwd)/tests/src/python:/tests_directory \
#                     -v $(pwd)/.docker/qgis_resources/test_runner:/usr/bin/test_runner \
#                     -v $(pwd)/.docker/qgis_resources/supervisor:/etc/supervisor \
#                     -e QGIS_BUILD_PATH=/root/QGIS/build/output/bin/qgis \
#                     -e TEST_RUNNER_PATH=/usr/bin/test_runner/qgis_testrunner.py \
#                     -e DISPLAY=:99 \
#                     qgis_image \
#                     /usr/bin/supervisord -c /etc/supervisor/supervisord.conf
#          # Wait for xvfb to finish starting
#          printf "Waiting for the docker...🐳..."
#          sleep 10
#          echo " done 🥩"
#
#          declare -A testrunners
#          # Passing cases:
#          testrunners["test_testrunner.run_passing"]=0
#          testrunners["test_testrunner.run_skipped_and_passing"]=0
#          # Failing cases:
#          testrunners["test_testrunner"]=1
#          testrunners["test_testrunner.run_all"]=1
#          testrunners["test_testrunner.run_failing"]=1
#          set +e # do not exit on error
#          # Run tests in the docker
#          for i in "${!testrunners[@]}"
#          do
#            echo "::group::docker_test_runner_${i}"
#            echo "test ${i}..."
#            docker exec -i qgis-testing-environment sh -c "cd /tests_directory && /usr/bin/test_runner/qgis_testrunner.sh ${i}"
#            [[ $? -eq "${testrunners[$i]}" ]] && echo "success" || exit 1
#            echo "::endgroup::"
#          done
#          set -e # switch back
#          docker stop qgis-testing-environment


#  tests-report-comment:
#    name: Write tests report in a comment
#    needs: build
#    runs-on: ubuntu-latest
#    if: always() && ( needs.build.result == 'failure' || needs.build.result == 'success' ) && github.event_name == 'pull_request'
#    steps:
#      - name: Find Comment
#        uses: peter-evans/find-comment@v1
#        id: find-comment
#        with:
#          issue-number: ${{ github.event.pull_request.number }}
#          comment-author: 'github-actions[bot]'
#          body-includes: Tests report
#
#      - name: Create comment
#        if: ${{ steps.find-comment.outputs.comment-id == 0 }}
#        uses: peter-evans/create-or-update-comment@v1
#        id: create-comment
#        with:
#          issue-number: ${{ github.event.pull_request.number }}
#          body: |
#            **Tests report**
#
#      - name: Process
#        id: process-vars
#        env:
#          COMMENT_FOUND: ${{ steps.find-comment.outputs.comment-id }}
#          COMMENT_CREATED: ${{ steps.create-comment.outputs.comment-id }}
#          COMMIT_SHA: ${{ github.event.pull_request.head.sha }}
#          CDASH_URL: ${{ needs.build.outputs.cdash_url }}
#          COMPILE_OUTCOME: ${{ needs.build.outputs.compile_outcome }}
#          TESTS_FAILING: ${{ needs.build.outputs.tests_failing }}
#          RUNNERS_OUTCOME: ${{ needs.build.outputs.runners_outcome }}
#        run: |
#          echo "::set-output name=COMMENT_ID::"$([[ "${COMMENT_FOUND}" -eq "0" ]] && echo ${COMMENT_CREATED} || echo ${COMMENT_FOUND})
#          if [[ ${COMPILE_OUTCOME} != "success" ]]; then
#            echo "::set-output name=COMMENT_BODY::${COMMIT_SHA} :scream: compilation failed"
#          elif [[ ${TESTS_FAILING} != "0" ]]; then
#            echo "::set-output name=COMMENT_BODY::${COMMIT_SHA} :fire: ${TESTS_FAILING} unit-tests are failing ${CDASH_URL}"
#          elif [[ ${RUNNERS_OUTCOME} != "success" ]]; then
#            echo "::set-output name=COMMENT_BODY::${COMMIT_SHA} :broken_heart: QGIS runners test failed"
#          else
#            echo "::set-output name=COMMENT_BODY::${COMMIT_SHA} :sunglasses: unit-tests succeeded"
#          fi
#
#      - name: Update comment
#        uses: peter-evans/create-or-update-comment@v1
#        with:
#          comment-id: ${{ steps.process-vars.outputs.COMMENT_ID }}
#          edit-mode: append
#          body: ${{ steps.process-vars.outputs.COMMENT_BODY }}

  run-tests:
    name: Run tests
    env:
      DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
      DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
      GH_WORKSPACE: ${{ github.workspace }} # used in docker compose

    runs-on: ubuntu-latest

    strategy:
      matrix:
        distro-version: [20.04]
        qt-version: [5]
        test-batch: [ALL_BUT_PROVIDERS, POSTGRES, HANA]
        docker-target: [binary-only]

        include:
          - distro-version: 20.04
            qt-version: 5
            test-batch: ORACLE
            docker-target: binary-for-oracle

          # - distro-version: rawhide
          #   qt-version: 6
          #   test-batch: ALL_BUT_PROVIDERS
          #   docker-target: single

      fail-fast: false

    services:
      # Run httpbin container for QgsNetwor* tests
      httpbin:
        image: kennethreitz/httpbin:latest
        ports:
          - 8082:80

    needs: build

    steps:

      - name: Print disk space
        run: |
          echo "DF -H"
          sudo df -h

      - name: Checkout
        uses: actions/checkout@v2

      - name: Set vars
        env:
          GITHUB_EVENT_NAME: ${{ github.event_name }}
          GITHUB_REF: ${{ github.ref }}
          GITHUB_PR_NUMBER: ${{github.event.number}}
        run: |
          # Be aware that these instructions are duplicated in build job
          CTEST_BUILD_NAME=$( [[ ${GITHUB_EVENT_NAME} =~ ^pull_request$ ]] && echo "PR${GITHUB_PR_NUMBER}" || echo ${GITHUB_REF##*/} )"_${GITHUB_SHA}_${{ matrix.test-batch }}"
          echo "CTEST_BUILD_NAME=${CTEST_BUILD_NAME}" >> $GITHUB_ENV

      - name: Print vars
        run: |
          echo CTEST_BUILD_NAME: ${CTEST_BUILD_NAME}

      - name: Build Docker Container with Testing Environment
        id: docker-build
        uses: whoan/docker-build-with-cache-action@v5
        with:
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}
          registry: docker.pkg.github.com
          image_name: qgis3-qt${{ matrix.qt-version }}-build-deps-bin-only
          dockerfile: .docker/qgis3-qt${{ matrix.qt-version }}-build-deps.dockerfile
          build_extra_args: "--target ${{ matrix.docker-target }} --build-arg=DISTRO_VERSION=${{ matrix.distro-version }}"
          push_git_tag: true
          push_image_and_stages: on:push
          pull_image_and_stages: ${{ github.event_name != 'workflow_dispatch' }}

      - name: Tag image
        run: |
          docker tag ${DOCKER_IMAGE} qgis3-build-deps-binary-image
        env:
          DOCKER_IMAGE: ${{ steps.docker-build.outputs.FULL_IMAGE_NAME }}

      - name: Print disk space
        run: |
          echo "DOCKER IMAGES"
          docker images
          echo "DF -H"
          sudo df -h

      - name: Download build artifact
        uses: actions/download-artifact@v2
        with:
          name: build.tgz
          path: .

      - name: Extract build artifact
        run: |
          tar xvzf build.tgz
          rm -Rf build.tgz

      - name: Print disk space
        run: |
          echo "DF -H"
          sudo df -h

      - name: Run tests
        id: tests
        env:
          TEST_BATCH: ${{matrix.test-batch}}
          QGIS_HTTPBIN_HOST: "localhost:8082"
        run: |
          DOCKERFILE=$( ( [[ ${{ matrix.test-batch }} == "ORACLE"   ]] && echo "docker-compose-testing-oracle.yml" ) \
                     || ( [[ ${{ matrix.test-batch }} == "POSTGRES" ]] && echo "docker-compose-testing-postgres.yml" ) \
                     || echo "docker-compose-testing.yml" )
          [[ ${{ matrix.test-batch }} == "ORACLE" ]] && sudo rm -rf /usr/share/dotnet/sdk
          echo "TEST_BATCH=$TEST_BATCH"
          echo "DOCKERFILE=$DOCKERFILE"
          docker-compose -f .docker/$DOCKERFILE run qgis-deps /root/QGIS/.docker/docker-qgis-test.sh $TEST_BATCH