add coverage report (#6045)

Co-authored-by: Dmytro Rezchykov <dmitry.rezchykov@zazmic.com>

.github/workflows/test-command.yml

@@ -195,6 +195,16 @@ jobs:
             **/normalization_test_output/**/build/compiled/airbyte_utils/**
             **/normalization_test_output/**/build/run/airbyte_utils/**
             **/normalization_test_output/**/models/generated/**
+
+      - name: Test coverage reports artifacts
+        if: github.event.inputs.comment-id && success()
+        uses: actions/upload-artifact@v2
+        with:
+          name: test-reports
+          path: |
+            **/${{ github.event.inputs.connector }}/htmlcov/**
+          retention-days: 3
+
       - name: Report Status
         if: github.ref == 'refs/heads/master' && always()
         run: ./tools/status/report.sh ${{ github.event.inputs.connector }} ${{github.repository}} ${{github.run_id}} ${{steps.test.outcome}}
@@ -209,6 +219,7 @@ jobs:
           comment-id: ${{ github.event.inputs.comment-id }}
           body: |
             > :white_check_mark: ${{github.event.inputs.connector}} https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}
+            ${{env.PYTHON_UNITTEST_COVERAGE_REPORT}}
       - name: Add Failure Comment
         if: github.event.inputs.comment-id && failure()
         uses: peter-evans/create-or-update-comment@v1

buildSrc/src/main/groovy/airbyte-python.gradle

@@ -13,28 +13,28 @@ class AirbytePythonConfiguration {
 class Helpers {
     static addTestTaskIfTestFilesFound(Project project, String testFilesDirectory, String taskName, taskDependencies) {
         """
-        This method verifies if there are test files in a directory before adding the pytest task to run tests on that directory. This is needed 
+        This method verifies if there are test files in a directory before adding the pytest task to run tests on that directory. This is needed
         because if there are no tests in that dir and we run pytest on it, it exits with exit code 5 which gradle takes to mean that the process
         failed, since it's non-zero. This means that if a module doesn't need a unit or integration test, it still needs to add a dummy test file
-        like: 
-        
+        like:
+
         ```
         def make_ci_pass_test():
             assert True
         ```
-        
+
         So we use this method to leverage pytest's test discovery rules (https://docs.pytest.org/en/6.2.x/goodpractices.html#conventions-for-python-test-discovery)
-        to selectively run pytest based on whether there seem to be test files in that directory. 
-        Namely, if the directory contains a file whose name is test_*.py or *_test.py then it's a test. 
-       
+        to selectively run pytest based on whether there seem to be test files in that directory.
+        Namely, if the directory contains a file whose name is test_*.py or *_test.py then it's a test.
+
         See https://github.com/airbytehq/airbyte/issues/4979 for original context
         """
         if (project.file(testFilesDirectory).exists()) {
 
             project.projectDir.toPath().resolve(testFilesDirectory).traverse(type: FileType.FILES, nameFilter: ~/(^test_.*|.*_test)\.py$/) { file ->
                 project.task(taskName, type: PythonTask, dependsOn: taskDependencies) {
                     module = "pytest"
-                    command = "-s ${testFilesDirectory}"
+                    command = "-s ${testFilesDirectory} --cov=./ --cov-config=${project.rootProject.file('tools/python/.coveragerc').absolutePath} --cov-report html --cov-report term"
                 }
                 // If a file is found, terminate the traversal, thus causing this task to be declared at most once
                 return FileVisitResult.TERMINATE
@@ -68,6 +68,7 @@ class AirbytePythonPlugin implements Plugin<Project> {
             pip 'mypy:0.812'
             pip 'isort:5.6.4'
             pip 'pytest:6.1.2'
+            pip 'pytest-cov:2.12.1'
             pip 'pip:21.1.3'
         }
 

tools/bin/ci_integration_test.sh

@@ -46,10 +46,27 @@ run | tee build.out
 # return status of "run" command, not "tee"
 # https://tldp.org/LDP/abs/html/internalvariables.html#PIPESTATUSREF
 run_status=${PIPESTATUS[0]}
+
 test $run_status == "0" || {
+   # Build failed
+   link=$(cat build.out | grep -A1 "Publishing build scan..." | tail -n1 | tr -d "\n")
    # Save gradle scan link to github GRADLE_SCAN_LINK variable for next job.
    # https://docs.github.com/en/actions/reference/workflow-commands-for-github-actions#setting-an-environment-variable
-   LINK=$(cat build.out | grep -A1 "Publishing build scan..." | tail -n1 | tr -d "\n")
-   echo "GRADLE_SCAN_LINK=$LINK" >> $GITHUB_ENV
+   echo "GRADLE_SCAN_LINK=$link" >> $GITHUB_ENV
+   exit $run_status
 }
-exit $run_status
+
+# Build successed
+coverage_report=`sed -n '/^[ \t]*-\+ coverage: /,/TOTAL   /p' build.out`
+
+if ! test -z "$coverage_report"
+then
+   echo "PYTHON_UNITTEST_COVERAGE_REPORT<<EOF" >> $GITHUB_ENV
+   echo "Python tests coverage:" >> $GITHUB_ENV
+   echo '```' >> $GITHUB_ENV
+   echo "$coverage_report" >> $GITHUB_ENV
+   echo '```' >> $GITHUB_ENV
+   echo "EOF" >> $GITHUB_ENV
+else
+   echo "PYTHON_UNITTEST_COVERAGE_REPORT=No Python unittests run" >> $GITHUB_ENV
+fi

tools/python/.coveragerc

@@ -0,0 +1,7 @@
+[run]
+omit =
+   .venv/*
+   main.py
+   setup.py
+   unit_tests/*
+   integration_tests/*