Upload test results into the database

This will help us find the flaky tests or the rare failures.
timescale · Nov 11, 2022 · 04a08cf · 04a08cf
1 parent f132148
commit 04a08cf
Show file tree

Hide file tree

Showing 16 changed files with 331 additions and 54 deletions.
diff --git a/.github/workflows/linux-build-and-test.yaml b/.github/workflows/linux-build-and-test.yaml
@@ -28,6 +28,7 @@ jobs:
         fi
     
   regress:
+    # Change the JOB_NAME variable below when changing the name.
     name: PG${{ matrix.pg }}${{ matrix.snapshot }} ${{ matrix.name }} ${{ matrix.os }}
     needs: matrixbuilder
     runs-on: ${{ matrix.os }}
@@ -59,7 +60,7 @@ jobs:
         # This is needed because GitHub image macos-10.15 version
         # 20210927.1 did not install OpenSSL so we install openssl
         # explicitly.
-        brew install openssl
+        brew install openssl gawk
         sudo perl -MCPAN -e "CPAN::Shell->notest('install', 'IPC::Run')"
         sudo perl -MCPAN -e "CPAN::Shell->notest('install', 'Test::Most')"
 
@@ -169,9 +170,11 @@ jobs:
       uses: actions/upload-artifact@v3
       with:
         name: Regression diff ${{ matrix.os }} ${{ matrix.name }} ${{ matrix.pg }}
-        path: regression.log
+        path: |
+          regression.log
+          installcheck.log
 
-    - name: Save postmaster.log
+    - name: Save PostgreSQL log
       if: always()
       uses: actions/upload-artifact@v3
       with:
@@ -181,14 +184,16 @@ jobs:
     - name: Stack trace
       if: always() && steps.collectlogs.outputs.coredumps == 'true'
       run: |
-        sudo coredumpctl gdb <<EOT
+        sudo coredumpctl gdb <<<"
           set verbose on
           show debug-file-directory
-          printf "%s\n\n", debug_query_string
+          printf "query = '%s'\n\n", debug_query_string
+          frame function ExceptionalCondition
+          printf "condition = '%s'\n", conditionName
           bt full
-        EOT
+        " | tee stacktrace.log
         ./scripts/bundle_coredumps.sh
-        false
+        grep -C40 "was terminated by signal" postgres.log > postgres-failure.log ||:
 
     - name: Coredumps
       if: always() && steps.collectlogs.outputs.coredumps == 'true'
@@ -206,3 +211,29 @@ jobs:
           build/test/tmp_check/log
           build/tsl/test/tmp_check/log
 
+    - name: Upload test results to the database
+      # Don't upload the results of the flaky check, because the db schema only
+      # supports running one test once per job.
+      if: always() && (! contains(matrix.name, 'Flaky'))
+      env:
+        # GitHub Actions allow you neither to use the env context for the job name,
+        # nor to access the job name from the step context, so we have to
+        # duplicate it to work around this nonsense.
+        JOB_NAME: PG${{ matrix.pg }}${{ matrix.snapshot }} ${{ matrix.name }} ${{ matrix.os }}
+        CI_STATS_DB: ${{ secrets.CI_STATS_DB }}
+        GITHUB_EVENT_NAME: ${{ github.event_name }}
+        GITHUB_REF_NAME: ${{ github.ref_name }}
+        GITHUB_REPOSITORY: ${{ github.repository }}
+        GITHUB_RUN_ATTEMPT: ${{ github.run_attempt }}
+        GITHUB_RUN_ID: ${{ github.run_id }}
+        GITHUB_RUN_NUMBER: ${{ github.run_number }}
+        JOB_STATUS: ${{ job.status }}
+      run: |
+        if [[ "${{ github.event_name }}" == "pull_request" ]] ;
+        then
+            GITHUB_PR_NUMBER="${{ github.event.number }}"
+        else
+            GITHUB_PR_NUMBER=0
+        fi
+        export GITHUB_PR_NUMBER
+        scripts/upload_ci_stats.sh
diff --git a/.github/workflows/sanitizer-build-and-test.yaml b/.github/workflows/sanitizer-build-and-test.yaml
@@ -48,6 +48,8 @@ jobs:
       run: python .github/gh_config_reader.py
 
   sanitizer:
+    # Change the JOB_NAME variable below when changing the name.
+    # Don't use the env variable here because the env context is not accessible.
     name: PG${{ matrix.pg }} Sanitizer ${{ matrix.os }}
     runs-on: ${{ matrix.os }}
     needs: config
@@ -130,9 +132,11 @@ jobs:
       uses: actions/upload-artifact@v3
       with:
         name: Regression diff ${{ matrix.os }} ${{ env.name }} ${{ matrix.pg }}
-        path: regression.log
+        path: |
+          regression.log
+          installcheck.log
 
-    - name: Save postmaster.log
+    - name: Save PostgreSQL log
       if: always()
       uses: actions/upload-artifact@v3
       with:
@@ -142,12 +146,16 @@ jobs:
     - name: Stack trace
       if: always() && steps.collectlogs.outputs.coredumps == 'true'
       run: |
-        sudo coredumpctl gdb <<EOT
-          printf "%s\n\n", debug_query_string
+        sudo coredumpctl gdb <<<"
+          set verbose on
+          show debug-file-directory
+          printf "query = '%s'\n\n", debug_query_string
+          frame function ExceptionalCondition
+          printf "condition = '%s'\n", conditionName
           bt full
-        EOT
+        " | tee stacktrace.log
         ./scripts/bundle_coredumps.sh
-        false
+        grep -C40 "was terminated by signal" postgres.log > postgres-failure.log ||:
 
     - name: Coredumps
       if: always() && steps.collectlogs.outputs.coredumps == 'true'
@@ -161,4 +169,29 @@ jobs:
       uses: actions/upload-artifact@v3
       with:
         name: sanitizer logs ${{ matrix.os }} ${{ env.name }} ${{ matrix.pg }}
-        path: ${{ github.workspace }}/sanitizer.log.*
+        path: ${{ github.workspace }}/sanitizer.log
+
+    - name: Upload test results to the database
+      if: always()
+      env:
+        # GitHub Actions allow you neither to use the env context for the job name,
+        # nor to access the job name from the step context, so we have to
+        # duplicate it to work around this nonsense.
+        JOB_NAME: PG${{ matrix.pg }} ${{ env.name }} ${{ matrix.os }}
+        CI_STATS_DB: ${{ secrets.CI_STATS_DB }}
+        GITHUB_EVENT_NAME: ${{ github.event_name }}
+        GITHUB_REF_NAME: ${{ github.ref_name }}
+        GITHUB_REPOSITORY: ${{ github.repository }}
+        GITHUB_RUN_ATTEMPT: ${{ github.run_attempt }}
+        GITHUB_RUN_ID: ${{ github.run_id }}
+        GITHUB_RUN_NUMBER: ${{ github.run_number }}
+        JOB_STATUS: ${{ job.status }}
+      run: |
+        if [[ "${{ github.event_name }}" == "pull_request" ]] ;
+        then
+            GITHUB_PR_NUMBER="${{ github.event.number }}"
+        else
+            GITHUB_PR_NUMBER=0
+        fi
+        export GITHUB_PR_NUMBER
+        scripts/upload_ci_stats.sh
diff --git a/.github/workflows/windows-build-and-test.yaml b/.github/workflows/windows-build-and-test.yaml
@@ -33,6 +33,7 @@ jobs:
         fi
 
   build:
+    # Change the JOB_NAME variable below when changing the name.
     name: PG${{ matrix.pg }} ${{ matrix.build_type }} ${{ matrix.os }}
     runs-on: ${{ matrix.os }}
     needs: config
@@ -61,13 +62,29 @@ jobs:
       TABLESPACE1: D:\tablespace1\
       TABLESPACE2: D:\tablespace2\
     steps:
+    - name: Setup WSL
+      if: matrix.pg != '12'
+      uses: Vampire/setup-wsl@v1
+      with:
+        additional-packages:
+          cmake
+          gawk
+          gcc
+          git
+          gnupg
+          make
+          postgresql-client
+          postgresql-common
+          tree
+
     - name: Configure git
       # Since we want to reuse the checkout in the WSL environment
       # we have to prevent git from changing the line ending in the
       # shell scripts as that would break them.
       run: |
         git config --global core.autocrlf false
         git config --global core.eol lf
+
     - name: Checkout TimescaleDB source
       uses: actions/checkout@v3
       # Use a cache for the PostgreSQL installation to speed things up
@@ -107,6 +124,7 @@ jobs:
         icacls ${{ env.TABLESPACE2 }} /grant runneradmin:F /T
         copy build_win/test/postgresql.conf ${{ env.PGDATA }}
         copy build_win/test/pg_hba.conf ${{ env.PGDATA }}
+        icacls . /grant runneradmin:F /T
         ~/PostgreSQL/${{ matrix.pg }}/bin/pg_ctl start -o "${{ matrix.pg_config }}" --log=postgres.log
         ~/PostgreSQL/${{ matrix.pg }}/bin/pg_isready -U postgres -d postgres --timeout=30
         ~/PostgreSQL/${{ matrix.pg }}/bin/psql -U postgres -d postgres -c 'CREATE USER root SUPERUSER LOGIN;'
@@ -120,17 +138,6 @@ jobs:
         echo "Data directory:"
         ~/PostgreSQL/${{ matrix.pg }}/bin/psql -U postgres -d postgres -c 'SHOW data_directory;'
 
-    - name: Setup WSL
-      if: matrix.pg != '12'
-      uses: Vampire/setup-wsl@v1
-      with:
-        additional-packages:
-          cmake
-          gcc
-          git
-          gnupg
-          make
-          postgresql-common
     - name: Install postgres for test runner
       if: matrix.pg != '12'
       shell: wsl-bash {0}
@@ -149,10 +156,10 @@ jobs:
         # isolationtester is only packaged with pg14+ so we would have to build our own postgres
         # to get it for earlier versions so we skip it for < 14.
         if [[ "${{ matrix.pg }}" == "14" ]]; then
-          make -C build_wsl isolationchecklocal
+          make -C build_wsl isolationchecklocal | tee -a installcheck.log
         fi
 
-        make -C build_wsl regresschecklocal IGNORES="${{ matrix.ignores }}"
+        make -C build_wsl regresschecklocal IGNORES="${{ matrix.ignores }}" | tee -a installcheck.log
 
     - name: Setup postgres cluster for TSL tests
       if: matrix.pg != '12'
@@ -177,43 +184,63 @@ jobs:
         # isolationtester is only packaged with pg14+ so we would have to build our own postgres
         # to get it for earlier versions so we skip it for < 14.
         if [[ "${{ matrix.pg }}" == "14" ]]; then
-          make -C build_wsl isolationchecklocal-t
+          make -C build_wsl isolationchecklocal-t | tee -a installcheck.log
         fi
 
-        make -C build_wsl -k regresschecklocal-t IGNORES="${{ matrix.tsl_ignores }}" SKIPS="${{ matrix.tsl_skips }} ${{ matrix.tsl_skips_version }}"
+        make -C build_wsl -k regresschecklocal-t IGNORES="${{ matrix.tsl_ignores }}" SKIPS="${{ matrix.tsl_skips }} ${{ matrix.tsl_skips_version }}" | tee -a installcheck.log
 
     - name: Show regression diffs
-      if: always() && matrix.pg != '12'
-      shell: python
       id: collectlogs
+      if: always() && matrix.pg != '12'
+      env:
+        WSLENV: GITHUB_OUTPUT
+      shell: wsl-bash {0}
       run: |
-        import re
-        import os
-        from pathlib import Path
-
-        for path in Path('build_wsl').rglob('regression.out'):
-          for line in path.open():
-            if re.search('failed', line, re.IGNORECASE):
-              print(line, end='')
-
-        for path in Path('build_wsl').rglob('regression.diffs'):
-          for line in path.open():
-            print(line, end='')
-        
-        with open(os.environ['GITHUB_OUTPUT'], 'a') as output:
-          print('regression_diff=true', file=output)
+        find . -name regression.diffs -exec cat {} + > regression.log
+        if [[ -s regression.log ]]; then echo "regression_diff=true" >>$GITHUB_OUTPUT; fi
+        grep -e 'FAILED' -e 'failed (ignored)' installcheck.log || true
+        cat regression.log
 
     - name: Save regression diffs
       if: always() && matrix.pg != '12' && steps.collectlogs.outputs.regression_diff == 'true'
       uses: actions/upload-artifact@v3
       with:
         name: Regression diff ${{ matrix.os }} ${{ matrix.name }} ${{ matrix.pg }}
-        path: regression.log
+        path: |
+          regression.log
+          installcheck.log
 
-    - name: Save postgres log
+    - name: Save PostgreSQL log
       if: always()
       uses: actions/upload-artifact@v3
       with:
-        name: Postgres log ${{ matrix.os }} ${{ matrix.name }} ${{ matrix.pg }}
+        name: PostgreSQL log ${{ matrix.os }} ${{ matrix.name }} ${{ matrix.pg }}
         path: postgres.log
 
+    - name: Upload test results to the database
+      if: always() && matrix.pg != '12'
+      shell: wsl-bash {0}
+      env:
+        # Update when adding new variables.
+        WSLENV: "JOB_NAME:CI_STATS_DB:GITHUB_EVENT_NAME:GITHUB_REF_NAME:GITHUB_REPOSITORY:GITHUB_RUN_ATTEMPT:GITHUB_RUN_ID:GITHUB_RUN_NUMBER:JOB_STATUS"
+        # GitHub Actions allow you neither to use the env context for the job name,
+        # nor to access the job name from the step context, so we have to
+        # duplicate it to work around this nonsense.
+        JOB_NAME: PG${{ matrix.pg }} ${{ matrix.build_type }} ${{ matrix.os }}
+        CI_STATS_DB: ${{ secrets.CI_STATS_DB }}
+        GITHUB_EVENT_NAME: ${{ github.event_name }}
+        GITHUB_REF_NAME: ${{ github.ref_name }}
+        GITHUB_REPOSITORY: ${{ github.repository }}
+        GITHUB_RUN_ATTEMPT: ${{ github.run_attempt }}
+        GITHUB_RUN_ID: ${{ github.run_id }}
+        GITHUB_RUN_NUMBER: ${{ github.run_number }}
+        JOB_STATUS: ${{ job.status }}
+      run: |
+        if [[ "${{ github.event_name }}" == "pull_request" ]] ;
+        then
+            GITHUB_PR_NUMBER="${{ github.event.number }}"
+        else
+            GITHUB_PR_NUMBER=0
+        fi
+        export GITHUB_PR_NUMBER
+        scripts/upload_ci_stats.sh