From eeaf16c263728c962f9c8f66b1d98a94bd38ab11 Mon Sep 17 00:00:00 2001 From: Matt Page Date: Tue, 7 May 2024 14:58:47 -0700 Subject: [PATCH 1/4] Log TSAN warnings to separate files and archive them This ensures we don't lose races that occur in subprocesses or interleave races from workers running in parallel. --- .github/workflows/build.yml | 2 ++ .github/workflows/reusable-tsan.yml | 12 +++++++++++- Tools/tsan/supressions.txt | 3 +++ 3 files changed, 16 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 8f966591c30235..6ac42a33905b92 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -494,6 +494,7 @@ jobs: config_hash: ${{ needs.check_source.outputs.config_hash }} options: ./configure --config-cache --with-thread-sanitizer --with-pydebug suppressions_path: Tools/tsan/supressions.txt + tsan_logs_artifact_name: tsan-logs-default build_tsan_free_threading: name: 'Thread sanitizer (free-threading)' @@ -504,6 +505,7 @@ jobs: config_hash: ${{ needs.check_source.outputs.config_hash }} options: ./configure --config-cache --disable-gil --with-thread-sanitizer --with-pydebug suppressions_path: Tools/tsan/suppressions_free_threading.txt + tsan_logs_artifact_name: tsan-logs-free-threading # CIFuzz job based on https://google.github.io/oss-fuzz/getting-started/continuous-integration/ cifuzz: diff --git a/.github/workflows/reusable-tsan.yml b/.github/workflows/reusable-tsan.yml index 48bd5b547e8cba..788a6082a7333a 100644 --- a/.github/workflows/reusable-tsan.yml +++ b/.github/workflows/reusable-tsan.yml @@ -11,6 +11,10 @@ on: description: 'A repo relative path to the suppressions file' required: true type: string + tsan_logs_artifact_name: + description: 'Name of the TSAN logs artifact. Must be unique for each job.' + required: true + type: string jobs: build_tsan_reusable: @@ -41,7 +45,7 @@ jobs: sudo sysctl -w vm.mmap_rnd_bits=28 - name: TSAN Option Setup run: | - echo "TSAN_OPTIONS=suppressions=${GITHUB_WORKSPACE}/${{ inputs.suppressions_path }}" >> $GITHUB_ENV + echo "TSAN_OPTIONS=log_path=${GITHUB_WORKSPACE}/tsan_log suppressions=${GITHUB_WORKSPACE}/${{ inputs.suppressions_path }} handle_segv=0" >> $GITHUB_ENV echo "CC=clang" >> $GITHUB_ENV echo "CXX=clang++" >> $GITHUB_ENV - name: Add ccache to PATH @@ -60,3 +64,9 @@ jobs: run: make pythoninfo - name: Tests run: ./python -m test --tsan -j4 + - name: Archive TSAN logs + if: always() + uses: actions/upload-artifact@v4 + with: + name: ${{ inputs.tsan_logs_artifact_name }} + path: tsan_log.* diff --git a/Tools/tsan/supressions.txt b/Tools/tsan/supressions.txt index c778c791eacce8..22ba9d6ba2ab4d 100644 --- a/Tools/tsan/supressions.txt +++ b/Tools/tsan/supressions.txt @@ -2,3 +2,6 @@ # reference: https://github.com/google/sanitizers/wiki/ThreadSanitizerSuppressions race:get_allocator_unlocked race:set_allocator_unlocked + +# https://gist.github.com/mpage/daaf32b39180c1989572957b943eb665 +thread:pthread_create From e6b45377774be18e60d17ecf11ea939059473b07 Mon Sep 17 00:00:00 2001 From: Matt Page Date: Thu, 9 May 2024 15:29:59 -0700 Subject: [PATCH 2/4] Don't emit a warning if no files are found --- .github/workflows/reusable-tsan.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/reusable-tsan.yml b/.github/workflows/reusable-tsan.yml index 788a6082a7333a..4061673f96590b 100644 --- a/.github/workflows/reusable-tsan.yml +++ b/.github/workflows/reusable-tsan.yml @@ -70,3 +70,4 @@ jobs: with: name: ${{ inputs.tsan_logs_artifact_name }} path: tsan_log.* + if-no-files-found: ignore From 8a77f0b88a0a37fc3ea0c4c918835fa3e940a318 Mon Sep 17 00:00:00 2001 From: Matt Page Date: Thu, 9 May 2024 15:47:13 -0700 Subject: [PATCH 3/4] Display the first 1000 lines of each tsan log --- .github/workflows/reusable-tsan.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/reusable-tsan.yml b/.github/workflows/reusable-tsan.yml index 4061673f96590b..691b0d90516a5f 100644 --- a/.github/workflows/reusable-tsan.yml +++ b/.github/workflows/reusable-tsan.yml @@ -64,6 +64,8 @@ jobs: run: make pythoninfo - name: Tests run: ./python -m test --tsan -j4 + - name: Display TSAN logs + run: find ${GITHUB_WORKSPACE} -name 'tsan_log.*' | xargs head -n 1000 - name: Archive TSAN logs if: always() uses: actions/upload-artifact@v4 From 30d578a3a0404af4bc01b0f9066b6921205f16fe Mon Sep 17 00:00:00 2001 From: Matt Page Date: Thu, 9 May 2024 16:04:20 -0700 Subject: [PATCH 4/4] Always show logs --- .github/workflows/reusable-tsan.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/reusable-tsan.yml b/.github/workflows/reusable-tsan.yml index 691b0d90516a5f..b6d5d8fa1c7157 100644 --- a/.github/workflows/reusable-tsan.yml +++ b/.github/workflows/reusable-tsan.yml @@ -65,6 +65,7 @@ jobs: - name: Tests run: ./python -m test --tsan -j4 - name: Display TSAN logs + if: always() run: find ${GITHUB_WORKSPACE} -name 'tsan_log.*' | xargs head -n 1000 - name: Archive TSAN logs if: always()