Speedup clang tidy (#3219)

We make ccache save a log file, from which we find out which files need a rebuild. Instead of running clang-tidy on every file, we now only run it on those files that had a miss in ccache. This significantly speeds up clang-tidy in CIs. Note that we treat clang-tidy warnings as errors in CIs. So if there is a hit in the ccache's cache, the file is free of clang-tidy warnings. Also note that we set CCACHE_EXTRAFILES to the clang-tidy configuration file. So any changes to the config file will invalidate the cache as it should. If a CI job successfully passes the compilation stage with ccache enabled, but fails at the clang-tidy stage, the ccache's cache has been updated. One might think this will cause issues because we now have files that have been cached by ccache and meanwhile have failed the clang-tidy checks. However, this is not an issue for Github CIs. This is because ccache's cache will not be saved as a Github cache, if a CI fails. Fix various clang-tidy warnings in Tests/. They were not reported before because clang-tidy was not used on the tests.
AMReX-Codes · Mar 30, 2023 · d781598 · d781598
1 parent b41c441
commit d781598
Show file tree

Hide file tree

Showing 81 changed files with 629 additions and 567 deletions.
diff --git a/.github/workflows/clang.yml b/.github/workflows/clang.yml
@@ -34,6 +34,8 @@ jobs:
         export CCACHE_COMPRESS=1
         export CCACHE_COMPRESSLEVEL=10
         export CCACHE_MAXSIZE=125M
+        export CCACHE_EXTRAFILES=${{ github.workspace }}/.clang-tidy
+        export CCACHE_LOGFILE=${{ github.workspace }}/ccache.log.txt
         ccache -z
 
         mkdir build
@@ -52,13 +54,16 @@ jobs:
             -DCMAKE_C_COMPILER=$(which clang-7)         \
             -DCMAKE_CXX_COMPILER=$(which clang++-7)     \
             -DCMAKE_Fortran_COMPILER=$(which gfortran)  \
-            -DAMReX_CLANG_TIDY=ON                     \
-            -DAMReX_CLANG_TIDY_WERROR=ON              \
             -DCMAKE_CXX_COMPILER_LAUNCHER=ccache
         make -j 2
         make install
         make test_install
 
+        ${{github.workspace}}/Tools/C_scripts/mmclt.py --input ${{github.workspace}}/ccache.log.txt
+        make -j2 -f clang-tidy-ccache-misses.mak \
+            CLANG_TIDY=clang-tidy-12 \
+            CLANG_TIDY_ARGS="--config-file=${{github.workspace}}/.clang-tidy --warnings-as-errors=*"
+
         export PATH=/tmp/my-amrex/bin:$PATH
         which fcompare
 
@@ -90,6 +95,8 @@ jobs:
         export CCACHE_COMPRESS=1
         export CCACHE_COMPRESSLEVEL=10
         export CCACHE_MAXSIZE=125M
+        export CCACHE_EXTRAFILES=${{ github.workspace }}/.clang-tidy
+        export CCACHE_LOGFILE=${{ github.workspace }}/ccache.log.txt
         ccache -z
 
         mkdir build
@@ -107,11 +114,14 @@ jobs:
             -DCMAKE_C_COMPILER=$(which clang-14)      \
             -DCMAKE_CXX_COMPILER=$(which clang++-14)  \
             -DCMAKE_Fortran_COMPILER=$(which gfortran)  \
-            -DAMReX_CLANG_TIDY=ON                     \
-            -DAMReX_CLANG_TIDY_WERROR=ON              \
             -DCMAKE_CXX_COMPILER_LAUNCHER=ccache
         make -j 2
 
+        ${{github.workspace}}/Tools/C_scripts/mmclt.py --input ${{github.workspace}}/ccache.log.txt
+        make -j2 -f clang-tidy-ccache-misses.mak \
+            CLANG_TIDY=clang-tidy-14 \
+            CLANG_TIDY_ARGS="--config-file=${{github.workspace}}/.clang-tidy --warnings-as-errors=*"
+
         ctest --output-on-failure -E GhostsAndVirtuals
 
         ccache -s
@@ -139,12 +149,18 @@ jobs:
         export CCACHE_COMPRESS=1
         export CCACHE_COMPRESSLEVEL=10
         export CCACHE_MAXSIZE=125M
+        export CCACHE_EXTRAFILES=${{ github.workspace }}/.clang-tidy
+        export CCACHE_LOGFILE=${{ github.workspace }}/ccache.log.txt
         ccache -z
 
         ./configure --dim 2 --with-fortran no --comp llvm --with-mpi no
         make -j2 WARN_ALL=TRUE WARN_ERROR=TRUE XTRA_CXXFLAGS="-fno-operator-names" \
-            USE_CLANG_TIDY=TRUE CLANG_TIDY=clang-tidy-14 CLANG_TIDY_WARN_ERROR=TRUE \
             CCACHE=ccache
         make install
 
+        ${{github.workspace}}/Tools/C_scripts/mmclt.py --input ${{github.workspace}}/ccache.log.txt
+        make -j2 -f clang-tidy-ccache-misses.mak \
+            CLANG_TIDY=clang-tidy-14 \
+            CLANG_TIDY_ARGS="--config-file=${{github.workspace}}/.clang-tidy --warnings-as-errors=*"
+
         ccache -s
diff --git a/.github/workflows/gcc.yml b/.github/workflows/gcc.yml
@@ -35,6 +35,8 @@ jobs:
         export CCACHE_COMPRESS=1
         export CCACHE_COMPRESSLEVEL=10
         export CCACHE_MAXSIZE=125M
+        export CCACHE_EXTRAFILES=${{ github.workspace }}/.clang-tidy
+        export CCACHE_LOGFILE=${{ github.workspace }}/ccache.log.txt
         ccache -z
 
         mkdir build
@@ -47,13 +49,16 @@ jobs:
             -DCMAKE_C_COMPILER=$(which gcc-8)     \
             -DCMAKE_CXX_COMPILER=$(which g++-8)   \
             -DCMAKE_Fortran_COMPILER=$(which gfortran-8) \
-            -DAMReX_CLANG_TIDY=ON                 \
-            -DAMReX_CLANG_TIDY_WERROR=ON          \
             -DCMAKE_CXX_COMPILER_LAUNCHER=ccache
         make -j 2
         make install
         make test_install
 
+        ${{github.workspace}}/Tools/C_scripts/mmclt.py --input ${{github.workspace}}/ccache.log.txt
+        make -j2 -f clang-tidy-ccache-misses.mak \
+            CLANG_TIDY=clang-tidy-12 \
+            CLANG_TIDY_ARGS="--config-file=${{github.workspace}}/.clang-tidy --warnings-as-errors=*"
+
         export PATH=/tmp/my-amrex/bin:$PATH
         which fcompare
 
@@ -86,6 +91,8 @@ jobs:
         export CCACHE_COMPRESS=1
         export CCACHE_COMPRESSLEVEL=10
         export CCACHE_MAXSIZE=500M
+        export CCACHE_EXTRAFILES=${{ github.workspace }}/.clang-tidy
+        export CCACHE_LOGFILE=${{ github.workspace }}/ccache.log.txt
         ccache -z
 
         cmake -S . -B build             \
@@ -96,11 +103,14 @@ jobs:
             -DAMReX_FORTRAN=ON          \
             -DAMReX_PARTICLES=ON        \
             -DAMReX_SPACEDIM=3          \
-            -DAMReX_CLANG_TIDY=ON       \
-            -DAMReX_CLANG_TIDY_WERROR=ON \
             -DCMAKE_CXX_COMPILER_LAUNCHER=ccache
         cmake --build build -j 2
 
+        ${{github.workspace}}/Tools/C_scripts/mmclt.py --input ${{github.workspace}}/ccache.log.txt
+        make -j2 -f clang-tidy-ccache-misses.mak \
+            CLANG_TIDY=clang-tidy-12 \
+            CLANG_TIDY_ARGS="--config-file=${{github.workspace}}/.clang-tidy --warnings-as-errors=*"
+
         ctest --test-dir build --output-on-failure
 
         ccache -s
@@ -129,6 +139,8 @@ jobs:
         export CCACHE_COMPRESS=1
         export CCACHE_COMPRESSLEVEL=10
         export CCACHE_MAXSIZE=500M
+        export CCACHE_EXTRAFILES=${{ github.workspace }}/.clang-tidy
+        export CCACHE_LOGFILE=${{ github.workspace }}/ccache.log.txt
         ccache -z
 
         cmake -S . -B build             \
@@ -139,11 +151,14 @@ jobs:
             -DAMReX_FORTRAN=ON          \
             -DAMReX_PARTICLES=ON        \
             -DAMReX_SPACEDIM=2          \
-            -DAMReX_CLANG_TIDY=ON       \
-            -DAMReX_CLANG_TIDY_WERROR=ON \
             -DCMAKE_CXX_COMPILER_LAUNCHER=ccache
         cmake --build build -j 2
 
+        ${{github.workspace}}/Tools/C_scripts/mmclt.py --input ${{github.workspace}}/ccache.log.txt
+        make -j2 -f clang-tidy-ccache-misses.mak \
+            CLANG_TIDY=clang-tidy-12 \
+            CLANG_TIDY_ARGS="--config-file=${{github.workspace}}/.clang-tidy --warnings-as-errors=*"
+
         ctest --test-dir build --output-on-failure
 
         ccache -s
@@ -173,21 +188,26 @@ jobs:
         export CCACHE_COMPRESS=1
         export CCACHE_COMPRESSLEVEL=10
         export CCACHE_MAXSIZE=250M
+        export CCACHE_EXTRAFILES=${{ github.workspace }}/.clang-tidy
+        export CCACHE_LOGFILE=${{ github.workspace }}/ccache.log.txt
         ccache -z
 
         cmake -S . -B build             \
             -DCMAKE_BUILD_TYPE=Debug    \
             -DCMAKE_VERBOSE_MAKEFILE=ON \
-            -DAMReX_EB=ON               \
+            -DAMReX_EB=OFF              \
             -DAMReX_ENABLE_TESTS=ON     \
             -DAMReX_FORTRAN=ON          \
             -DAMReX_PARTICLES=ON        \
             -DAMReX_SPACEDIM=1          \
-            -DAMReX_CLANG_TIDY=ON       \
-            -DAMReX_CLANG_TIDY_WERROR=ON \
             -DCMAKE_CXX_COMPILER_LAUNCHER=ccache
         cmake --build build -j 2
 
+        ${{github.workspace}}/Tools/C_scripts/mmclt.py --input ${{github.workspace}}/ccache.log.txt
+        make -j2 -f clang-tidy-ccache-misses.mak \
+            CLANG_TIDY=clang-tidy-12 \
+            CLANG_TIDY_ARGS="--config-file=${{github.workspace}}/.clang-tidy --warnings-as-errors=*"
+
         ctest --test-dir build --output-on-failure
 
         ccache -s
@@ -216,6 +236,8 @@ jobs:
         export CCACHE_COMPRESS=1
         export CCACHE_COMPRESSLEVEL=10
         export CCACHE_MAXSIZE=500M
+        export CCACHE_EXTRAFILES=${{ github.workspace }}/.clang-tidy
+        export CCACHE_LOGFILE=${{ github.workspace }}/ccache.log.txt
         ccache -z
 
         mkdir build
@@ -236,11 +258,14 @@ jobs:
             -DCMAKE_C_COMPILER=$(which gcc-10)              \
             -DCMAKE_CXX_COMPILER=$(which g++-10)            \
             -DCMAKE_Fortran_COMPILER=$(which gfortran-10)   \
-            -DAMReX_CLANG_TIDY=ON       \
-            -DCMAKE_CXX_COMPILER_LAUNCHER=ccache
             # Cannot use C++20 yet -DAMReX_CLANG_TIDY_WERROR=ON
         make -j 2
 
+        ${{github.workspace}}/Tools/C_scripts/mmclt.py --input ${{github.workspace}}/ccache.log.txt
+        make -j2 -f clang-tidy-ccache-misses.mak \
+            CLANG_TIDY=clang-tidy-12 \
+            CLANG_TIDY_ARGS="--config-file=${{github.workspace}}/.clang-tidy --warnings-as-errors=*"
+
         ctest --output-on-failure
 
         ccache -s
@@ -269,6 +294,8 @@ jobs:
         export CCACHE_COMPRESS=1
         export CCACHE_COMPRESSLEVEL=10
         export CCACHE_MAXSIZE=125M
+        export CCACHE_EXTRAFILES=${{ github.workspace }}/.clang-tidy
+        export CCACHE_LOGFILE=${{ github.workspace }}/ccache.log.txt
         ccache -z
 
         mkdir build
@@ -288,11 +315,14 @@ jobs:
             -DCMAKE_C_COMPILER=$(which gcc-8)     \
             -DCMAKE_CXX_COMPILER=$(which g++-8)   \
             -DCMAKE_Fortran_COMPILER=$(which gfortran-8) \
-            -DAMReX_CLANG_TIDY=ON       \
-            -DAMReX_CLANG_TIDY_WERROR=ON \
             -DCMAKE_CXX_COMPILER_LAUNCHER=ccache
         make -j 2
 
+        ${{github.workspace}}/Tools/C_scripts/mmclt.py --input ${{github.workspace}}/ccache.log.txt
+        make -j2 -f clang-tidy-ccache-misses.mak \
+            CLANG_TIDY=clang-tidy-12 \
+            CLANG_TIDY_ARGS="--config-file=${{github.workspace}}/.clang-tidy --warnings-as-errors=*"
+
         ctest --output-on-failure
 
         ccache -s
@@ -328,6 +358,8 @@ jobs:
         export CCACHE_COMPRESS=1
         export CCACHE_COMPRESSLEVEL=10
         export CCACHE_MAXSIZE=500M
+        export CCACHE_EXTRAFILES=${{ github.workspace }}/.clang-tidy
+        export CCACHE_LOGFILE=${{ github.workspace }}/ccache.log.txt
         ccache -z
 
         mkdir build
@@ -347,11 +379,14 @@ jobs:
             -DCMAKE_C_COMPILER=$(which gcc-12)     \
             -DCMAKE_CXX_COMPILER=$(which g++-12)   \
             -DCMAKE_CXX_STANDARD=17     \
-            -DAMReX_CLANG_TIDY=ON       \
-            -DAMReX_CLANG_TIDY_WERROR=ON \
             -DCMAKE_CXX_COMPILER_LAUNCHER=ccache
         make -j 2
 
+        ${{github.workspace}}/Tools/C_scripts/mmclt.py --input ${{github.workspace}}/ccache.log.txt
+        make -j2 -f clang-tidy-ccache-misses.mak \
+            CLANG_TIDY=clang-tidy-14 \
+            CLANG_TIDY_ARGS="--config-file=${{github.workspace}}/.clang-tidy --warnings-as-errors=*"
+
         ctest --output-on-failure
 
         ccache -s
@@ -379,14 +414,20 @@ jobs:
         export CCACHE_COMPRESS=1
         export CCACHE_COMPRESSLEVEL=10
         export CCACHE_MAXSIZE=250M
+        export CCACHE_EXTRAFILES=${{ github.workspace }}/.clang-tidy
+        export CCACHE_LOGFILE=${{ github.workspace }}/ccache.log.txt
         ccache -z
 
         ./configure --dim 1
         make -j2 XTRA_CXXFLAGS=-fno-operator-names \
-            USE_CLANG_TIDY=TRUE CLANG_TIDY=clang-tidy-12 CLANG_TIDY_WARN_ERROR=TRUE \
             CCACHE=ccache
         make install
 
+        ${{github.workspace}}/Tools/C_scripts/mmclt.py --input ${{github.workspace}}/ccache.log.txt
+        make -j2 -f clang-tidy-ccache-misses.mak \
+            CLANG_TIDY=clang-tidy-12 \
+            CLANG_TIDY_ARGS="--config-file=${{github.workspace}}/.clang-tidy --warnings-as-errors=*"
+
         ccache -s
 
   # Build 3D libamrex with configure
@@ -412,14 +453,20 @@ jobs:
         export CCACHE_COMPRESS=1
         export CCACHE_COMPRESSLEVEL=10
         export CCACHE_MAXSIZE=250M
+        export CCACHE_EXTRAFILES=${{ github.workspace }}/.clang-tidy
+        export CCACHE_LOGFILE=${{ github.workspace }}/ccache.log.txt
         ccache -z
 
         ./configure --dim 3 --enable-eb yes --enable-xsdk-defaults yes
         make -j2 WARN_ALL=TRUE WARN_ERROR=TRUE XTRA_CXXFLAGS=-fno-operator-names \
-            USE_CLANG_TIDY=TRUE CLANG_TIDY=clang-tidy-12 CLANG_TIDY_WARN_ERROR=TRUE \
             CCACHE=ccache
         make install
 
+        ${{github.workspace}}/Tools/C_scripts/mmclt.py --input ${{github.workspace}}/ccache.log.txt
+        make -j2 -f clang-tidy-ccache-misses.mak \
+            CLANG_TIDY=clang-tidy-12 \
+            CLANG_TIDY_ARGS="--config-file=${{github.workspace}}/.clang-tidy --warnings-as-errors=*"
+
         ccache -s
 
   # Build 3D libamrex with single precision and tiny profiler
@@ -445,14 +492,20 @@ jobs:
         export CCACHE_COMPRESS=1
         export CCACHE_COMPRESSLEVEL=10
         export CCACHE_MAXSIZE=250M
+        export CCACHE_EXTRAFILES=${{ github.workspace }}/.clang-tidy
+        export CCACHE_LOGFILE=${{ github.workspace }}/ccache.log.txt
         ccache -z
 
         ./configure --dim 3 --enable-eb no --enable-xsdk-defaults no --single-precision yes --single-precision-particles yes --enable-tiny-profile yes
         make -j2 WARN_ALL=TRUE WARN_ERROR=TRUE XTRA_CXXFLAGS=-fno-operator-names \
-            USE_CLANG_TIDY=TRUE CLANG_TIDY=clang-tidy-12 CLANG_TIDY_WARN_ERROR=TRUE \
             CCACHE=ccache
         make install
 
+        ${{github.workspace}}/Tools/C_scripts/mmclt.py --input ${{github.workspace}}/ccache.log.txt
+        make -j2 -f clang-tidy-ccache-misses.mak \
+            CLANG_TIDY=clang-tidy-12 \
+            CLANG_TIDY_ARGS="--config-file=${{github.workspace}}/.clang-tidy --warnings-as-errors=*"
+
         ccache -s
 
   # Build 3D libamrex debug omp build with configure
@@ -478,14 +531,20 @@ jobs:
         export CCACHE_COMPRESS=1
         export CCACHE_COMPRESSLEVEL=10
         export CCACHE_MAXSIZE=250M
+        export CCACHE_EXTRAFILES=${{ github.workspace }}/.clang-tidy
+        export CCACHE_LOGFILE=${{ github.workspace }}/ccache.log.txt
         ccache -z
 
         ./configure --dim 3 --enable-eb yes --enable-xsdk-defaults yes --with-omp yes --debug yes
         make -j2 WARN_ALL=TRUE WARN_ERROR=TRUE XTRA_CXXFLAGS=-fno-operator-names \
-            USE_CLANG_TIDY=TRUE CLANG_TIDY=clang-tidy-12 CLANG_TIDY_WARN_ERROR=TRUE \
             CCACHE=ccache
         make install
 
+        ${{github.workspace}}/Tools/C_scripts/mmclt.py --input ${{github.workspace}}/ccache.log.txt
+        make -j2 -f clang-tidy-ccache-misses.mak \
+            CLANG_TIDY=clang-tidy-12 \
+            CLANG_TIDY_ARGS="--config-file=${{github.workspace}}/.clang-tidy --warnings-as-errors=*"
+
         ccache -s
 
   # Build Tools/Plotfile
@@ -511,13 +570,19 @@ jobs:
         export CCACHE_COMPRESS=1
         export CCACHE_COMPRESSLEVEL=10
         export CCACHE_MAXSIZE=125M
+        export CCACHE_EXTRAFILES=${{ github.workspace }}/.clang-tidy
+        export CCACHE_LOGFILE=${{ github.workspace }}/ccache.log.txt
         ccache -z
 
         cd Tools/Plotfile
         make -j2 USE_MPI=FALSE USE_OMP=FALSE WARN_ALL=TRUE WARN_ERROR=TRUE XTRA_CXXFLAGS=-fno-operator-names \
-            USE_CLANG_TIDY=TRUE CLANG_TIDY=clang-tidy-12 CLANG_TIDY_WARN_ERROR=TRUE \
             CCACHE=ccache
 
+        ${{github.workspace}}/Tools/C_scripts/mmclt.py --input ${{github.workspace}}/ccache.log.txt
+        make -j2 -f clang-tidy-ccache-misses.mak \
+            CLANG_TIDY=clang-tidy-12 \
+            CLANG_TIDY_ARGS="--config-file=${{github.workspace}}/.clang-tidy --warnings-as-errors=*"
+
         ccache -s
 
   # Build libamrex and run all tests
@@ -544,6 +609,8 @@ jobs:
         export CCACHE_COMPRESS=1
         export CCACHE_COMPRESSLEVEL=10
         export CCACHE_MAXSIZE=125M
+        export CCACHE_EXTRAFILES=${{ github.workspace }}/.clang-tidy
+        export CCACHE_LOGFILE=${{ github.workspace }}/ccache.log.txt
         ccache -z
 
         mkdir build
@@ -553,11 +620,14 @@ jobs:
             -DCMAKE_VERBOSE_MAKEFILE=ON \
             -DAMReX_ENABLE_TESTS=ON     \
             -DAMReX_PARTICLES=ON        \
-            -DAMReX_CLANG_TIDY=ON       \
-            -DAMReX_CLANG_TIDY_WERROR=ON \
             -DCMAKE_CXX_COMPILER_LAUNCHER=ccache
         make -j 2
 
+        ${{github.workspace}}/Tools/C_scripts/mmclt.py --input ${{github.workspace}}/ccache.log.txt
+        make -j2 -f clang-tidy-ccache-misses.mak \
+            CLANG_TIDY=clang-tidy-12 \
+            CLANG_TIDY_ARGS="--config-file=${{github.workspace}}/.clang-tidy --warnings-as-errors=*"
+
         ccache -s
 
     - name: Run tests