ci: Build physmon summary from explicit report information (#2350)

Co-authored-by: Andreas Stefl <stefl.andreas@gmail.com>
Co-authored-by: kodiakhq[bot] <49736102+kodiakhq[bot]@users.noreply.github.com>

.github/workflows/builds.yml

@@ -309,7 +309,7 @@ jobs:
         run: >
           echo "::group::Dependencies"
           && git config --global safe.directory "$GITHUB_WORKSPACE"
-          && pip3 install histcmp==0.6.2 spyral-cli==1.1.0 matplotlib
+          && pip3 install histcmp==0.6.3 spyral-cli==1.1.1 matplotlib
           && pip3 install -r Examples/Scripts/requirements.txt
           && /usr/local/bin/geant4-config --install-datasets
           && source /usr/local/bin/thisroot.sh

CI/physmon/phys_perf_mon.sh

@@ -18,54 +18,96 @@ refcommit=$(cat $refdir/commit)
 commit=$(git rev-parse --short HEAD)
 SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}"  )" &> /dev/null && pwd  )
 
-SPYRAL_BIN="spyral"
-SPYRAL="${SPYRAL_BIN} run -i 0.1 --summary"
+# File to accumulate the histcmp results
+histcmp_results=$outdir/histcmp_results.csv
+echo -n "" > $histcmp_results
 
-mkdir -p "${outdir}/memory"
+SPYRAL_BIN=${SPYRAL_BIN:=spyral}
+SPYRAL="${SPYRAL_BIN} run -i 0.05 --summary"
+
+mkdir -p ${outdir}/memory
+
+set +e
+ec=0
 
 source $SCRIPT_DIR/setup.sh
+
+function run_physmon_gen() {
+    title=$1
+    slug=$2
+
+    script=CI/physmon/workflows/physmon_${slug}.py
+
+    $SPYRAL -l "$title" -o "$outdir/memory/mem_${slug}.csv" -- ${script} $outdir 2>&1 > $outdir/run_${slug}.log
+
+    this_ec=$?
+    ec=$(($ec | $this_ec))
+
+    if [ $this_ec -ne 0 ]; then
+      echo "::error::🟥 Dataset generation failed: ${script} -> ec=$this_ec"
+    else
+      echo "::notice::✅ Dataset generation succeeded: ${script}"   
+    fi
+
+    $SPYRAL_BIN plot $outdir/memory/mem_${slug}.csv --output $outdir/memory
+}
+
 echo "::group::Generate validation dataset"
 if [[ "$mode" == "all" || "$mode" == "kalman" ]]; then
-    $SPYRAL -l "Truth Tracking KF" -o "$outdir/memory/mem_truth_tracking_kalman.csv" -- CI/physmon/workflows/physmon_truth_tracking_kalman.py $outdir 2>&1 > $outdir/run_truth_tracking_kalman.log
-    $SPYRAL_BIN plot $outdir/memory/mem_truth_tracking_kalman.csv --output $outdir/memory
+    run_physmon_gen "Truth Tracking KF" "truth_tracking_kalman"
 fi
 if [[ "$mode" == "all" || "$mode" == "gsf" ]]; then
-    $SPYRAL -l "Truth Tracking GSF" -o "$outdir/memory/mem_truth_tracking_gsf.csv" -- CI/physmon/workflows/physmon_truth_tracking_gsf.py $outdir 2>&1 > $outdir/run_truth_tracking_gsf.log
-    $SPYRAL_BIN plot $outdir/memory/mem_truth_tracking_gsf.csv --output $outdir/memory
+    run_physmon_gen "Truth Tracking GSF" "truth_tracking_gsf"
 fi
 if [[ "$mode" == "all" || "$mode" == "fullchains" ]]; then
-    $SPYRAL -l "CKF Tracking" -o "$outdir/memory/mem_ckf_tracking.csv" -- CI/physmon/workflows/physmon_ckf_tracking.py $outdir 2>&1 > $outdir/run_ckf_tracking.log
-    $SPYRAL -l "Track finding ttbar" -o "$outdir/memory/mem_ttbar.csv" -- CI/physmon/workflows/physmon_track_finding_ttbar.py $outdir 2>&1 > $outdir/run_track_finding_ttbar.log
-    $SPYRAL_BIN plot $outdir/memory/mem_ckf_tracking.csv --output $outdir/memory
-    $SPYRAL_BIN plot $outdir/memory/mem_ttbar.csv --output $outdir/memory
+    run_physmon_gen "CKF Tracking" "ckf_tracking"
+    run_physmon_gen "Track finding ttbar" "track_finding_ttbar"
+
 fi
 if [[ "$mode" == "all" || "$mode" == "vertexing" ]]; then
-    $SPYRAL -l "Vertexing" -o "$outdir/memory/mem_vertexing.csv" -- CI/physmon/workflows/physmon_vertexing.py $outdir 2>&1 > $outdir/run_vertexing.log
-    $SPYRAL_BIN plot $outdir/memory/mem_vertexing.csv --output $outdir/memory
+    run_physmon_gen "Vertexing" "vertexing"
 fi
 if [[ "$mode" == "all" || "$mode" == "simulation" ]]; then
-    $SPYRAL -l "Simulation" -o "$outdir/memory/mem_simulation.csv" -- CI/physmon/workflows/physmon_simulation.py $outdir 2>&1 > $outdir/run_simulation.log
-    $SPYRAL_BIN plot $outdir/memory/mem_simulation.csv --output $outdir/memory
+    run_physmon_gen "Simulation" "simulation"
 fi
 echo "::endgroup::"
 
 
-set +e
-
-ec=0
-
-function run() {
+function run_histcmp() {
     a=$1
     b=$2
+    title=$3
+    slug=$4
+    shift 4
 
     echo "::group::Comparing $a vs. $b"
 
-    histcmp \
+    if [ ! -f "$a" ]; then
+      echo "::error::histcmp failed: File $a does not exist"
+      ec=1
+    fi
+
+    if [ ! -f "$b" ]; then
+      echo "::error::histcmp failed: File $b does not exist"
+      ec=1
+    fi
+
+    histcmp $a $b \
         --label-reference=reference \
         --label-monitored=monitored \
+        --title="$title" \
+        -o $outdir/${slug}.html \
+        -p $outdir/${slug}_plots \
         "$@"
 
-    ec=$(($ec | $?))
+    this_ec=$?
+    ec=$(($ec | $this_ec))
+
+    if [ $this_ec -ne 0 ]; then
+      echo "::error::histcmp failed (${slug}): ec=$this_ec"
+    fi
+
+    echo "\"${title}\",${slug},${this_ec}" >> $histcmp_results
 
     echo "::endgroup::"
 }
@@ -81,22 +123,20 @@ function full_chain() {
     echo $config
 
     if [ $suffix != truth_smeared ]; then
-	    run \
-  	      $outdir/performance_seeding_${suffix}.root \
-    	    $refdir/performance_seeding_${suffix}.root \
-      	  --title "Seeding ${suffix}" \
-        	-c $config \
-        	-o $outdir/seeding_${suffix}.html \
-        	-p $outdir/seeding_${suffix}_plots
+      run_histcmp \
+        $outdir/performance_seeding_${suffix}.root \
+        $refdir/performance_seeding_${suffix}.root \
+        "Seeding ${suffix}" \
+        seeding_${suffix} \
+        -c $config
     fi
 
-    run \
+    run_histcmp \
         $outdir/performance_ckf_${suffix}.root \
         $refdir/performance_ckf_${suffix}.root \
-        --title "CKF ${suffix}" \
-        -c $config \
-        -o $outdir/ckf_${suffix}.html \
-        -p $outdir/ckf_${suffix}_plots
+        "CKF ${suffix}" \
+        ckf_${suffix} \
+        -c $config
 
     Examples/Scripts/generic_plotter.py \
         $outdir/performance_ivf_${suffix}.root \
@@ -109,12 +149,11 @@ function full_chain() {
     # remove ntuple file because it's large
     rm $outdir/performance_ivf_${suffix}.root
 
-    run \
+    run_histcmp \
         $outdir/performance_ivf_${suffix}_hist.root \
         $refdir/performance_ivf_${suffix}_hist.root \
-        --title "IVF ${suffix}" \
-        -o $outdir/ivf_${suffix}.html \
-        -p $outdir/ivf_${suffix}_plots
+        "IVF ${suffix}" \
+        ivf_${suffix}
 
     Examples/Scripts/generic_plotter.py \
         $outdir/performance_amvf_${suffix}.root \
@@ -127,12 +166,11 @@ function full_chain() {
     # remove ntuple file because it's large
     rm $outdir/performance_amvf_${suffix}.root
 
-    run \
+    run_histcmp \
         $outdir/performance_amvf_${suffix}_hist.root \
         $refdir/performance_amvf_${suffix}_hist.root \
-        --title "AMVF ${suffix}" \
-        -o $outdir/amvf_${suffix}.html \
-        -p $outdir/amvf_${suffix}_plots
+        "AMVF ${suffix}" \
+        amvf_${suffix}
 
     Examples/Scripts/generic_plotter.py \
         $outdir/tracksummary_ckf_${suffix}.root \
@@ -145,12 +183,12 @@ function full_chain() {
     # remove ntuple file because it's large
     rm $outdir/tracksummary_ckf_${suffix}.root
 
-    run \
+    run_histcmp \
         $outdir/tracksummary_ckf_${suffix}_hist.root \
         $refdir/tracksummary_ckf_${suffix}_hist.root \
-        --title "Track Summary CKF ${suffix}" \
-        -o $outdir/tracksummary_ckf_${suffix}.html \
-        -p $outdir/tracksummary_ckf_${suffix}_plots
+        "Track Summary CKF ${suffix}" \
+        tracksummary_ckf_${suffix}
+
 }
 
 function simulation() {
@@ -169,12 +207,11 @@ function simulation() {
     # remove ntuple file because it's large
     rm $outdir/particles_initial_${suffix}.root
 
-    run \
+    run_histcmp \
         $outdir/particles_initial_${suffix}_hist.root \
         $refdir/particles_initial_${suffix}_hist.root \
-        --title "Particles inital ${suffix}" \
-        -o $outdir/particles_initial_${suffix}.html \
-        -p $outdir/particles_initial_${suffix}_plots
+        "Particles inital ${suffix}" \
+        particles_initial_${suffix}
 
     Examples/Scripts/generic_plotter.py \
         $outdir/particles_final_${suffix}.root \
@@ -187,12 +224,11 @@ function simulation() {
     # remove ntuple file because it's large
     rm $outdir/particles_final_${suffix}.root
 
-    run \
+    run_histcmp \
         $outdir/particles_final_${suffix}_hist.root \
         $refdir/particles_final_${suffix}_hist.root \
-        --title "Particles final ${suffix}" \
-        -o $outdir/particles_final_${suffix}.html \
-        -p $outdir/particles_final_${suffix}_plots
+        "Particles final ${suffix}" \
+        particles_final_${suffix}
 }
 
 if [[ "$mode" == "all" || "$mode" == "fullchains" ]]; then
@@ -201,42 +237,37 @@ if [[ "$mode" == "all" || "$mode" == "fullchains" ]]; then
     full_chain seeded
     full_chain orthogonal
 
-    run \
+    run_histcmp \
         $outdir/performance_ambi_seeded.root \
         $refdir/performance_ambi_seeded.root \
-        --title "Ambisolver seeded" \
-        -o $outdir/ambi_seeded.html \
-        -p $outdir/ambi_seeded_plots
+        "Ambisolver seeded" \
+        ambi_seeded
 
-    run \
+    run_histcmp \
         $outdir/performance_ambi_orthogonal.root \
         $refdir/performance_ambi_orthogonal.root \
-        --title "Ambisolver orthogonal" \
-        -o $outdir/ambi_orthogonal.html \
-        -p $outdir/ambi_orthogonal_plots
+        "Ambisolver orthogonal" \
+        ambi_orthogonal
 
-    run \
+    run_histcmp \
         $outdir/performance_seeding_ttbar.root \
         $refdir/performance_seeding_ttbar.root \
-        --title "Seeding ttbar" \
-        -c $config \
-        -o $outdir/seeding_ttbar.html \
-        -p $outdir/seeding_ttbar_plots
+        "Seeding ttbar" \
+        seeding_ttbar \
+        -c $config
 
-    run \
+    run_histcmp \
         $outdir/performance_ckf_ttbar.root \
         $refdir/performance_ckf_ttbar.root \
-        --title "CKF ttbar" \
-        -c $config \
-        -o $outdir/ckf_ttbar.html \
-        -p $outdir/ckf_ttbar_plots
+        "CKF ttbar" \
+        ckf_ttbar \
+        -c $config
 
-    run \
+    run_histcmp \
         $outdir/performance_ambi_ttbar.root \
         $refdir/performance_ambi_ttbar.root \
-        --title "Ambisolver " \
-        -o $outdir/ambi_ttbar.html \
-        -p $outdir/ambi_ttbar_plots
+        "Ambisolver " \
+        ambi_ttbar
 
     Examples/Scripts/generic_plotter.py \
         $outdir/performance_amvf_ttbar.root \
@@ -249,32 +280,29 @@ if [[ "$mode" == "all" || "$mode" == "fullchains" ]]; then
     # remove ntuple file because it's large
     rm $outdir/performance_amvf_ttbar.root
 
-    run \
+    run_histcmp \
         $outdir/performance_amvf_ttbar_hist.root \
         $refdir/performance_amvf_ttbar_hist.root \
-        --title "AMVF ttbar" \
-        -o $outdir/amvf_ttbar.html \
-        -p $outdir/amvf_ttbar_plots
+        "AMVF ttbar" \
+        amvf_ttbar
 fi
 
 if [[ "$mode" == "all" || "$mode" == "gsf" ]]; then
-    run \
+    run_histcmp \
         $outdir/performance_gsf.root \
         $refdir/performance_gsf.root \
-        --title "Truth tracking (GSF)" \
-        -c CI/physmon/gsf.yml \
-        -o $outdir/gsf.html \
-        -p $outdir/gsf_plots
+        "Truth tracking (GSF)" \
+        gsf \
+        -c CI/physmon/gsf.yml
 fi
 
 if [[ "$mode" == "all" || "$mode" == "kalman" ]]; then
-    run \
+    run_histcmp \
         $outdir/performance_truth_tracking.root \
         $refdir/performance_truth_tracking.root \
-        --title "Truth tracking" \
-        -c CI/physmon/truth_tracking.yml \
-        -o $outdir/truth_tracking.html \
-        -p $outdir/truth_tracking_plots
+        "Truth tracking" \
+        truth_tracking \
+        -c CI/physmon/truth_tracking.yml
 fi
 
 if [[ "$mode" == "all" || "$mode" == "vertexing" ]]; then
@@ -290,8 +318,7 @@ if [[ "$mode" == "all" || "$mode" == "simulation" ]]; then
     simulation geant4
 fi
 
-CI/physmon/summary.py $outdir/*.html \
-  --base $outdir \
+CI/physmon/summary.py $histcmp_results \
   --md $outdir/summary.md \
   --html $outdir/summary.html
 ec=$(($ec | $?))