diff --git a/.github/workflows/builds.yml b/.github/workflows/builds.yml index 3dacbeaa801..a75420befd9 100644 --- a/.github/workflows/builds.yml +++ b/.github/workflows/builds.yml @@ -161,7 +161,7 @@ jobs: steps: - name: Install git lfs - run: apt-get update && apt-get install -y git-lfs + run: apt-get update && apt-get install -y git-lfs time - uses: actions/checkout@v3 with: diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 3f504e902c4..ec01be27d66 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -221,7 +221,7 @@ linux_physmon: expire_in: 1 week script: - - apt-get update && apt-get install -y git-lfs + - apt-get update && apt-get install -y git-lfs time - git clone $CLONE_URL src - cd src diff --git a/CI/physmon/phys_perf_mon.sh b/CI/physmon/phys_perf_mon.sh index 81288e902a9..7eb7933bf01 100755 --- a/CI/physmon/phys_perf_mon.sh +++ b/CI/physmon/phys_perf_mon.sh @@ -1,7 +1,18 @@ #!/bin/bash set -e -set -x + +# helper function to selectively print and run commands without a subshell +function run() { + set -x + "$@" + { set +x; } 2> /dev/null +} + +export run + + +shopt -s extglob mode=${1:-all} @@ -22,12 +33,74 @@ SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd histcmp_results=$outdir/histcmp_results.csv echo -n "" > $histcmp_results -SPYRAL_BIN=${SPYRAL_BIN:=spyral} -SPYRAL="${SPYRAL_BIN} run -i 0.05 --summary" - -mkdir -p ${outdir}/memory +memory_dir=${outdir}/memory +mkdir -p "$memory_dir" + +if [ "$(uname)" == "Darwin" ]; then + function measure { + label=$1 + slug=$2 + shift + shift + echo "Measure Darwin $label ($slug)" >&2 + tmp=$(mktemp) + echo "+ $@" >&2 + /usr/bin/time -l -o "$tmp" "$@" + + of="${memory_dir}/mem_${slug}.csv" + { + echo "# spyral-label: $label" + echo "# spyral-cmd: $*" + echo "time,rss,vms" + # in bytes + grep -E "real" "$tmp" | awk '{printf $1}' + printf "," + grep -E "maximum resident set size" "$tmp" | awk '{printf $1}' + printf ",0\n" + } > "$of" + } + export measure +elif [ "$(uname)" == "Linux" ]; then + function measure { + label=$1 + slug=$2 + shift + shift + echo "Measure Linux $label ($slug)" >&2 + tmp=$(mktemp) + echo "+ $@" >&2 + /usr/bin/time -v -o "$tmp" "$@" + # in kbytes + max_rss=$(grep "Maximum resident set size (kbytes):" "$tmp" | awk '{printf $(NF)}') + max_rss=$(( 1000*max_rss )) + wall_time=$(grep "Elapsed (wall clock)" "$tmp" | awk '{printf $(NF)}') + echo $max_rss + wall_time=$(python3 -c "i='${wall_time}';p=i.split(':');p = p if len(p) == 3 else ['0', *p];t=float(p[0])*60*60 + float(p[1])*60 + float(p[2]);print(t)") + echo $wall_time + + of="${memory_dir}/mem_${slug}.csv" + { + echo "# spyral-label: $label" + echo "# spyral-cmd: $*" + echo "time,rss,vms" + echo "${wall_time},${max_rss},0" + } > "$of" + } + export measure +else + function measure { + echo "Not measuring because unknown environment" + shift + shift + "$@" + } + export measure +fi -set +e +if [ -n "$CI" ]; then + echo "CI mode, do not abort immediately on failure" + set +e +fi ec=0 source $SCRIPT_DIR/setup.sh @@ -38,7 +111,7 @@ function run_physmon_gen() { script=CI/physmon/workflows/physmon_${slug}.py - $SPYRAL -l "$title" -o "$outdir/memory/mem_${slug}.csv" -- ${script} $outdir 2>&1 > $outdir/run_${slug}.log + measure "$title" "$slug" ${script} $outdir 2>&1 > $outdir/run_${slug}.log this_ec=$? ec=$(($ec | $this_ec)) @@ -46,10 +119,8 @@ function run_physmon_gen() { if [ $this_ec -ne 0 ]; then echo "::error::🟥 Dataset generation failed: ${script} -> ec=$this_ec" else - echo "::notice::✅ Dataset generation succeeded: ${script}" + echo "::notice::✅ Dataset generation succeeded: ${script}" fi - - $SPYRAL_BIN plot $outdir/memory/mem_${slug}.csv --output $outdir/memory } echo "::group::Generate validation dataset" @@ -95,7 +166,7 @@ function run_histcmp() { ec=1 fi - histcmp $a $b \ + run histcmp $a $b \ --label-reference=reference \ --label-monitored=monitored \ --title="$title" \ @@ -124,7 +195,7 @@ function full_chain() { config="CI/physmon/default.yml" fi echo $config - + if [ $suffix != truth_smeared ]; then run_histcmp \ $outdir/performance_seeding_${suffix}.root \ @@ -133,7 +204,7 @@ function full_chain() { seeding_${suffix} \ -c $config fi - + run_histcmp \ $outdir/performance_ckf_${suffix}.root \ $refdir/performance_ckf_${suffix}.root \ @@ -141,7 +212,7 @@ function full_chain() { ckf_${suffix} \ -c $config - Examples/Scripts/generic_plotter.py \ + run Examples/Scripts/generic_plotter.py \ $outdir/performance_ivf_${suffix}.root \ vertexing \ $outdir/performance_ivf_${suffix}_hist.root \ @@ -158,7 +229,7 @@ function full_chain() { "IVF ${suffix}" \ ivf_${suffix} - Examples/Scripts/generic_plotter.py \ + run Examples/Scripts/generic_plotter.py \ $outdir/performance_amvf_${suffix}.root \ vertexing \ $outdir/performance_amvf_${suffix}_hist.root \ @@ -176,7 +247,7 @@ function full_chain() { amvf_${suffix} if [ $suffix == seeded ]; then - Examples/Scripts/generic_plotter.py \ + run Examples/Scripts/generic_plotter.py \ $outdir/performance_amvf_gridseeder_${suffix}.root \ vertexing \ $outdir/performance_amvf_gridseeder_${suffix}_hist.root \ @@ -194,7 +265,7 @@ function full_chain() { amvf_gridseeder_${suffix} fi - Examples/Scripts/generic_plotter.py \ + run Examples/Scripts/generic_plotter.py \ $outdir/tracksummary_ckf_${suffix}.root \ tracksummary \ $outdir/tracksummary_ckf_${suffix}_hist.root \ @@ -218,7 +289,7 @@ function simulation() { config="CI/physmon/simulation_config.yml" - Examples/Scripts/generic_plotter.py \ + run Examples/Scripts/generic_plotter.py \ $outdir/particles_${suffix}.root \ particles \ $outdir/particles_${suffix}_hist.root \ @@ -274,7 +345,7 @@ if [[ "$mode" == "all" || "$mode" == "fullchains" ]]; then "Ambisolver " \ ambi_ttbar - Examples/Scripts/generic_plotter.py \ + run Examples/Scripts/generic_plotter.py \ $outdir/performance_amvf_ttbar.root \ vertexing \ $outdir/performance_amvf_ttbar_hist.root \ @@ -282,7 +353,7 @@ if [[ "$mode" == "all" || "$mode" == "fullchains" ]]; then --config CI/physmon/vertexing_ttbar_config.yml ec=$(($ec | $?)) - Examples/Scripts/generic_plotter.py \ + run Examples/Scripts/generic_plotter.py \ $outdir/tracksummary_ckf_ttbar.root \ tracksummary \ $outdir/tracksummary_ckf_ttbar_hist.root \ @@ -307,7 +378,7 @@ if [[ "$mode" == "all" || "$mode" == "fullchains" ]]; then "AMVF ttbar" \ amvf_ttbar - Examples/Scripts/generic_plotter.py \ + run Examples/Scripts/generic_plotter.py \ $outdir/performance_amvf_gridseeder_ttbar.root \ vertexing \ $outdir/performance_amvf_gridseeder_ttbar_hist.root \ @@ -353,7 +424,7 @@ if [[ "$mode" == "all" || "$mode" == "gx2f" ]]; then fi if [[ "$mode" == "all" || "$mode" == "vertexing" ]]; then - Examples/Scripts/vertex_mu_scan.py \ + run Examples/Scripts/vertex_mu_scan.py \ $outdir/performance_vertexing_*mu*.root \ $outdir/vertexing_mu_scan.pdf @@ -365,7 +436,7 @@ if [[ "$mode" == "all" || "$mode" == "simulation" ]]; then simulation geant4 fi -CI/physmon/summary.py $histcmp_results \ +run CI/physmon/summary.py $histcmp_results \ --md $outdir/summary.md \ --html $outdir/summary.html ec=$(($ec | $?))