Skip to content

Parallelise o2dpg generator tests #2250

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
jackal1-66 wants to merge 2 commits into
base: master
Choose a base branch
from
+82 −24
Open

Parallelise o2dpg generator tests #2250

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
50b1310
First implementation of parallel testing
jackal1-66 Jan 23, 2026
907d77e
Test with many generators
jackal1-66 Jan 28, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions MC/config/PWGHF/external/generator/generator_pythia8_embed_hf.C
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -420,3 +420,4 @@ FairGenerator * GeneratorPythia8EmbedHFCharmAndBeauty(bool usePtHardBins = false
return myGen;
}


1 change: 1 addition & 0 deletions MC/config/PWGHF/external/generator/generator_pythia8_gaptriggered_hf.C
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -438,3 +438,4 @@ FairGenerator *GeneratorPythia8GapHF(int inputTriggerRatio, float yQuarkMin = -1

return myGen;
}

104 changes: 80 additions & 24 deletions test/run_generator_tests.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,37 +46,35 @@ TEST_COUNTER=1
# whether or not to delete everything except logs (default is to delete)
KEEP_ONLY_LOGS=1

# Number of workers for parallel test runs
JOBS=${JOBS:-8}
echo "Running tests with up to ${JOBS} parallel jobs"
# Prepare some colored output
SRED="\033[0;31m"
SGREEN="\033[0;32m"
SEND="\033[0m"


echo_green()
{
echo -e "${SGREEN}${*}${SEND}"
}


echo_red()
{
echo -e "${SRED}${*}${SEND}"
}


# Prevent the script from being soured to omit unexpected surprises when exit is used
SCRIPT_NAME="$(basename "$(test -L "$0" && readlink "$0" || echo "$0")")"
if [ "${SCRIPT_NAME}" != "$(basename ${BASH_SOURCE[0]})" ] ; then
echo_red "This script cannot not be sourced" >&2
return 1
fi


##################################
# Core and utility functionality #
##################################


get_test_script_path_for_ini()
{
local ini_path=${1}
Expand Down Expand Up @@ -132,7 +130,7 @@ exec_test()
[[ "${RET}" != "0" ]] && { remove_artifacts ; return ${RET} ; }
# run the simulation, fail if not successful
echo "### Testing base o2-sim executable ###" >> ${LOG_FILE_SIM}
o2-sim -g ${generator_lower} ${trigger} --noGeant -n ${nev} -j 4 --configFile ${ini_path} --configKeyValues "GeneratorPythia8.includePartonEvent=true" >> ${LOG_FILE_SIM} 2>&1
o2-sim -g ${generator_lower} ${trigger} --noGeant -n ${nev} -j 1 --configFile ${ini_path} --configKeyValues "GeneratorPythia8.includePartonEvent=true" >> ${LOG_FILE_SIM} 2>&1
RET=${?}
[[ "${RET}" != "0" ]] && { remove_artifacts ; return ${RET} ; }

Expand All @@ -151,6 +149,13 @@ exec_test()
return ${RET}
}

wait_for_slot()
{
# Wait until the number of background jobs is within the limit
while (( $(jobs -r | wc -l) >= JOBS )) ; do
sleep 0.1
done
}

check_generators()
{
Expand All @@ -169,31 +174,37 @@ check_generators()
local look_for=$(grep " ${g}.*\(\)" ${test_script})
local has_trigger="$(grep Trigger${g} ${ini_path})"
[[ -z "${look_for}" ]] && continue
echo -n "Test ${TEST_COUNTER}: ${ini_path} with generator ${g}"
tested_any=1
# prepare the test directory
local test_dir=${TEST_COUNTER}_$(basename ${ini})_${g}_dir
rm -rf ${test_dir} 2> /dev/null
mkdir ${test_dir}
pushd ${test_dir} > /dev/null
# one single test
exec_test ${ini_path} ${g} ${has_trigger}
RET=${?}
popd > /dev/null
if [[ "${RET}" != "0" ]] ; then
echo_red " -> FAILED"
ret_this=${RET}
else
echo_green " -> PASSED"
fi
local test_num=${TEST_COUNTER}
((TEST_COUNTER++))

# Wait for an available slot before starting a new test
wait_for_slot

echo "Test ${test_num}: ${ini_path} with generator ${g} - STARTED"
# Run test in background
(
cd ${test_dir}
exec_test ${ini_path} ${g} ${has_trigger}
exit $?
) &
local pid=$!

# Store test information in global arrays
test_pids+=(${pid})
test_numbers+=(${test_num})
test_generators+=(${g})
test_ini_paths+=("${ini_path}")
fi
done
[[ -z "${tested_any}" ]] && { echo_red "No test scenario was found for any generator. There must be at least one generator to be tested." ; ret_this=1 ; }
return ${ret_this}
}


add_ini_files_from_macros()
{
# given a list of macros, collect all INI files which contain at least one of them
Expand All @@ -216,7 +227,6 @@ add_ini_files_from_macros()
done
}


get_root_includes()
{
# check if some R__ADD_INCLUDE_PATH is used in the including macro and check the included file against that
Expand All @@ -236,7 +246,6 @@ get_root_includes()
echo ${full_includes}
}


find_including_macros()
{
# figure out the macros that INCLUDE macros that have changed, so that in turn we can check
Expand Down Expand Up @@ -283,7 +292,6 @@ find_including_macros()
echo ${including_macros}
}


add_ini_files_from_tests()
{
# Collect also those INI files for which the test has been changed
Expand All @@ -302,7 +310,6 @@ add_ini_files_from_tests()
done
}


collect_ini_files()
{
# Collect all INI files which have changed
Expand Down Expand Up @@ -336,7 +343,6 @@ collect_ini_files()
add_ini_files_from_tests ${macros}
}


get_git_repo_directory()
{
local repo=
Expand Down Expand Up @@ -469,6 +475,12 @@ pushd ${TEST_PARENT_DIR} > /dev/null
# global return code to be returned at the end
ret_global=0

# Global arrays to track all test jobs (across all INI files)
declare -a test_pids
declare -a test_numbers
declare -a test_generators
declare -a test_ini_paths

# check each of the INI files
for ini in ${ini_files_full_paths} ; do

Expand All @@ -495,6 +507,50 @@ for ini in ${ini_files_full_paths} ; do
[[ "${fail_immediately}" == "1" ]] && break
fi
done

# Wait for all test jobs to complete and collect results
total_tests=${#test_pids[@]}
completed=0
while (( completed < total_tests )) ; do
# Wait for any background job to complete
wait -n
RET=$?

# Find which job completed by checking which PID no longer exists
for idx in "${!test_pids[@]}" ; do
pid="${test_pids[$idx]}"
if ! kill -0 ${pid} 2>/dev/null ; then
# This job has completed, get its actual exit status
wait ${pid} 2>/dev/null
RET=$?

test_num="${test_numbers[$idx]}"
generator="${test_generators[$idx]}"
ini_path="${test_ini_paths[$idx]}"

if [[ "${RET}" != "0" ]] ; then
echo_red "Test ${test_num}: ${ini_path} with generator ${generator} -> FAILED"
ret_global=${RET}
if [[ "${fail_immediately}" == "1" ]] ; then
# Kill remaining background jobs
for remaining_pid in "${test_pids[@]}" ; do
kill ${remaining_pid} 2>/dev/null
done
completed=${total_tests}
break
fi
else
echo_green "Test ${test_num}: ${ini_path} with generator ${generator} -> PASSED"
fi

# Remove this job from tracking
unset test_pids[$idx]
((completed++))
break
fi
done
done

# return to where we came from
popd > /dev/null

Expand Down