From 60b6f71afb30d4b73ecd82694b3f6da449b279fb Mon Sep 17 00:00:00 2001 From: Nathan Baltzell Date: Mon, 7 Oct 2024 13:00:56 -0400 Subject: [PATCH 01/14] add option to merge outputs (requires yq for trains) --- bin/run-clara | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/bin/run-clara b/bin/run-clara index 4fe4d00945..62fef9ef30 100755 --- a/bin/run-clara +++ b/bin/run-clara @@ -64,7 +64,7 @@ input=. output=. threads=2 prefix=rec_ -while getopts i:o:p:c:t:n:h opt +while getopts i:o:p:c:t:n:mh opt do case $opt in i) input=$OPTARG ;; @@ -73,6 +73,7 @@ do c) CLARA_HOME=$OPTARG ;; t) threads=$OPTARG && echo $threads | grep -q -E '^[0-9]+$' || error "-t must be an integer, threads" ;; n) nevents="-e $OPTARG" && echo $nevents | grep -q -E '^-e [0-9]+$' || error "-n must be an integer, events" ;; + m) merge=1 ;; h) echo -e $usage && echo -e $info && exit 0 ;; esac done @@ -86,6 +87,7 @@ shift $((OPTIND-1)) [ $threads -eq 0 ] && threads=`grep -c ^processor /proc/cpuinfo` ! test -r $input && echo "WARNING: Interpreting -i as a glob: $input" ! test -r $input && ! compgen -G $input && error "Invalid glob -i: $input" +! [ -z ${merge+x} ] && ! command -v hipo-utils >& /dev/null && error "Merging requested, but hipo-utils is not in \$PATH" # Create the environment variables and directories required by CLARA: [ -e $output ] && echo "WARNING: Using existing directory: $output." @@ -137,3 +139,17 @@ else $yaml $CLARA_USER_DATA/filelist.txt fi +# Merge outputs: +if ! [ -z ${merge+x} ] +then + if grep -q org.jlab.jnp.grapes $yaml >& /dev/null + then + for id in $(yq .configuration.services.*.id $yaml | uniq) + do + hipo-utils -merge -o $output/$prefix$id.hipo $output/$prefix$id*.hipo + done + else + outfiles=$(sed "s#^#$output/$prefix#" $CLARA_USER_DATA/filelist.txt) + hipo-utils -merge -o $output/$prefix.hipo $outfiles + fi +fi From a9a3eee5b9c681cad33ab646012e18935fa5945c Mon Sep 17 00:00:00 2001 From: Nathan Baltzell Date: Mon, 7 Oct 2024 16:54:23 -0400 Subject: [PATCH 02/14] document dependencies --- bin/run-clara | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/bin/run-clara b/bin/run-clara index 62fef9ef30..1087fd00db 100755 --- a/bin/run-clara +++ b/bin/run-clara @@ -3,17 +3,21 @@ ulimit -u 49152 >& /dev/null export JAVA_OPTS="${JAVA_OPTS} -XX:+IgnoreUnrecognizedVMOptions" -usage="Usage: run-clara [-i IN] [-o OUT] [-c CLARA] [-t #] [-n #] YAML" -info='\nOptions:\n -YAML - path to CLARA YAML steering file\n --i input HIPO file, directory of *.hipo files, or glob of HIPO files (default=.)\n --o output directory (default=.)\n --p output prefix (default=rec_)\n --c CLARA installation (default=$CLARA_HOME)\n --t number of threads (default=2)\n --n number of events (default=-1)\n\n +usage="\nUsage: run-clara [-i IN] [-o OUT] [-p PREFIX] [-c CLARA_HOME] [-t #] [-n #] [-m] [-h] YAML" +info='\nRequired Arguments:\n +\tYAML = path to CLARA YAML steering file\n +Options:\n +\t-i directory or shell glob of input HIPO files (default=.)\n +\t-o output directory (default=.)\n +\t-p output prefix (default=rec_)\n +\t-c CLARA installation (default=$CLARA_HOME)\n +\t-t number of threads (default=2)\n +\t-n number of events (default=-1)\n +\t-m merge output files (see dependencies below)\n +\t-h print this help and exit\n\n Defaults will use $CLARA_HOME to read all *.hipo files in $PWD,\n -with all output written to $PWD.' +with all output written to $PWD. Merging outputs (-m) requires\n +hipo-utils and yq (https://github.com/mikefarah/yq) in $PATH.' function error() { echo -e "\n$usage\n\nERROR: $@." && exit 1 From d1084e42141a9b617b9fedc0bdcb4028a3d0859c Mon Sep 17 00:00:00 2001 From: Nathan Baltzell Date: Tue, 8 Oct 2024 08:22:06 -0400 Subject: [PATCH 03/14] support unsorted YAMLs --- bin/run-clara | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/run-clara b/bin/run-clara index 1087fd00db..02b60b6bd7 100755 --- a/bin/run-clara +++ b/bin/run-clara @@ -148,7 +148,7 @@ if ! [ -z ${merge+x} ] then if grep -q org.jlab.jnp.grapes $yaml >& /dev/null then - for id in $(yq .configuration.services.*.id $yaml | uniq) + for id in $(yq .configuration.services.*.id $yaml | sort -n | uniq) do hipo-utils -merge -o $output/$prefix$id.hipo $output/$prefix$id*.hipo done From 54cf29001f63fb8ab12426112f7fac727e518f95 Mon Sep 17 00:00:00 2001 From: Nathan Baltzell Date: Tue, 8 Oct 2024 15:41:49 -0400 Subject: [PATCH 04/14] fix glob --- bin/run-clara | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/run-clara b/bin/run-clara index 02b60b6bd7..c97226a2cf 100755 --- a/bin/run-clara +++ b/bin/run-clara @@ -108,7 +108,7 @@ export CLARA_HOME=$(abspath $CLARA_HOME) export CLAS12DIR=$CLARA_HOME/plugins/clas12 # Generate the file for CLARA containing a file list (of relative paths, not absolute): -! test -r $input && compgen -G $input > $CLARA_USER_DATA/filelist.txt +! test -r $input && compgen -G $input | xargs basename > $CLARA_USER_DATA/filelist.txt test -d $input && find $input -maxdepth 1 -name "*.hipo" -exec basename {} \; > $CLARA_USER_DATA/filelist.txt test -f $input && echo $(basename $input) > $CLARA_USER_DATA/filelist.txt [ $(cat $CLARA_USER_DATA/filelist.txt | wc -l) -gt 0 ] || error "Found no input files." From 258b81ef34be2ef3656d5b66e34f3db6fdbc28b2 Mon Sep 17 00:00:00 2001 From: Nathan Baltzell Date: Wed, 9 Oct 2024 14:29:35 -0400 Subject: [PATCH 05/14] switch to standard input list (and -y YAML) and symlink them for CLARA --- bin/run-clara | 41 +++++++++---------- .../advanced-tests/run-advanced-tests.sh | 2 +- 2 files changed, 21 insertions(+), 22 deletions(-) diff --git a/bin/run-clara b/bin/run-clara index c97226a2cf..263e31c335 100755 --- a/bin/run-clara +++ b/bin/run-clara @@ -3,11 +3,11 @@ ulimit -u 49152 >& /dev/null export JAVA_OPTS="${JAVA_OPTS} -XX:+IgnoreUnrecognizedVMOptions" -usage="\nUsage: run-clara [-i IN] [-o OUT] [-p PREFIX] [-c CLARA_HOME] [-t #] [-n #] [-m] [-h] YAML" +usage="\nUsage: run-clara -y YAML [-h] [-m] [-t #] [-n #] [-o DIR] [-p PREFIX] [-c CLARA_HOME] FILE..." info='\nRequired Arguments:\n -\tYAML = path to CLARA YAML steering file\n +\tFILE... (input data files)\n +\t-y YAML file\n Options:\n -\t-i directory or shell glob of input HIPO files (default=.)\n \t-o output directory (default=.)\n \t-p output prefix (default=rec_)\n \t-c CLARA installation (default=$CLARA_HOME)\n @@ -15,9 +15,7 @@ Options:\n \t-n number of events (default=-1)\n \t-m merge output files (see dependencies below)\n \t-h print this help and exit\n\n -Defaults will use $CLARA_HOME to read all *.hipo files in $PWD,\n -with all output written to $PWD. Merging outputs (-m) requires\n -hipo-utils and yq (https://github.com/mikefarah/yq) in $PATH.' +Merging outputs (-m) requires hipo-utils and yq (https://github.com/mikefarah/yq).' function error() { echo -e "\n$usage\n\nERROR: $@." && exit 1 @@ -64,14 +62,13 @@ function get_dpe_port() { set -e # Check user command-line options: -input=. output=. threads=2 prefix=rec_ -while getopts i:o:p:c:t:n:mh opt +while getopts y:o:p:c:t:n:mh opt do case $opt in - i) input=$OPTARG ;; + y) yaml=$OPTARG ;; o) output=$OPTARG ;; p) prefix=$OPTARG ;; c) CLARA_HOME=$OPTARG ;; @@ -83,14 +80,14 @@ do done shift $((OPTIND-1)) -[ $# -gt 1 ] && error "Extra arguments: ${@:2} (options must come before positionals)" -[ $# -lt 1 ] && error "YAML file argument is required" -[ -r $1 ] && yaml=$1 || error "YAML file does not exist: $yaml" +inputs=$@ + +[ $# -lt 1 ] && error "Input data files are required." +[ -z ${yaml+x} ] && error "-y YAML is required" +[ -r $yaml ] || error "YAML file does not exist: $yaml" [ -z ${CLARA_HOME+x} ] && error "-c must be specified or \$CLARA_HOME set" [ -d $CLARA_HOME ] || error "Invalid CLARA_HOME: $CLARA_HOME" [ $threads -eq 0 ] && threads=`grep -c ^processor /proc/cpuinfo` -! test -r $input && echo "WARNING: Interpreting -i as a glob: $input" -! test -r $input && ! compgen -G $input && error "Invalid glob -i: $input" ! [ -z ${merge+x} ] && ! command -v hipo-utils >& /dev/null && error "Merging requested, but hipo-utils is not in \$PATH" # Create the environment variables and directories required by CLARA: @@ -103,18 +100,20 @@ unset CLARA_MONITOR_FE # Normalize all paths: output=$(abspath $output) yaml=$(abspath $yaml) -test -r $input && input=$(abspath $input) export CLARA_HOME=$(abspath $CLARA_HOME) export CLAS12DIR=$CLARA_HOME/plugins/clas12 # Generate the file for CLARA containing a file list (of relative paths, not absolute): -! test -r $input && compgen -G $input | xargs basename > $CLARA_USER_DATA/filelist.txt -test -d $input && find $input -maxdepth 1 -name "*.hipo" -exec basename {} \; > $CLARA_USER_DATA/filelist.txt -test -f $input && echo $(basename $input) > $CLARA_USER_DATA/filelist.txt +touch $CLARA_USER_DATA/filelist.txt +for x in $inputs +do + test -f $x && test -r $x || error "Invalid input file: $x" + echo $(basename $x) >> $CLARA_USER_DATA/filelist.txt + ln -s $(abspath $(dirname $x))/$(basename $x) $CLARA_USER_DATA +done [ $(cat $CLARA_USER_DATA/filelist.txt | wc -l) -gt 0 ] || error "Found no input files." # Finally, run CLARA: -[ -f $input ] || [ -h $input ] && input=$(dirname $input) if [ $(uname) == "Darwin" ] then ip=$(get_host_ip) || error "Unknown IP address" @@ -128,12 +127,12 @@ then unset JAVA_OPTS $CLARA_HOME/bin/clara-orchestrator \ -F -f ${ip}%${port}_java -s recon \ - -i $input -o $output -z $prefix \ + -i $CLARA_USER_DATA -o $output -z $prefix \ -p $threads -t $threads \ $yaml $CLARA_USER_DATA/filelist.txt else $CLARA_HOME/lib/clara/run-clara \ - -i $input \ + -i $CLARA_USER_DATA \ -o $CLARA_USER_DATA \ -z $prefix \ -x $CLARA_USER_DATA/log \ diff --git a/validation/advanced-tests/run-advanced-tests.sh b/validation/advanced-tests/run-advanced-tests.sh index c5eb2fd224..9f46454014 100755 --- a/validation/advanced-tests/run-advanced-tests.sh +++ b/validation/advanced-tests/run-advanced-tests.sh @@ -22,7 +22,7 @@ $COAT/bin/decoder -t -0.5 -s 0.0 -i ./twoTrackEvents_809_raw.evio -o ./twoTrackE [ $? -ne 0 ] && echo "decoder failure" && exit 3 # run clara -$COAT/bin/run-clara $COAT/etc/services/kpp.yaml +$COAT/bin/run-clara -y $COAT/etc/services/kpp.yaml *.hipo [ $? -ne 0 ] && echo "reconstruction with clara failure" && exit 4 # compile test codes From 5bfa1117d61a8e131cfe7601e28ffb20cb412ecf Mon Sep 17 00:00:00 2001 From: Nathan Baltzell Date: Wed, 9 Oct 2024 15:39:42 -0400 Subject: [PATCH 06/14] cleanup printout --- bin/run-clara | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/run-clara b/bin/run-clara index 263e31c335..c064ba681f 100755 --- a/bin/run-clara +++ b/bin/run-clara @@ -91,7 +91,7 @@ inputs=$@ ! [ -z ${merge+x} ] && ! command -v hipo-utils >& /dev/null && error "Merging requested, but hipo-utils is not in \$PATH" # Create the environment variables and directories required by CLARA: -[ -e $output ] && echo "WARNING: Using existing directory: $output." +[ -e $output ] && echo "WARNING: Using existing directory: $output" mkdir -p -v $output || error "Cannot create -o output directory: $output" mkdir -p $output/log $output/config $output/data/output export CLARA_USER_DATA=$output From 705305efdeaccb5e0d25ab2887188b80e5576788 Mon Sep 17 00:00:00 2001 From: Nathan Baltzell Date: Wed, 9 Oct 2024 15:40:45 -0400 Subject: [PATCH 07/14] use bash --- validation/advanced-tests/run-advanced-tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/validation/advanced-tests/run-advanced-tests.sh b/validation/advanced-tests/run-advanced-tests.sh index 9f46454014..f6e3572d4b 100755 --- a/validation/advanced-tests/run-advanced-tests.sh +++ b/validation/advanced-tests/run-advanced-tests.sh @@ -1,4 +1,4 @@ -#!/bin/sh -f +#!/bin/bash -f # coatjava must already be built at ../../coatjava/ From 0ea1483fa3dff88ca923dc51d7d6522ca5dc17a0 Mon Sep 17 00:00:00 2001 From: Nathan Baltzell Date: Wed, 9 Oct 2024 17:12:26 -0400 Subject: [PATCH 08/14] cleanup --- bin/run-clara | 137 +++++++++++++++++++++++++------------------------- 1 file changed, 68 insertions(+), 69 deletions(-) diff --git a/bin/run-clara b/bin/run-clara index c064ba681f..814cc4338a 100755 --- a/bin/run-clara +++ b/bin/run-clara @@ -2,8 +2,9 @@ ulimit -u 49152 >& /dev/null export JAVA_OPTS="${JAVA_OPTS} -XX:+IgnoreUnrecognizedVMOptions" +set -e -usage="\nUsage: run-clara -y YAML [-h] [-m] [-t #] [-n #] [-o DIR] [-p PREFIX] [-c CLARA_HOME] FILE..." +usage="Usage: run-clara -y YAML [-h] [-m] [-t #] [-n #] [-o DIR] [-p PREFIX] [-c CLARA_HOME] FILE..." info='\nRequired Arguments:\n \tFILE... (input data files)\n \t-y YAML file\n @@ -20,11 +21,57 @@ Merging outputs (-m) requires hipo-utils and yq (https://github.com/mikefarah/yq function error() { echo -e "\n$usage\n\nERROR: $@." && exit 1 } -function abspath() { - [ -d $1 ] && echo $(cd $1 && pwd) && return 0 - [ -r $1 ] && echo $(cd $(dirname $1) && pwd)/$(basename $1) && return 0 - return 1 -} + +# Interpret command line: +threads=2 +prefix=rec_ +CLARA_USER_DATA=. +while getopts y:o:p:c:t:n:mh opt +do + case $opt in + y) yaml=$OPTARG ;; + o) CLARA_USER_DATA=$OPTARG ;; + p) prefix=$OPTARG ;; + c) CLARA_HOME=$OPTARG ;; + t) threads=$OPTARG && echo $threads | grep -q -E '^[0-9]+$' || error "-t must be an integer, threads" ;; + n) nevents="-e $OPTARG" && echo $nevents | grep -q -E '^-e [0-9]+$' || error "-n must be an integer, events" ;; + m) merge=1 ;; + h) echo -e "\n$usage" && echo -e $info && exit 0 ;; + esac +done +shift $((OPTIND-1)) +inputs=("${@}") + +# Check configuration: +[ ${#inputs[@]} -lt 1 ] && error "Input data files are required" +[ -z ${yaml+x} ] && error "-y YAML is required" +[ -f $yaml ] && [ -r $yaml ] || error "YAML file does not exist: $yaml" +[ -z ${CLARA_HOME+x} ] && error "-c must be specified or \$CLARA_HOME set" +[ -d $CLARA_HOME ] || error "Invalid CLARA_HOME: $CLARA_HOME" +[ $threads -eq 0 ] && threads=`grep -c ^processor /proc/cpuinfo` +! [ -z ${merge+x} ] && [ "${#inputs[@]}" -eq 1 ] && echo "WARNING: merging disabled for single file" +! [ -z ${merge+x} ] && ! command -v hipo-utils >& /dev/null && error "Merging requested, but hipo-utils is not in \$PATH" +yaml=$(cd $(dirname $yaml) && pwd)/$(basename $yaml) + +# Create the environment variables and directories required by CLARA: +[ -e $CLARA_USER_DATA ] && echo "WARNING: Using existing directory: $CLARA_USER_DATA" +mkdir -p -v $CLARA_USER_DATA || error "Cannot create -o output directory: $CLARA_USER_DATA" +mkdir -p $CLARA_USER_DATA/log $CLARA_USER_DATA/config $CLARA_USER_DATA/data/output +export CLARA_USER_DATA=$(cd $CLARA_USER_DATA && pwd) +export CLARA_HOME=$(cd $CLARA_HOME && pwd) +export CLAS12DIR=$CLARA_HOME/plugins/clas12 +unset CLARA_MONITOR_FE + +# Generate the file for CLARA containing a list of file basenames: +rm -f $CLARA_USER_DATA/filelist.txt && touch $CLARA_USER_DATA/filelist.txt +for x in $inputs +do + test -f $x && test -r $x || error "Invalid input file: $x" + echo $(basename $x) >> $CLARA_USER_DATA/filelist.txt + ln -sf $(cd $(dirname $x) && pwd)/$(basename $x) $CLARA_USER_DATA +done +[ $(cat $CLARA_USER_DATA/filelist.txt | wc -l) -gt 0 ] || error "Found no input files" + function get_host_ip() { if command -v ip >/dev/null 2>&1 then @@ -59,79 +106,30 @@ function get_dpe_port() { return 1 } -set -e - -# Check user command-line options: -output=. -threads=2 -prefix=rec_ -while getopts y:o:p:c:t:n:mh opt -do - case $opt in - y) yaml=$OPTARG ;; - o) output=$OPTARG ;; - p) prefix=$OPTARG ;; - c) CLARA_HOME=$OPTARG ;; - t) threads=$OPTARG && echo $threads | grep -q -E '^[0-9]+$' || error "-t must be an integer, threads" ;; - n) nevents="-e $OPTARG" && echo $nevents | grep -q -E '^-e [0-9]+$' || error "-n must be an integer, events" ;; - m) merge=1 ;; - h) echo -e $usage && echo -e $info && exit 0 ;; - esac -done - -shift $((OPTIND-1)) -inputs=$@ - -[ $# -lt 1 ] && error "Input data files are required." -[ -z ${yaml+x} ] && error "-y YAML is required" -[ -r $yaml ] || error "YAML file does not exist: $yaml" -[ -z ${CLARA_HOME+x} ] && error "-c must be specified or \$CLARA_HOME set" -[ -d $CLARA_HOME ] || error "Invalid CLARA_HOME: $CLARA_HOME" -[ $threads -eq 0 ] && threads=`grep -c ^processor /proc/cpuinfo` -! [ -z ${merge+x} ] && ! command -v hipo-utils >& /dev/null && error "Merging requested, but hipo-utils is not in \$PATH" - -# Create the environment variables and directories required by CLARA: -[ -e $output ] && echo "WARNING: Using existing directory: $output" -mkdir -p -v $output || error "Cannot create -o output directory: $output" -mkdir -p $output/log $output/config $output/data/output -export CLARA_USER_DATA=$output -unset CLARA_MONITOR_FE - -# Normalize all paths: -output=$(abspath $output) -yaml=$(abspath $yaml) -export CLARA_HOME=$(abspath $CLARA_HOME) -export CLAS12DIR=$CLARA_HOME/plugins/clas12 - -# Generate the file for CLARA containing a file list (of relative paths, not absolute): -touch $CLARA_USER_DATA/filelist.txt -for x in $inputs -do - test -f $x && test -r $x || error "Invalid input file: $x" - echo $(basename $x) >> $CLARA_USER_DATA/filelist.txt - ln -s $(abspath $(dirname $x))/$(basename $x) $CLARA_USER_DATA -done -[ $(cat $CLARA_USER_DATA/filelist.txt | wc -l) -gt 0 ] || error "Found no input files." - # Finally, run CLARA: if [ $(uname) == "Darwin" ] then ip=$(get_host_ip) || error "Unknown IP address" port=$(get_dpe_port) || error "Unknown DPE port" - $CLARA_HOME/bin/j_dpe \ + set -v + echo $CLARA_HOME/bin/j_dpe \ --host $ip --port $port \ --session recon --max-cores $threads \ --max-sockets 5120 --report 5 \ 2>&1 | tee $CLARA_USER_DATA/log/dpe.log & - echo "Sleeping 7 ......." && sleep 7 + set +v + #echo "Sleeping 7 ......." && sleep 7 unset JAVA_OPTS - $CLARA_HOME/bin/clara-orchestrator \ + set -v + echo $CLARA_HOME/bin/clara-orchestrator \ -F -f ${ip}%${port}_java -s recon \ - -i $CLARA_USER_DATA -o $output -z $prefix \ + -i $CLARA_USER_DATA -o $CLARA_USER_DATA -z $prefix \ -p $threads -t $threads \ $yaml $CLARA_USER_DATA/filelist.txt + set +v else - $CLARA_HOME/lib/clara/run-clara \ + set -v + echo $CLARA_HOME/lib/clara/run-clara \ -i $CLARA_USER_DATA \ -o $CLARA_USER_DATA \ -z $prefix \ @@ -140,19 +138,20 @@ else $nevents \ -s recon \ $yaml $CLARA_USER_DATA/filelist.txt + set +v fi # Merge outputs: -if ! [ -z ${merge+x} ] +if [ ${#inputs[@]} -gt 1 ] && ! [ -z ${merge+x} ] then if grep -q org.jlab.jnp.grapes $yaml >& /dev/null then for id in $(yq .configuration.services.*.id $yaml | sort -n | uniq) do - hipo-utils -merge -o $output/$prefix$id.hipo $output/$prefix$id*.hipo + echo hipo-utils -merge -o $CLARA_USER_DATA/$prefix$id.hipo $CLARA_USER_DATA/$prefix$id*.hipo done else - outfiles=$(sed "s#^#$output/$prefix#" $CLARA_USER_DATA/filelist.txt) - hipo-utils -merge -o $output/$prefix.hipo $outfiles + outfiles=$(sed "s#^#$CLARA_USER_DATA/$prefix#" $CLARA_USER_DATA/filelist.txt) + echo hipo-utils -merge -o $CLARA_USER_DATA/$prefix.hipo $outfiles fi fi From b9d50c5b2bfa92f2eb665b04314bd70d53d5863a Mon Sep 17 00:00:00 2001 From: Nathan Baltzell Date: Wed, 9 Oct 2024 17:31:44 -0400 Subject: [PATCH 09/14] avoid platform dependence --- bin/run-clara | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/bin/run-clara b/bin/run-clara index 814cc4338a..6293ca75f1 100755 --- a/bin/run-clara +++ b/bin/run-clara @@ -40,16 +40,15 @@ do esac done shift $((OPTIND-1)) -inputs=("${@}") +inputs=$@ # Check configuration: -[ ${#inputs[@]} -lt 1 ] && error "Input data files are required" +[ $# -lt 1 ] && error "Input data files are required" [ -z ${yaml+x} ] && error "-y YAML is required" [ -f $yaml ] && [ -r $yaml ] || error "YAML file does not exist: $yaml" [ -z ${CLARA_HOME+x} ] && error "-c must be specified or \$CLARA_HOME set" [ -d $CLARA_HOME ] || error "Invalid CLARA_HOME: $CLARA_HOME" [ $threads -eq 0 ] && threads=`grep -c ^processor /proc/cpuinfo` -! [ -z ${merge+x} ] && [ "${#inputs[@]}" -eq 1 ] && echo "WARNING: merging disabled for single file" ! [ -z ${merge+x} ] && ! command -v hipo-utils >& /dev/null && error "Merging requested, but hipo-utils is not in \$PATH" yaml=$(cd $(dirname $yaml) && pwd)/$(basename $yaml) @@ -142,7 +141,7 @@ else fi # Merge outputs: -if [ ${#inputs[@]} -gt 1 ] && ! [ -z ${merge+x} ] +if ! [ -z ${merge+x} ] then if grep -q org.jlab.jnp.grapes $yaml >& /dev/null then From f81fa29cd42cd0db7bcf7ec6e389c788cbec268f Mon Sep 17 00:00:00 2001 From: Nathan Baltzell Date: Wed, 9 Oct 2024 17:37:15 -0400 Subject: [PATCH 10/14] reenable --- bin/run-clara | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/bin/run-clara b/bin/run-clara index 6293ca75f1..8de32a22f4 100755 --- a/bin/run-clara +++ b/bin/run-clara @@ -111,7 +111,7 @@ then ip=$(get_host_ip) || error "Unknown IP address" port=$(get_dpe_port) || error "Unknown DPE port" set -v - echo $CLARA_HOME/bin/j_dpe \ + $CLARA_HOME/bin/j_dpe \ --host $ip --port $port \ --session recon --max-cores $threads \ --max-sockets 5120 --report 5 \ @@ -120,7 +120,7 @@ then #echo "Sleeping 7 ......." && sleep 7 unset JAVA_OPTS set -v - echo $CLARA_HOME/bin/clara-orchestrator \ + $CLARA_HOME/bin/clara-orchestrator \ -F -f ${ip}%${port}_java -s recon \ -i $CLARA_USER_DATA -o $CLARA_USER_DATA -z $prefix \ -p $threads -t $threads \ @@ -128,7 +128,7 @@ then set +v else set -v - echo $CLARA_HOME/lib/clara/run-clara \ + $CLARA_HOME/lib/clara/run-clara \ -i $CLARA_USER_DATA \ -o $CLARA_USER_DATA \ -z $prefix \ @@ -147,10 +147,10 @@ then then for id in $(yq .configuration.services.*.id $yaml | sort -n | uniq) do - echo hipo-utils -merge -o $CLARA_USER_DATA/$prefix$id.hipo $CLARA_USER_DATA/$prefix$id*.hipo + hipo-utils -merge -o $CLARA_USER_DATA/$prefix$id.hipo $CLARA_USER_DATA/$prefix$id*.hipo done else outfiles=$(sed "s#^#$CLARA_USER_DATA/$prefix#" $CLARA_USER_DATA/filelist.txt) - echo hipo-utils -merge -o $CLARA_USER_DATA/$prefix.hipo $outfiles + hipo-utils -merge -o $CLARA_USER_DATA/$prefix.hipo $outfiles fi fi From 5030db36230c2f6c76f652f4a86df4a07d1bfa71 Mon Sep 17 00:00:00 2001 From: Nathan Baltzell Date: Wed, 9 Oct 2024 17:46:04 -0400 Subject: [PATCH 11/14] debug --- validation/advanced-tests/run-advanced-tests.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/validation/advanced-tests/run-advanced-tests.sh b/validation/advanced-tests/run-advanced-tests.sh index f6e3572d4b..30f249178b 100755 --- a/validation/advanced-tests/run-advanced-tests.sh +++ b/validation/advanced-tests/run-advanced-tests.sh @@ -22,6 +22,7 @@ $COAT/bin/decoder -t -0.5 -s 0.0 -i ./twoTrackEvents_809_raw.evio -o ./twoTrackE [ $? -ne 0 ] && echo "decoder failure" && exit 3 # run clara +ls -l $COAT/bin/run-clara -y $COAT/etc/services/kpp.yaml *.hipo [ $? -ne 0 ] && echo "reconstruction with clara failure" && exit 4 From 296a2a53210d86a8f887313771f1f10e7ed558b9 Mon Sep 17 00:00:00 2001 From: Nathan Baltzell Date: Wed, 9 Oct 2024 17:49:11 -0400 Subject: [PATCH 12/14] -f disables expansion, donot need it anyway --- validation/advanced-tests/run-advanced-tests.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/validation/advanced-tests/run-advanced-tests.sh b/validation/advanced-tests/run-advanced-tests.sh index 30f249178b..4922e0c48f 100755 --- a/validation/advanced-tests/run-advanced-tests.sh +++ b/validation/advanced-tests/run-advanced-tests.sh @@ -22,8 +22,7 @@ $COAT/bin/decoder -t -0.5 -s 0.0 -i ./twoTrackEvents_809_raw.evio -o ./twoTrackE [ $? -ne 0 ] && echo "decoder failure" && exit 3 # run clara -ls -l -$COAT/bin/run-clara -y $COAT/etc/services/kpp.yaml *.hipo +$COAT/bin/run-clara -y $COAT/etc/services/kpp.yaml ./twoTrackEvents_809.hipo [ $? -ne 0 ] && echo "reconstruction with clara failure" && exit 4 # compile test codes From 8ba094639faae55375725b2ac42b27564fa32af0 Mon Sep 17 00:00:00 2001 From: Nathan Baltzell Date: Wed, 9 Oct 2024 18:21:06 -0400 Subject: [PATCH 13/14] only link if necessary --- bin/run-clara | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/run-clara b/bin/run-clara index 8de32a22f4..36a830a29a 100755 --- a/bin/run-clara +++ b/bin/run-clara @@ -67,7 +67,7 @@ for x in $inputs do test -f $x && test -r $x || error "Invalid input file: $x" echo $(basename $x) >> $CLARA_USER_DATA/filelist.txt - ln -sf $(cd $(dirname $x) && pwd)/$(basename $x) $CLARA_USER_DATA + test -f $CLARA_USER_DATA/$(basename $x) || ln -sf $(cd $(dirname $x) && pwd)/$(basename $x) $CLARA_USER_DATA done [ $(cat $CLARA_USER_DATA/filelist.txt | wc -l) -gt 0 ] || error "Found no input files" From 95863ac9996ad61e64dacabc60da87f396b02e2f Mon Sep 17 00:00:00 2001 From: Nathan Baltzell Date: Thu, 10 Oct 2024 12:03:35 -0400 Subject: [PATCH 14/14] fix glob --- bin/run-clara | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/run-clara b/bin/run-clara index 36a830a29a..7973a39cb0 100755 --- a/bin/run-clara +++ b/bin/run-clara @@ -147,7 +147,7 @@ then then for id in $(yq .configuration.services.*.id $yaml | sort -n | uniq) do - hipo-utils -merge -o $CLARA_USER_DATA/$prefix$id.hipo $CLARA_USER_DATA/$prefix$id*.hipo + hipo-utils -merge -o $CLARA_USER_DATA/$prefix$id.hipo $CLARA_USER_DATA/$prefix*$id.hipo done else outfiles=$(sed "s#^#$CLARA_USER_DATA/$prefix#" $CLARA_USER_DATA/filelist.txt)