add option to merge outputs

bin/run-clara

@@ -2,27 +2,75 @@
 
 ulimit -u 49152 >& /dev/null
 export JAVA_OPTS="${JAVA_OPTS} -XX:+IgnoreUnrecognizedVMOptions"
+set -e
 
-usage="Usage:  run-clara [-i IN] [-o OUT] [-c CLARA] [-t #] [-n #] YAML"
-info='\nOptions:\n
-YAML - path to CLARA YAML steering file\n
--i   input HIPO file, directory of *.hipo files, or glob of HIPO files (default=.)\n
--o   output directory (default=.)\n
--p   output prefix (default=rec_)\n
--c   CLARA installation (default=$CLARA_HOME)\n
--t   number of threads (default=2)\n
--n   number of events (default=-1)\n\n
-Defaults will use $CLARA_HOME to read all *.hipo files in $PWD,\n
-with all output written to $PWD.'
+usage="Usage:  run-clara -y YAML [-h] [-m] [-t #] [-n #] [-o DIR] [-p PREFIX] [-c CLARA_HOME] FILE..."
+info='\nRequired Arguments:\n
+\tFILE... (input data files)\n
+\t-y   YAML file\n
+Options:\n
+\t-o   output directory (default=.)\n
+\t-p   output prefix (default=rec_)\n
+\t-c   CLARA installation (default=$CLARA_HOME)\n
+\t-t   number of threads (default=2)\n
+\t-n   number of events (default=-1)\n
+\t-m   merge output files (see dependencies below)\n
+\t-h   print this help and exit\n\n
+Merging outputs (-m) requires hipo-utils and yq (https://github.com/mikefarah/yq).'
 
 function error() {
     echo -e "\n$usage\n\nERROR:  $@." && exit 1
 }
-function abspath() {
-    [ -d $1 ] && echo $(cd $1 && pwd) && return 0
-    [ -r $1 ] && echo $(cd $(dirname $1) && pwd)/$(basename $1) && return 0
-    return 1
-}
+
+# Interpret command line:
+threads=2
+prefix=rec_
+CLARA_USER_DATA=.
+while getopts y:o:p:c:t:n:mh opt
+do
+    case $opt in
+        y) yaml=$OPTARG ;;
+        o) CLARA_USER_DATA=$OPTARG ;;
+        p) prefix=$OPTARG ;;
+        c) CLARA_HOME=$OPTARG ;;
+        t) threads=$OPTARG && echo $threads | grep -q -E '^[0-9]+$' || error "-t must be an integer, threads" ;;
+        n) nevents="-e $OPTARG" && echo $nevents | grep -q -E '^-e [0-9]+$' || error "-n must be an integer, events" ;;
+        m) merge=1 ;;
+        h) echo -e "\n$usage" && echo -e $info && exit 0 ;;
+    esac
+done
+shift $((OPTIND-1))
+inputs=$@
+
+# Check configuration:
+[ $# -lt 1 ] && error "Input data files are required"
+[ -z ${yaml+x} ] && error "-y YAML is required"
+[ -f $yaml ] && [ -r $yaml ] || error "YAML file does not exist:  $yaml"
+[ -z ${CLARA_HOME+x} ] && error "-c must be specified or \$CLARA_HOME set"
+[ -d $CLARA_HOME ] || error "Invalid CLARA_HOME:  $CLARA_HOME"
+[ $threads -eq 0 ] && threads=`grep -c ^processor /proc/cpuinfo`
+! [ -z ${merge+x} ] && ! command -v hipo-utils >& /dev/null && error "Merging requested, but hipo-utils is not in \$PATH"
+yaml=$(cd $(dirname $yaml) && pwd)/$(basename $yaml)
+
+# Create the environment variables and directories required by CLARA:
+[ -e $CLARA_USER_DATA ] && echo "WARNING:  Using existing directory:  $CLARA_USER_DATA"
+mkdir -p -v $CLARA_USER_DATA || error "Cannot create -o output directory: $CLARA_USER_DATA"
+mkdir -p $CLARA_USER_DATA/log $CLARA_USER_DATA/config $CLARA_USER_DATA/data/output
+export CLARA_USER_DATA=$(cd $CLARA_USER_DATA && pwd)
+export CLARA_HOME=$(cd $CLARA_HOME && pwd)
+export CLAS12DIR=$CLARA_HOME/plugins/clas12
+unset CLARA_MONITOR_FE
+
+# Generate the file for CLARA containing a list of file basenames:
+rm -f $CLARA_USER_DATA/filelist.txt && touch $CLARA_USER_DATA/filelist.txt
+for x in $inputs
+do
+    test -f $x && test -r $x || error "Invalid input file:  $x"
+    echo $(basename $x) >> $CLARA_USER_DATA/filelist.txt
+    test -f $CLARA_USER_DATA/$(basename $x) || ln -sf $(cd $(dirname $x) && pwd)/$(basename $x) $CLARA_USER_DATA
+done
+[ $(cat $CLARA_USER_DATA/filelist.txt | wc -l) -gt 0 ] || error "Found no input files"
+
 function get_host_ip() {
     if command -v ip >/dev/null 2>&1
     then
@@ -57,83 +105,52 @@ function get_dpe_port() {
     return 1
 }
 
-set -e
-
-# Check user command-line options:
-input=.
-output=.
-threads=2
-prefix=rec_
-while getopts i:o:p:c:t:n:h opt
-do
-    case $opt in
-        i) input=$OPTARG ;;
-        o) output=$OPTARG ;;
-        p) prefix=$OPTARG ;;
-        c) CLARA_HOME=$OPTARG ;;
-        t) threads=$OPTARG && echo $threads | grep -q -E '^[0-9]+$' || error "-t must be an integer, threads" ;;
-        n) nevents="-e $OPTARG" && echo $nevents | grep -q -E '^-e [0-9]+$' || error "-n must be an integer, events" ;;
-        h) echo -e $usage && echo -e $info && exit 0 ;;
-    esac
-done
-
-shift $((OPTIND-1))
-[ $# -gt 1 ] && error "Extra arguments:  ${@:2} (options must come before positionals)"
-[ $# -lt 1 ] && error "YAML file argument is required"
-[ -r $1 ] && yaml=$1 || error "YAML file does not exist:  $yaml"
-[ -z ${CLARA_HOME+x} ] && error "-c must be specified or \$CLARA_HOME set"
-[ -d $CLARA_HOME ] || error "Invalid CLARA_HOME:  $CLARA_HOME"
-[ $threads -eq 0 ] && threads=`grep -c ^processor /proc/cpuinfo`
-! test -r $input && echo "WARNING:  Interpreting -i as a glob:  $input"
-! test -r $input && ! compgen -G $input && error "Invalid glob -i:  $input"
-
-# Create the environment variables and directories required by CLARA:
-[ -e $output ] && echo "WARNING:  Using existing directory:  $output."
-mkdir -p -v $output || error "Cannot create -o output directory: $output"
-mkdir -p $output/log $output/config $output/data/output
-export CLARA_USER_DATA=$output
-unset CLARA_MONITOR_FE
-
-# Normalize all paths:
-output=$(abspath $output)
-yaml=$(abspath $yaml)
-test -r $input && input=$(abspath $input)
-export CLARA_HOME=$(abspath $CLARA_HOME)
-export CLAS12DIR=$CLARA_HOME/plugins/clas12
-
-# Generate the file for CLARA containing a file list (of relative paths, not absolute):
-! test -r $input && compgen -G $input > $CLARA_USER_DATA/filelist.txt
-test -d $input && find $input -maxdepth 1 -name "*.hipo" -exec basename {} \; > $CLARA_USER_DATA/filelist.txt
-test -f $input && echo $(basename $input) > $CLARA_USER_DATA/filelist.txt
-[ $(cat $CLARA_USER_DATA/filelist.txt | wc -l) -gt 0 ] || error "Found no input files."
-
 # Finally, run CLARA:
-[ -f $input ] || [ -h $input ] && input=$(dirname $input)
 if [ $(uname) == "Darwin" ]
 then
     ip=$(get_host_ip) || error "Unknown IP address"
     port=$(get_dpe_port) || error "Unknown DPE port"
+    set -v
     $CLARA_HOME/bin/j_dpe \
         --host $ip --port $port \
         --session recon --max-cores $threads \
         --max-sockets 5120 --report 5 \
         2>&1 | tee $CLARA_USER_DATA/log/dpe.log &
-    echo "Sleeping 7 ......." && sleep 7
+    set +v
+    #echo "Sleeping 7 ......." && sleep 7
     unset JAVA_OPTS
+    set -v
     $CLARA_HOME/bin/clara-orchestrator \
         -F -f ${ip}%${port}_java -s recon \
-            -i $input -o $output -z $prefix \
+            -i $CLARA_USER_DATA -o $CLARA_USER_DATA -z $prefix \
             -p $threads -t $threads \
             $yaml $CLARA_USER_DATA/filelist.txt
+    set +v
 else
+    set -v
     $CLARA_HOME/lib/clara/run-clara \
-    -i $input \
+    -i $CLARA_USER_DATA \
     -o $CLARA_USER_DATA \
     -z $prefix \
     -x $CLARA_USER_DATA/log \
     -t $threads \
     $nevents \
     -s recon \
     $yaml $CLARA_USER_DATA/filelist.txt
+    set +v
 fi
 
+# Merge outputs:
+if ! [ -z ${merge+x} ]
+then
+    if grep -q org.jlab.jnp.grapes $yaml >& /dev/null
+    then
+        for id in $(yq .configuration.services.*.id $yaml | sort -n | uniq)
+        do
+            hipo-utils -merge -o $CLARA_USER_DATA/$prefix$id.hipo $CLARA_USER_DATA/$prefix*$id.hipo
+        done
+    else
+        outfiles=$(sed "s#^#$CLARA_USER_DATA/$prefix#" $CLARA_USER_DATA/filelist.txt)
+        hipo-utils -merge -o $CLARA_USER_DATA/$prefix.hipo $outfiles
+    fi
+fi

validation/advanced-tests/run-advanced-tests.sh

@@ -1,4 +1,4 @@
-#!/bin/sh -f
+#!/bin/bash -f
 
 # coatjava must already be built at ../../coatjava/
 
@@ -22,7 +22,7 @@ $COAT/bin/decoder -t -0.5 -s 0.0 -i ./twoTrackEvents_809_raw.evio -o ./twoTrackE
 [ $? -ne 0 ] && echo "decoder failure" && exit 3
 
 # run clara
-$COAT/bin/run-clara $COAT/etc/services/kpp.yaml
+$COAT/bin/run-clara -y $COAT/etc/services/kpp.yaml ./twoTrackEvents_809.hipo
 [ $? -ne 0 ] && echo "reconstruction with clara failure" && exit 4
 
 # compile test codes