diff --git a/DATA/common/gen_topo_helper_functions.sh b/DATA/common/gen_topo_helper_functions.sh index 6a3e84149..00fc1f118 100755 --- a/DATA/common/gen_topo_helper_functions.sh +++ b/DATA/common/gen_topo_helper_functions.sh @@ -9,9 +9,19 @@ has_detector() [[ $WORKFLOW_DETECTORS =~ (^|,)"$1"(,|$) ]] } +has_detector_from_global_reader_clusters() +{ + [[ $WORKFLOW_DETECTORS_USE_GLOBAL_READER_CLUSTERS =~ (^|,)"$1"(,|$) ]] +} + +has_detector_from_global_reader_tracks() +{ + [[ $WORKFLOW_DETECTORS_USE_GLOBAL_READER_TRACKS =~ (^|,)"$1"(,|$) ]] +} + has_detector_from_global_reader() { - [[ $WORKFLOW_DETECTORS_USE_GLOBAL_READER =~ (^|,)"$1"(,|$) ]] + has_detector_from_global_reader_tracks $1 || has_detector_from_global_reader_clusters $1 } has_detector_calib() diff --git a/DATA/common/setenv.sh b/DATA/common/setenv.sh index 6c31539c9..f7342e553 100755 --- a/DATA/common/setenv.sh +++ b/DATA/common/setenv.sh @@ -49,6 +49,8 @@ if [[ -z "${WORKFLOW_DETECTORS_RECO+x}" ]] || [[ "0$WORKFLOW_DETECTORS_RECO" == if [[ -z "${WORKFLOW_DETECTORS_CTF+x}" ]] || [[ "0$WORKFLOW_DETECTORS_CTF" == "0ALL" ]]; then export WORKFLOW_DETECTORS_CTF=$WORKFLOW_DETECTORS; fi if [[ "0${WORKFLOW_DETECTORS_FLP_PROCESSING:-}" == "0ALL" ]]; then export WORKFLOW_DETECTORS_FLP_PROCESSING=$WORKFLOW_DETECTORS; fi if [[ "0${WORKFLOW_DETECTORS_USE_GLOBAL_READER:-}" == "0ALL" ]]; then export WORKFLOW_DETECTORS_USE_GLOBAL_READER=$WORKFLOW_DETECTORS; else export WORKFLOW_DETECTORS_USE_GLOBAL_READER=${WORKFLOW_DETECTORS_USE_GLOBAL_READER:-}; fi +if [[ "0${WORKFLOW_DETECTORS_USE_GLOBAL_READER_TRACKS:-}" == "0ALL" ]]; then export WORKFLOW_DETECTORS_USE_GLOBAL_READER_TRACKS=$WORKFLOW_DETECTORS; else export WORKFLOW_DETECTORS_USE_GLOBAL_READER_TRACKS=${WORKFLOW_DETECTORS_USE_GLOBAL_READER_TRACKS:-}; fi +if [[ "0${WORKFLOW_DETECTORS_USE_GLOBAL_READER_CLUSTERS:-}" == "0ALL" ]]; then export WORKFLOW_DETECTORS_USE_GLOBAL_READER_CLUSTERS=$WORKFLOW_DETECTORS; else export WORKFLOW_DETECTORS_USE_GLOBAL_READER_CLUSTERS=${WORKFLOW_DETECTORS_USE_GLOBAL_READER_CLUSTERS:-}; fi if [[ -z "${WORKFLOW_PARAMETERS:-}" ]]; then export WORKFLOW_PARAMETERS=; fi if [[ ! -z ${WORKFLOW_DETECTORS_EXCLUDE_QC:-} ]]; then @@ -61,6 +63,16 @@ if [[ ! -z ${WORKFLOW_DETECTORS_EXCLUDE_CALIB:-} ]]; then export WORKFLOW_DETECTORS_CALIB=$(echo $WORKFLOW_DETECTORS_CALIB | sed -e "s/,$i,/,/g" -e "s/^$i,//" -e "s/,$i"'$'"//" -e "s/^$i"'$'"//") done fi +if [[ ! -z ${WORKFLOW_DETECTORS_EXCLUDE_GLOBAL_READER_TRACKS:-} ]]; then + for i in ${WORKFLOW_DETECTORS_EXCLUDE_GLOBAL_READER_TRACKS//,/ }; do + export WORKFLOW_DETECTORS_USE_GLOBAL_READER_TRACKS=$(echo $WORKFLOW_DETECTORS_USE_GLOBAL_READER_TRACKS | sed -e "s/,$i,/,/g" -e "s/^$i,//" -e "s/,$i"'$'"//" -e "s/^$i"'$'"//") + done +fi +if [[ ! -z ${WORKFLOW_DETECTORS_EXCLUDE_GLOBAL_READER_CLUSTERS:-} ]]; then + for i in ${WORKFLOW_DETECTORS_EXCLUDE_GLOBAL_READER_CLUSTERS//,/ }; do + export WORKFLOW_DETECTORS_USE_GLOBAL_READER_CLUSTERS=$(echo $WORKFLOW_DETECTORS_USE_GLOBAL_READER_CLUSTERS | sed -e "s/,$i,/,/g" -e "s/^$i,//" -e "s/,$i"'$'"//" -e "s/^$i"'$'"//") + done +fi if [[ -z "${TFLOOP:-}" ]]; then export TFLOOP=0; fi # loop over timeframes if [[ -z "${NTIMEFRAMES:-}" ]]; then export NTIMEFRAMES=-1; fi # max number of time frames to process, <=0 : unlimited diff --git a/DATA/common/setenv_calib.sh b/DATA/common/setenv_calib.sh index 87fe461ea..a1d2e7692 100755 --- a/DATA/common/setenv_calib.sh +++ b/DATA/common/setenv_calib.sh @@ -30,6 +30,8 @@ if has_detector_calib FDD && has_processing_step FDD_RECO; then CAN_DO_CALIB_FDD if has_detector_calib ZDC && has_processing_step ZDC_RECO; then CAN_DO_CALIB_ZDC_TDC=1; else CAN_DO_CALIB_ZDC_TDC=0; fi # for async recalibration if has_detector_calib EMC && has_detector_reco EMC && [[ $SYNCMODE != 1 ]]; then CAN_DO_CALIB_EMC_ASYNC_RECALIB=1; else CAN_DO_CALIB_EMC_ASYNC_RECALIB=0; fi +if [[ $SYNCMODE != 1 ]] && has_detector_reco TPC; then CAN_DO_CALIB_ASYNC_EXTRACTTPCCURRENTS=1; else CAN_DO_CALIB_ASYNC_EXTRACTTPCCURRENTS=0; fi +if [[ $SYNCMODE != 1 ]] && has_detector_reco TPC && has_detector_reco ITS && has_detector_reco FT0; then CAN_DO_CALIB_ASYNC_EXTRACTTIMESERIES=1; else CAN_DO_CALIB_ASYNC_EXTRACTTIMESERIES=0; fi # additional individual settings for calibration workflows if has_detector CTP; then export CALIB_TPC_SCDCALIB_CTP_INPUT="--enable-ctp"; else export CALIB_TPC_SCDCALIB_CTP_INPUT=""; fi @@ -174,6 +176,11 @@ fi ( [[ -z ${CALIB_ZDC_TDC:-} ]] || [[ $CAN_DO_CALIB_ZDC_TDC == 0 ]] ) && CALIB_ZDC_TDC=0 # for async: ( [[ -z ${CALIB_EMC_ASYNC_RECALIB:-} ]] || [[ $CAN_DO_CALIB_EMC_ASYNC_RECALIB == 0 ]] ) && CALIB_EMC_ASYNC_RECALIB=0 +( [[ -z ${CALIB_ASYNC_EXTRACTTPCCURRENTS:-} ]] || [[ $CAN_DO_CALIB_ASYNC_EXTRACTTPCCURRENTS == 0 ]] ) && CALIB_ASYNC_EXTRACTTPCCURRENTS=0 +( [[ -z ${CALIB_ASYNC_DISABLE3DCURRENTS:-} ]] || [[ $CAN_DO_CALIB_ASYNC_DISABLE3DCURRENTS == 0 ]] ) && CALIB_ASYNC_DISABLE3DCURRENTS=0 +: ${ON_SKIMMED_DATA:=0} +( [[ -z ${CALIB_ASYNC_EXTRACTTIMESERIES:-} ]] || [[ $CAN_DO_CALIB_ASYNC_EXTRACTTIMESERIES == 0 ]] ) && CALIB_ASYNC_EXTRACTTIMESERIES=0 + if [[ "0${GEN_TOPO_VERBOSE:-}" == "01" ]]; then echo "CALIB_PRIMVTX_MEANVTX = $CALIB_PRIMVTX_MEANVTX" 1>&2 diff --git a/DATA/production/configurations/asyncReco/async_pass.sh b/DATA/production/configurations/asyncReco/async_pass.sh index c65ac7b49..05ed0dea2 100755 --- a/DATA/production/configurations/asyncReco/async_pass.sh +++ b/DATA/production/configurations/asyncReco/async_pass.sh @@ -149,6 +149,14 @@ if [[ -n "$ALIEN_JDL_DOMEANVTXCALIB" ]]; then export ADD_CALIB=1 fi +if [[ "$ALIEN_JDL_EXTRACTCURRENTS" == "1" ]]; then + export ADD_CALIB=1 +fi + +if [[ "$ALIEN_JDL_EXTRACTTIMESERIES" == "1" ]]; then + export ADD_CALIB=1 +fi + # AOD file size if [[ -n "$ALIEN_JDL_AODFILESIZE" ]]; then export AOD_FILE_SIZE="$ALIEN_JDL_AODFILESIZE" @@ -577,18 +585,19 @@ else if ([[ -z "$ALIEN_JDL_SSPLITSTEP" ]] && [[ -z "$ALIEN_JDL_SSPLITSTEP" ]]) || [[ "$ALIEN_JDL_SSPLITSTEP" -eq 3 ]] || ( [[ -n $ALIEN_JDL_STARTSPLITSTEP ]] && [[ "$ALIEN_JDL_STARTSPLITSTEP" -le 3 ]]) || [[ "$ALIEN_JDL_SSPLITSTEP" -eq "all" ]]; then # 3. matching, QC, calib, AOD WORKFLOW_PARAMETERS=$WORKFLOW_PARAMETERS_START + echo "WORKFLOW_PARAMETERS=$WORKFLOW_PARAMETERS" echo "Step 3) matching, QC, calib, AOD" echo -e "\nStep 3) matching, QC, calib, AOD" >> workflowconfig.log export TIMEFRAME_RATE_LIMIT=0 echo "Removing detectors $DETECTORS_EXCLUDE" READER_DELAY=${ALIEN_JDL_READERDELAY:-30} export ARGS_EXTRA_PROCESS_o2_global_track_cluster_reader+=" --reader-delay $READER_DELAY " - echo "extra args are $ARGS_EXTRA_PROCESS_o2_global_track_cluster_reader_workflow" - env $SETTING_ROOT_OUTPUT IS_SIMULATED_DATA=0 WORKFLOWMODE=print TFDELAY=$TFDELAYSECONDS WORKFLOW_DETECTORS=ALL WORKFLOW_DETECTORS_EXCLUDE=$DETECTORS_EXCLUDE WORKFLOW_DETECTORS_USE_GLOBAL_READER=ALL WORKFLOW_DETECTORS_EXCLUDE_QC=CPV,$DETECTORS_EXCLUDE ./run-workflow-on-inputlist.sh $INPUT_TYPE list.list >> workflowconfig.log + echo "extra args are $ARGS_EXTRA_PROCESS_o2_global_track_cluster_reader" + env $SETTING_ROOT_OUTPUT IS_SIMULATED_DATA=0 WORKFLOWMODE=print TFDELAY=$TFDELAYSECONDS WORKFLOW_DETECTORS=ALL WORKFLOW_DETECTORS_EXCLUDE=$DETECTORS_EXCLUDE WORKFLOW_DETECTORS_USE_GLOBAL_READER_TRACKS=ALL WORKFLOW_DETECTORS_USE_GLOBAL_READER_CLUSTERS=ALL WORKFLOW_DETECTORS_EXCLUDE_GLOBAL_READER_TRACKS=HMP WORKFLOW_DETECTORS_EXCLUDE_QC=CPV,$DETECTORS_EXCLUDE ./run-workflow-on-inputlist.sh $INPUT_TYPE list.list >> workflowconfig.log # run it if [[ "0$RUN_WORKFLOW" != "00" ]]; then timeStart=`date +%s` - time env $SETTING_ROOT_OUTPUT IS_SIMULATED_DATA=0 WORKFLOWMODE=run TFDELAY=$TFDELAYSECONDS WORKFLOW_DETECTORS=ALL WORKFLOW_DETECTORS_USE_GLOBAL_READER=ALL WORKFLOW_DETECTORS_EXCLUDE=$DETECTORS_EXCLUDE WORKFLOW_DETECTORS_EXCLUDE_QC=CPV,$DETECTORS_EXCLUDE ./run-workflow-on-inputlist.sh $INPUT_TYPE list.list + time env $SETTING_ROOT_OUTPUT IS_SIMULATED_DATA=0 WORKFLOWMODE=run TFDELAY=$TFDELAYSECONDS WORKFLOW_DETECTORS=ALL WORKFLOW_DETECTORS_EXCLUDE=$DETECTORS_EXCLUDE WORKFLOW_DETECTORS_USE_GLOBAL_READER_TRACKS=ALL WORKFLOW_DETECTORS_USE_GLOBAL_READER_CLUSTERS=ALL WORKFLOW_DETECTORS_EXCLUDE_GLOBAL_READER_TRACKS=HMP WORKFLOW_DETECTORS_EXCLUDE_QC=CPV,$DETECTORS_EXCLUDE ./run-workflow-on-inputlist.sh $INPUT_TYPE list.list exitcode=$? timeEnd=`date +%s` timeUsed=$(( $timeUsed+$timeEnd-$timeStart )) diff --git a/DATA/production/configurations/asyncReco/setenv_extra.sh b/DATA/production/configurations/asyncReco/setenv_extra.sh index a34d8ffce..1036128d6 100644 --- a/DATA/production/configurations/asyncReco/setenv_extra.sh +++ b/DATA/production/configurations/asyncReco/setenv_extra.sh @@ -553,6 +553,10 @@ if [[ $ADD_CALIB == "1" ]]; then export CALIB_ZDC_TDC=0 export CALIB_FT0_TIMEOFFSET=0 export CALIB_TPC_SCDCALIB=0 + export CALIB_FT0_INTEGRATEDCURR=0 + export CALIB_FV0_INTEGRATEDCURR=0 + export CALIB_FDD_INTEGRATEDCURR=0 + export CALIB_TOF_INTEGRATEDCURR=0 if [[ $DO_TPC_RESIDUAL_EXTRACTION == "1" ]]; then export CALIB_TPC_SCDCALIB=1 export CALIB_TPC_SCDCALIB_SENDTRKDATA=1 @@ -582,38 +586,31 @@ if [[ $ADD_CALIB == "1" ]]; then export ARGS_EXTRA_PROCESS_o2_calibration_trd_workflow="$ARGS_EXTRA_PROCESS_o2_calibration_trd_workflow --enable-root-output" export CALIB_TRD_GAIN=1 fi - if [[ $ALIEN_JDL_DOUPLOADSLOCALLY == 1 ]]; then - export CCDB_POPULATOR_UPLOAD_PATH="file://$PWD" + # extra workflows in case we want to process the currents for FT0, FV0, TOF, TPC + if [[ -n $ALIEN_JDL_EXTRACTCURRENTS ]] ; then + export CALIB_FT0_INTEGRATEDCURR=$ALIEN_JDL_EXTRACTCURRENTS + export CALIB_FV0_INTEGRATEDCURR=$ALIEN_JDL_EXTRACTCURRENTS + export CALIB_FDD_INTEGRATEDCURR=$ALIEN_JDL_EXTRACTCURRENTS + export CALIB_TOF_INTEGRATEDCURR=$ALIEN_JDL_EXTRACTCURRENTS + export CALIB_ASYNC_EXTRACTTPCCURRENTS=$ALIEN_JDL_EXTRACTCURRENTS fi -fi - -# extra workflows in case we want to process the currents for FT0, FV0, TOF, TPC -if [[ $ALIEN_JDL_EXTRACTCURRENTS == 1 ]]; then - if [[ -z "${WORKFLOW_DETECTORS_RECO+x}" ]] || [[ "0$WORKFLOW_DETECTORS_RECO" == "0ALL" ]]; then export WORKFLOW_DETECTORS_RECO=$WORKFLOW_DETECTORS; fi - has_detector_reco FT0 && add_comma_separated ADD_EXTRA_WORKFLOW "o2-ft0-integrate-cluster-workflow" - has_detector_reco FV0 && add_comma_separated ADD_EXTRA_WORKFLOW "o2-fv0-integrate-cluster-workflow" - has_detector_reco TOF && add_comma_separated ADD_EXTRA_WORKFLOW "o2-tof-integrate-cluster-workflow" - if [[ $ALIEN_JDL_DISABLE3DCURRENTS != 1 ]]; then - export ARGS_EXTRA_PROCESS_o2_tpc_integrate_cluster_workflow="$ARGS_EXTRA_PROCESS_o2_tpc_integrate_cluster_workflow--process-3D-currents --nSlicesTF 1" + if [[ -n $ALIEN_JDL_DISABLE3DCURRENTS ]]; then + export CALIB_ASYNC_DISABLE3DCURRENTS=$ALIEN_JDL_DISABLE3DCURRENTS fi - has_detector_reco TPC && add_comma_separated ADD_EXTRA_WORKFLOW "o2-tpc-integrate-cluster-workflow" -fi -# extra workflows in case we want to process the currents for time series -if [[ $ALIEN_JDL_EXTRACTTIMESERIES == 1 ]]; then - if [[ -z "${WORKFLOW_DETECTORS_RECO+x}" ]] || [[ "0$WORKFLOW_DETECTORS_RECO" == "0ALL" ]]; then export WORKFLOW_DETECTORS_RECO=$WORKFLOW_DETECTORS; fi - has_detector_reco TPC && has_detector_reco ITS && has_detector_reco FT0 && add_comma_separated ADD_EXTRA_WORKFLOW "o2-tpc-time-series-workflow" - if [[ ! -z "$ALIEN_JDL_ENABLEUNBINNEDTIMESERIES" ]]; then - export ARGS_EXTRA_PROCESS_o2_tpc_time_series_workflow="$ARGS_EXTRA_PROCESS_o2_tpc_time_series_workflow --enable-unbinned-root-output --sample-unbinned-tsallis --threads 1" - fi - if [[ $ON_SKIMMED_DATA == 1 ]] || [[ ! -z "$ALIEN_JDL_SAMPLINGFACTORTIMESERIES" ]] ; then - if [[ $ON_SKIMMED_DATA == 1 ]] ; then - SAMPLINGFACTORTIMESERIES=0.1 + # extra workflows in case we want to process the currents for time series + if [[ -n $ALIEN_JDL_EXTRACTTIMESERIES ]] ; then + echo "Adding timeseries in setenv_extra.sh" + export CALIB_ASYNC_EXTRACTTIMESERIES=$ALIEN_JDL_EXTRACTTIMESERIES + if [[ -n $ALIEN_JDL_ENABLEUNBINNEDTIMESERIES ]]; then + export CALIB_ASYNC_ENABLEUNBINNEDTIMESERIES=$ALIEN_JDL_ENABLEUNBINNEDTIMESERIES fi - if [[ ! -z "$ALIEN_JDL_SAMPLINGFACTORTIMESERIES" ]]; then # this takes priority - export SAMPLINGFACTORTIMESERIES=${ALIEN_JDL_SAMPLINGFACTORTIMESERIES} + if [[ -n $ALIEN_JDL_SAMPLINGFACTORTIMESERIES ]]; then + export CALIB_ASYNC_SAMPLINGFACTORTIMESERIES=$ALIEN_JDL_SAMPLINGFACTORTIMESERIES fi - export ARGS_EXTRA_PROCESS_o2_tpc_time_series_workflow="$ARGS_EXTRA_PROCESS_o2_tpc_time_series_workflow --sampling-factor ${SAMPLINGFACTORTIMESERIES}" + fi + if [[ $ALIEN_JDL_DOUPLOADSLOCALLY == 1 ]]; then + export CCDB_POPULATOR_UPLOAD_PATH="file://$PWD" fi fi