Add configurable detector version support for datasets

.github/workflows/mirror.yaml

@@ -4,6 +4,27 @@ on:
   delete:
   push:
   workflow_dispatch:
+    inputs:
+      container_name:
+        description: 'Container name'
+        required: false
+        default: 'eic_ci'
+        type: string
+      image_tag:
+        description: 'Container image tag'
+        required: false
+        default: 'nightly'
+        type: string
+      detector_config:
+        description: 'Detector configuration'
+        required: false
+        default: 'epic_craterlake'
+        type: string
+      detector_version:
+        description: 'Detector version'
+        required: false
+        default: 'main'
+        type: string
 
 concurrency:
   group: mirror-${{ github.event_name }}
@@ -43,6 +64,10 @@ jobs:
           GITHUB_SHA=${{ github.event.pull_request.head.sha || github.sha }}
           GITHUB_PR=${{ github.event.pull_request.number }}
           PIPELINE_NAME=${{ github.repository }}: ${{ github.event.pull_request.title || github.ref_name }}
+          CONTAINER_NAME=${{ inputs.container_name || 'eic_ci' }}
+          IMAGE_TAG=${{ inputs.image_tag || 'nightly' }}
+          DETECTOR_CONFIG=${{ inputs.detector_config || 'epic_craterlake' }}
+          DETECTOR_VERSION=${{ inputs.detector_version || 'main' }}
     - name: Set pending EICweb status
       if: ${{ github.event_name != 'delete' }}
       env:

.gitlab-ci.yml

@@ -1,4 +1,13 @@
-image: eicweb.phy.anl.gov:4567/containers/eic_container/eic_xl:nightly
+variables:
+  CONTAINER_NAME: "eic_ci"
+  IMAGE_TAG: "nightly"
+  DETECTOR_CONFIG: "epic_craterlake"
+  DETECTOR_VERSION: "main"
+  EBEAM: "18"
+  PBEAM: "275"
+  NEVENTS_PER_TEST: "100"
+
+image: eicweb.phy.anl.gov:4567/containers/eic_container/${CONTAINER_NAME}:${IMAGE_TAG}
 
 default:
   timeout: 3 hours
@@ -18,50 +27,74 @@ stages:
 .nevents:
   stage: nevents
   script:
-    - mkdir -p $(dirname results/datasets/glob/$DATA)
-    - grep -v "^\#" $DATA | parallel scripts/glob.sh results/datasets/glob/$DATA {}
-    - sort -o results/datasets/glob/$DATA results/datasets/glob/$DATA
-    - mkdir -p $(dirname results/datasets/nevents/$DATA)
-    - grep -v "^\#" results/datasets/glob/$DATA | parallel scripts/count_events.sh results/datasets/nevents/$DATA {}
-    - sort -o results/datasets/nevents/$DATA results/datasets/nevents/$DATA
+    - export DETECTOR_CONFIG="${DETECTOR_CONFIG_OVERRIDE:-${DETECTOR_CONFIG}}"
+    - export DETECTOR_VERSION="${DETECTOR_VERSION_OVERRIDE:-${DETECTOR_VERSION}}"
+    - export RESULTS_BASE="results/${CONTAINER_NAME}/${IMAGE_TAG}/${DETECTOR_CONFIG}/${DETECTOR_VERSION}"
+    - echo "INFO [nevents] - CONTAINER_NAME = ${CONTAINER_NAME}"
+    - echo "INFO [nevents] - IMAGE_TAG = ${IMAGE_TAG}"
+    - echo "INFO [nevents] - DETECTOR_CONFIG = ${DETECTOR_CONFIG}"
+    - echo "INFO [nevents] - DETECTOR_VERSION = ${DETECTOR_VERSION}"
+    - echo "INFO [nevents] - RESULTS_BASE = ${RESULTS_BASE}"
+    - mkdir -p $(dirname ${RESULTS_BASE}/datasets/glob/$DATA)
+    - grep -v "^\#" $DATA | parallel scripts/glob.sh ${RESULTS_BASE}/datasets/glob/$DATA {}
+    - sort -o ${RESULTS_BASE}/datasets/glob/$DATA ${RESULTS_BASE}/datasets/glob/$DATA
+    - mkdir -p $(dirname ${RESULTS_BASE}/datasets/nevents/$DATA)
+    - grep -v "^\#" ${RESULTS_BASE}/datasets/glob/$DATA | parallel scripts/count_events.sh ${RESULTS_BASE}/datasets/nevents/$DATA {}
+    - sort -o ${RESULTS_BASE}/datasets/nevents/$DATA ${RESULTS_BASE}/datasets/nevents/$DATA
     - sort -o $DATA $DATA
 
 .timings:
   stage: timings
   script:
-    - mkdir -p $(dirname results/datasets/timings/$DATA)
+    - export DETECTOR_CONFIG="${DETECTOR_CONFIG_OVERRIDE:-${DETECTOR_CONFIG}}"
+    - export DETECTOR_VERSION="${DETECTOR_VERSION_OVERRIDE:-${DETECTOR_VERSION}}"
+    - export EBEAM="${EBEAM_OVERRIDE:-${EBEAM}}"
+    - export PBEAM="${PBEAM_OVERRIDE:-${PBEAM}}"
+    - export NEVENTS_PER_TEST="${NEVENTS_PER_TEST_OVERRIDE:-${NEVENTS_PER_TEST}}"
+    - export RESULTS_BASE="results/${CONTAINER_NAME}/${IMAGE_TAG}/${DETECTOR_CONFIG}/${DETECTOR_VERSION}"
+    - echo "INFO [timings] - CONTAINER_NAME = ${CONTAINER_NAME}"
+    - echo "INFO [timings] - IMAGE_TAG = ${IMAGE_TAG}"
+    - echo "INFO [timings] - DETECTOR_CONFIG = ${DETECTOR_CONFIG}"
+    - echo "INFO [timings] - DETECTOR_VERSION = ${DETECTOR_VERSION}"
+    - echo "INFO [timings] - EBEAM = ${EBEAM}"
+    - echo "INFO [timings] - PBEAM = ${PBEAM}"
+    - echo "INFO [timings] - NEVENTS_PER_TEST = ${NEVENTS_PER_TEST}"
+    - echo "INFO [timings] - RESULTS_BASE = ${RESULTS_BASE}"
+    - mkdir -p $(dirname ${RESULTS_BASE}/datasets/timings/$DATA)
     # Use sed '1!d1' instead of head -n 1 to avoid pipefail issues
-    - grep -v "^\#" results/datasets/nevents/$DATA | sed '1!d' | parallel scripts/determine_timing.sh results/datasets/timings/$DATA {}
+    - grep -v "^\#" ${RESULTS_BASE}/datasets/nevents/$DATA | sed '1!d' | parallel scripts/determine_timing.sh ${RESULTS_BASE}/datasets/timings/$DATA {}
     - |
-      IFS="," read file ext nevents dt0 dt1 < results/datasets/timings/$DATA
+      IFS="," read file ext nevents dt0 dt1 < ${RESULTS_BASE}/datasets/timings/$DATA
       export dt0 dt1
-      grep -v "^\#" results/datasets/nevents/$DATA | sed '1d' | parallel scripts/determine_timing.sh results/datasets/timings/$DATA {}
-    - sort -o results/datasets/timings/$DATA results/datasets/timings/$DATA
+      grep -v "^\#" ${RESULTS_BASE}/datasets/nevents/$DATA | sed '1d' | parallel scripts/determine_timing.sh ${RESULTS_BASE}/datasets/timings/$DATA {}
+    - sort -o ${RESULTS_BASE}/datasets/timings/$DATA ${RESULTS_BASE}/datasets/timings/$DATA
 
 .timings_all:
   stage: timings
   script:
-    - mkdir -p $(dirname results/datasets/timings/$DATA)
-    - grep -v "^\#" results/datasets/nevents/$DATA | parallel scripts/determine_timing.sh results/datasets/timings/$DATA {}
-    - sort -o results/datasets/timings/$DATA results/datasets/timings/$DATA
+    - export RESULTS_BASE="results/${CONTAINER_NAME}/${IMAGE_TAG}/${DETECTOR_CONFIG}/${DETECTOR_VERSION}"
+    - mkdir -p $(dirname ${RESULTS_BASE}/datasets/timings/$DATA)
+    - grep -v "^\#" ${RESULTS_BASE}/datasets/nevents/$DATA | parallel scripts/determine_timing.sh ${RESULTS_BASE}/datasets/timings/$DATA {}
+    - sort -o ${RESULTS_BASE}/datasets/timings/$DATA ${RESULTS_BASE}/datasets/timings/$DATA
 
 .collect:
   stage: collect
   script:
+    - export RESULTS_BASE="results/${CONTAINER_NAME}/${IMAGE_TAG}/${DETECTOR_CONFIG}/${DETECTOR_VERSION}"
     - rm -rf results/logs/
-    - find results/datasets/
-    - find results/datasets/timings/ -name "*.csv" -print0 -exec awk 'BEGIN {FS=","} {sum+=$3*$5+$4} END {print(":",sum/3600,"core-hours")}' {} \;
-    - for d in `find results/datasets/timings/ -type d` ; do
+    - find ${RESULTS_BASE}/datasets/
+    - find ${RESULTS_BASE}/datasets/timings/ -name "*.csv" -print0 -exec awk 'BEGIN {FS=","} {sum+=$3*$5+$4} END {print(":",sum/3600,"core-hours")}' {} \;
+    - for d in `find ${RESULTS_BASE}/datasets/timings/ -type d` ; do
         echo -n $d ;
         find $d -name "*.csv" -exec cat {} \; | awk 'BEGIN {FS=","} {sum+=$3*$5+$4} END {print(":",sum/3600,"core-hours")}' ;
       done
-    - find results/datasets/timings/ -name "*.csv" -print0 -exec awk 'BEGIN {FS=","} {sum+=$3*$7+$6} END {print(":",sum/1048576,"GB (full)")}' {} \;
-    - for d in `find results/datasets/timings/ -type d` ; do
+    - find ${RESULTS_BASE}/datasets/timings/ -name "*.csv" -print0 -exec awk 'BEGIN {FS=","} {sum+=$3*$7+$6} END {print(":",sum/1048576,"GB (full)")}' {} \;
+    - for d in `find ${RESULTS_BASE}/datasets/timings/ -type d` ; do
         echo -n $d ;
         find $d -name "*.csv" -exec cat {} \; | awk 'BEGIN {FS=","} {sum+=$3*$7+$6} END {print(":",sum/1048576,"GB (full)")}' ;
       done
-    - find results/datasets/timings/ -name "*.csv" -print0 -exec awk 'BEGIN {FS=","} {sum+=$3*$9+$8} END {print(":",sum/1048576,"GB (reco)")}' {} \;
-    - for d in `find results/datasets/timings/ -type d` ; do
+    - find ${RESULTS_BASE}/datasets/timings/ -name "*.csv" -print0 -exec awk 'BEGIN {FS=","} {sum+=$3*$9+$8} END {print(":",sum/1048576,"GB (reco)")}' {} \;
+    - for d in `find ${RESULTS_BASE}/datasets/timings/ -type d` ; do
         echo -n $d ;
         find $d -name "*.csv" -exec cat {} \; | awk 'BEGIN {FS=","} {sum+=$3*$9+$8} END {print(":",sum/1048576,"GB (reco)")}' ;
       done

DIS/eA/config.yml

@@ -1,4 +1,58 @@
-DIS:eA:nevents:
+.eAu_10x100_variables: &eAu_10x100_variables
+  DETECTOR_CONFIG_OVERRIDE: "epic_craterlake_without_zdc"
+  EBEAM_OVERRIDE: "10"
+  PBEAM_OVERRIDE: "100_Au197"
+
+DIS:eA:eAu:10x100:nevents:
+  variables:
+    <<: *eAu_10x100_variables
+  extends: .nevents
+  parallel:
+    matrix:
+      - DATA:
+        - "DIS/eA/BeAGLE1.03.02-1.0_DIS_eAu_10x100_q2_1to10.csv"
+        - "DIS/eA/BeAGLE1.03.02-1.0_DIS_eAu_10x100_q2_10to100.csv"
+
+DIS:eA:eAu:10x100:timings:
+  variables:
+    <<: *eAu_10x100_variables
+  extends: .timings
+  needs:
+    - DIS:eA:eAu:10x100:nevents
+  parallel:
+    matrix:
+      - DATA:
+        - "DIS/eA/BeAGLE1.03.02-1.0_DIS_eAu_10x100_q2_1to10.csv"
+        - "DIS/eA/BeAGLE1.03.02-1.0_DIS_eAu_10x100_q2_10to100.csv"
+
+.eAu_5x41_variables: &eAu_5x41_variables
+  DETECTOR_CONFIG_OVERRIDE: "epic_craterlake_without_zdc"
+  EBEAM_OVERRIDE: "5"
+  PBEAM_OVERRIDE: "41_Au197"
+
+DIS:eA:eAu:5x41:nevents:
+  variables:
+    <<: *eAu_5x41_variables
+  extends: .nevents
+  parallel:
+    matrix:
+      - DATA:
+        - "DIS/eA/BeAGLE1.03.02-1.0_DIS_eAu_5x41_q2_1to10.csv"
+        - "DIS/eA/BeAGLE1.03.02-1.0_DIS_eAu_5x41_q2_10to100.csv"
+
+DIS:eA:eAu:5x41:timings:
+  variables:
+    <<: *eAu_5x41_variables
+  extends: .timings
+  needs:
+    - DIS:eA:eAu:5x41:nevents
+  parallel:
+    matrix:
+      - DATA:
+        - "DIS/eA/BeAGLE1.03.02-1.0_DIS_eAu_5x41_q2_1to10.csv"
+        - "DIS/eA/BeAGLE1.03.02-1.0_DIS_eAu_5x41_q2_10to100.csv"
+
+DIS:eA:other:nevents:
   extends: .nevents
   parallel:
     matrix:
@@ -8,10 +62,6 @@ DIS:eA:nevents:
         - "DIS/eA/BeAGLE1.03.02-1.0_DIS_eRu_en_10x115.csv"
         - "DIS/eA/BeAGLE1.03.02-1.0_DIS_eRu_ep_10x115.csv"
         - "DIS/eA/BeAGLE1.03.02-1.0_DIS_eHe3_10x166.csv"
-        - "DIS/eA/BeAGLE1.03.02-1.0_DIS_eAu_10x100_q2_1to10.csv"
-        - "DIS/eA/BeAGLE1.03.02-1.0_DIS_eAu_10x100_q2_10to100.csv"
-        - "DIS/eA/BeAGLE1.03.02-1.0_DIS_eAu_5x41_q2_1to10.csv"
-        - "DIS/eA/BeAGLE1.03.02-1.0_DIS_eAu_5x41_q2_10to100.csv"
         - "DIS/eA/BeAGLE1.03.02-1.2_DIS_eHe3_10x166_q2_2to10.csv"
         - "DIS/eA/BeAGLE1.03.02-1.2_DIS_eHe3_10x166_q2_10to100.csv"
         - "DIS/eA/BeAGLE1.03.02-1.2_DIS_eHe3_10x166_q2_100to10000.csv"
@@ -25,10 +75,10 @@ DIS:eA:nevents:
         - "DIS/eA/BeAGLE1.03.02-1.2_DIS_eHe3_5x41_q2_10to100.csv"
         - "DIS/eA/BeAGLE1.03.02-1.2_DIS_eHe3_5x41_q2_100to10000.csv"
 
-DIS:eA:timings:
+DIS:eA:other:timings:
   extends: .timings
-  needs: 
-    - DIS:eA:nevents
+  needs:
+    - DIS:eA:other:nevents
   parallel:
     matrix:
       - DATA:
@@ -37,10 +87,6 @@ DIS:eA:timings:
         - "DIS/eA/BeAGLE1.03.02-1.0_DIS_eRu_en_10x115.csv"
         - "DIS/eA/BeAGLE1.03.02-1.0_DIS_eRu_ep_10x115.csv"
         - "DIS/eA/BeAGLE1.03.02-1.0_DIS_eHe3_10x166.csv"
-        - "DIS/eA/BeAGLE1.03.02-1.0_DIS_eAu_10x100_q2_1to10.csv"
-        - "DIS/eA/BeAGLE1.03.02-1.0_DIS_eAu_10x100_q2_10to100.csv"
-        - "DIS/eA/BeAGLE1.03.02-1.0_DIS_eAu_5x41_q2_1to10.csv"
-        - "DIS/eA/BeAGLE1.03.02-1.0_DIS_eAu_5x41_q2_10to100.csv"
         - "DIS/eA/BeAGLE1.03.02-1.2_DIS_eHe3_10x166_q2_2to10.csv"
         - "DIS/eA/BeAGLE1.03.02-1.2_DIS_eHe3_10x166_q2_10to100.csv"
         - "DIS/eA/BeAGLE1.03.02-1.2_DIS_eHe3_10x166_q2_100to10000.csv"
@@ -56,5 +102,7 @@ DIS:eA:timings:
 
 DIS:eA:collect:
   extends: .collect
-  needs: 
-    - DIS:eA:timings
+  needs:
+    - DIS:eA:eAu:10x100:timings
+    - DIS:eA:eAu:5x41:timings
+    - DIS:eA:other:timings

SIDIS/config.yml

@@ -1,11 +1,62 @@
-SIDIS:nevents:
+.eAu_10x100_variables: &eAu_10x100_variables
+  NEVENTS_PER_TEST_OVERRIDE: 25
+  DETECTOR_CONFIG_OVERRIDE: "epic_craterlake_without_zdc"
+  EBEAM_OVERRIDE: "10"
+  PBEAM_OVERRIDE: "100_Au197"
+
+SIDIS:eAu:10x100:nevents:
+  variables:
+    <<: *eAu_10x100_variables
   extends: .nevents
   parallel:
     matrix:
       - DATA:
         - "SIDIS/eA/BeAGLE1.03.01-2.0_D0_eAu_10x100_q2_1to10000.csv"
-        - "SIDIS/eA/BeAGLE1.03.01-2.0_D0_eAu_5x41_q2_1to10000.csv"
         - "SIDIS/eA/BeAGLE1.03.01-2.0_Lc_eAu_10x100_q2_1to10000.csv"
+
+SIDIS:eAu:10x100:timings:
+  variables:
+    <<: *eAu_10x100_variables
+  extends: .timings
+  needs:
+    - SIDIS:eAu:10x100:nevents
+  parallel:
+    matrix:
+      - DATA:
+        - "SIDIS/eA/BeAGLE1.03.01-2.0_D0_eAu_10x100_q2_1to10000.csv"
+        - "SIDIS/eA/BeAGLE1.03.01-2.0_Lc_eAu_10x100_q2_1to10000.csv"
+
+.eAu_5x41_variables: &eAu_5x41_variables
+  NEVENTS_PER_TEST_OVERRIDE: 25
+  DETECTOR_CONFIG_OVERRIDE: "epic_craterlake_without_zdc"
+  EBEAM_OVERRIDE: "5"
+  PBEAM_OVERRIDE: "41_Au197"
+
+SIDIS:eAu:5x41:nevents:
+  variables:
+    <<: *eAu_5x41_variables
+  extends: .nevents
+  parallel:
+    matrix:
+      - DATA:
+        - "SIDIS/eA/BeAGLE1.03.01-2.0_D0_eAu_5x41_q2_1to10000.csv"
+
+SIDIS:eAu:5x41:timings:
+  variables:
+    <<: *eAu_5x41_variables
+  extends: .timings
+  needs:
+    - SIDIS:eAu:5x41:nevents
+  parallel:
+    matrix:
+      - DATA:
+        - "SIDIS/eA/BeAGLE1.03.01-2.0_D0_eAu_5x41_q2_1to10000.csv"
+
+SIDIS:ep:nevents:
+  extends: .nevents
+  parallel:
+    matrix:
+      - DATA:
         - "SIDIS/ep/pythia8.306-1.2_D0_ep_5x41_q2_1to10000.csv"
         - "SIDIS/ep/pythia8.306-1.2_Lc_ep_10x100_q2_1to10000.csv"
         - "SIDIS/pythia6-eic-1.0.0_5x41_q2_0to1_ep_noradcor.csv"
@@ -18,18 +69,15 @@ SIDIS:nevents:
         - "SIDIS/pythia8.306-1.0_Lambda_18x275_hiAcc.csv"
         - "SIDIS/pythia8.306-1.0_Lambda_18x275_hiDiv.csv"
 
-SIDIS:timings:
+SIDIS:ep:timings:
   extends: .timings
   needs:
-    - SIDIS:nevents
+    - SIDIS:ep:nevents
   variables:
-    NEVENTS_PER_TEST: 15
+    NEVENTS_PER_TEST_OVERRIDE: 25
   parallel:
     matrix:
       - DATA:
-        - "SIDIS/eA/BeAGLE1.03.01-2.0_D0_eAu_10x100_q2_1to10000.csv"
-        - "SIDIS/eA/BeAGLE1.03.01-2.0_D0_eAu_5x41_q2_1to10000.csv"
-        - "SIDIS/eA/BeAGLE1.03.01-2.0_Lc_eAu_10x100_q2_1to10000.csv"
         - "SIDIS/ep/pythia8.306-1.2_D0_ep_5x41_q2_1to10000.csv"
         - "SIDIS/ep/pythia8.306-1.2_Lc_ep_10x100_q2_1to10000.csv"
         - "SIDIS/pythia6-eic-1.0.0_5x41_q2_0to1_ep_noradcor.csv"
@@ -42,8 +90,9 @@ SIDIS:timings:
         - "SIDIS/pythia8.306-1.0_Lambda_18x275_hiAcc.csv"
         - "SIDIS/pythia8.306-1.0_Lambda_18x275_hiDiv.csv"
 
-
 SIDIS:collect:
   extends: .collect
-  needs: 
-    - SIDIS:timings
+  needs:
+    - SIDIS:eAu:10x100:timings
+    - SIDIS:eAu:5x41:timings
+    - SIDIS:ep:timings

scripts/determine_timing.sh

@@ -29,9 +29,16 @@ nlines=$((2*n_events_test*n_lines_per_event))
 dir=$(dirname EVGEN/${file}.${ext})
 mkdir -p ${dir}
 
-logfile=results/logs/${file}.out
+logfile=${RESULTS_BASE:-results}/logs/${file}.out
 mkdir -p $(dirname ${logfile})
 
+echo "INFO [determine_timing.sh] - DETECTOR_CONFIG = ${DETECTOR_CONFIG:-epic_craterlake}"
+echo "INFO [determine_timing.sh] - DETECTOR_VERSION = ${DETECTOR_VERSION:-main}"
+echo "INFO [determine_timing.sh] - EBEAM = ${EBEAM:-18}"
+echo "INFO [determine_timing.sh] - PBEAM = ${PBEAM:-275}"
+echo "INFO [determine_timing.sh] - NEVENTS_PER_TEST = ${n_events_test}"
+echo "INFO [determine_timing.sh] - RESULTS_BASE = ${RESULTS_BASE:-results}"
+
 # time for 1 event (first)
 t1=$(date +%s.%N)
 /opt/campaigns/hepmc3/scripts/run.sh EVGEN/${file} ${ext} 1 2>&1 | tee ${logfile}.1