From cc95b5f1072ddd89eb6d9786b4cbd67b0f596ce5 Mon Sep 17 00:00:00 2001 From: Sakib Rahman Date: Thu, 13 Nov 2025 17:43:14 -0500 Subject: [PATCH 01/11] Add configurable detector version support for datasets MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add DETECTOR_CONFIG and DETECTOR_VERSION environment variables - Set global defaults to epic_craterlake and main in .gitlab-ci.yml - Configure eAu BeAGLE samples to use epic_craterlake_without_zdc - Split SIDIS jobs into eAu and ep categories - Split DIS eA jobs into eAu and other nuclei categories - Export detector variables in determine_timing.sh before run.sh calls 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- .gitlab-ci.yml | 4 +++ DIS/eA/config.yml | 45 +++++++++++++++-------- SIDIS/config.yml | 71 +++++++++++++++++++++++++++++++------ scripts/determine_timing.sh | 4 +++ 4 files changed, 99 insertions(+), 25 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 1334a00..7de20b7 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,5 +1,9 @@ image: eicweb.phy.anl.gov:4567/containers/eic_container/eic_xl:nightly +variables: + DETECTOR_CONFIG: "epic_craterlake" + DETECTOR_VERSION: "main" + default: timeout: 3 hours artifacts: diff --git a/DIS/eA/config.yml b/DIS/eA/config.yml index 38ab83e..a839a42 100644 --- a/DIS/eA/config.yml +++ b/DIS/eA/config.yml @@ -1,4 +1,28 @@ -DIS:eA:nevents: +DIS:eA:eAu:nevents: + extends: .nevents + parallel: + matrix: + - DATA: + - "DIS/eA/BeAGLE1.03.02-1.0_DIS_eAu_10x100_q2_1to10.csv" + - "DIS/eA/BeAGLE1.03.02-1.0_DIS_eAu_10x100_q2_10to100.csv" + - "DIS/eA/BeAGLE1.03.02-1.0_DIS_eAu_5x41_q2_1to10.csv" + - "DIS/eA/BeAGLE1.03.02-1.0_DIS_eAu_5x41_q2_10to100.csv" + +DIS:eA:eAu:timings: + extends: .timings + needs: + - DIS:eA:eAu:nevents + variables: + DETECTOR_CONFIG: "epic_craterlake_without_zdc" + parallel: + matrix: + - DATA: + - "DIS/eA/BeAGLE1.03.02-1.0_DIS_eAu_10x100_q2_1to10.csv" + - "DIS/eA/BeAGLE1.03.02-1.0_DIS_eAu_10x100_q2_10to100.csv" + - "DIS/eA/BeAGLE1.03.02-1.0_DIS_eAu_5x41_q2_1to10.csv" + - "DIS/eA/BeAGLE1.03.02-1.0_DIS_eAu_5x41_q2_10to100.csv" + +DIS:eA:other:nevents: extends: .nevents parallel: matrix: @@ -8,10 +32,6 @@ DIS:eA:nevents: - "DIS/eA/BeAGLE1.03.02-1.0_DIS_eRu_en_10x115.csv" - "DIS/eA/BeAGLE1.03.02-1.0_DIS_eRu_ep_10x115.csv" - "DIS/eA/BeAGLE1.03.02-1.0_DIS_eHe3_10x166.csv" - - "DIS/eA/BeAGLE1.03.02-1.0_DIS_eAu_10x100_q2_1to10.csv" - - "DIS/eA/BeAGLE1.03.02-1.0_DIS_eAu_10x100_q2_10to100.csv" - - "DIS/eA/BeAGLE1.03.02-1.0_DIS_eAu_5x41_q2_1to10.csv" - - "DIS/eA/BeAGLE1.03.02-1.0_DIS_eAu_5x41_q2_10to100.csv" - "DIS/eA/BeAGLE1.03.02-1.2_DIS_eHe3_10x166_q2_2to10.csv" - "DIS/eA/BeAGLE1.03.02-1.2_DIS_eHe3_10x166_q2_10to100.csv" - "DIS/eA/BeAGLE1.03.02-1.2_DIS_eHe3_10x166_q2_100to10000.csv" @@ -25,10 +45,10 @@ DIS:eA:nevents: - "DIS/eA/BeAGLE1.03.02-1.2_DIS_eHe3_5x41_q2_10to100.csv" - "DIS/eA/BeAGLE1.03.02-1.2_DIS_eHe3_5x41_q2_100to10000.csv" -DIS:eA:timings: +DIS:eA:other:timings: extends: .timings - needs: - - DIS:eA:nevents + needs: + - DIS:eA:other:nevents parallel: matrix: - DATA: @@ -37,10 +57,6 @@ DIS:eA:timings: - "DIS/eA/BeAGLE1.03.02-1.0_DIS_eRu_en_10x115.csv" - "DIS/eA/BeAGLE1.03.02-1.0_DIS_eRu_ep_10x115.csv" - "DIS/eA/BeAGLE1.03.02-1.0_DIS_eHe3_10x166.csv" - - "DIS/eA/BeAGLE1.03.02-1.0_DIS_eAu_10x100_q2_1to10.csv" - - "DIS/eA/BeAGLE1.03.02-1.0_DIS_eAu_10x100_q2_10to100.csv" - - "DIS/eA/BeAGLE1.03.02-1.0_DIS_eAu_5x41_q2_1to10.csv" - - "DIS/eA/BeAGLE1.03.02-1.0_DIS_eAu_5x41_q2_10to100.csv" - "DIS/eA/BeAGLE1.03.02-1.2_DIS_eHe3_10x166_q2_2to10.csv" - "DIS/eA/BeAGLE1.03.02-1.2_DIS_eHe3_10x166_q2_10to100.csv" - "DIS/eA/BeAGLE1.03.02-1.2_DIS_eHe3_10x166_q2_100to10000.csv" @@ -56,5 +72,6 @@ DIS:eA:timings: DIS:eA:collect: extends: .collect - needs: - - DIS:eA:timings \ No newline at end of file + needs: + - DIS:eA:eAu:timings + - DIS:eA:other:timings \ No newline at end of file diff --git a/SIDIS/config.yml b/SIDIS/config.yml index fe54ecb..eb70545 100644 --- a/SIDIS/config.yml +++ b/SIDIS/config.yml @@ -1,11 +1,62 @@ -SIDIS:nevents: +.eAu_10x100_variables: &eAu_10x100_variables + NEVENTS_PER_TEST: 25 + DETECTOR_CONFIG_OVERRIDE: "epic_craterlake_without_zdc" + EBEAM_OVERRIDE: "10" + PBEAM_OVERRIDE: "100_Au197" + +SIDIS:eAu:10x100:nevents: + variables: + <<: *eAu_10x100_variables extends: .nevents parallel: matrix: - DATA: - "SIDIS/eA/BeAGLE1.03.01-2.0_D0_eAu_10x100_q2_1to10000.csv" - - "SIDIS/eA/BeAGLE1.03.01-2.0_D0_eAu_5x41_q2_1to10000.csv" - "SIDIS/eA/BeAGLE1.03.01-2.0_Lc_eAu_10x100_q2_1to10000.csv" + +SIDIS:eAu:10x100:timings: + variables: + <<: *eAu_10x100_variables + extends: .timings + needs: + - SIDIS:eAu:10x100:nevents + parallel: + matrix: + - DATA: + - "SIDIS/eA/BeAGLE1.03.01-2.0_D0_eAu_10x100_q2_1to10000.csv" + - "SIDIS/eA/BeAGLE1.03.01-2.0_Lc_eAu_10x100_q2_1to10000.csv" + +.eAu_5x41_variables: &eAu_5x41_variables + NEVENTS_PER_TEST: 25 + DETECTOR_CONFIG_OVERRIDE: "epic_craterlake_without_zdc" + EBEAM_OVERRIDE: "5" + PBEAM_OVERRIDE: "41_Au197" + +SIDIS:eAu:5x41:nevents: + variables: + <<: *eAu_5x41_variables + extends: .nevents + parallel: + matrix: + - DATA: + - "SIDIS/eA/BeAGLE1.03.01-2.0_D0_eAu_5x41_q2_1to10000.csv" + +SIDIS:eAu:5x41:timings: + variables: + <<: *eAu_5x41_variables + extends: .timings + needs: + - SIDIS:eAu:5x41:nevents + parallel: + matrix: + - DATA: + - "SIDIS/eA/BeAGLE1.03.01-2.0_D0_eAu_5x41_q2_1to10000.csv" + +SIDIS:ep:nevents: + extends: .nevents + parallel: + matrix: + - DATA: - "SIDIS/ep/pythia8.306-1.2_D0_ep_5x41_q2_1to10000.csv" - "SIDIS/ep/pythia8.306-1.2_Lc_ep_10x100_q2_1to10000.csv" - "SIDIS/pythia6-eic-1.0.0_5x41_q2_0to1_ep_noradcor.csv" @@ -18,18 +69,15 @@ SIDIS:nevents: - "SIDIS/pythia8.306-1.0_Lambda_18x275_hiAcc.csv" - "SIDIS/pythia8.306-1.0_Lambda_18x275_hiDiv.csv" -SIDIS:timings: +SIDIS:ep:timings: extends: .timings needs: - - SIDIS:nevents + - SIDIS:ep:nevents variables: - NEVENTS_PER_TEST: 15 + NEVENTS_PER_TEST: 25 parallel: matrix: - DATA: - - "SIDIS/eA/BeAGLE1.03.01-2.0_D0_eAu_10x100_q2_1to10000.csv" - - "SIDIS/eA/BeAGLE1.03.01-2.0_D0_eAu_5x41_q2_1to10000.csv" - - "SIDIS/eA/BeAGLE1.03.01-2.0_Lc_eAu_10x100_q2_1to10000.csv" - "SIDIS/ep/pythia8.306-1.2_D0_ep_5x41_q2_1to10000.csv" - "SIDIS/ep/pythia8.306-1.2_Lc_ep_10x100_q2_1to10000.csv" - "SIDIS/pythia6-eic-1.0.0_5x41_q2_0to1_ep_noradcor.csv" @@ -42,8 +90,9 @@ SIDIS:timings: - "SIDIS/pythia8.306-1.0_Lambda_18x275_hiAcc.csv" - "SIDIS/pythia8.306-1.0_Lambda_18x275_hiDiv.csv" - SIDIS:collect: extends: .collect - needs: - - SIDIS:timings + needs: + - SIDIS:eAu:10x100:timings + - SIDIS:eAu:5x41:timings + - SIDIS:ep:timings diff --git a/scripts/determine_timing.sh b/scripts/determine_timing.sh index bc13c27..d0c9762 100755 --- a/scripts/determine_timing.sh +++ b/scripts/determine_timing.sh @@ -32,6 +32,10 @@ mkdir -p ${dir} logfile=results/logs/${file}.out mkdir -p $(dirname ${logfile}) +# Export detector configuration if provided +export DETECTOR_CONFIG="${DETECTOR_CONFIG:-epic_craterlake}" +export DETECTOR_VERSION="${DETECTOR_VERSION:-main}" + # time for 1 event (first) t1=$(date +%s.%N) /opt/campaigns/hepmc3/scripts/run.sh EVGEN/${file} ${ext} 1 2>&1 | tee ${logfile}.1 From 2106ac8b9daaa1936053d561b8cf296ec0d2a2e5 Mon Sep 17 00:00:00 2001 From: Sakib Rahman Date: Thu, 13 Nov 2025 18:00:37 -0500 Subject: [PATCH 02/11] Add workflow dispatch support for custom pipeline variables MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add workflow_dispatch inputs for image_tag, detector_config, and detector_version - Make container image tag configurable via IMAGE_TAG variable in GitLab CI - Pass workflow inputs as variables to GitLab pipeline trigger - Set sensible defaults for all configurable variables This allows manual workflow triggers with custom container images and detector configurations while maintaining backward compatibility. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- .github/workflows/mirror.yaml | 19 +++++++++++++++++++ .gitlab-ci.yml | 5 +++-- 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/.github/workflows/mirror.yaml b/.github/workflows/mirror.yaml index bcae56c..c300c44 100644 --- a/.github/workflows/mirror.yaml +++ b/.github/workflows/mirror.yaml @@ -4,6 +4,22 @@ on: delete: push: workflow_dispatch: + inputs: + image_tag: + description: 'Container image tag' + required: false + default: 'nightly' + type: string + detector_config: + description: 'Detector configuration' + required: false + default: 'epic_craterlake' + type: string + detector_version: + description: 'Detector version' + required: false + default: 'main' + type: string concurrency: group: mirror-${{ github.event_name }} @@ -43,6 +59,9 @@ jobs: GITHUB_SHA=${{ github.event.pull_request.head.sha || github.sha }} GITHUB_PR=${{ github.event.pull_request.number }} PIPELINE_NAME=${{ github.repository }}: ${{ github.event.pull_request.title || github.ref_name }} + IMAGE_TAG=${{ inputs.image_tag || 'nightly' }} + DETECTOR_CONFIG=${{ inputs.detector_config || 'epic_craterlake' }} + DETECTOR_VERSION=${{ inputs.detector_version || 'main' }} - name: Set pending EICweb status if: ${{ github.event_name != 'delete' }} env: diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 7de20b7..cdf0e81 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,9 +1,10 @@ -image: eicweb.phy.anl.gov:4567/containers/eic_container/eic_xl:nightly - variables: + IMAGE_TAG: "nightly" DETECTOR_CONFIG: "epic_craterlake" DETECTOR_VERSION: "main" +image: eicweb.phy.anl.gov:4567/containers/eic_container/eic_xl:${IMAGE_TAG} + default: timeout: 3 hours artifacts: From bf01059b16aabf8fb38f450a2bd11740692bdb62 Mon Sep 17 00:00:00 2001 From: Sakib Rahman Date: Thu, 13 Nov 2025 18:24:27 -0500 Subject: [PATCH 03/11] Add configurable container name support MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add workflow_dispatch input for container_name (default: eic_xl) - Add CONTAINER_NAME variable to GitLab CI (default: eic_xl) - Update image path to use ${CONTAINER_NAME} variable - Pass container_name from workflow to GitLab pipeline This allows switching between different container images like eic_xl and eic_ci via workflow dispatch. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- .github/workflows/mirror.yaml | 6 ++++++ .gitlab-ci.yml | 3 ++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/workflows/mirror.yaml b/.github/workflows/mirror.yaml index c300c44..b7295dd 100644 --- a/.github/workflows/mirror.yaml +++ b/.github/workflows/mirror.yaml @@ -5,6 +5,11 @@ on: push: workflow_dispatch: inputs: + container_name: + description: 'Container name' + required: false + default: 'eic_xl' + type: string image_tag: description: 'Container image tag' required: false @@ -59,6 +64,7 @@ jobs: GITHUB_SHA=${{ github.event.pull_request.head.sha || github.sha }} GITHUB_PR=${{ github.event.pull_request.number }} PIPELINE_NAME=${{ github.repository }}: ${{ github.event.pull_request.title || github.ref_name }} + CONTAINER_NAME=${{ inputs.container_name || 'eic_xl' }} IMAGE_TAG=${{ inputs.image_tag || 'nightly' }} DETECTOR_CONFIG=${{ inputs.detector_config || 'epic_craterlake' }} DETECTOR_VERSION=${{ inputs.detector_version || 'main' }} diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index cdf0e81..0410665 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,9 +1,10 @@ variables: + CONTAINER_NAME: "eic_xl" IMAGE_TAG: "nightly" DETECTOR_CONFIG: "epic_craterlake" DETECTOR_VERSION: "main" -image: eicweb.phy.anl.gov:4567/containers/eic_container/eic_xl:${IMAGE_TAG} +image: eicweb.phy.anl.gov:4567/containers/eic_container/${CONTAINER_NAME}:${IMAGE_TAG} default: timeout: 3 hours From 2a0d94257f33fa0c4fb3cf22a06d911491fcefc7 Mon Sep 17 00:00:00 2001 From: Sakib Rahman Date: Thu, 13 Nov 2025 20:51:30 -0500 Subject: [PATCH 04/11] Add configurable artifact directory structure MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add RESULTS_BASE variable combining all 4 config variables - Update all artifact paths to use ${RESULTS_BASE} - New structure: results/${CONTAINER_NAME}/${IMAGE_TAG}/${DETECTOR_CONFIG}/${DETECTOR_VERSION}/datasets/ This allows multiple configurations to be run in parallel with separate artifact directories for each combination of settings. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- .gitlab-ci.yml | 43 ++++++++++++++++++++++--------------------- 1 file changed, 22 insertions(+), 21 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 0410665..eed1a5a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -3,6 +3,7 @@ variables: IMAGE_TAG: "nightly" DETECTOR_CONFIG: "epic_craterlake" DETECTOR_VERSION: "main" + RESULTS_BASE: "results/${CONTAINER_NAME}/${IMAGE_TAG}/${DETECTOR_CONFIG}/${DETECTOR_VERSION}" image: eicweb.phy.anl.gov:4567/containers/eic_container/${CONTAINER_NAME}:${IMAGE_TAG} @@ -24,50 +25,50 @@ stages: .nevents: stage: nevents script: - - mkdir -p $(dirname results/datasets/glob/$DATA) - - grep -v "^\#" $DATA | parallel scripts/glob.sh results/datasets/glob/$DATA {} - - sort -o results/datasets/glob/$DATA results/datasets/glob/$DATA - - mkdir -p $(dirname results/datasets/nevents/$DATA) - - grep -v "^\#" results/datasets/glob/$DATA | parallel scripts/count_events.sh results/datasets/nevents/$DATA {} - - sort -o results/datasets/nevents/$DATA results/datasets/nevents/$DATA + - mkdir -p $(dirname ${RESULTS_BASE}/datasets/glob/$DATA) + - grep -v "^\#" $DATA | parallel scripts/glob.sh ${RESULTS_BASE}/datasets/glob/$DATA {} + - sort -o ${RESULTS_BASE}/datasets/glob/$DATA ${RESULTS_BASE}/datasets/glob/$DATA + - mkdir -p $(dirname ${RESULTS_BASE}/datasets/nevents/$DATA) + - grep -v "^\#" ${RESULTS_BASE}/datasets/glob/$DATA | parallel scripts/count_events.sh ${RESULTS_BASE}/datasets/nevents/$DATA {} + - sort -o ${RESULTS_BASE}/datasets/nevents/$DATA ${RESULTS_BASE}/datasets/nevents/$DATA - sort -o $DATA $DATA .timings: stage: timings script: - - mkdir -p $(dirname results/datasets/timings/$DATA) + - mkdir -p $(dirname ${RESULTS_BASE}/datasets/timings/$DATA) # Use sed '1!d1' instead of head -n 1 to avoid pipefail issues - - grep -v "^\#" results/datasets/nevents/$DATA | sed '1!d' | parallel scripts/determine_timing.sh results/datasets/timings/$DATA {} + - grep -v "^\#" ${RESULTS_BASE}/datasets/nevents/$DATA | sed '1!d' | parallel scripts/determine_timing.sh ${RESULTS_BASE}/datasets/timings/$DATA {} - | - IFS="," read file ext nevents dt0 dt1 < results/datasets/timings/$DATA + IFS="," read file ext nevents dt0 dt1 < ${RESULTS_BASE}/datasets/timings/$DATA export dt0 dt1 - grep -v "^\#" results/datasets/nevents/$DATA | sed '1d' | parallel scripts/determine_timing.sh results/datasets/timings/$DATA {} - - sort -o results/datasets/timings/$DATA results/datasets/timings/$DATA + grep -v "^\#" ${RESULTS_BASE}/datasets/nevents/$DATA | sed '1d' | parallel scripts/determine_timing.sh ${RESULTS_BASE}/datasets/timings/$DATA {} + - sort -o ${RESULTS_BASE}/datasets/timings/$DATA ${RESULTS_BASE}/datasets/timings/$DATA .timings_all: stage: timings script: - - mkdir -p $(dirname results/datasets/timings/$DATA) - - grep -v "^\#" results/datasets/nevents/$DATA | parallel scripts/determine_timing.sh results/datasets/timings/$DATA {} - - sort -o results/datasets/timings/$DATA results/datasets/timings/$DATA + - mkdir -p $(dirname ${RESULTS_BASE}/datasets/timings/$DATA) + - grep -v "^\#" ${RESULTS_BASE}/datasets/nevents/$DATA | parallel scripts/determine_timing.sh ${RESULTS_BASE}/datasets/timings/$DATA {} + - sort -o ${RESULTS_BASE}/datasets/timings/$DATA ${RESULTS_BASE}/datasets/timings/$DATA .collect: stage: collect script: - rm -rf results/logs/ - - find results/datasets/ - - find results/datasets/timings/ -name "*.csv" -print0 -exec awk 'BEGIN {FS=","} {sum+=$3*$5+$4} END {print(":",sum/3600,"core-hours")}' {} \; - - for d in `find results/datasets/timings/ -type d` ; do + - find ${RESULTS_BASE}/datasets/ + - find ${RESULTS_BASE}/datasets/timings/ -name "*.csv" -print0 -exec awk 'BEGIN {FS=","} {sum+=$3*$5+$4} END {print(":",sum/3600,"core-hours")}' {} \; + - for d in `find ${RESULTS_BASE}/datasets/timings/ -type d` ; do echo -n $d ; find $d -name "*.csv" -exec cat {} \; | awk 'BEGIN {FS=","} {sum+=$3*$5+$4} END {print(":",sum/3600,"core-hours")}' ; done - - find results/datasets/timings/ -name "*.csv" -print0 -exec awk 'BEGIN {FS=","} {sum+=$3*$7+$6} END {print(":",sum/1048576,"GB (full)")}' {} \; - - for d in `find results/datasets/timings/ -type d` ; do + - find ${RESULTS_BASE}/datasets/timings/ -name "*.csv" -print0 -exec awk 'BEGIN {FS=","} {sum+=$3*$7+$6} END {print(":",sum/1048576,"GB (full)")}' {} \; + - for d in `find ${RESULTS_BASE}/datasets/timings/ -type d` ; do echo -n $d ; find $d -name "*.csv" -exec cat {} \; | awk 'BEGIN {FS=","} {sum+=$3*$7+$6} END {print(":",sum/1048576,"GB (full)")}' ; done - - find results/datasets/timings/ -name "*.csv" -print0 -exec awk 'BEGIN {FS=","} {sum+=$3*$9+$8} END {print(":",sum/1048576,"GB (reco)")}' {} \; - - for d in `find results/datasets/timings/ -type d` ; do + - find ${RESULTS_BASE}/datasets/timings/ -name "*.csv" -print0 -exec awk 'BEGIN {FS=","} {sum+=$3*$9+$8} END {print(":",sum/1048576,"GB (reco)")}' {} \; + - for d in `find ${RESULTS_BASE}/datasets/timings/ -type d` ; do echo -n $d ; find $d -name "*.csv" -exec cat {} \; | awk 'BEGIN {FS=","} {sum+=$3*$9+$8} END {print(":",sum/1048576,"GB (reco)")}' ; done From 36926ab6f91cb367a648626564649b63dc4927de Mon Sep 17 00:00:00 2001 From: Sakib Rahman Date: Thu, 13 Nov 2025 21:20:02 -0500 Subject: [PATCH 05/11] Change default container to eic_ci MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Update default CONTAINER_NAME from eic_xl to eic_ci in both GitLab CI variables and GitHub workflow dispatch inputs. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- .github/workflows/mirror.yaml | 4 ++-- .gitlab-ci.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/mirror.yaml b/.github/workflows/mirror.yaml index b7295dd..41edb83 100644 --- a/.github/workflows/mirror.yaml +++ b/.github/workflows/mirror.yaml @@ -8,7 +8,7 @@ on: container_name: description: 'Container name' required: false - default: 'eic_xl' + default: 'eic_ci' type: string image_tag: description: 'Container image tag' @@ -64,7 +64,7 @@ jobs: GITHUB_SHA=${{ github.event.pull_request.head.sha || github.sha }} GITHUB_PR=${{ github.event.pull_request.number }} PIPELINE_NAME=${{ github.repository }}: ${{ github.event.pull_request.title || github.ref_name }} - CONTAINER_NAME=${{ inputs.container_name || 'eic_xl' }} + CONTAINER_NAME=${{ inputs.container_name || 'eic_ci' }} IMAGE_TAG=${{ inputs.image_tag || 'nightly' }} DETECTOR_CONFIG=${{ inputs.detector_config || 'epic_craterlake' }} DETECTOR_VERSION=${{ inputs.detector_version || 'main' }} diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index eed1a5a..e184b41 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,5 +1,5 @@ variables: - CONTAINER_NAME: "eic_xl" + CONTAINER_NAME: "eic_ci" IMAGE_TAG: "nightly" DETECTOR_CONFIG: "epic_craterlake" DETECTOR_VERSION: "main" From 1dcfa9b8b44a14aea53c0c5270636c4b8c890ae4 Mon Sep 17 00:00:00 2001 From: Sakib Rahman Date: Thu, 13 Nov 2025 21:47:50 -0500 Subject: [PATCH 06/11] Add debug output for detector configuration variables MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add echo statements to .timings template to debug whether DETECTOR_CONFIG and DETECTOR_VERSION are being set correctly from job variables. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- .gitlab-ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index e184b41..e32fc9a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -36,6 +36,8 @@ stages: .timings: stage: timings script: + - echo "DEBUG - DETECTOR_CONFIG from CI = ${DETECTOR_CONFIG}" + - echo "DEBUG - DETECTOR_VERSION from CI = ${DETECTOR_VERSION}" - mkdir -p $(dirname ${RESULTS_BASE}/datasets/timings/$DATA) # Use sed '1!d1' instead of head -n 1 to avoid pipefail issues - grep -v "^\#" ${RESULTS_BASE}/datasets/nevents/$DATA | sed '1!d' | parallel scripts/determine_timing.sh ${RESULTS_BASE}/datasets/timings/$DATA {} From cabc3a214284442490dfc95a9fd12050fecf1909 Mon Sep 17 00:00:00 2001 From: Sakib Rahman Date: Thu, 13 Nov 2025 22:38:23 -0500 Subject: [PATCH 07/11] Fix RESULTS_BASE to respect job-specific variable overrides MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Move RESULTS_BASE calculation from global variables to runtime in each job template's script section. This ensures RESULTS_BASE is calculated using the actual DETECTOR_CONFIG and other variables after job-specific overrides are applied. Previously, RESULTS_BASE was evaluated once globally with the default DETECTOR_CONFIG value and never recalculated when jobs overrode DETECTOR_CONFIG. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- .gitlab-ci.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index e32fc9a..ff0c191 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -3,7 +3,6 @@ variables: IMAGE_TAG: "nightly" DETECTOR_CONFIG: "epic_craterlake" DETECTOR_VERSION: "main" - RESULTS_BASE: "results/${CONTAINER_NAME}/${IMAGE_TAG}/${DETECTOR_CONFIG}/${DETECTOR_VERSION}" image: eicweb.phy.anl.gov:4567/containers/eic_container/${CONTAINER_NAME}:${IMAGE_TAG} @@ -25,6 +24,7 @@ stages: .nevents: stage: nevents script: + - export RESULTS_BASE="results/${CONTAINER_NAME}/${IMAGE_TAG}/${DETECTOR_CONFIG}/${DETECTOR_VERSION}" - mkdir -p $(dirname ${RESULTS_BASE}/datasets/glob/$DATA) - grep -v "^\#" $DATA | parallel scripts/glob.sh ${RESULTS_BASE}/datasets/glob/$DATA {} - sort -o ${RESULTS_BASE}/datasets/glob/$DATA ${RESULTS_BASE}/datasets/glob/$DATA @@ -36,8 +36,10 @@ stages: .timings: stage: timings script: + - export RESULTS_BASE="results/${CONTAINER_NAME}/${IMAGE_TAG}/${DETECTOR_CONFIG}/${DETECTOR_VERSION}" - echo "DEBUG - DETECTOR_CONFIG from CI = ${DETECTOR_CONFIG}" - echo "DEBUG - DETECTOR_VERSION from CI = ${DETECTOR_VERSION}" + - echo "DEBUG - RESULTS_BASE = ${RESULTS_BASE}" - mkdir -p $(dirname ${RESULTS_BASE}/datasets/timings/$DATA) # Use sed '1!d1' instead of head -n 1 to avoid pipefail issues - grep -v "^\#" ${RESULTS_BASE}/datasets/nevents/$DATA | sed '1!d' | parallel scripts/determine_timing.sh ${RESULTS_BASE}/datasets/timings/$DATA {} @@ -50,6 +52,7 @@ stages: .timings_all: stage: timings script: + - export RESULTS_BASE="results/${CONTAINER_NAME}/${IMAGE_TAG}/${DETECTOR_CONFIG}/${DETECTOR_VERSION}" - mkdir -p $(dirname ${RESULTS_BASE}/datasets/timings/$DATA) - grep -v "^\#" ${RESULTS_BASE}/datasets/nevents/$DATA | parallel scripts/determine_timing.sh ${RESULTS_BASE}/datasets/timings/$DATA {} - sort -o ${RESULTS_BASE}/datasets/timings/$DATA ${RESULTS_BASE}/datasets/timings/$DATA @@ -57,6 +60,7 @@ stages: .collect: stage: collect script: + - export RESULTS_BASE="results/${CONTAINER_NAME}/${IMAGE_TAG}/${DETECTOR_CONFIG}/${DETECTOR_VERSION}" - rm -rf results/logs/ - find ${RESULTS_BASE}/datasets/ - find ${RESULTS_BASE}/datasets/timings/ -name "*.csv" -print0 -exec awk 'BEGIN {FS=","} {sum+=$3*$5+$4} END {print(":",sum/3600,"core-hours")}' {} \; From e6414fc795488aba5f579882d77bebaee96c3cf2 Mon Sep 17 00:00:00 2001 From: Sakib Rahman Date: Thu, 13 Nov 2025 23:53:32 -0500 Subject: [PATCH 08/11] Add debug output in determine_timing.sh and use RESULTS_BASE for logs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add debug echo statements to show DETECTOR_CONFIG, DETECTOR_VERSION, and RESULTS_BASE - Update logfile path to use RESULTS_BASE instead of hardcoded results/ 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- scripts/determine_timing.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/scripts/determine_timing.sh b/scripts/determine_timing.sh index d0c9762..f6a8411 100755 --- a/scripts/determine_timing.sh +++ b/scripts/determine_timing.sh @@ -29,13 +29,17 @@ nlines=$((2*n_events_test*n_lines_per_event)) dir=$(dirname EVGEN/${file}.${ext}) mkdir -p ${dir} -logfile=results/logs/${file}.out +logfile=${RESULTS_BASE:-results}/logs/${file}.out mkdir -p $(dirname ${logfile}) # Export detector configuration if provided export DETECTOR_CONFIG="${DETECTOR_CONFIG:-epic_craterlake}" export DETECTOR_VERSION="${DETECTOR_VERSION:-main}" +echo "DEBUG [determine_timing.sh] - DETECTOR_CONFIG = ${DETECTOR_CONFIG}" +echo "DEBUG [determine_timing.sh] - DETECTOR_VERSION = ${DETECTOR_VERSION}" +echo "DEBUG [determine_timing.sh] - RESULTS_BASE = ${RESULTS_BASE:-results}" + # time for 1 event (first) t1=$(date +%s.%N) /opt/campaigns/hepmc3/scripts/run.sh EVGEN/${file} ${ext} 1 2>&1 | tee ${logfile}.1 From dcdc451a2a0e4460cd79ed008ee31b0362c43a6b Mon Sep 17 00:00:00 2001 From: Sakib Rahman Date: Fri, 14 Nov 2025 14:29:01 -0500 Subject: [PATCH 09/11] Add override support to nevents jobs and use YAML anchors for shared variables MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add DETECTOR_CONFIG_OVERRIDE and DETECTOR_VERSION_OVERRIDE support to .nevents template - Use YAML anchors to define variables once and reuse in both nevents and timings jobs - Ensures both job types write to and read from the same RESULTS_BASE path 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- .gitlab-ci.yml | 2 ++ DIS/eA/config.yml | 47 +++++++++++++++++++++++++++++++++++++++-------- 2 files changed, 41 insertions(+), 8 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index ff0c191..aa7bcb9 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -24,6 +24,8 @@ stages: .nevents: stage: nevents script: + - export DETECTOR_CONFIG="${DETECTOR_CONFIG_OVERRIDE:-${DETECTOR_CONFIG}}" + - export DETECTOR_VERSION="${DETECTOR_VERSION_OVERRIDE:-${DETECTOR_VERSION}}" - export RESULTS_BASE="results/${CONTAINER_NAME}/${IMAGE_TAG}/${DETECTOR_CONFIG}/${DETECTOR_VERSION}" - mkdir -p $(dirname ${RESULTS_BASE}/datasets/glob/$DATA) - grep -v "^\#" $DATA | parallel scripts/glob.sh ${RESULTS_BASE}/datasets/glob/$DATA {} diff --git a/DIS/eA/config.yml b/DIS/eA/config.yml index a839a42..4955676 100644 --- a/DIS/eA/config.yml +++ b/DIS/eA/config.yml @@ -1,24 +1,54 @@ -DIS:eA:eAu:nevents: +.eAu_10x100_variables: &eAu_10x100_variables + DETECTOR_CONFIG_OVERRIDE: "epic_craterlake_without_zdc" + EBEAM_OVERRIDE: "10" + PBEAM_OVERRIDE: "100_Au197" + +DIS:eA:eAu:10x100:nevents: + variables: + <<: *eAu_10x100_variables extends: .nevents parallel: matrix: - DATA: - "DIS/eA/BeAGLE1.03.02-1.0_DIS_eAu_10x100_q2_1to10.csv" - "DIS/eA/BeAGLE1.03.02-1.0_DIS_eAu_10x100_q2_10to100.csv" - - "DIS/eA/BeAGLE1.03.02-1.0_DIS_eAu_5x41_q2_1to10.csv" - - "DIS/eA/BeAGLE1.03.02-1.0_DIS_eAu_5x41_q2_10to100.csv" -DIS:eA:eAu:timings: +DIS:eA:eAu:10x100:timings: + variables: + <<: *eAu_10x100_variables extends: .timings needs: - - DIS:eA:eAu:nevents - variables: - DETECTOR_CONFIG: "epic_craterlake_without_zdc" + - DIS:eA:eAu:10x100:nevents parallel: matrix: - DATA: - "DIS/eA/BeAGLE1.03.02-1.0_DIS_eAu_10x100_q2_1to10.csv" - "DIS/eA/BeAGLE1.03.02-1.0_DIS_eAu_10x100_q2_10to100.csv" + +.eAu_5x41_variables: &eAu_5x41_variables + DETECTOR_CONFIG_OVERRIDE: "epic_craterlake_without_zdc" + EBEAM_OVERRIDE: "5" + PBEAM_OVERRIDE: "41_Au197" + +DIS:eA:eAu:5x41:nevents: + variables: + <<: *eAu_5x41_variables + extends: .nevents + parallel: + matrix: + - DATA: + - "DIS/eA/BeAGLE1.03.02-1.0_DIS_eAu_5x41_q2_1to10.csv" + - "DIS/eA/BeAGLE1.03.02-1.0_DIS_eAu_5x41_q2_10to100.csv" + +DIS:eA:eAu:5x41:timings: + variables: + <<: *eAu_5x41_variables + extends: .timings + needs: + - DIS:eA:eAu:5x41:nevents + parallel: + matrix: + - DATA: - "DIS/eA/BeAGLE1.03.02-1.0_DIS_eAu_5x41_q2_1to10.csv" - "DIS/eA/BeAGLE1.03.02-1.0_DIS_eAu_5x41_q2_10to100.csv" @@ -73,5 +103,6 @@ DIS:eA:other:timings: DIS:eA:collect: extends: .collect needs: - - DIS:eA:eAu:timings + - DIS:eA:eAu:10x100:timings + - DIS:eA:eAu:5x41:timings - DIS:eA:other:timings \ No newline at end of file From 644c0512708700745f0161139381b1eecf6ce39b Mon Sep 17 00:00:00 2001 From: Sakib Rahman Date: Fri, 14 Nov 2025 20:12:50 -0500 Subject: [PATCH 10/11] Add NEVENTS_PER_TEST global variable with override support MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add NEVENTS_PER_TEST as a global variable (default: 100) with override support via NEVENTS_PER_TEST_OVERRIDE. Also re-add EBEAM and PBEAM global variables with override support that were lost during rebase. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- .gitlab-ci.yml | 8 ++++++++ SIDIS/config.yml | 6 +++--- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index aa7bcb9..d83b2fb 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -3,6 +3,9 @@ variables: IMAGE_TAG: "nightly" DETECTOR_CONFIG: "epic_craterlake" DETECTOR_VERSION: "main" + EBEAM: "18" + PBEAM: "275" + NEVENTS_PER_TEST: "100" image: eicweb.phy.anl.gov:4567/containers/eic_container/${CONTAINER_NAME}:${IMAGE_TAG} @@ -38,6 +41,11 @@ stages: .timings: stage: timings script: + - export DETECTOR_CONFIG="${DETECTOR_CONFIG_OVERRIDE:-${DETECTOR_CONFIG}}" + - export DETECTOR_VERSION="${DETECTOR_VERSION_OVERRIDE:-${DETECTOR_VERSION}}" + - export EBEAM="${EBEAM_OVERRIDE:-${EBEAM}}" + - export PBEAM="${PBEAM_OVERRIDE:-${PBEAM}}" + - export NEVENTS_PER_TEST="${NEVENTS_PER_TEST_OVERRIDE:-${NEVENTS_PER_TEST}}" - export RESULTS_BASE="results/${CONTAINER_NAME}/${IMAGE_TAG}/${DETECTOR_CONFIG}/${DETECTOR_VERSION}" - echo "DEBUG - DETECTOR_CONFIG from CI = ${DETECTOR_CONFIG}" - echo "DEBUG - DETECTOR_VERSION from CI = ${DETECTOR_VERSION}" diff --git a/SIDIS/config.yml b/SIDIS/config.yml index eb70545..78b8dde 100644 --- a/SIDIS/config.yml +++ b/SIDIS/config.yml @@ -1,5 +1,5 @@ .eAu_10x100_variables: &eAu_10x100_variables - NEVENTS_PER_TEST: 25 + NEVENTS_PER_TEST_OVERRIDE: 25 DETECTOR_CONFIG_OVERRIDE: "epic_craterlake_without_zdc" EBEAM_OVERRIDE: "10" PBEAM_OVERRIDE: "100_Au197" @@ -27,7 +27,7 @@ SIDIS:eAu:10x100:timings: - "SIDIS/eA/BeAGLE1.03.01-2.0_Lc_eAu_10x100_q2_1to10000.csv" .eAu_5x41_variables: &eAu_5x41_variables - NEVENTS_PER_TEST: 25 + NEVENTS_PER_TEST_OVERRIDE: 25 DETECTOR_CONFIG_OVERRIDE: "epic_craterlake_without_zdc" EBEAM_OVERRIDE: "5" PBEAM_OVERRIDE: "41_Au197" @@ -74,7 +74,7 @@ SIDIS:ep:timings: needs: - SIDIS:ep:nevents variables: - NEVENTS_PER_TEST: 25 + NEVENTS_PER_TEST_OVERRIDE: 25 parallel: matrix: - DATA: From 8af54a9d080e07daa53de85660a9ab60cd7a0cbe Mon Sep 17 00:00:00 2001 From: Sakib Rahman Date: Fri, 14 Nov 2025 20:20:30 -0500 Subject: [PATCH 11/11] Replace DEBUG with INFO and print all relevant variables in nevents/timings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add INFO logging to .nevents and .timings templates showing all variables - Replace DEBUG with INFO in determine_timing.sh - Remove redundant exports from determine_timing.sh (variables come from CI) - Print EBEAM, PBEAM, and NEVENTS_PER_TEST in addition to existing variables 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- .gitlab-ci.yml | 16 +++++++++++++--- scripts/determine_timing.sh | 13 ++++++------- 2 files changed, 19 insertions(+), 10 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d83b2fb..c53cccc 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -30,6 +30,11 @@ stages: - export DETECTOR_CONFIG="${DETECTOR_CONFIG_OVERRIDE:-${DETECTOR_CONFIG}}" - export DETECTOR_VERSION="${DETECTOR_VERSION_OVERRIDE:-${DETECTOR_VERSION}}" - export RESULTS_BASE="results/${CONTAINER_NAME}/${IMAGE_TAG}/${DETECTOR_CONFIG}/${DETECTOR_VERSION}" + - echo "INFO [nevents] - CONTAINER_NAME = ${CONTAINER_NAME}" + - echo "INFO [nevents] - IMAGE_TAG = ${IMAGE_TAG}" + - echo "INFO [nevents] - DETECTOR_CONFIG = ${DETECTOR_CONFIG}" + - echo "INFO [nevents] - DETECTOR_VERSION = ${DETECTOR_VERSION}" + - echo "INFO [nevents] - RESULTS_BASE = ${RESULTS_BASE}" - mkdir -p $(dirname ${RESULTS_BASE}/datasets/glob/$DATA) - grep -v "^\#" $DATA | parallel scripts/glob.sh ${RESULTS_BASE}/datasets/glob/$DATA {} - sort -o ${RESULTS_BASE}/datasets/glob/$DATA ${RESULTS_BASE}/datasets/glob/$DATA @@ -47,9 +52,14 @@ stages: - export PBEAM="${PBEAM_OVERRIDE:-${PBEAM}}" - export NEVENTS_PER_TEST="${NEVENTS_PER_TEST_OVERRIDE:-${NEVENTS_PER_TEST}}" - export RESULTS_BASE="results/${CONTAINER_NAME}/${IMAGE_TAG}/${DETECTOR_CONFIG}/${DETECTOR_VERSION}" - - echo "DEBUG - DETECTOR_CONFIG from CI = ${DETECTOR_CONFIG}" - - echo "DEBUG - DETECTOR_VERSION from CI = ${DETECTOR_VERSION}" - - echo "DEBUG - RESULTS_BASE = ${RESULTS_BASE}" + - echo "INFO [timings] - CONTAINER_NAME = ${CONTAINER_NAME}" + - echo "INFO [timings] - IMAGE_TAG = ${IMAGE_TAG}" + - echo "INFO [timings] - DETECTOR_CONFIG = ${DETECTOR_CONFIG}" + - echo "INFO [timings] - DETECTOR_VERSION = ${DETECTOR_VERSION}" + - echo "INFO [timings] - EBEAM = ${EBEAM}" + - echo "INFO [timings] - PBEAM = ${PBEAM}" + - echo "INFO [timings] - NEVENTS_PER_TEST = ${NEVENTS_PER_TEST}" + - echo "INFO [timings] - RESULTS_BASE = ${RESULTS_BASE}" - mkdir -p $(dirname ${RESULTS_BASE}/datasets/timings/$DATA) # Use sed '1!d1' instead of head -n 1 to avoid pipefail issues - grep -v "^\#" ${RESULTS_BASE}/datasets/nevents/$DATA | sed '1!d' | parallel scripts/determine_timing.sh ${RESULTS_BASE}/datasets/timings/$DATA {} diff --git a/scripts/determine_timing.sh b/scripts/determine_timing.sh index f6a8411..f14d33d 100755 --- a/scripts/determine_timing.sh +++ b/scripts/determine_timing.sh @@ -32,13 +32,12 @@ mkdir -p ${dir} logfile=${RESULTS_BASE:-results}/logs/${file}.out mkdir -p $(dirname ${logfile}) -# Export detector configuration if provided -export DETECTOR_CONFIG="${DETECTOR_CONFIG:-epic_craterlake}" -export DETECTOR_VERSION="${DETECTOR_VERSION:-main}" - -echo "DEBUG [determine_timing.sh] - DETECTOR_CONFIG = ${DETECTOR_CONFIG}" -echo "DEBUG [determine_timing.sh] - DETECTOR_VERSION = ${DETECTOR_VERSION}" -echo "DEBUG [determine_timing.sh] - RESULTS_BASE = ${RESULTS_BASE:-results}" +echo "INFO [determine_timing.sh] - DETECTOR_CONFIG = ${DETECTOR_CONFIG:-epic_craterlake}" +echo "INFO [determine_timing.sh] - DETECTOR_VERSION = ${DETECTOR_VERSION:-main}" +echo "INFO [determine_timing.sh] - EBEAM = ${EBEAM:-18}" +echo "INFO [determine_timing.sh] - PBEAM = ${PBEAM:-275}" +echo "INFO [determine_timing.sh] - NEVENTS_PER_TEST = ${n_events_test}" +echo "INFO [determine_timing.sh] - RESULTS_BASE = ${RESULTS_BASE:-results}" # time for 1 event (first) t1=$(date +%s.%N)