diff --git a/.github/workflows/mirror.yaml b/.github/workflows/mirror.yaml index bcae56c..41edb83 100644 --- a/.github/workflows/mirror.yaml +++ b/.github/workflows/mirror.yaml @@ -4,6 +4,27 @@ on: delete: push: workflow_dispatch: + inputs: + container_name: + description: 'Container name' + required: false + default: 'eic_ci' + type: string + image_tag: + description: 'Container image tag' + required: false + default: 'nightly' + type: string + detector_config: + description: 'Detector configuration' + required: false + default: 'epic_craterlake' + type: string + detector_version: + description: 'Detector version' + required: false + default: 'main' + type: string concurrency: group: mirror-${{ github.event_name }} @@ -43,6 +64,10 @@ jobs: GITHUB_SHA=${{ github.event.pull_request.head.sha || github.sha }} GITHUB_PR=${{ github.event.pull_request.number }} PIPELINE_NAME=${{ github.repository }}: ${{ github.event.pull_request.title || github.ref_name }} + CONTAINER_NAME=${{ inputs.container_name || 'eic_ci' }} + IMAGE_TAG=${{ inputs.image_tag || 'nightly' }} + DETECTOR_CONFIG=${{ inputs.detector_config || 'epic_craterlake' }} + DETECTOR_VERSION=${{ inputs.detector_version || 'main' }} - name: Set pending EICweb status if: ${{ github.event_name != 'delete' }} env: diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 1334a00..c53cccc 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,4 +1,13 @@ -image: eicweb.phy.anl.gov:4567/containers/eic_container/eic_xl:nightly +variables: + CONTAINER_NAME: "eic_ci" + IMAGE_TAG: "nightly" + DETECTOR_CONFIG: "epic_craterlake" + DETECTOR_VERSION: "main" + EBEAM: "18" + PBEAM: "275" + NEVENTS_PER_TEST: "100" + +image: eicweb.phy.anl.gov:4567/containers/eic_container/${CONTAINER_NAME}:${IMAGE_TAG} default: timeout: 3 hours @@ -18,50 +27,74 @@ stages: .nevents: stage: nevents script: - - mkdir -p $(dirname results/datasets/glob/$DATA) - - grep -v "^\#" $DATA | parallel scripts/glob.sh results/datasets/glob/$DATA {} - - sort -o results/datasets/glob/$DATA results/datasets/glob/$DATA - - mkdir -p $(dirname results/datasets/nevents/$DATA) - - grep -v "^\#" results/datasets/glob/$DATA | parallel scripts/count_events.sh results/datasets/nevents/$DATA {} - - sort -o results/datasets/nevents/$DATA results/datasets/nevents/$DATA + - export DETECTOR_CONFIG="${DETECTOR_CONFIG_OVERRIDE:-${DETECTOR_CONFIG}}" + - export DETECTOR_VERSION="${DETECTOR_VERSION_OVERRIDE:-${DETECTOR_VERSION}}" + - export RESULTS_BASE="results/${CONTAINER_NAME}/${IMAGE_TAG}/${DETECTOR_CONFIG}/${DETECTOR_VERSION}" + - echo "INFO [nevents] - CONTAINER_NAME = ${CONTAINER_NAME}" + - echo "INFO [nevents] - IMAGE_TAG = ${IMAGE_TAG}" + - echo "INFO [nevents] - DETECTOR_CONFIG = ${DETECTOR_CONFIG}" + - echo "INFO [nevents] - DETECTOR_VERSION = ${DETECTOR_VERSION}" + - echo "INFO [nevents] - RESULTS_BASE = ${RESULTS_BASE}" + - mkdir -p $(dirname ${RESULTS_BASE}/datasets/glob/$DATA) + - grep -v "^\#" $DATA | parallel scripts/glob.sh ${RESULTS_BASE}/datasets/glob/$DATA {} + - sort -o ${RESULTS_BASE}/datasets/glob/$DATA ${RESULTS_BASE}/datasets/glob/$DATA + - mkdir -p $(dirname ${RESULTS_BASE}/datasets/nevents/$DATA) + - grep -v "^\#" ${RESULTS_BASE}/datasets/glob/$DATA | parallel scripts/count_events.sh ${RESULTS_BASE}/datasets/nevents/$DATA {} + - sort -o ${RESULTS_BASE}/datasets/nevents/$DATA ${RESULTS_BASE}/datasets/nevents/$DATA - sort -o $DATA $DATA .timings: stage: timings script: - - mkdir -p $(dirname results/datasets/timings/$DATA) + - export DETECTOR_CONFIG="${DETECTOR_CONFIG_OVERRIDE:-${DETECTOR_CONFIG}}" + - export DETECTOR_VERSION="${DETECTOR_VERSION_OVERRIDE:-${DETECTOR_VERSION}}" + - export EBEAM="${EBEAM_OVERRIDE:-${EBEAM}}" + - export PBEAM="${PBEAM_OVERRIDE:-${PBEAM}}" + - export NEVENTS_PER_TEST="${NEVENTS_PER_TEST_OVERRIDE:-${NEVENTS_PER_TEST}}" + - export RESULTS_BASE="results/${CONTAINER_NAME}/${IMAGE_TAG}/${DETECTOR_CONFIG}/${DETECTOR_VERSION}" + - echo "INFO [timings] - CONTAINER_NAME = ${CONTAINER_NAME}" + - echo "INFO [timings] - IMAGE_TAG = ${IMAGE_TAG}" + - echo "INFO [timings] - DETECTOR_CONFIG = ${DETECTOR_CONFIG}" + - echo "INFO [timings] - DETECTOR_VERSION = ${DETECTOR_VERSION}" + - echo "INFO [timings] - EBEAM = ${EBEAM}" + - echo "INFO [timings] - PBEAM = ${PBEAM}" + - echo "INFO [timings] - NEVENTS_PER_TEST = ${NEVENTS_PER_TEST}" + - echo "INFO [timings] - RESULTS_BASE = ${RESULTS_BASE}" + - mkdir -p $(dirname ${RESULTS_BASE}/datasets/timings/$DATA) # Use sed '1!d1' instead of head -n 1 to avoid pipefail issues - - grep -v "^\#" results/datasets/nevents/$DATA | sed '1!d' | parallel scripts/determine_timing.sh results/datasets/timings/$DATA {} + - grep -v "^\#" ${RESULTS_BASE}/datasets/nevents/$DATA | sed '1!d' | parallel scripts/determine_timing.sh ${RESULTS_BASE}/datasets/timings/$DATA {} - | - IFS="," read file ext nevents dt0 dt1 < results/datasets/timings/$DATA + IFS="," read file ext nevents dt0 dt1 < ${RESULTS_BASE}/datasets/timings/$DATA export dt0 dt1 - grep -v "^\#" results/datasets/nevents/$DATA | sed '1d' | parallel scripts/determine_timing.sh results/datasets/timings/$DATA {} - - sort -o results/datasets/timings/$DATA results/datasets/timings/$DATA + grep -v "^\#" ${RESULTS_BASE}/datasets/nevents/$DATA | sed '1d' | parallel scripts/determine_timing.sh ${RESULTS_BASE}/datasets/timings/$DATA {} + - sort -o ${RESULTS_BASE}/datasets/timings/$DATA ${RESULTS_BASE}/datasets/timings/$DATA .timings_all: stage: timings script: - - mkdir -p $(dirname results/datasets/timings/$DATA) - - grep -v "^\#" results/datasets/nevents/$DATA | parallel scripts/determine_timing.sh results/datasets/timings/$DATA {} - - sort -o results/datasets/timings/$DATA results/datasets/timings/$DATA + - export RESULTS_BASE="results/${CONTAINER_NAME}/${IMAGE_TAG}/${DETECTOR_CONFIG}/${DETECTOR_VERSION}" + - mkdir -p $(dirname ${RESULTS_BASE}/datasets/timings/$DATA) + - grep -v "^\#" ${RESULTS_BASE}/datasets/nevents/$DATA | parallel scripts/determine_timing.sh ${RESULTS_BASE}/datasets/timings/$DATA {} + - sort -o ${RESULTS_BASE}/datasets/timings/$DATA ${RESULTS_BASE}/datasets/timings/$DATA .collect: stage: collect script: + - export RESULTS_BASE="results/${CONTAINER_NAME}/${IMAGE_TAG}/${DETECTOR_CONFIG}/${DETECTOR_VERSION}" - rm -rf results/logs/ - - find results/datasets/ - - find results/datasets/timings/ -name "*.csv" -print0 -exec awk 'BEGIN {FS=","} {sum+=$3*$5+$4} END {print(":",sum/3600,"core-hours")}' {} \; - - for d in `find results/datasets/timings/ -type d` ; do + - find ${RESULTS_BASE}/datasets/ + - find ${RESULTS_BASE}/datasets/timings/ -name "*.csv" -print0 -exec awk 'BEGIN {FS=","} {sum+=$3*$5+$4} END {print(":",sum/3600,"core-hours")}' {} \; + - for d in `find ${RESULTS_BASE}/datasets/timings/ -type d` ; do echo -n $d ; find $d -name "*.csv" -exec cat {} \; | awk 'BEGIN {FS=","} {sum+=$3*$5+$4} END {print(":",sum/3600,"core-hours")}' ; done - - find results/datasets/timings/ -name "*.csv" -print0 -exec awk 'BEGIN {FS=","} {sum+=$3*$7+$6} END {print(":",sum/1048576,"GB (full)")}' {} \; - - for d in `find results/datasets/timings/ -type d` ; do + - find ${RESULTS_BASE}/datasets/timings/ -name "*.csv" -print0 -exec awk 'BEGIN {FS=","} {sum+=$3*$7+$6} END {print(":",sum/1048576,"GB (full)")}' {} \; + - for d in `find ${RESULTS_BASE}/datasets/timings/ -type d` ; do echo -n $d ; find $d -name "*.csv" -exec cat {} \; | awk 'BEGIN {FS=","} {sum+=$3*$7+$6} END {print(":",sum/1048576,"GB (full)")}' ; done - - find results/datasets/timings/ -name "*.csv" -print0 -exec awk 'BEGIN {FS=","} {sum+=$3*$9+$8} END {print(":",sum/1048576,"GB (reco)")}' {} \; - - for d in `find results/datasets/timings/ -type d` ; do + - find ${RESULTS_BASE}/datasets/timings/ -name "*.csv" -print0 -exec awk 'BEGIN {FS=","} {sum+=$3*$9+$8} END {print(":",sum/1048576,"GB (reco)")}' {} \; + - for d in `find ${RESULTS_BASE}/datasets/timings/ -type d` ; do echo -n $d ; find $d -name "*.csv" -exec cat {} \; | awk 'BEGIN {FS=","} {sum+=$3*$9+$8} END {print(":",sum/1048576,"GB (reco)")}' ; done diff --git a/DIS/eA/config.yml b/DIS/eA/config.yml index 38ab83e..4955676 100644 --- a/DIS/eA/config.yml +++ b/DIS/eA/config.yml @@ -1,4 +1,58 @@ -DIS:eA:nevents: +.eAu_10x100_variables: &eAu_10x100_variables + DETECTOR_CONFIG_OVERRIDE: "epic_craterlake_without_zdc" + EBEAM_OVERRIDE: "10" + PBEAM_OVERRIDE: "100_Au197" + +DIS:eA:eAu:10x100:nevents: + variables: + <<: *eAu_10x100_variables + extends: .nevents + parallel: + matrix: + - DATA: + - "DIS/eA/BeAGLE1.03.02-1.0_DIS_eAu_10x100_q2_1to10.csv" + - "DIS/eA/BeAGLE1.03.02-1.0_DIS_eAu_10x100_q2_10to100.csv" + +DIS:eA:eAu:10x100:timings: + variables: + <<: *eAu_10x100_variables + extends: .timings + needs: + - DIS:eA:eAu:10x100:nevents + parallel: + matrix: + - DATA: + - "DIS/eA/BeAGLE1.03.02-1.0_DIS_eAu_10x100_q2_1to10.csv" + - "DIS/eA/BeAGLE1.03.02-1.0_DIS_eAu_10x100_q2_10to100.csv" + +.eAu_5x41_variables: &eAu_5x41_variables + DETECTOR_CONFIG_OVERRIDE: "epic_craterlake_without_zdc" + EBEAM_OVERRIDE: "5" + PBEAM_OVERRIDE: "41_Au197" + +DIS:eA:eAu:5x41:nevents: + variables: + <<: *eAu_5x41_variables + extends: .nevents + parallel: + matrix: + - DATA: + - "DIS/eA/BeAGLE1.03.02-1.0_DIS_eAu_5x41_q2_1to10.csv" + - "DIS/eA/BeAGLE1.03.02-1.0_DIS_eAu_5x41_q2_10to100.csv" + +DIS:eA:eAu:5x41:timings: + variables: + <<: *eAu_5x41_variables + extends: .timings + needs: + - DIS:eA:eAu:5x41:nevents + parallel: + matrix: + - DATA: + - "DIS/eA/BeAGLE1.03.02-1.0_DIS_eAu_5x41_q2_1to10.csv" + - "DIS/eA/BeAGLE1.03.02-1.0_DIS_eAu_5x41_q2_10to100.csv" + +DIS:eA:other:nevents: extends: .nevents parallel: matrix: @@ -8,10 +62,6 @@ DIS:eA:nevents: - "DIS/eA/BeAGLE1.03.02-1.0_DIS_eRu_en_10x115.csv" - "DIS/eA/BeAGLE1.03.02-1.0_DIS_eRu_ep_10x115.csv" - "DIS/eA/BeAGLE1.03.02-1.0_DIS_eHe3_10x166.csv" - - "DIS/eA/BeAGLE1.03.02-1.0_DIS_eAu_10x100_q2_1to10.csv" - - "DIS/eA/BeAGLE1.03.02-1.0_DIS_eAu_10x100_q2_10to100.csv" - - "DIS/eA/BeAGLE1.03.02-1.0_DIS_eAu_5x41_q2_1to10.csv" - - "DIS/eA/BeAGLE1.03.02-1.0_DIS_eAu_5x41_q2_10to100.csv" - "DIS/eA/BeAGLE1.03.02-1.2_DIS_eHe3_10x166_q2_2to10.csv" - "DIS/eA/BeAGLE1.03.02-1.2_DIS_eHe3_10x166_q2_10to100.csv" - "DIS/eA/BeAGLE1.03.02-1.2_DIS_eHe3_10x166_q2_100to10000.csv" @@ -25,10 +75,10 @@ DIS:eA:nevents: - "DIS/eA/BeAGLE1.03.02-1.2_DIS_eHe3_5x41_q2_10to100.csv" - "DIS/eA/BeAGLE1.03.02-1.2_DIS_eHe3_5x41_q2_100to10000.csv" -DIS:eA:timings: +DIS:eA:other:timings: extends: .timings - needs: - - DIS:eA:nevents + needs: + - DIS:eA:other:nevents parallel: matrix: - DATA: @@ -37,10 +87,6 @@ DIS:eA:timings: - "DIS/eA/BeAGLE1.03.02-1.0_DIS_eRu_en_10x115.csv" - "DIS/eA/BeAGLE1.03.02-1.0_DIS_eRu_ep_10x115.csv" - "DIS/eA/BeAGLE1.03.02-1.0_DIS_eHe3_10x166.csv" - - "DIS/eA/BeAGLE1.03.02-1.0_DIS_eAu_10x100_q2_1to10.csv" - - "DIS/eA/BeAGLE1.03.02-1.0_DIS_eAu_10x100_q2_10to100.csv" - - "DIS/eA/BeAGLE1.03.02-1.0_DIS_eAu_5x41_q2_1to10.csv" - - "DIS/eA/BeAGLE1.03.02-1.0_DIS_eAu_5x41_q2_10to100.csv" - "DIS/eA/BeAGLE1.03.02-1.2_DIS_eHe3_10x166_q2_2to10.csv" - "DIS/eA/BeAGLE1.03.02-1.2_DIS_eHe3_10x166_q2_10to100.csv" - "DIS/eA/BeAGLE1.03.02-1.2_DIS_eHe3_10x166_q2_100to10000.csv" @@ -56,5 +102,7 @@ DIS:eA:timings: DIS:eA:collect: extends: .collect - needs: - - DIS:eA:timings \ No newline at end of file + needs: + - DIS:eA:eAu:10x100:timings + - DIS:eA:eAu:5x41:timings + - DIS:eA:other:timings \ No newline at end of file diff --git a/SIDIS/config.yml b/SIDIS/config.yml index fe54ecb..78b8dde 100644 --- a/SIDIS/config.yml +++ b/SIDIS/config.yml @@ -1,11 +1,62 @@ -SIDIS:nevents: +.eAu_10x100_variables: &eAu_10x100_variables + NEVENTS_PER_TEST_OVERRIDE: 25 + DETECTOR_CONFIG_OVERRIDE: "epic_craterlake_without_zdc" + EBEAM_OVERRIDE: "10" + PBEAM_OVERRIDE: "100_Au197" + +SIDIS:eAu:10x100:nevents: + variables: + <<: *eAu_10x100_variables extends: .nevents parallel: matrix: - DATA: - "SIDIS/eA/BeAGLE1.03.01-2.0_D0_eAu_10x100_q2_1to10000.csv" - - "SIDIS/eA/BeAGLE1.03.01-2.0_D0_eAu_5x41_q2_1to10000.csv" - "SIDIS/eA/BeAGLE1.03.01-2.0_Lc_eAu_10x100_q2_1to10000.csv" + +SIDIS:eAu:10x100:timings: + variables: + <<: *eAu_10x100_variables + extends: .timings + needs: + - SIDIS:eAu:10x100:nevents + parallel: + matrix: + - DATA: + - "SIDIS/eA/BeAGLE1.03.01-2.0_D0_eAu_10x100_q2_1to10000.csv" + - "SIDIS/eA/BeAGLE1.03.01-2.0_Lc_eAu_10x100_q2_1to10000.csv" + +.eAu_5x41_variables: &eAu_5x41_variables + NEVENTS_PER_TEST_OVERRIDE: 25 + DETECTOR_CONFIG_OVERRIDE: "epic_craterlake_without_zdc" + EBEAM_OVERRIDE: "5" + PBEAM_OVERRIDE: "41_Au197" + +SIDIS:eAu:5x41:nevents: + variables: + <<: *eAu_5x41_variables + extends: .nevents + parallel: + matrix: + - DATA: + - "SIDIS/eA/BeAGLE1.03.01-2.0_D0_eAu_5x41_q2_1to10000.csv" + +SIDIS:eAu:5x41:timings: + variables: + <<: *eAu_5x41_variables + extends: .timings + needs: + - SIDIS:eAu:5x41:nevents + parallel: + matrix: + - DATA: + - "SIDIS/eA/BeAGLE1.03.01-2.0_D0_eAu_5x41_q2_1to10000.csv" + +SIDIS:ep:nevents: + extends: .nevents + parallel: + matrix: + - DATA: - "SIDIS/ep/pythia8.306-1.2_D0_ep_5x41_q2_1to10000.csv" - "SIDIS/ep/pythia8.306-1.2_Lc_ep_10x100_q2_1to10000.csv" - "SIDIS/pythia6-eic-1.0.0_5x41_q2_0to1_ep_noradcor.csv" @@ -18,18 +69,15 @@ SIDIS:nevents: - "SIDIS/pythia8.306-1.0_Lambda_18x275_hiAcc.csv" - "SIDIS/pythia8.306-1.0_Lambda_18x275_hiDiv.csv" -SIDIS:timings: +SIDIS:ep:timings: extends: .timings needs: - - SIDIS:nevents + - SIDIS:ep:nevents variables: - NEVENTS_PER_TEST: 15 + NEVENTS_PER_TEST_OVERRIDE: 25 parallel: matrix: - DATA: - - "SIDIS/eA/BeAGLE1.03.01-2.0_D0_eAu_10x100_q2_1to10000.csv" - - "SIDIS/eA/BeAGLE1.03.01-2.0_D0_eAu_5x41_q2_1to10000.csv" - - "SIDIS/eA/BeAGLE1.03.01-2.0_Lc_eAu_10x100_q2_1to10000.csv" - "SIDIS/ep/pythia8.306-1.2_D0_ep_5x41_q2_1to10000.csv" - "SIDIS/ep/pythia8.306-1.2_Lc_ep_10x100_q2_1to10000.csv" - "SIDIS/pythia6-eic-1.0.0_5x41_q2_0to1_ep_noradcor.csv" @@ -42,8 +90,9 @@ SIDIS:timings: - "SIDIS/pythia8.306-1.0_Lambda_18x275_hiAcc.csv" - "SIDIS/pythia8.306-1.0_Lambda_18x275_hiDiv.csv" - SIDIS:collect: extends: .collect - needs: - - SIDIS:timings + needs: + - SIDIS:eAu:10x100:timings + - SIDIS:eAu:5x41:timings + - SIDIS:ep:timings diff --git a/scripts/determine_timing.sh b/scripts/determine_timing.sh index bc13c27..f14d33d 100755 --- a/scripts/determine_timing.sh +++ b/scripts/determine_timing.sh @@ -29,9 +29,16 @@ nlines=$((2*n_events_test*n_lines_per_event)) dir=$(dirname EVGEN/${file}.${ext}) mkdir -p ${dir} -logfile=results/logs/${file}.out +logfile=${RESULTS_BASE:-results}/logs/${file}.out mkdir -p $(dirname ${logfile}) +echo "INFO [determine_timing.sh] - DETECTOR_CONFIG = ${DETECTOR_CONFIG:-epic_craterlake}" +echo "INFO [determine_timing.sh] - DETECTOR_VERSION = ${DETECTOR_VERSION:-main}" +echo "INFO [determine_timing.sh] - EBEAM = ${EBEAM:-18}" +echo "INFO [determine_timing.sh] - PBEAM = ${PBEAM:-275}" +echo "INFO [determine_timing.sh] - NEVENTS_PER_TEST = ${n_events_test}" +echo "INFO [determine_timing.sh] - RESULTS_BASE = ${RESULTS_BASE:-results}" + # time for 1 event (first) t1=$(date +%s.%N) /opt/campaigns/hepmc3/scripts/run.sh EVGEN/${file} ${ext} 1 2>&1 | tee ${logfile}.1