diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 22d5e885fe..f46eba9093 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -42,8 +42,6 @@ jobs: with: key: raw_test_data # fixed key will always hit; clear cache to trigger cache miss path: | - clas_005038.evio.00000 - clas_021559.evio.00001 clas_018779.evio.00001 lookup-only: true - name: install xrootd-client @@ -53,10 +51,12 @@ jobs: sudo apt -y install xrootd-client - name: download if: ${{ steps.cache.outputs.cache-hit != 'true' }} +# run: | +# xrdcp xroot://sci-xrootd.jlab.org///osgpool/hallb/clas12/validation/raw/rg-d/clas_018779.evio.00001 ./ run: | - xrdcp xroot://sci-xrootd.jlab.org///osgpool/hallb/clas12/validation/clas_005038.evio.00000 ./ - xrdcp xroot://sci-xrootd.jlab.org///osgpool/hallb/clas12/validation/raw/rg-l/clas_021559.evio.00001 ./ - xrdcp xroot://sci-xrootd.jlab.org///osgpool/hallb/clas12/validation/raw/rg-d/clas_018779.evio.00001 ./ + git lfs install + git clone --depth 1 https://code.jlab.org/hallb/clas12/validation-data + cp validation-data/raw/rg-d/clas_018779.evio.00001 . # build ############################################################################# @@ -76,14 +76,6 @@ jobs: java-version: ${{ env.JAVA_VERSION }} distribution: ${{ env.java_distribution }} cache: maven - - name: install xrootd-client (linux) - if: ${{ matrix.runner == 'ubuntu-latest' }} - run: | - sudo apt -y update - sudo apt -y install xrootd-client - - name: install xrootd-client (macos) - if: ${{ matrix.runner == 'macos-latest' }} - run: brew install xrootd - name: setup cvmfs uses: cvmfs-contrib/github-action-cvmfs@v5 with: @@ -94,7 +86,11 @@ jobs: if: ${{ github.ref_type == 'tag' }} run: libexec/version-bump.sh ${{ github.ref_name }} - name: build - run: ./build-coatjava.sh --clara --xrootd --no-progress -T${{ env.nthreads }} + run: | + git lfs install + git submodule update --init ./etc/data/magfield + ./build-coatjava.sh --cvmfs --no-progress -T${{ env.nthreads }} + ./install-clara -b -c ./coatjava ./clara - name: tar # tarball to preserve permissions run: | tar czvf coatjava.tar.gz coatjava @@ -134,15 +130,11 @@ jobs: java-version: ${{ matrix.JAVA_VERSION }} distribution: ${{ env.java_distribution }} cache: maven - - name: install xrootd-client - run: | - sudo apt -y update - sudo apt -y install xrootd-client - uses: cvmfs-contrib/github-action-cvmfs@v5 with: cvmfs_repositories: 'oasis.opensciencegrid.org' - name: unit tests - run: ./build-coatjava.sh --xrootd --unittests --no-progress -T${{ env.nthreads }} + run: ./build-coatjava.sh --cvmfs --unittests --no-progress -T${{ env.nthreads }} - name: collect jacoco report if: ${{ matrix.JAVA_VERSION == env.JAVA_VERSION }} run: validation/jacoco-aggregate.sh @@ -190,10 +182,6 @@ jobs: - uses: cvmfs-contrib/github-action-cvmfs@v5 with: cvmfs_repositories: 'oasis.opensciencegrid.org' - - name: install xrootd-client - run: | - sudo apt -y update - sudo apt -y install xrootd-client - uses: actions/download-artifact@v6 with: name: build_ubuntu-latest @@ -201,15 +189,13 @@ jobs: with: key: raw_test_data path: | - clas_005038.evio.00000 - clas_021559.evio.00001 clas_018779.evio.00001 - name: untar build run: tar xzvf coatjava.tar.gz - name: run test run: | ls -lhtr - ./coatjava/bin/decoder -n 10000 -o dog.hipo ./clas_005038.evio.00000 + ./coatjava/bin/decoder -n 10000 -o dog.hipo ./clas_018779.evio.00001 test_clara: needs: [ build, download_test_data ] @@ -235,8 +221,6 @@ jobs: with: key: raw_test_data path: | - clas_005038.evio.00000 - clas_021559.evio.00001 clas_018779.evio.00001 - name: untar build run: | @@ -287,9 +271,13 @@ jobs: with: cvmfs_repositories: 'oasis.opensciencegrid.org' - name: untar build - run: tar xzvf coatjava.tar.gz + run: | + tar xzvf coatjava.tar.gz + tar xzvf clara.tar.gz - name: run test run: | + git lfs install + git submodule update --init validation/advanced-tests/data cd validation/advanced-tests echo "COMMAND: ${{ matrix.cmd }}" ${{ matrix.cmd }} diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 7cb5764b5b..d2ac1c03a2 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -49,6 +49,8 @@ build: - job: mirror optional: true script: + - git lfs install + - git submodule update --init etc/data/magfield - ./build-coatjava.sh --clara -T$JL_RUNNER_AVAIL_CPU --quiet --no-progress - tar -czf coatjava.tar.gz coatjava - tar -czf clara.tar.gz clara @@ -69,7 +71,9 @@ build: download: stage: build script: - - xrdcp xroot://sci-xrootd.jlab.org///osgpool/hallb/clas12/validation/raw/rg-d/clas_018779.evio.00001 ./ +# - xrdcp xroot://sci-xrootd.jlab.org///osgpool/hallb/clas12/validation/raw/rg-d/clas_018779.evio.00001 ./ + - git clone https://code.jlab.org/hallb/clas12/validation-data + - cp validation-data/raw/rg-d/clas_018779.evio.00001 . artifacts: when: always expire_in: 1 day @@ -124,6 +128,9 @@ eb: dependencies: [build] script: - tar -xzf coatjava.tar.gz + - tar -xzf clara.tar.gz + - git lfs install + - git submodule update --init validation/advanced-tests/data - cd validation/advanced-tests - ./run-eb-tests.sh -100 ${ARG} parallel: diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000000..b5a2f825c4 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,6 @@ +[submodule "etc/data/magfield"] + path = etc/data/magfield + url = https://code.jlab.org/hallb/clas12/magfield +[submodule "validation/advanced-tests/data"] + path = validation/advanced-tests/data + url = https://code.jlab.org/hallb/clas12/coatjava/advanced-tests diff --git a/build-coatjava.sh b/build-coatjava.sh index 22559520f0..c29676e1d4 100755 --- a/build-coatjava.sh +++ b/build-coatjava.sh @@ -23,8 +23,10 @@ usage='''build-coatjava.sh [OPTIONS]... [MAVEN_OPTIONS]... --xrootd use xrootd to download field maps --cvmfs use cvmfs to download field maps + --lfs use lfs for field maps and test data --clara install clara too + --data download test data (requires lfs) --help show this message @@ -40,7 +42,9 @@ downloadMaps="yes" runUnitTests="no" useXrootd=false useCvmfs=false +useLfs=false installClara=false +downloadData=false mvnArgs=() wgetArgs=() for xx in $@ @@ -61,8 +65,10 @@ do wgetArgs+=(--no-verbose) ;; --xrootd) useXrootd=true ;; - --cvmfs) useCvmfs=true ;; - --clara) installClara=true ;; + --cvmfs) useCvmfs=true ;; + --lfs) useLfs=true ;; + --clara) installClara=true ;; + --data) downloadData=true ;; -h|--help) echo "$usage" exit 2 @@ -71,6 +77,18 @@ do esac done +if $downloadData && ! $useLfs; then + echo "$usage" + echo "ERROR::::::::::: --data requires --lfs" + exit 2 +fi + +# Currently only git-lfs works from offsite: +if ! [[ $(hostname) == *.jlab.org ]]; then + echo "INFO: using --lfs for offsite usage" + useLfs=true +fi + src_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" &> /dev/null && pwd)" prefix_dir=$src_dir/coatjava clara_home=$src_dir/clara @@ -91,8 +109,13 @@ download () { if $useXrootd; then xrdcp $1 ./ ret=$? + elif $useLfs; then + cd $src_dir + git submodule update --init etc/data/magfield + if $downloadData; then git submodule update --init validation/advanced-tests/data; fi + cd - > /dev/null elif $useCvmfs; then - cp $1 ./ + cp -v $1 ./ ret=$? elif command_exists wget ; then $wget $1 @@ -116,6 +139,7 @@ magfield_dir=$src_dir/etc/data/magfield if [ $cleanBuild == "no" ] && [ $downloadMaps == "yes" ]; then echo 'Retrieving field maps ...' webDir=https://clasweb.jlab.org/clas12offline/magfield + if $useLfs; then webDir=${magfield_dir##$src_dir}; fi if $useXrootd; then webDir=xroot://sci-xrootd.jlab.org//osgpool/hallb/clas12/coatjava/magfield; fi if $useCvmfs; then webDir=/cvmfs/oasis.opensciencegrid.org/jlab/hallb/clas12/sw/noarch/data/magfield; fi mkdir -p $magfield_dir diff --git a/etc/data/magfield b/etc/data/magfield new file mode 160000 index 0000000000..a0e899ea96 --- /dev/null +++ b/etc/data/magfield @@ -0,0 +1 @@ +Subproject commit a0e899ea96c86609ea36f5111ed7c319e8463673 diff --git a/install-clara b/install-clara index 876d0c408a..ec5d65f5f0 100755 --- a/install-clara +++ b/install-clara @@ -56,10 +56,17 @@ function cleanup() { } function build_clara { - git clone --branch $clara --depth 1 https://code.jlab.org/hallb/clas12/clara-java + #git clone --branch $clara --depth 1 https://code.jlab.org/hallb/clas12/clara-java + git clone --depth 1 https://code.jlab.org/hallb/clas12/clara-java cd clara-java && CLARA_HOME=$1 ./gradlew deploy && cd - } +function build_grapes { + #git clone --branch $grapes --depth 1 https://code.jlab.org/hallb/clas12/coatjava/grapes + git clone --depth 1 https://code.jlab.org/hallb/clas12/coatjava/grapes + cd grapes && mvn package && cd - +} + trap cleanup EXIT # Configure: @@ -133,14 +140,14 @@ fi if ! compgen -G "$coatjava/lib/clas/coat-libs-*.jar" > /dev/null then echo -e "\nRetrieving COATJAVA version $coatjava ..." - get https://clasweb.jlab.org/clas12offline/distribution/coatjava/coatjava-$coatjava.tar.gz - coatjava=./coatjava + get https://github.com/JeffersonLab/coatjava/releases/download/$coatjava/coatjava-$coatjava.tar.gz + coatjava=./coatjava-$coatjava fi mkdir -p clara-cre/plugins/clas12/config cp -Lr $coatjava/etc $coatjava/bin $coatjava/lib* clara-cre/plugins/clas12 # GRAPES: echo -e "\nRetrieving GRAPES version $grapes ..." -get https://clasweb.jlab.org/clas12offline/distribution/grapes/grapes-$grapes.tar.gz -mv grapes-$grapes clara-cre/plugins/grapes - +build_grapes $PWD/grapes +mkdir -p clara-cre/plugins/grapes/lib/core/ +cp grapes/target/grapes-*core*.jar clara-cre/plugins/grapes/lib/core/ diff --git a/validation/advanced-tests/data b/validation/advanced-tests/data new file mode 160000 index 0000000000..cebf632bbd --- /dev/null +++ b/validation/advanced-tests/data @@ -0,0 +1 @@ +Subproject commit cebf632bbd0a5b45ee3e1f34ee1b24903a0a1013 diff --git a/validation/advanced-tests/run-advanced-tests.sh b/validation/advanced-tests/run-advanced-tests.sh index d763f7ec12..240822351e 100755 --- a/validation/advanced-tests/run-advanced-tests.sh +++ b/validation/advanced-tests/run-advanced-tests.sh @@ -1,27 +1,24 @@ #!/bin/bash -f -# coatjava must already be built at ../../coatjava/ +# coatjava and clara must already be built at ../../coatjava/ +# and input data files at ./data # set up environment JAVA_OPTS="-Djava.util.logging.config.file=$PWD/../../etc/logging/debug.properties" -CLARA_HOME=$PWD/clara_installation/ ; export CLARA_HOME +CLARA_HOME=$PWD/../../clara/ ; export CLARA_HOME COAT=$CLARA_HOME/plugins/clas12/ -# install clara -../../install-clara -c ../../coatjava $CLARA_HOME -[ $? -ne 0 ] && echo "clara installation error" && exit 1 - # source coatjava environment source $COAT/libexec/env.sh classPath="${COATJAVA_CLASSPATH}:../lib/*:src/" # download test files -wget --no-check-certificate http://clasweb.jlab.org/clas12offline/distribution/coatjava/validation_files/twoTrackEvents_809_raw.evio.tar.gz -[ $? -ne 0 ] && echo "wget validation files failure" && exit 2 -tar -zxvf twoTrackEvents_809_raw.evio.tar.gz +#wget --no-check-certificate http://clasweb.jlab.org/clas12offline/distribution/coatjava/validation_files/twoTrackEvents_809_raw.evio.tar.gz +#[ $? -ne 0 ] && echo "wget validation files failure" && exit 2 +#tar -zxvf twoTrackEvents_809_raw.evio.tar.gz # run decoder -$COAT/bin/decoder -t -0.5 -s 0.0 -i ./twoTrackEvents_809_raw.evio -o ./twoTrackEvents_809.hipo -c 2 +$COAT/bin/decoder -t -0.5 -s 0.0 -i ./data/twoTrackEvents_809_raw.evio -o ./twoTrackEvents_809.hipo -c 2 [ $? -ne 0 ] && echo "decoder failure" && exit 3 # take a peek diff --git a/validation/advanced-tests/run-eb-tests.sh b/validation/advanced-tests/run-eb-tests.sh index 748d7a5b97..43ca6e6ed4 100755 --- a/validation/advanced-tests/run-eb-tests.sh +++ b/validation/advanced-tests/run-eb-tests.sh @@ -3,8 +3,10 @@ webDir=http://clasweb.jlab.org/clas12offline/distribution/coatjava/validation_files/eb webVersion=5.11-fid-tm-dc2-r11 webDir=$webDir/$webVersion +webDir=./data/$webVersion # coatjava must already be built at ../../coatjava/ +# and input test data files at ./data # whether to use CLARA (0=no) useClara=0 @@ -121,8 +123,8 @@ then fi # download test files, if necessary: - wget -N --no-check-certificate $webDir/${stub}.hipo - if [ $? != 0 ] ; then echo "wget validation files failure" ; exit 1 ; fi + #wget -N --no-check-certificate $webDir/${stub}.hipo + #if [ $? != 0 ] ; then echo "wget validation files failure" ; exit 1 ; fi # update the schema dictionary: (no longer necessary now that recon-util does it) #rm -f up_${stub}.hipo @@ -134,7 +136,7 @@ then then GEOMDBVAR=$geoDbVariation export GEOMDBVAR - ../../coatjava/bin/recon-util -i ${stub}.hipo -o out_${stub}.hipo -c 2 + ../../coatjava/bin/recon-util -i ${webDir}/${stub}.hipo -o out_${stub}.hipo -c 2 else echo "set inputDir $PWD/" > cook.clara echo "set outputDir $PWD/" >> cook.clara