From 5e4484ca693fbfbb7cb0d91b13f1d214631dcb43 Mon Sep 17 00:00:00 2001 From: Nathan Baltzell Date: Thu, 14 Aug 2025 20:01:59 -0400 Subject: [PATCH 01/11] relax --- .../src/main/java/org/jlab/detector/decode/CLASDecoder4.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/common-tools/clas-detector/src/main/java/org/jlab/detector/decode/CLASDecoder4.java b/common-tools/clas-detector/src/main/java/org/jlab/detector/decode/CLASDecoder4.java index 407c21c4de..f4f3f2f46d 100644 --- a/common-tools/clas-detector/src/main/java/org/jlab/detector/decode/CLASDecoder4.java +++ b/common-tools/clas-detector/src/main/java/org/jlab/detector/decode/CLASDecoder4.java @@ -41,14 +41,14 @@ */ public class CLASDecoder4 { + protected DetectorEventDecoder detectorDecoder = null; + protected SchemaFactory schemaFactory = new SchemaFactory(); private CodaEventDecoder codaDecoder = null; - private DetectorEventDecoder detectorDecoder = null; private List dataList = new ArrayList<>(); private HipoDataSync writer = null; private HipoDataEvent hipoEvent = null; private boolean isRunNumberFixed = false; private int decoderDebugMode = 0; - private SchemaFactory schemaFactory = new SchemaFactory(); private ModeAHDC ahdcExtractor = new ModeAHDC(); private RCDBManager rcdbManager = new RCDBManager(); From badd0c87b31a22b3607d083ee9c40f489d36d0f0 Mon Sep 17 00:00:00 2001 From: Nathan Baltzell Date: Thu, 14 Aug 2025 20:02:11 -0400 Subject: [PATCH 02/11] add decoder wrapper --- .../jlab/detector/decode/CLASDecoder4U.java | 160 ++++++++++++++++++ 1 file changed, 160 insertions(+) create mode 100644 common-tools/clas-detector/src/main/java/org/jlab/detector/decode/CLASDecoder4U.java diff --git a/common-tools/clas-detector/src/main/java/org/jlab/detector/decode/CLASDecoder4U.java b/common-tools/clas-detector/src/main/java/org/jlab/detector/decode/CLASDecoder4U.java new file mode 100644 index 0000000000..f4b3a141ef --- /dev/null +++ b/common-tools/clas-detector/src/main/java/org/jlab/detector/decode/CLASDecoder4U.java @@ -0,0 +1,160 @@ +package org.jlab.detector.decode; + +import java.io.File; +import java.util.ArrayList; +import java.util.TreeSet; +import org.jlab.detector.helicity.HelicitySequence; +import org.jlab.detector.helicity.HelicityState; +import org.jlab.io.evio.EvioDataEvent; +import org.jlab.io.evio.EvioSource; +import org.jlab.jnp.hipo4.data.Bank; +import org.jlab.jnp.hipo4.data.Event; +import org.jlab.jnp.hipo4.io.HipoWriterSorted; +import org.jlab.utils.benchmark.ProgressPrintout; +import org.jlab.utils.options.OptionParser; +import org.jlab.utils.system.ClasUtilsFile; + +/** + * Wrapper of everything in CLAS12Decoder4's main method. + * + * @author baltzell + */ +public class CLASDecoder4U extends CLASDecoder4 { + + private EvioSource reader; + private HipoWriterSorted writer; + private Bank config; + private Bank helicity; + private ArrayList filenames; + private TreeSet helicities; + private ProgressPrintout progress; + private Double torus; + private Double solenoid; + private int runNumber; + private int maxEvents; + private int eventCounter; + + public CLASDecoder4U(OptionParser o) { + super(); + init(o); + } + + public static OptionParser getOptionParser() { + OptionParser p = new OptionParser("decoder"); + p.setDescription("CLAS12 Data Decoder"); + p.addOption("-n", "-1", "maximum number of events to process"); + p.addOption("-c", "2", "compression type (0-NONE, 1-LZ4 Fast, 2-LZ4 Best, 3-GZIP)"); + p.addOption("-r", "-1","run number in the header bank (-1 means use CODA run)"); + p.addOption("-t", null,"torus current in the header bank (null means use RCDB)"); + p.addOption("-s", null,"solenoid current in the header bank (null means use RCDB)"); + p.addOption("-x", null,"CCDB timestamp (MM/DD/YYYY-HH:MM:SS)"); + p.addOption("-V","default","CCDB variation"); + p.addOption("-o",null,"output HIPO filename"); + p.setRequiresInputList(true); + return p; + } + + public void disableProgressMeter() { progress=null; } + + /** + * @return whether there's another event to get + */ + public boolean hasNext() { + if (maxEvents > 0 && eventCounter > maxEvents) return false; + if (reader != null && reader.hasEvent()) return true; + return !filenames.isEmpty(); + } + + /** + * @return the next event + */ + public Event getNext() { + EvioDataEvent evio = getNextEvioEvent(); + Event event = getDecodedEvent(evio, runNumber, eventCounter, torus, solenoid); + if (writer != null) process(event); + if (progress != null) progress.updateStatus(); + if (++eventCounter%25000 == 0) System.gc(); + return event; + } + + /** + * Close the writer, after writing HEL::flip banks. + */ + public void close() { + if (writer != null) { + HelicitySequence.writeFlips(writer, helicities); + writer.close(); + } + } + + /** + * The command-line "decoder" program. + * @param args + */ + public static void main(String[] args) { + + // hijack arguments, when run from an IDE: + if (System.console() == null && args.length == 0) { + // delete output file, if necessary: + File f = new File("tmp.hipo"); + if (f.exists()) f.delete(); + // setup decoder command-line options: + args = new String[]{"-o","tmp.hipo",System.getenv("HOME")+"/data/clas_005038.evio.00001"}; + // try to find bankdefs: + System.setProperty("CLAS12DIR", System.getenv("HOME")+"/sw/coatjava/dev/coatjava"); + } + + // parse command-line options: + OptionParser opts = CLASDecoder4U.getOptionParser(); + opts.parse(args); + + // run the decoder: + CLASDecoder4U decoder = new CLASDecoder4U(opts); + while (decoder.hasNext()) decoder.getNext(); + decoder.close(); + } + + private EvioDataEvent getNextEvioEvent() { + if (reader == null || !reader.hasEvent()) { + reader = new EvioSource(); + reader.open(filenames.remove(0)); + } + return (EvioDataEvent)reader.getNextEvent(); + } + + private void process(Event event) { + event.read(config); + event.read(helicity); + helicities.add(HelicityState.createFromFadcBank(helicity, config, detectorDecoder.scalerManager)); + Event tag = createTaggedEvent(event, "RAW::epics","RAW::scaler","RUN::scaler","HEL::scaler"); + if (!tag.isEmpty()) + writer.addEvent(tag, 1); + writer.addEvent(event,0); + } + + private void init(OptionParser o){ + eventCounter = 0; + writer = null; + filenames = new ArrayList<>(o.getInputList()); + config = new Bank(schemaFactory.getSchema("RUN::config")); + helicity = new Bank(schemaFactory.getSchema("HEL::adc")); + helicities = new TreeSet<>(); + progress = new ProgressPrintout(); + torus = o.getOption("-t").getValue()==null?null:o.getOption("-t").doubleValue(); + solenoid = o.getOption("-s").getValue()==null?null:o.getOption("-s").doubleValue(); + runNumber = o.getOption("-r").intValue(); + maxEvents = o.getOption("-n").intValue(); + if (runNumber > 0) setRunNumber(runNumber, true); + if (o.getOption("-x").getValue() != null) + detectorDecoder.setTimestamp(o.getOption("-x").stringValue()); + if (o.getOption("-v").getValue() != null) + detectorDecoder.setVariation(o.getOption("-v").stringValue()); + if (o.getOption("-o").getValue() != null) { + writer = new HipoWriterSorted(); + writer.setCompressionType(o.getOption("-c").intValue()); + writer.getSchemaFactory().initFromDirectory(ClasUtilsFile.getResourceDir("CLAS12DIR", "etc/bankdefs/hipo4")); + writer.open(o.getOption("-o").stringValue()); + } + } + +} \ No newline at end of file From 8e52d59d3a3e6218878703c30ea07a96d3d23fab Mon Sep 17 00:00:00 2001 From: Nathan Baltzell Date: Fri, 15 Aug 2025 10:36:24 -0400 Subject: [PATCH 03/11] cache data from xrootd --- .github/workflows/ci.yml | 29 ++++++++++++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 15cd747565..46af131e64 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -30,6 +30,29 @@ env: jobs: + # download & cache + ############################################################################# + + download_test_data: + runs-on: ubuntu-latest + steps: + - uses: actions/cache@v4 + id: cache + with: + key: raw_test_data # fixed key will always hit; clear cache to trigger cache miss + path: clas_005038.evio.00000 + lookup-only: true + - name: install xrootd-client + if: ${{ steps.cache.outputs.cache-hit != 'true' }} + run: | + sudo apt -y update + sudo apt -y upgrade + sudo apt -y install xrootd-client + - name: download + if: ${{ steps.cache.outputs.cache-hit != 'true' }} + run: | + xrdcp xroot://sci-xrootd.jlab.org//osgpool/hallb/clas12/validation/raw/rg-a/clas_005038.evio.00000 ./ + # build ############################################################################# @@ -104,7 +127,7 @@ jobs: retention-days: 1 test_decoder: - needs: [ build ] + needs: [ build, download_test_data ] runs-on: ubuntu-latest steps: - uses: actions/checkout@v5 @@ -123,6 +146,10 @@ jobs: - uses: actions/download-artifact@v5 with: name: build_ubuntu-latest + - uses: actions/cache/restore@v4 + with: + key: raw_test_data + path: clas_005038.evio.00000 - name: untar build run: tar xzvf coatjava.tar.gz - name: run test From d9f50c8ccae2515fdb8b971bf7f9fffdb55975bc Mon Sep 17 00:00:00 2001 From: Nathan Baltzell Date: Fri, 15 Aug 2025 10:47:42 -0400 Subject: [PATCH 04/11] fix xrootd path, reduce job length --- .github/workflows/ci.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 46af131e64..b810b7ec90 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -51,7 +51,7 @@ jobs: - name: download if: ${{ steps.cache.outputs.cache-hit != 'true' }} run: | - xrdcp xroot://sci-xrootd.jlab.org//osgpool/hallb/clas12/validation/raw/rg-a/clas_005038.evio.00000 ./ + xrdcp xroot://sci-xrootd.jlab.org///osgpool/hallb/clas12/validation/raw/rg-a/clas_005038.evio.00000 ./ # build ############################################################################# @@ -153,9 +153,7 @@ jobs: - name: untar build run: tar xzvf coatjava.tar.gz - name: run test - run: | - xrdcp xroot://sci-xrootd.jlab.org///osgpool/hallb/clas12/validation/clas_005038.evio.00000 . - ./coatjava/bin/decoder -n 100000 -o dog.hipo ./clas_005038.evio.00000 + run: ./coatjava/bin/decoder -n 10000 -o dog.hipo ./clas_005038.evio.00000 test_coatjava: needs: [ build ] From dc4e5484f597e34f6dd8088ae99a036544387bc7 Mon Sep 17 00:00:00 2001 From: Nathan Baltzell Date: Fri, 15 Aug 2025 11:04:30 -0400 Subject: [PATCH 05/11] use old, working path for now --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b810b7ec90..562c1ee497 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -51,7 +51,7 @@ jobs: - name: download if: ${{ steps.cache.outputs.cache-hit != 'true' }} run: | - xrdcp xroot://sci-xrootd.jlab.org///osgpool/hallb/clas12/validation/raw/rg-a/clas_005038.evio.00000 ./ + xrdcp xroot://sci-xrootd.jlab.org///osgpool/hallb/clas12/validation/clas_005038.evio.00000 ./ # build ############################################################################# From 2936f0321fae47008cff934cf125c848189a0b31 Mon Sep 17 00:00:00 2001 From: Nathan Baltzell Date: Fri, 29 Aug 2025 21:50:38 -0400 Subject: [PATCH 06/11] add constructor --- .../main/java/org/jlab/detector/decode/CLASDecoder4U.java | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/common-tools/clas-detector/src/main/java/org/jlab/detector/decode/CLASDecoder4U.java b/common-tools/clas-detector/src/main/java/org/jlab/detector/decode/CLASDecoder4U.java index f4b3a141ef..c7dde4111b 100644 --- a/common-tools/clas-detector/src/main/java/org/jlab/detector/decode/CLASDecoder4U.java +++ b/common-tools/clas-detector/src/main/java/org/jlab/detector/decode/CLASDecoder4U.java @@ -39,6 +39,13 @@ public CLASDecoder4U(OptionParser o) { init(o); } + public CLASDecoder4U(String... filename) { + super(); + OptionParser o = getOptionParser(); + o.parse(filename); + init(o); + } + public static OptionParser getOptionParser() { OptionParser p = new OptionParser("decoder"); p.setDescription("CLAS12 Data Decoder"); From 65629e95bd65004410be0e2e4129920a6b6e0f0a Mon Sep 17 00:00:00 2001 From: Nathan Baltzell Date: Fri, 29 Aug 2025 22:05:17 -0400 Subject: [PATCH 07/11] reuse parser --- .../org/jlab/detector/decode/CLASDecoder4.java | 16 +--------------- 1 file changed, 1 insertion(+), 15 deletions(-) diff --git a/common-tools/clas-detector/src/main/java/org/jlab/detector/decode/CLASDecoder4.java b/common-tools/clas-detector/src/main/java/org/jlab/detector/decode/CLASDecoder4.java index f4f3f2f46d..7d2af0c547 100644 --- a/common-tools/clas-detector/src/main/java/org/jlab/detector/decode/CLASDecoder4.java +++ b/common-tools/clas-detector/src/main/java/org/jlab/detector/decode/CLASDecoder4.java @@ -815,21 +815,7 @@ public Event createTaggedEvent(Event e, String... banks) { public static void main(String[] args){ - OptionParser parser = new OptionParser("decoder"); - parser.setDescription("CLAS12 Data Decoder"); - parser.addOption("-n", "-1", "maximum number of events to process"); - parser.addOption("-c", "2", "compression type (0-NONE, 1-LZ4 Fast, 2-LZ4 Best, 3-GZIP)"); - parser.addOption("-d", "0","debug mode, set >0 for more verbose output"); - parser.addOption("-m", "run","translation tables source (use -m devel for development tables)"); - parser.addOption("-b", "16","record buffer size in MB"); - parser.addOption("-r", "-1","run number in the header bank (-1 means use CODA run)"); - parser.addOption("-t", null,"torus current in the header bank (null means use RCDB)"); - parser.addOption("-s", null,"solenoid current in the header bank (null means use RCDB)"); - parser.addOption("-x", null,"CCDB timestamp (MM/DD/YYYY-HH:MM:SS)"); - parser.addOption("-v","default","CCDB variation"); - parser.addRequired("-o","output.hipo"); - parser.parse(args); - + OptionParser parser = CLASDecoder4U.getOptionParser(); List inputList = parser.getInputList(); if(inputList.isEmpty()==true){ From 84b9ca21abfdc167a84ef595d93cd5005123bd29 Mon Sep 17 00:00:00 2001 From: Nathan Baltzell Date: Wed, 3 Sep 2025 17:52:22 -0400 Subject: [PATCH 08/11] fix up test/main method --- .../java/org/jlab/utils/options/OptionParser.java | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/common-tools/clas-utils/src/main/java/org/jlab/utils/options/OptionParser.java b/common-tools/clas-utils/src/main/java/org/jlab/utils/options/OptionParser.java index 77a8fed72f..8ae1566223 100644 --- a/common-tools/clas-utils/src/main/java/org/jlab/utils/options/OptionParser.java +++ b/common-tools/clas-utils/src/main/java/org/jlab/utils/options/OptionParser.java @@ -215,16 +215,15 @@ public static String getVersion(){ } public static void main(String[] args){ - OptionParser parser = new OptionParser(); + if (args.length == 0) args = new String[]{"-o","out.dat","in.dat"}; + OptionParser parser = new OptionParser("PotionParser"); parser.addRequired("-o"); parser.addOption("-r", "10"); parser.addOption("-t", "25.0"); parser.addOption("-d", "35"); - parser.addOption("-h","helpless"); - parser.addOption("-v","versionless"); - if (args.length == 0) parser.parse("-o","out.dat","in.dat"); - else parser.parse(args); - parser.show(); + parser.show(); + parser.parse(args); + System.out.println("INPUTLIST: "+parser.getInputList()); parser.parse("-h"); } } From 1c31d354cd8ae7b3e14d5343ae927168e0bb8e5a Mon Sep 17 00:00:00 2001 From: Nathan Baltzell Date: Wed, 3 Sep 2025 17:56:55 -0400 Subject: [PATCH 09/11] add missing parse --- .../src/main/java/org/jlab/detector/decode/CLASDecoder4.java | 1 + 1 file changed, 1 insertion(+) diff --git a/common-tools/clas-detector/src/main/java/org/jlab/detector/decode/CLASDecoder4.java b/common-tools/clas-detector/src/main/java/org/jlab/detector/decode/CLASDecoder4.java index 7d2af0c547..fdae069d19 100644 --- a/common-tools/clas-detector/src/main/java/org/jlab/detector/decode/CLASDecoder4.java +++ b/common-tools/clas-detector/src/main/java/org/jlab/detector/decode/CLASDecoder4.java @@ -816,6 +816,7 @@ public Event createTaggedEvent(Event e, String... banks) { public static void main(String[] args){ OptionParser parser = CLASDecoder4U.getOptionParser(); + parser.parse(args); List inputList = parser.getInputList(); if(inputList.isEmpty()==true){ From 01efddf4da11c52b9a0be9c6b7ba15e27abdb751 Mon Sep 17 00:00:00 2001 From: Nathan Baltzell Date: Wed, 3 Sep 2025 18:21:20 -0400 Subject: [PATCH 10/11] restore not-really-supported options --- .../src/main/java/org/jlab/detector/decode/CLASDecoder4.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/common-tools/clas-detector/src/main/java/org/jlab/detector/decode/CLASDecoder4.java b/common-tools/clas-detector/src/main/java/org/jlab/detector/decode/CLASDecoder4.java index fdae069d19..5821f8df29 100644 --- a/common-tools/clas-detector/src/main/java/org/jlab/detector/decode/CLASDecoder4.java +++ b/common-tools/clas-detector/src/main/java/org/jlab/detector/decode/CLASDecoder4.java @@ -816,6 +816,9 @@ public Event createTaggedEvent(Event e, String... banks) { public static void main(String[] args){ OptionParser parser = CLASDecoder4U.getOptionParser(); + parser.addOption("-d", "0","debug mode, set >0 for more verbose output"); + parser.addOption("-m", "run","translation tables source (use -m devel for development tables)"); + parser.addOption("-b", "16","record buffer size in MB"); parser.parse(args); List inputList = parser.getInputList(); From a4428230a6d09c9d9a4520bbf2cfc9619bad03d8 Mon Sep 17 00:00:00 2001 From: Nathan Baltzell Date: Thu, 4 Sep 2025 18:30:47 -0400 Subject: [PATCH 11/11] Update .github/workflows/ci.yml Co-authored-by: Christopher Dilks --- .github/workflows/ci.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 562c1ee497..8e0109c0d0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -46,7 +46,6 @@ jobs: if: ${{ steps.cache.outputs.cache-hit != 'true' }} run: | sudo apt -y update - sudo apt -y upgrade sudo apt -y install xrootd-client - name: download if: ${{ steps.cache.outputs.cache-hit != 'true' }}