From a510f3fac157807e00e68e8933ee6922d121cbc8 Mon Sep 17 00:00:00 2001 From: Mun Jung Jung Date: Thu, 17 Jul 2025 20:25:37 -0500 Subject: [PATCH 01/14] add dnn roi only workflow for mc --- sbndcode/JobConfigurations/CMakeLists.txt | 1 + .../wcls-sim-drift-depoflux-nf-sp.jsonnet | 88 +++++++++++++++++-- sbndcode/WireCell/wcsimsp_sbnd.fcl | 23 ++++- sbndcode/WireCell/wcsp_data_sbnd.fcl | 10 +++ 4 files changed, 111 insertions(+), 11 deletions(-) diff --git a/sbndcode/JobConfigurations/CMakeLists.txt b/sbndcode/JobConfigurations/CMakeLists.txt index 86aebe83e..37a3b8b4d 100644 --- a/sbndcode/JobConfigurations/CMakeLists.txt +++ b/sbndcode/JobConfigurations/CMakeLists.txt @@ -1,3 +1,4 @@ add_subdirectory(base) add_subdirectory(standard) +add_subdirectory(dnnroi) diff --git a/sbndcode/WireCell/cfg/pgrapher/experiment/sbnd/wcls-sim-drift-depoflux-nf-sp.jsonnet b/sbndcode/WireCell/cfg/pgrapher/experiment/sbnd/wcls-sim-drift-depoflux-nf-sp.jsonnet index 88d6e1ce7..95af5e19f 100644 --- a/sbndcode/WireCell/cfg/pgrapher/experiment/sbnd/wcls-sim-drift-depoflux-nf-sp.jsonnet +++ b/sbndcode/WireCell/cfg/pgrapher/experiment/sbnd/wcls-sim-drift-depoflux-nf-sp.jsonnet @@ -5,7 +5,7 @@ local reality = std.extVar('reality'); local sigoutform = std.extVar('signal_output_form'); // eg "sparse" or "dense" local savetid = std.extVar("save_track_id"); -local use_dnnroi = std.extVar('use_dnnroi'); +local roi = std.extVar('roi'); local nchunks = std.extVar('nchunks'); local tick_per_slice = std.extVar('tick_per_slice'); local dnnroi_model_p0 = std.extVar('dnnroi_model_p0'); @@ -138,7 +138,29 @@ local wcls_depoflux_writer = g.pnode({ }, nin=1, nout=1, uses=tools.anodes + [tools.field]); local sp_maker = import 'pgrapher/experiment/sbnd/sp.jsonnet'; -local sp_override = if use_dnnroi then { + +local sp_override = +if roi == "dnn" then { + sparse: true, + use_roi_debug_mode: true, + save_negative_charge: false, // TODO: no negative charge in gauss, default is false + use_multi_plane_protection: true, + do_not_mp_protect_traditional: false, // TODO: do_not_mp_protect_traditional to make a clear ref, defualt is false + mp_tick_resolution:4, + tight_lf_tag: "", + // loose_lf_tag: "", // comment to use as input to dnnsp + cleanup_roi_tag: "", + break_roi_loop1_tag: "", + break_roi_loop2_tag: "", + shrink_roi_tag: "", + extend_roi_tag: "", + // mp3_roi_tag: "", + // mp2_roi_tag: "", + decon_charge_tag: "", + gauss_tag: "", + wiener_tag: "", +} +else if roi == "both" then { sparse: true, use_roi_debug_mode: true, save_negative_charge: false, // TODO: no negative charge in gauss, default is false @@ -152,9 +174,11 @@ local sp_override = if use_dnnroi then { break_roi_loop2_tag: "", shrink_roi_tag: "", extend_roi_tag: "", -} else { +} +else if roi == "trad" then { sparse: true, }; + local sp = sp_maker(params, tools, sp_override); local sp_pipes = [sp.make_sigproc(a) for a in tools.anodes]; @@ -340,7 +364,27 @@ local chsel_pipes = [ ]; -local nfsp_pipes = if use_dnnroi then +local nfsp_pipes = +if roi == "dnn" then +// oports: 0: dnnroi, 1: traditional sp +[ + g.intern( + innodes=[chsel_pipes[n]], + outnodes=[dnnroi_pipes[n],sp_fans[n]], + centernodes=[nf_pipes[n], sp_pipes[n], sp_fans[n]], + edges=[ + g.edge(chsel_pipes[n], nf_pipes[n], 0, 0), + g.edge(nf_pipes[n], sp_pipes[n], 0, 0), + g.edge(sp_pipes[n], sp_fans[n], 0, 0), + g.edge(sp_fans[n], dnnroi_pipes[n], 0, 0), + ], + iports=chsel_pipes[n].iports, + oports=dnnroi_pipes[n].oports+[sp_fans[n].oports[1]], + name='nfsp_pipe_%d' % n, + ) + for n in std.range(0, std.length(tools.anodes) - 1) +] +else if roi == "both" then // oports: 0: dnnroi, 1: traditional sp [ g.intern( @@ -359,7 +403,7 @@ local nfsp_pipes = if use_dnnroi then ) for n in std.range(0, std.length(tools.anodes) - 1) ] -else +else if roi == "trad" then [ g.pipeline([ chsel_pipes[n], @@ -474,15 +518,32 @@ wcls_output_sp.sp_signals, //sp sink_sp //sp ]); -local graph_sim_dnn = g.pipeline([ +local graph_sim = g.pipeline([ wcls_input_sim.deposet, //sim setdrifter, //sim wcls_depoflux_writer, //sim bi_manifold1, //sim ]); - local graph_sp_dnn = +g.intern( + innodes=[retagger_sim], + outnodes=[], + centernodes=nfsp_pipes+[wcls_output_sim.sim_digits, fanout_apa, retagger_dnnroi, fanin_apa_dnnroi, wcls_output_sp.dnnsp_signals, sink_dnnroi], + edges=[ + g.edge(retagger_sim, wcls_output_sim.sim_digits, 0, 0), + g.edge(wcls_output_sim.sim_digits, fanout_apa, 0, 0), + g.edge(fanout_apa, nfsp_pipes[0], 0, 0), + g.edge(fanout_apa, nfsp_pipes[1], 1, 0), + g.edge(nfsp_pipes[0], fanin_apa_dnnroi, 0, 0), + g.edge(nfsp_pipes[1], fanin_apa_dnnroi, 0, 1), + g.edge(fanin_apa_dnnroi, retagger_dnnroi, 0, 0), + g.edge(retagger_dnnroi, wcls_output_sp.dnnsp_signals, 0, 0), + g.edge(wcls_output_sp.dnnsp_signals, sink_dnnroi, 0, 0), + ] +); + +local graph_sp_both = g.intern( innodes=[retagger_sim], outnodes=[], @@ -506,13 +567,22 @@ g.intern( ); local graph_dnn = g.pipeline([ -graph_sim_dnn, +graph_sim, graph_sp_dnn, ]); +local graph_both = g.pipeline([ +graph_sim, +graph_sp_both, +]); + local save_simdigits = std.extVar('save_simdigits'); -local graph = if use_dnnroi then graph_dnn else if save_simdigits == "true" then graph1_trad else graph2_trad; +local graph = +if roi == "dnn" then graph_dnn +else if roi == "both" then graph_both +else if save_simdigits == "true" then graph1_trad +else graph2_trad; local app = { type: 'TbbFlow', diff --git a/sbndcode/WireCell/wcsimsp_sbnd.fcl b/sbndcode/WireCell/wcsimsp_sbnd.fcl index 03d5f429a..843ae0af9 100644 --- a/sbndcode/WireCell/wcsimsp_sbnd.fcl +++ b/sbndcode/WireCell/wcsimsp_sbnd.fcl @@ -29,7 +29,7 @@ sbnd_wcls: { # ------------------------------------------------------------------------------------ # -## Configuration for 2D Sim + Signal Processing +## Configuration for 2D Sim + Signal Processing -- default: run only traditional ROI finding sbnd_wcls_simsp: @local::sbnd_wcls sbnd_wcls_simsp.wcls_main.configs: ["pgrapher/experiment/sbnd/wcls-sim-drift-depoflux-nf-sp.jsonnet"] sbnd_wcls_simsp.wcls_main.inputers: ["wclsSimDepoSetSource:"] @@ -61,6 +61,7 @@ sbnd_wcls_simsp.wcls_main.params: { dnnroi_model_p0: "DNN_ROI/plane0.ts" dnnroi_model_p1: "DNN_ROI/plane1.ts" + roi: "trad" } sbnd_wcls_simsp.wcls_main.structs: { @@ -75,11 +76,29 @@ sbnd_wcls_simsp.wcls_main.structs: { ## simulated front porch size [us] tick0_time: @local::sbnd_detectorclocks.TriggerOffsetTPC nticks: 3427 - use_dnnroi: false nchunks: 2 tick_per_slice: 4 } +# ------------------------------------------------------------------------------------ # + +## Configuration for 2D Sim + Signal Processing -- run both traditional and DNN ROI finding +sbnd_wcls_simsp_bothrois: @local::sbnd_wcls_simsp +sbnd_wcls_simsp_bothrois.wcls_main.outputers: ["wclsDepoFluxWriter:postdrift", "wclsFrameSaver:spsaver", "wclsFrameSaver:dnnsaver"] # "wclsFrameSaver:simdigits" <- by default, do not save RawDigits. Uncomment this line to save RawDigits and set +sbnd_wcls_simsp_bothrois.wcls_main.params.roi: "both" +sbnd_wcls_simsp_bothrois.wcls_main.structs.nchunks: 2 # should match training config +sbnd_wcls_simsp_bothrois.wcls_main.structs.tick_per_slice: 4 # should match training config + +# ------------------------------------------------------------------------------------ # +## Configuration for 2D Sim + Signal Processing -- run DNN ROI finding only +sbnd_wcls_simsp_dnnroi: @local::sbnd_wcls_simsp +sbnd_wcls_simsp_dnnroi.wcls_main.outputers: ["wclsDepoFluxWriter:postdrift", "wclsFrameSaver:dnnsaver"] # "wclsFrameSaver:simdigits" <- by default, do not save RawDigits. Uncomment this line to save RawDigits and set +sbnd_wcls_simsp_dnnroi.wcls_main.params.roi: "dnn" +sbnd_wcls_simsp_dnnroi.wcls_main.structs.nchunks: 2 # should match training config +sbnd_wcls_simsp_dnnroi.wcls_main.structs.tick_per_slice: 4 # should match training config + + + # ------------------------------------------------------------------------------------ # ## Configuration for **Signal processing ONLY** diff --git a/sbndcode/WireCell/wcsp_data_sbnd.fcl b/sbndcode/WireCell/wcsp_data_sbnd.fcl index cf8b0d6c9..c926f8835 100644 --- a/sbndcode/WireCell/wcsp_data_sbnd.fcl +++ b/sbndcode/WireCell/wcsp_data_sbnd.fcl @@ -66,4 +66,14 @@ sbnd_wcls_sp_data.wcls_main.structs: { tick_per_slice: 4 } +# ------------------------------------------------------------------------------------ # + +## Configuration for **Signal processing ONLY** for data -- run both traditional and DNN ROI finding + +sbnd_wcls_sp_data_bothrois: @local::sbnd_wcls_sp_data +sbnd_wcls_sp_data_bothrois.wcls_main.outputers: ["wclsDepoFluxWriter:postdrift", "wclsFrameSaver:spsaver", "wclsFrameSaver:dnnsaver"] # "wclsFrameSaver:simdigits" <- by default, do not save RawDigits. Uncomment this line to save RawDigits and set +sbnd_wcls_sp_data_bothrois.wcls_main.structs.use_dnnroi: true +sbnd_wcls_sp_data_bothrois.wcls_main.structs.nchunks: 2 # should match training config +sbnd_wcls_sp_data_bothrois.wcls_main.structs.tick_per_slice: 4 # should match training config + END_PROLOG From 5d93e0e2d396e4ccd87c697f6637522ae8191685 Mon Sep 17 00:00:00 2001 From: Mun Jung Jung Date: Thu, 17 Jul 2025 21:11:02 -0500 Subject: [PATCH 02/14] dnnroi only fcls for data --- .../experiment/sbnd/wcls-nf-sp-data.jsonnet | 76 ++++++++++++++++--- sbndcode/WireCell/wcsimsp_sbnd.fcl | 4 +- sbndcode/WireCell/wcsp_data_sbnd.fcl | 20 +++-- 3 files changed, 82 insertions(+), 18 deletions(-) diff --git a/sbndcode/WireCell/cfg/pgrapher/experiment/sbnd/wcls-nf-sp-data.jsonnet b/sbndcode/WireCell/cfg/pgrapher/experiment/sbnd/wcls-nf-sp-data.jsonnet index 875c5bfd8..e2fee758b 100644 --- a/sbndcode/WireCell/cfg/pgrapher/experiment/sbnd/wcls-nf-sp-data.jsonnet +++ b/sbndcode/WireCell/cfg/pgrapher/experiment/sbnd/wcls-nf-sp-data.jsonnet @@ -24,7 +24,7 @@ local sigoutform = std.extVar('signal_output_form'); // eg "sparse" or "dense" local raw_input_label = std.extVar('raw_input_label'); // eg "daq" local use_paramresp = std.extVar('use_paramresp'); // eg "true" or "false" -local use_dnnroi = std.extVar('use_dnnroi'); +local roi = std.extVar('roi'); local nchunks = std.extVar('nchunks'); local tick_per_slice = std.extVar('tick_per_slice'); local dnnroi_model_p0 = std.extVar('dnnroi_model_p0'); @@ -172,21 +172,39 @@ local nf_maker = import 'pgrapher/experiment/sbnd/nf-data.jsonnet'; //added Ewer local nf_pipes = [nf_maker(params, tools.anodes[n], chndb[n], n, name='nf%d' % n) for n in std.range(0, std.length(tools.anodes) - 1)]; local sp_maker = import 'pgrapher/experiment/sbnd/sp.jsonnet'; -local sp_override = if use_dnnroi then { +local sp_override = +if roi == "dnn" then { sparse: true, use_roi_debug_mode: true, - save_negtive_charge: true, // TODO: no negative charge in gauss, default is false + save_negative_charge: false, // TODO: no negative charge in gauss, default is false use_multi_plane_protection: true, - mp_tick_resolution: 4, + do_not_mp_protect_traditional: false, // TODO: do_not_mp_protect_traditional to make a clear ref, defualt is false + mp_tick_resolution:4, tight_lf_tag: "", - // loose_lf_tag: "", cleanup_roi_tag: "", break_roi_loop1_tag: "", break_roi_loop2_tag: "", shrink_roi_tag: "", extend_roi_tag: "", - // m_decon_charge_tag: "", -} else { + decon_charge_tag: "", + gauss_tag: "", + wiener_tag: "", +} +else if roi == "both" then { + sparse: true, + use_roi_debug_mode: true, + save_negative_charge: false, // TODO: no negative charge in gauss, default is false + use_multi_plane_protection: true, + do_not_mp_protect_traditional: false, // TODO: do_not_mp_protect_traditional to make a clear ref, defualt is false + mp_tick_resolution:4, + tight_lf_tag: "", + cleanup_roi_tag: "", + break_roi_loop1_tag: "", + break_roi_loop2_tag: "", + shrink_roi_tag: "", + extend_roi_tag: "", +} +else { sparse: true, }; //local sp = sp_maker(params, tools, { sparse: sigoutform == 'sparse' }); @@ -230,7 +248,26 @@ local fanout = function (name, multiplicity=2) local sp_fans = [fanout("sp_fan_%d" % n) for n in std.range(0, std.length(tools.anodes) - 1)]; local dnnroi_pipes = [ dnnroi(tools.anodes[n], ts_p0, ts_p1, output_scale=1, nchunks=nchunks) for n in std.range(0, std.length(tools.anodes) - 1) ]; -local nfsp_pipes = if use_dnnroi then +local nfsp_pipes = +if roi == "dnn" then +[ + g.intern( + innodes=[chsel_pipes[n]], + outnodes=[dnnroi_pipes[n],sp_fans[n]], + centernodes=[nf_pipes[n], sp_pipes[n], sp_fans[n]], + edges=[ + g.edge(chsel_pipes[n], nf_pipes[n], 0, 0), + g.edge(nf_pipes[n], sp_pipes[n], 0, 0), + g.edge(sp_pipes[n], sp_fans[n], 0, 0), + g.edge(sp_fans[n], dnnroi_pipes[n], 0, 0), + ], + iports=chsel_pipes[n].iports, + oports=dnnroi_pipes[n].oports+[sp_fans[n].oports[1]], + name='nfsp_pipe_%d' % n, + ) + for n in std.range(0, std.length(tools.anodes) - 1) +] +else if roi == "both" then [ g.intern( innodes=[chsel_pipes[n]], @@ -249,7 +286,7 @@ local nfsp_pipes = if use_dnnroi then ) for n in std.range(0, std.length(tools.anodes) - 1) ] -else +else if roi == "trad" then [ g.pipeline([ chsel_pipes[n], @@ -339,7 +376,24 @@ local framefanin = function(name) g.pnode({ local fanin_apa_dnnroi = framefanin('fanin_apa_dnnroi'); local fanin_apa_sp = framefanin('fanin_apa_sp'); -local graph = if use_dnnroi then +local graph = +if roi == "dnn" then + g.intern( + innodes=[wcls_input.adc_digits], + outnodes=[], + centernodes=nfsp_pipes+[fanout_apa, retag_dnnroi, fanin_apa_dnnroi, wcls_output.dnnsp_signals, sink_dnnroi], + edges=[ + g.edge(wcls_input.adc_digits, fanout_apa, 0, 0), + g.edge(fanout_apa, nfsp_pipes[0], 0, 0), + g.edge(fanout_apa, nfsp_pipes[1], 1, 0), + g.edge(nfsp_pipes[0], fanin_apa_dnnroi, 0, 0), + g.edge(nfsp_pipes[1], fanin_apa_dnnroi, 0, 1), + g.edge(fanin_apa_dnnroi, retag_dnnroi, 0, 0), + g.edge(retag_dnnroi, wcls_output.dnnsp_signals, 0, 0), + g.edge(wcls_output.dnnsp_signals, sink_dnnroi, 0, 0), + ] +) +else if roi == "both" then g.intern( innodes=[wcls_input.adc_digits], outnodes=[], @@ -360,7 +414,7 @@ local graph = if use_dnnroi then g.edge(wcls_output.sp_signals, sink_sp, 0, 0), ] ) -else +else if roi == "trad" then g.pipeline([wcls_input.adc_digits, fanpipe, retag_sp, diff --git a/sbndcode/WireCell/wcsimsp_sbnd.fcl b/sbndcode/WireCell/wcsimsp_sbnd.fcl index 843ae0af9..9968f1a4c 100644 --- a/sbndcode/WireCell/wcsimsp_sbnd.fcl +++ b/sbndcode/WireCell/wcsimsp_sbnd.fcl @@ -76,8 +76,8 @@ sbnd_wcls_simsp.wcls_main.structs: { ## simulated front porch size [us] tick0_time: @local::sbnd_detectorclocks.TriggerOffsetTPC nticks: 3427 - nchunks: 2 - tick_per_slice: 4 + nchunks: 2 # dummy for trad + tick_per_slice: 4 # dummy for trad } # ------------------------------------------------------------------------------------ # diff --git a/sbndcode/WireCell/wcsp_data_sbnd.fcl b/sbndcode/WireCell/wcsp_data_sbnd.fcl index c926f8835..7f35074f5 100644 --- a/sbndcode/WireCell/wcsp_data_sbnd.fcl +++ b/sbndcode/WireCell/wcsp_data_sbnd.fcl @@ -56,14 +56,14 @@ sbnd_wcls_sp_data.wcls_main.params : { dnnroi_model_p0: "DNN_ROI/plane0.ts" dnnroi_model_p1: "DNN_ROI/plane1.ts" + roi: "trad" } sbnd_wcls_sp_data.wcls_main.structs: { # Set the waveform sample length, eg, 6000, 15000, "auto" nticks: 3427 - use_dnnroi: false - nchunks: 2 - tick_per_slice: 4 + nchunks: 2 # dummy for trad + tick_per_slice: 4 # dummy for trad } # ------------------------------------------------------------------------------------ # @@ -71,9 +71,19 @@ sbnd_wcls_sp_data.wcls_main.structs: { ## Configuration for **Signal processing ONLY** for data -- run both traditional and DNN ROI finding sbnd_wcls_sp_data_bothrois: @local::sbnd_wcls_sp_data -sbnd_wcls_sp_data_bothrois.wcls_main.outputers: ["wclsDepoFluxWriter:postdrift", "wclsFrameSaver:spsaver", "wclsFrameSaver:dnnsaver"] # "wclsFrameSaver:simdigits" <- by default, do not save RawDigits. Uncomment this line to save RawDigits and set -sbnd_wcls_sp_data_bothrois.wcls_main.structs.use_dnnroi: true +sbnd_wcls_sp_data_bothrois.wcls_main.outputers: ["wclsFrameSaver:spsaver", "wclsFrameSaver:dnnsaver"] # "wclsFrameSaver:simdigits" <- by default, do not save RawDigits. Uncomment this line to save RawDigits and set +sbnd_wcls_sp_data_bothrois.wcls_main.params.roi: "both" sbnd_wcls_sp_data_bothrois.wcls_main.structs.nchunks: 2 # should match training config sbnd_wcls_sp_data_bothrois.wcls_main.structs.tick_per_slice: 4 # should match training config +# ------------------------------------------------------------------------------------ # + +## Configuration for **Signal processing ONLY** for data -- run DNN ROI finding only + +sbnd_wcls_sp_data_dnnroi: @local::sbnd_wcls_sp_data +sbnd_wcls_sp_data_dnnroi.wcls_main.outputers: ["wclsFrameSaver:dnnsaver"] # "wclsFrameSaver:simdigits" <- by default, do not save RawDigits. Uncomment this line to save RawDigits and set +sbnd_wcls_sp_data_dnnroi.wcls_main.params.roi: "dnn" +sbnd_wcls_sp_data_dnnroi.wcls_main.structs.nchunks: 2 # should match training config +sbnd_wcls_sp_data_dnnroi.wcls_main.structs.tick_per_slice: 4 # should match training config + END_PROLOG From cee18ddbd8f5e46634b281b4274a4a9de7cf42a0 Mon Sep 17 00:00:00 2001 From: Mun Jung Jung Date: Thu, 17 Jul 2025 21:11:31 -0500 Subject: [PATCH 03/14] don't save trad sp images for dnnroi only workflow --- .../experiment/sbnd/wcls-sim-drift-depoflux-nf-sp.jsonnet | 4 ---- 1 file changed, 4 deletions(-) diff --git a/sbndcode/WireCell/cfg/pgrapher/experiment/sbnd/wcls-sim-drift-depoflux-nf-sp.jsonnet b/sbndcode/WireCell/cfg/pgrapher/experiment/sbnd/wcls-sim-drift-depoflux-nf-sp.jsonnet index 95af5e19f..4624261f3 100644 --- a/sbndcode/WireCell/cfg/pgrapher/experiment/sbnd/wcls-sim-drift-depoflux-nf-sp.jsonnet +++ b/sbndcode/WireCell/cfg/pgrapher/experiment/sbnd/wcls-sim-drift-depoflux-nf-sp.jsonnet @@ -148,14 +148,11 @@ if roi == "dnn" then { do_not_mp_protect_traditional: false, // TODO: do_not_mp_protect_traditional to make a clear ref, defualt is false mp_tick_resolution:4, tight_lf_tag: "", - // loose_lf_tag: "", // comment to use as input to dnnsp cleanup_roi_tag: "", break_roi_loop1_tag: "", break_roi_loop2_tag: "", shrink_roi_tag: "", extend_roi_tag: "", - // mp3_roi_tag: "", - // mp2_roi_tag: "", decon_charge_tag: "", gauss_tag: "", wiener_tag: "", @@ -168,7 +165,6 @@ else if roi == "both" then { do_not_mp_protect_traditional: false, // TODO: do_not_mp_protect_traditional to make a clear ref, defualt is false mp_tick_resolution:4, tight_lf_tag: "", - // loose_lf_tag: "", // comment to use as input to dnnsp cleanup_roi_tag: "", break_roi_loop1_tag: "", break_roi_loop2_tag: "", From 6847440aec727814f7d4e47b4562c010913836bf Mon Sep 17 00:00:00 2001 From: Mun Jung Jung Date: Thu, 17 Jul 2025 21:12:05 -0500 Subject: [PATCH 04/14] put dnnroi fcls in a separate job fcl dir --- .../JobConfigurations/dnnroi/CMakeLists.txt | 1 + .../dnnroi/sp_data_bothrois.fcl | 40 +++++++++++++++++++ .../dnnroi/sp_data_dnnroi.fcl | 40 +++++++++++++++++++ .../dnnroi/standard_detsim_sbnd_bothrois.fcl | 6 +++ .../dnnroi/standard_detsim_sbnd_dnnroi.fcl | 6 +++ 5 files changed, 93 insertions(+) create mode 100644 sbndcode/JobConfigurations/dnnroi/CMakeLists.txt create mode 100644 sbndcode/JobConfigurations/dnnroi/sp_data_bothrois.fcl create mode 100644 sbndcode/JobConfigurations/dnnroi/sp_data_dnnroi.fcl create mode 100644 sbndcode/JobConfigurations/dnnroi/standard_detsim_sbnd_bothrois.fcl create mode 100644 sbndcode/JobConfigurations/dnnroi/standard_detsim_sbnd_dnnroi.fcl diff --git a/sbndcode/JobConfigurations/dnnroi/CMakeLists.txt b/sbndcode/JobConfigurations/dnnroi/CMakeLists.txt new file mode 100644 index 000000000..13355789a --- /dev/null +++ b/sbndcode/JobConfigurations/dnnroi/CMakeLists.txt @@ -0,0 +1 @@ +install_fhicl() diff --git a/sbndcode/JobConfigurations/dnnroi/sp_data_bothrois.fcl b/sbndcode/JobConfigurations/dnnroi/sp_data_bothrois.fcl new file mode 100644 index 000000000..a15cdddc4 --- /dev/null +++ b/sbndcode/JobConfigurations/dnnroi/sp_data_bothrois.fcl @@ -0,0 +1,40 @@ +#include "services_sbnd.fcl" +#include "messages_sbnd.fcl" +#include "rootoutput_sbnd.fcl" +#include "wcsp_data_sbnd.fcl" + +### fcl to run SP and save Wire products based on both the tradition ROI finding algorithm and the DNN-based ROI finding +### munjung@uchicago.edu + +process_name: WCLS + +services: +{ + @table::services + @table::sbnd_data_services +} + +source: { + module_type: RootInput +} + +outputs:{ + out1: + { + @table::sbnd_rootoutput # inherit shared settings + dataTier: "reconstructed" + } + +} + +physics : { + producers: { + sptpc2d: @local::sbnd_wcls_sp_data_bothrois + } + + sp : [ sptpc2d ] + trigger_paths : [ sp ] + + o1 : [ out1 ] + end_paths: [ o1 ] +} diff --git a/sbndcode/JobConfigurations/dnnroi/sp_data_dnnroi.fcl b/sbndcode/JobConfigurations/dnnroi/sp_data_dnnroi.fcl new file mode 100644 index 000000000..cbfb3c9fd --- /dev/null +++ b/sbndcode/JobConfigurations/dnnroi/sp_data_dnnroi.fcl @@ -0,0 +1,40 @@ +#include "services_sbnd.fcl" +#include "messages_sbnd.fcl" +#include "rootoutput_sbnd.fcl" +#include "wcsp_data_sbnd.fcl" + +### fcl to run the DNN-based ROI finding +### munjung@uchicago.edu + +process_name: WCLS + +services: +{ + @table::services + @table::sbnd_data_services +} + +source: { + module_type: RootInput +} + +outputs:{ + out1: + { + @table::sbnd_rootoutput # inherit shared settings + dataTier: "reconstructed" + } + +} + +physics : { + producers: { + sptpc2d: @local::sbnd_wcls_sp_data_dnnroi + } + + sp : [ sptpc2d ] + trigger_paths : [ sp ] + + o1 : [ out1 ] + end_paths: [ o1 ] +} diff --git a/sbndcode/JobConfigurations/dnnroi/standard_detsim_sbnd_bothrois.fcl b/sbndcode/JobConfigurations/dnnroi/standard_detsim_sbnd_bothrois.fcl new file mode 100644 index 000000000..36ad76984 --- /dev/null +++ b/sbndcode/JobConfigurations/dnnroi/standard_detsim_sbnd_bothrois.fcl @@ -0,0 +1,6 @@ +#include "standard_detsim_sbnd.fcl" + +### fcl to run SP and save Wire products based on both the tradition ROI finding algorithm and the DNN-based ROI finding +### munjung@uchicago.edu + +physics.producers.simtpc2d: @local::sbnd_wcls_simsp_bothrois \ No newline at end of file diff --git a/sbndcode/JobConfigurations/dnnroi/standard_detsim_sbnd_dnnroi.fcl b/sbndcode/JobConfigurations/dnnroi/standard_detsim_sbnd_dnnroi.fcl new file mode 100644 index 000000000..325b9a485 --- /dev/null +++ b/sbndcode/JobConfigurations/dnnroi/standard_detsim_sbnd_dnnroi.fcl @@ -0,0 +1,6 @@ +#include "standard_detsim_sbnd.fcl" + +### fcl to run SP and save Wire products based on the DNN-based ROI finding +### munjung@uchicago.edu + +physics.producers.simtpc2d: @local::sbnd_wcls_simsp_dnnroi \ No newline at end of file From f6b01644d9a53f77dca18e3c31bd8add8c2e0ba0 Mon Sep 17 00:00:00 2001 From: Mun Jung Jung Date: Fri, 18 Jul 2025 12:12:07 -0500 Subject: [PATCH 05/14] add Linyan's fcls --- .../dnnroi/reco1_nosp_data.fcl | 43 +++++++++++++++++ .../dnnroi/reco1_nosp_data_dnnroi.fcl | 43 +++++++++++++++++ .../dnnroi/reco1_postscrub_data_dnnroi.fcl | 47 +++++++++++++++++++ .../dnnroi/scrub_gaushit_data.fcl | 34 ++++++++++++++ .../dnnroi/sp_data_bothrois.fcl | 9 ++-- .../dnnroi/sp_data_dnnroi.fcl | 9 ++-- .../dnnroi/standard_detsim_sbnd_bothrois.fcl | 9 ++-- .../dnnroi/standard_detsim_sbnd_dnnroi.fcl | 10 ++-- .../dnnroi/standard_reco1_sbnd_dnnroi.fcl | 8 ++++ 9 files changed, 200 insertions(+), 12 deletions(-) create mode 100644 sbndcode/JobConfigurations/dnnroi/reco1_nosp_data.fcl create mode 100644 sbndcode/JobConfigurations/dnnroi/reco1_nosp_data_dnnroi.fcl create mode 100644 sbndcode/JobConfigurations/dnnroi/reco1_postscrub_data_dnnroi.fcl create mode 100644 sbndcode/JobConfigurations/dnnroi/scrub_gaushit_data.fcl create mode 100644 sbndcode/JobConfigurations/dnnroi/standard_reco1_sbnd_dnnroi.fcl diff --git a/sbndcode/JobConfigurations/dnnroi/reco1_nosp_data.fcl b/sbndcode/JobConfigurations/dnnroi/reco1_nosp_data.fcl new file mode 100644 index 000000000..612ca09f2 --- /dev/null +++ b/sbndcode/JobConfigurations/dnnroi/reco1_nosp_data.fcl @@ -0,0 +1,43 @@ +# File: reco1_nosp_data.fcl +# Authors: Mun Jung Jung (munjung@uchicago.edu) + +# runs everything downstream of sptpc2d in the reco1 chain for data +# runs gaushitfinder on sptpc2d:guass + +#include "wcsp_data_sbnd.fcl" +#include "opdeconvolution_sbnd_data.fcl" +#include "sbnd_ophitfinder_deco_data.fcl" +#include "sbnd_flashfinder_deco_data.fcl" +#include "crt_channel_map_service.fcl" +#include "crt_calib_service.fcl" +#include "standard_reco1_sbnd.fcl" + +services: +{ + @table::services + @table::sbnd_data_services + CRTChannelMapService: @local::crt_channel_map_standard + CRTCalibService: @local::crt_calib_service +} + +physics.producers: +{ + @table::physics.producers + sptpc2d: @local::sbnd_wcls_sp_data + crtstrips: @local::crtstriphitproducer_data_sbnd + crtclustering: @local::crtclusterproducer_data_sbnd + crtspacepoints: @local::crtspacepointproducer_data_sbnd + crttracks: @local::crttrackproducer_data_sbnd + opdecopmt: @local::SBNDOpDeconvolutionPMT_data + ophitpmt: @local::SBNDDecoOpHitFinderPMT_data + opflashtpc0: @local::SBNDDecoSimpleFlashTPC0_data + opflashtpc1: @local::SBNDDecoSimpleFlashTPC1_data +} + +physics.reco1: [gaushit, numberofhitsfilter, cluster3d, crtstrips, + crtclustering, crtspacepoints, crttracks, opdecopmt, ophitpmt, opflashtpc0, opflashtpc1] +physics.ana: [superadata] + +outputs.out1.SelectEvents: [ "reco1" ] + +physics.producers.gaushit.CalDataModuleLabel: "sptpc2d:gauss" diff --git a/sbndcode/JobConfigurations/dnnroi/reco1_nosp_data_dnnroi.fcl b/sbndcode/JobConfigurations/dnnroi/reco1_nosp_data_dnnroi.fcl new file mode 100644 index 000000000..4eca7cc0b --- /dev/null +++ b/sbndcode/JobConfigurations/dnnroi/reco1_nosp_data_dnnroi.fcl @@ -0,0 +1,43 @@ +# File: reco1_nosp_data_dnnroi.fcl +# Authors: Mun Jung Jung (munjung@uchicago.edu) + +# runs everything downstream of sptpc2d in the reco1 chain for data +# runs gaushitfinder on sptpc2d:dnnsp + +#include "wcsp_data_sbnd.fcl" +#include "opdeconvolution_sbnd_data.fcl" +#include "sbnd_ophitfinder_deco_data.fcl" +#include "sbnd_flashfinder_deco_data.fcl" +#include "crt_channel_map_service.fcl" +#include "crt_calib_service.fcl" +#include "standard_reco1_sbnd.fcl" + +services: +{ + @table::services + @table::sbnd_data_services + CRTChannelMapService: @local::crt_channel_map_standard + CRTCalibService: @local::crt_calib_service +} + +physics.producers: +{ + @table::physics.producers + sptpc2d: @local::sbnd_wcls_sp_data + crtstrips: @local::crtstriphitproducer_data_sbnd + crtclustering: @local::crtclusterproducer_data_sbnd + crtspacepoints: @local::crtspacepointproducer_data_sbnd + crttracks: @local::crttrackproducer_data_sbnd + opdecopmt: @local::SBNDOpDeconvolutionPMT_data + ophitpmt: @local::SBNDDecoOpHitFinderPMT_data + opflashtpc0: @local::SBNDDecoSimpleFlashTPC0_data + opflashtpc1: @local::SBNDDecoSimpleFlashTPC1_data +} + +physics.reco1: [gaushit, numberofhitsfilter, cluster3d, crtstrips, + crtclustering, crtspacepoints, crttracks, opdecopmt, ophitpmt, opflashtpc0, opflashtpc1] +physics.ana: [superadata] + +outputs.out1.SelectEvents: [ "reco1" ] + +physics.producers.gaushit.CalDataModuleLabel: "sptpc2d:dnnsp" diff --git a/sbndcode/JobConfigurations/dnnroi/reco1_postscrub_data_dnnroi.fcl b/sbndcode/JobConfigurations/dnnroi/reco1_postscrub_data_dnnroi.fcl new file mode 100644 index 000000000..91845106a --- /dev/null +++ b/sbndcode/JobConfigurations/dnnroi/reco1_postscrub_data_dnnroi.fcl @@ -0,0 +1,47 @@ +# File: reco1_postscrub_data_dnnroi.fcl +# Authors: Linyan Wan (lwan@fnal.gov) and Mun Jung Jung (munjung@uchicago.edu) +# +# to be used for the 2025 DNN ROI validation production +# MUST be run after running "scrub_gaushit_data.fcl" on Reco1 files +# +# runs gaushit, numberofhitsfilter, cluster3d, superadata on sptpc2d:dnnsp Wire products. + +#include "wcsp_data_sbnd.fcl" +#include "opdeconvolution_sbnd_data.fcl" +#include "sbnd_ophitfinder_deco_data.fcl" +#include "sbnd_flashfinder_deco_data.fcl" +#include "crt_channel_map_service.fcl" +#include "crt_calib_service.fcl" +#include "standard_reco1_sbnd.fcl" + +process_name: Reco1onDNN + +services: +{ + @table::services + @table::sbnd_data_services + CRTChannelMapService: @local::crt_channel_map_standard + CRTCalibService: @local::crt_calib_service +} + +physics.producers: +{ + @table::physics.producers + sptpc2d: @local::sbnd_wcls_sp_data + crtstrips: @local::crtstriphitproducer_data_sbnd + crtclustering: @local::crtclusterproducer_data_sbnd + crtspacepoints: @local::crtspacepointproducer_data_sbnd + crttracks: @local::crttrackproducer_data_sbnd + opdecopmt: @local::SBNDOpDeconvolutionPMT_data + ophitpmt: @local::SBNDDecoOpHitFinderPMT_data + opflashtpc0: @local::SBNDDecoSimpleFlashTPC0_data + opflashtpc1: @local::SBNDDecoSimpleFlashTPC1_data +} + +physics.reco1: [gaushit, numberofhitsfilter, cluster3d] +physics.ana: [superadata] + +outputs.out1.SelectEvents: [ "reco1" ] + +# uncomment below line to run DNN ROI finding SP for reco +physics.producers.gaushit.CalDataModuleLabel: "sptpc2d:dnnsp" \ No newline at end of file diff --git a/sbndcode/JobConfigurations/dnnroi/scrub_gaushit_data.fcl b/sbndcode/JobConfigurations/dnnroi/scrub_gaushit_data.fcl new file mode 100644 index 000000000..b5bbcf4eb --- /dev/null +++ b/sbndcode/JobConfigurations/dnnroi/scrub_gaushit_data.fcl @@ -0,0 +1,34 @@ +# File: scrub_gaushit_data.fcl +# Authors: Linyan Wan (lwan@fnal.gov) and Mun Jung Jung (munjung@uchicago.edu) +# +# to be used for the 2025 DNN ROI validation production +# +# removes gaushit products made in Reco 1 for data +# allows gaushitfinder to be run again on different choice of Wire products + +#include "rootoutput_sbnd.fcl" +#include "sam_sbnd.fcl" + +process_name: Scrub + +source: +{ + module_type: RootInput + inputCommands: [ "keep *_*_*_*", + "drop *_gaushit_*_*"] +} + +outputs: +{ + out1: + { + @table::sbnd_rootoutput + dataTier: "reconstructed" + } +} + +physics: +{ + stream1: [ out1 ] + end_paths: [ stream1 ] +} \ No newline at end of file diff --git a/sbndcode/JobConfigurations/dnnroi/sp_data_bothrois.fcl b/sbndcode/JobConfigurations/dnnroi/sp_data_bothrois.fcl index a15cdddc4..7c66c9d5e 100644 --- a/sbndcode/JobConfigurations/dnnroi/sp_data_bothrois.fcl +++ b/sbndcode/JobConfigurations/dnnroi/sp_data_bothrois.fcl @@ -1,11 +1,14 @@ +# File: sp_data_bothrois.fcl +# Authors: Mun Jung Jung (munjung@uchicago.edu) + +# runs WC 2D signal processing on data using both traditional and DNN-based ROI finding +# create three Wire products: sptpc2d:gauss, sptpc2d:wiener (from traditional), sptc2d:dnnsp (from DNN) + #include "services_sbnd.fcl" #include "messages_sbnd.fcl" #include "rootoutput_sbnd.fcl" #include "wcsp_data_sbnd.fcl" -### fcl to run SP and save Wire products based on both the tradition ROI finding algorithm and the DNN-based ROI finding -### munjung@uchicago.edu - process_name: WCLS services: diff --git a/sbndcode/JobConfigurations/dnnroi/sp_data_dnnroi.fcl b/sbndcode/JobConfigurations/dnnroi/sp_data_dnnroi.fcl index cbfb3c9fd..37dba8c96 100644 --- a/sbndcode/JobConfigurations/dnnroi/sp_data_dnnroi.fcl +++ b/sbndcode/JobConfigurations/dnnroi/sp_data_dnnroi.fcl @@ -1,11 +1,14 @@ +# File: sp_data_dnnroi.fcl +# Authors: Mun Jung Jung (munjung@uchicago.edu) + +# runs WC 2D signal processing using only DNN-based ROI finding +# create one Wire product: sptc2d:dnnsp + #include "services_sbnd.fcl" #include "messages_sbnd.fcl" #include "rootoutput_sbnd.fcl" #include "wcsp_data_sbnd.fcl" -### fcl to run the DNN-based ROI finding -### munjung@uchicago.edu - process_name: WCLS services: diff --git a/sbndcode/JobConfigurations/dnnroi/standard_detsim_sbnd_bothrois.fcl b/sbndcode/JobConfigurations/dnnroi/standard_detsim_sbnd_bothrois.fcl index 36ad76984..6e542e5f9 100644 --- a/sbndcode/JobConfigurations/dnnroi/standard_detsim_sbnd_bothrois.fcl +++ b/sbndcode/JobConfigurations/dnnroi/standard_detsim_sbnd_bothrois.fcl @@ -1,6 +1,9 @@ -#include "standard_detsim_sbnd.fcl" +# File: standard_detsim_sbnd_bothrois.fcl +# Authors: Mun Jung Jung (munjung@uchicago.edu) + +# runs WC 2D signal processing during detsim using both traditional and DNN-based ROI finding +# create three Wire products: simtpc2d:gauss, simtpc2d:wiener (from traditional), simtpc2d:dnnsp (from DNN) -### fcl to run SP and save Wire products based on both the tradition ROI finding algorithm and the DNN-based ROI finding -### munjung@uchicago.edu +#include "standard_detsim_sbnd.fcl" physics.producers.simtpc2d: @local::sbnd_wcls_simsp_bothrois \ No newline at end of file diff --git a/sbndcode/JobConfigurations/dnnroi/standard_detsim_sbnd_dnnroi.fcl b/sbndcode/JobConfigurations/dnnroi/standard_detsim_sbnd_dnnroi.fcl index 325b9a485..b454e4009 100644 --- a/sbndcode/JobConfigurations/dnnroi/standard_detsim_sbnd_dnnroi.fcl +++ b/sbndcode/JobConfigurations/dnnroi/standard_detsim_sbnd_dnnroi.fcl @@ -1,6 +1,10 @@ -#include "standard_detsim_sbnd.fcl" +# File: standard_detsim_sbnd_dnnroi.fcl +# Authors: Mun Jung Jung (munjung@uchicago.edu) + +# runs WC 2D signal processing during detsim using only DNN-based ROI finding +# create one Wire product: simtpc2d:dnnsp -### fcl to run SP and save Wire products based on the DNN-based ROI finding -### munjung@uchicago.edu + +#include "standard_detsim_sbnd.fcl" physics.producers.simtpc2d: @local::sbnd_wcls_simsp_dnnroi \ No newline at end of file diff --git a/sbndcode/JobConfigurations/dnnroi/standard_reco1_sbnd_dnnroi.fcl b/sbndcode/JobConfigurations/dnnroi/standard_reco1_sbnd_dnnroi.fcl new file mode 100644 index 000000000..baefcd9e2 --- /dev/null +++ b/sbndcode/JobConfigurations/dnnroi/standard_reco1_sbnd_dnnroi.fcl @@ -0,0 +1,8 @@ +# File: standard_reco1_sbnd_dnnroi.fcl +# Authors: Mun Jung Jung (munjung@uchicago.edu) + +# run MC reco1 on simtpc2d:dnnsp + +#include "standard_reco1_sbnd.fcl" + +physics.producers.gaushit.CalDataModuleLabel: "simtpc2d:dnnsp" From 080d41add2d21ade49b791ce41b814c7a9f8738b Mon Sep 17 00:00:00 2001 From: Mun Jung Jung Date: Fri, 18 Jul 2025 12:12:27 -0500 Subject: [PATCH 06/14] put SP sinks back in --- .../cfg/pgrapher/experiment/sbnd/wcls-nf-sp-data.jsonnet | 6 +++++- .../experiment/sbnd/wcls-sim-drift-depoflux-nf-sp.jsonnet | 6 +++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/sbndcode/WireCell/cfg/pgrapher/experiment/sbnd/wcls-nf-sp-data.jsonnet b/sbndcode/WireCell/cfg/pgrapher/experiment/sbnd/wcls-nf-sp-data.jsonnet index e2fee758b..ba52ceaa9 100644 --- a/sbndcode/WireCell/cfg/pgrapher/experiment/sbnd/wcls-nf-sp-data.jsonnet +++ b/sbndcode/WireCell/cfg/pgrapher/experiment/sbnd/wcls-nf-sp-data.jsonnet @@ -381,7 +381,7 @@ if roi == "dnn" then g.intern( innodes=[wcls_input.adc_digits], outnodes=[], - centernodes=nfsp_pipes+[fanout_apa, retag_dnnroi, fanin_apa_dnnroi, wcls_output.dnnsp_signals, sink_dnnroi], + centernodes=nfsp_pipes+[fanout_apa, retag_dnnroi, retag_sp, fanin_apa_dnnroi, fanin_apa_sp, wcls_output.dnnsp_signals, sink_dnnroi, sink_sp], edges=[ g.edge(wcls_input.adc_digits, fanout_apa, 0, 0), g.edge(fanout_apa, nfsp_pipes[0], 0, 0), @@ -391,6 +391,10 @@ if roi == "dnn" then g.edge(fanin_apa_dnnroi, retag_dnnroi, 0, 0), g.edge(retag_dnnroi, wcls_output.dnnsp_signals, 0, 0), g.edge(wcls_output.dnnsp_signals, sink_dnnroi, 0, 0), + g.edge(nfsp_pipes[0], fanin_apa_sp, 1, 0), + g.edge(nfsp_pipes[1], fanin_apa_sp, 1, 1), + g.edge(fanin_apa_sp, retag_sp, 0, 0), + g.edge(retag_sp, sink_sp, 0, 0), ] ) else if roi == "both" then diff --git a/sbndcode/WireCell/cfg/pgrapher/experiment/sbnd/wcls-sim-drift-depoflux-nf-sp.jsonnet b/sbndcode/WireCell/cfg/pgrapher/experiment/sbnd/wcls-sim-drift-depoflux-nf-sp.jsonnet index 4624261f3..3535696b0 100644 --- a/sbndcode/WireCell/cfg/pgrapher/experiment/sbnd/wcls-sim-drift-depoflux-nf-sp.jsonnet +++ b/sbndcode/WireCell/cfg/pgrapher/experiment/sbnd/wcls-sim-drift-depoflux-nf-sp.jsonnet @@ -525,7 +525,7 @@ local graph_sp_dnn = g.intern( innodes=[retagger_sim], outnodes=[], - centernodes=nfsp_pipes+[wcls_output_sim.sim_digits, fanout_apa, retagger_dnnroi, fanin_apa_dnnroi, wcls_output_sp.dnnsp_signals, sink_dnnroi], + centernodes=nfsp_pipes+[wcls_output_sim.sim_digits, fanout_apa, retagger_dnnroi, retagger_sp, fanin_apa_dnnroi, fanin_apa_sp, wcls_output_sp.dnnsp_signals, sink_dnnroi, sink_sp], edges=[ g.edge(retagger_sim, wcls_output_sim.sim_digits, 0, 0), g.edge(wcls_output_sim.sim_digits, fanout_apa, 0, 0), @@ -536,6 +536,10 @@ g.intern( g.edge(fanin_apa_dnnroi, retagger_dnnroi, 0, 0), g.edge(retagger_dnnroi, wcls_output_sp.dnnsp_signals, 0, 0), g.edge(wcls_output_sp.dnnsp_signals, sink_dnnroi, 0, 0), + g.edge(nfsp_pipes[0], fanin_apa_sp, 1, 0), + g.edge(nfsp_pipes[1], fanin_apa_sp, 1, 1), + g.edge(fanin_apa_sp, retagger_sp, 0, 0), + g.edge(retagger_sp, sink_sp, 0, 0), ] ); From 3a10ab524c7292fd80afe18dfead18b487391a6a Mon Sep 17 00:00:00 2001 From: Mun Jung Jung Date: Fri, 18 Jul 2025 15:32:02 -0500 Subject: [PATCH 07/14] inherit fcls when possible --- .../dnnroi/reco1_data_bothrois.fcl | 12 +++++++ .../dnnroi/reco1_nosp_data.fcl | 32 ++----------------- .../dnnroi/reco1_nosp_data_dnnroi.fcl | 30 +---------------- .../dnnroi/reco1_postscrub_data_dnnroi.fcl | 30 +---------------- 4 files changed, 16 insertions(+), 88 deletions(-) create mode 100644 sbndcode/JobConfigurations/dnnroi/reco1_data_bothrois.fcl diff --git a/sbndcode/JobConfigurations/dnnroi/reco1_data_bothrois.fcl b/sbndcode/JobConfigurations/dnnroi/reco1_data_bothrois.fcl new file mode 100644 index 000000000..e384e91a9 --- /dev/null +++ b/sbndcode/JobConfigurations/dnnroi/reco1_data_bothrois.fcl @@ -0,0 +1,12 @@ +# File: reco1_data_bothrois.fcl +# Authors: Mun Jung Jung (munjung@uchicago.edu) + +# runs Reco1 on data +# for SP, use both traditional and DNN-based ROI finding, creating three Wire products: simtpc2d:gauss, simtpc2d:wiener (from traditional), simtpc2d:dnnsp (from DNN) +# for downstream reco1, run gaushitfinder on sptpc2d:gauss, from traditional ROI finding +# to generate matching files with downstream reco1 run on sptpc2d:dnnsp, gaushits should be scrubbed and run again +# +#include "reco1_data.fcl" + +physics.producers.sptpc2d: @local::sbnd_wcls_sp_data_bothrois + diff --git a/sbndcode/JobConfigurations/dnnroi/reco1_nosp_data.fcl b/sbndcode/JobConfigurations/dnnroi/reco1_nosp_data.fcl index 612ca09f2..02453d931 100644 --- a/sbndcode/JobConfigurations/dnnroi/reco1_nosp_data.fcl +++ b/sbndcode/JobConfigurations/dnnroi/reco1_nosp_data.fcl @@ -2,37 +2,9 @@ # Authors: Mun Jung Jung (munjung@uchicago.edu) # runs everything downstream of sptpc2d in the reco1 chain for data -# runs gaushitfinder on sptpc2d:guass +# runs gaushitfinder on sptpc2d:gauss -#include "wcsp_data_sbnd.fcl" -#include "opdeconvolution_sbnd_data.fcl" -#include "sbnd_ophitfinder_deco_data.fcl" -#include "sbnd_flashfinder_deco_data.fcl" -#include "crt_channel_map_service.fcl" -#include "crt_calib_service.fcl" -#include "standard_reco1_sbnd.fcl" - -services: -{ - @table::services - @table::sbnd_data_services - CRTChannelMapService: @local::crt_channel_map_standard - CRTCalibService: @local::crt_calib_service -} - -physics.producers: -{ - @table::physics.producers - sptpc2d: @local::sbnd_wcls_sp_data - crtstrips: @local::crtstriphitproducer_data_sbnd - crtclustering: @local::crtclusterproducer_data_sbnd - crtspacepoints: @local::crtspacepointproducer_data_sbnd - crttracks: @local::crttrackproducer_data_sbnd - opdecopmt: @local::SBNDOpDeconvolutionPMT_data - ophitpmt: @local::SBNDDecoOpHitFinderPMT_data - opflashtpc0: @local::SBNDDecoSimpleFlashTPC0_data - opflashtpc1: @local::SBNDDecoSimpleFlashTPC1_data -} +#include "reco1_data.fcl" physics.reco1: [gaushit, numberofhitsfilter, cluster3d, crtstrips, crtclustering, crtspacepoints, crttracks, opdecopmt, ophitpmt, opflashtpc0, opflashtpc1] diff --git a/sbndcode/JobConfigurations/dnnroi/reco1_nosp_data_dnnroi.fcl b/sbndcode/JobConfigurations/dnnroi/reco1_nosp_data_dnnroi.fcl index 4eca7cc0b..ae908de84 100644 --- a/sbndcode/JobConfigurations/dnnroi/reco1_nosp_data_dnnroi.fcl +++ b/sbndcode/JobConfigurations/dnnroi/reco1_nosp_data_dnnroi.fcl @@ -4,35 +4,7 @@ # runs everything downstream of sptpc2d in the reco1 chain for data # runs gaushitfinder on sptpc2d:dnnsp -#include "wcsp_data_sbnd.fcl" -#include "opdeconvolution_sbnd_data.fcl" -#include "sbnd_ophitfinder_deco_data.fcl" -#include "sbnd_flashfinder_deco_data.fcl" -#include "crt_channel_map_service.fcl" -#include "crt_calib_service.fcl" -#include "standard_reco1_sbnd.fcl" - -services: -{ - @table::services - @table::sbnd_data_services - CRTChannelMapService: @local::crt_channel_map_standard - CRTCalibService: @local::crt_calib_service -} - -physics.producers: -{ - @table::physics.producers - sptpc2d: @local::sbnd_wcls_sp_data - crtstrips: @local::crtstriphitproducer_data_sbnd - crtclustering: @local::crtclusterproducer_data_sbnd - crtspacepoints: @local::crtspacepointproducer_data_sbnd - crttracks: @local::crttrackproducer_data_sbnd - opdecopmt: @local::SBNDOpDeconvolutionPMT_data - ophitpmt: @local::SBNDDecoOpHitFinderPMT_data - opflashtpc0: @local::SBNDDecoSimpleFlashTPC0_data - opflashtpc1: @local::SBNDDecoSimpleFlashTPC1_data -} +#include "reco1_data.fcl" physics.reco1: [gaushit, numberofhitsfilter, cluster3d, crtstrips, crtclustering, crtspacepoints, crttracks, opdecopmt, ophitpmt, opflashtpc0, opflashtpc1] diff --git a/sbndcode/JobConfigurations/dnnroi/reco1_postscrub_data_dnnroi.fcl b/sbndcode/JobConfigurations/dnnroi/reco1_postscrub_data_dnnroi.fcl index 91845106a..21e645eb5 100644 --- a/sbndcode/JobConfigurations/dnnroi/reco1_postscrub_data_dnnroi.fcl +++ b/sbndcode/JobConfigurations/dnnroi/reco1_postscrub_data_dnnroi.fcl @@ -6,38 +6,10 @@ # # runs gaushit, numberofhitsfilter, cluster3d, superadata on sptpc2d:dnnsp Wire products. -#include "wcsp_data_sbnd.fcl" -#include "opdeconvolution_sbnd_data.fcl" -#include "sbnd_ophitfinder_deco_data.fcl" -#include "sbnd_flashfinder_deco_data.fcl" -#include "crt_channel_map_service.fcl" -#include "crt_calib_service.fcl" -#include "standard_reco1_sbnd.fcl" +#include "reco1_data.fcl" process_name: Reco1onDNN -services: -{ - @table::services - @table::sbnd_data_services - CRTChannelMapService: @local::crt_channel_map_standard - CRTCalibService: @local::crt_calib_service -} - -physics.producers: -{ - @table::physics.producers - sptpc2d: @local::sbnd_wcls_sp_data - crtstrips: @local::crtstriphitproducer_data_sbnd - crtclustering: @local::crtclusterproducer_data_sbnd - crtspacepoints: @local::crtspacepointproducer_data_sbnd - crttracks: @local::crttrackproducer_data_sbnd - opdecopmt: @local::SBNDOpDeconvolutionPMT_data - ophitpmt: @local::SBNDDecoOpHitFinderPMT_data - opflashtpc0: @local::SBNDDecoSimpleFlashTPC0_data - opflashtpc1: @local::SBNDDecoSimpleFlashTPC1_data -} - physics.reco1: [gaushit, numberofhitsfilter, cluster3d] physics.ana: [superadata] From 51e8976dd099fd2b6111b8191efc9c447bb40475 Mon Sep 17 00:00:00 2001 From: Mun Jung Jung Date: Fri, 18 Jul 2025 16:02:46 -0500 Subject: [PATCH 08/14] remove unnecessary comment --- .../JobConfigurations/dnnroi/reco1_postscrub_data_dnnroi.fcl | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/sbndcode/JobConfigurations/dnnroi/reco1_postscrub_data_dnnroi.fcl b/sbndcode/JobConfigurations/dnnroi/reco1_postscrub_data_dnnroi.fcl index 21e645eb5..6a6357d99 100644 --- a/sbndcode/JobConfigurations/dnnroi/reco1_postscrub_data_dnnroi.fcl +++ b/sbndcode/JobConfigurations/dnnroi/reco1_postscrub_data_dnnroi.fcl @@ -15,5 +15,4 @@ physics.ana: [superadata] outputs.out1.SelectEvents: [ "reco1" ] -# uncomment below line to run DNN ROI finding SP for reco -physics.producers.gaushit.CalDataModuleLabel: "sptpc2d:dnnsp" \ No newline at end of file +physics.producers.gaushit.CalDataModuleLabel: "sptpc2d:dnnsp" From 27b14505c4b58111d40c365670ea898ba335e94a Mon Sep 17 00:00:00 2001 From: Mun Jung Jung Date: Fri, 18 Jul 2025 16:06:10 -0500 Subject: [PATCH 09/14] remove dnn related comments from standard fcls --- sbndcode/JobConfigurations/standard/reco/reco1_data.fcl | 9 --------- .../JobConfigurations/standard/standard_detsim_sbnd.fcl | 6 ------ .../JobConfigurations/standard/standard_reco1_sbnd.fcl | 3 --- 3 files changed, 18 deletions(-) diff --git a/sbndcode/JobConfigurations/standard/reco/reco1_data.fcl b/sbndcode/JobConfigurations/standard/reco/reco1_data.fcl index 563a4b64a..896b52c6c 100644 --- a/sbndcode/JobConfigurations/standard/reco/reco1_data.fcl +++ b/sbndcode/JobConfigurations/standard/reco/reco1_data.fcl @@ -35,12 +35,3 @@ physics.ana: [superadata] outputs.out1.SelectEvents: [ "reco1" ] physics.producers.gaushit.CalDataModuleLabel: "sptpc2d:gauss" - -# uncomment below 4 lines to run DNN ROI finding SP -#physics.producers.sptpc2d.wcls_main.outputers: ["wclsFrameSaver:spsaver" , "wclsFrameSaver:dnnsaver"] -#physics.producers.sptpc2d.wcls_main.structs.use_dnnroi: true -#physics.producers.sptpc2d.wcls_main.structs.nchunks: 2 # should match training config -#physics.producers.sptpc2d.wcls_main.structs.tick_per_slice: 4 # should match training config - -# uncomment below line to run DNN ROI finding SP for reco -#physics.producers.gaushit.CalDataModuleLabel: "sptpc2d:dnnsp" diff --git a/sbndcode/JobConfigurations/standard/standard_detsim_sbnd.fcl b/sbndcode/JobConfigurations/standard/standard_detsim_sbnd.fcl index b13b52cb0..1bb03bcc0 100644 --- a/sbndcode/JobConfigurations/standard/standard_detsim_sbnd.fcl +++ b/sbndcode/JobConfigurations/standard/standard_detsim_sbnd.fcl @@ -92,9 +92,3 @@ outputs: ] } } - -# uncomment below 4 lines to run DNN ROI finding SP -# physics.producers.simtpc2d.wcls_main.outputers: ["wclsDepoFluxWriter:postdrift", "wclsFrameSaver:spsaver", "wclsFrameSaver:dnnsaver"] # "wclsFrameSaver:simdigits" <- by default, do not save RawDigits. Uncomment this line to save RawDigits and set -# physics.producers.simtpc2d.wcls_main.structs.use_dnnroi: true -# physics.producers.simtpc2d.wcls_main.structs.nchunks: 2 # should match training config -# physics.producers.simtpc2d.wcls_main.structs.tick_per_slice: 4 # should match training config diff --git a/sbndcode/JobConfigurations/standard/standard_reco1_sbnd.fcl b/sbndcode/JobConfigurations/standard/standard_reco1_sbnd.fcl index 90dd8eb13..b6a95ce30 100644 --- a/sbndcode/JobConfigurations/standard/standard_reco1_sbnd.fcl +++ b/sbndcode/JobConfigurations/standard/standard_reco1_sbnd.fcl @@ -35,6 +35,3 @@ physics.end_paths: [ @sequence::physics.end_paths, ana ] #outputs table overrides outputs.out1.dataTier: "reconstructed" outputs.out1.outputCommands: [@sequence::outputs.out1.outputCommands, @sequence::sbnd_reco1_drops] - -# uncomment below line to run DNN ROI finding SP for reco -- must have run DNN ROI finding in detsim stage -#physics.producers.gaushit.CalDataModuleLabel: "simtpc2d:dnnsp" From 4a1735f5f2e5b3990d2f141637aefa2cd7dc18af Mon Sep 17 00:00:00 2001 From: Mun Jung Jung Date: Fri, 18 Jul 2025 16:24:20 -0500 Subject: [PATCH 10/14] update sp only block --- sbndcode/WireCell/wcsimsp_sbnd.fcl | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/sbndcode/WireCell/wcsimsp_sbnd.fcl b/sbndcode/WireCell/wcsimsp_sbnd.fcl index 9968f1a4c..8a2e2e66c 100644 --- a/sbndcode/WireCell/wcsimsp_sbnd.fcl +++ b/sbndcode/WireCell/wcsimsp_sbnd.fcl @@ -127,14 +127,13 @@ sbnd_wcls_sp.wcls_main.params : { # Save output signal waveforms (recob::Wire) in "sparse" or "dense" form signal_output_form: "sparse" + roi: "trad" dnnroi_model_p0: "DNN_ROI/plane0.ts" dnnroi_model_p1: "DNN_ROI/plane1.ts" - } sbnd_wcls_sp.wcls_main.structs: { nticks: 3427 - use_dnnroi: false nchunks: 2 tick_per_slice: 4 } From b3258003fe1e223f64ed8c2e572968823d39da15 Mon Sep 17 00:00:00 2001 From: nathanielerowe Date: Mon, 21 Jul 2025 19:21:32 +0000 Subject: [PATCH 11/14] Initial commit --- .../dnnroi/standard_detsim_sbnd_bothrois.fcl | 3 ++- .../dnnroi/standard_detsim_sbnd_bothrois_gpu.fcl | 11 +++++++++++ .../dnnroi/standard_detsim_sbnd_dnnroi.fcl | 3 ++- .../dnnroi/standard_detsim_sbnd_dnnroi_gpu.fcl | 12 ++++++++++++ .../pgrapher/experiment/sbnd/wcls-nf-sp-data.jsonnet | 11 ++++++----- .../cfg/pgrapher/experiment/sbnd/wcls-nf-sp.jsonnet | 9 ++++++--- .../sbnd/wcls-sim-drift-depoflux-nf-sp.jsonnet | 9 ++++++--- sbndcode/WireCell/wcsimsp_sbnd.fcl | 3 +++ sbndcode/WireCell/wcsp_data_sbnd.fcl | 2 ++ 9 files changed, 50 insertions(+), 13 deletions(-) create mode 100644 sbndcode/JobConfigurations/dnnroi/standard_detsim_sbnd_bothrois_gpu.fcl create mode 100644 sbndcode/JobConfigurations/dnnroi/standard_detsim_sbnd_dnnroi_gpu.fcl diff --git a/sbndcode/JobConfigurations/dnnroi/standard_detsim_sbnd_bothrois.fcl b/sbndcode/JobConfigurations/dnnroi/standard_detsim_sbnd_bothrois.fcl index 6e542e5f9..94af3d0b5 100644 --- a/sbndcode/JobConfigurations/dnnroi/standard_detsim_sbnd_bothrois.fcl +++ b/sbndcode/JobConfigurations/dnnroi/standard_detsim_sbnd_bothrois.fcl @@ -6,4 +6,5 @@ #include "standard_detsim_sbnd.fcl" -physics.producers.simtpc2d: @local::sbnd_wcls_simsp_bothrois \ No newline at end of file +physics.producers.simtpc2d: @local::sbnd_wcls_simsp_bothrois +physics.producers.simtpc2d.wcls_main.params.wc_device: "cpu" diff --git a/sbndcode/JobConfigurations/dnnroi/standard_detsim_sbnd_bothrois_gpu.fcl b/sbndcode/JobConfigurations/dnnroi/standard_detsim_sbnd_bothrois_gpu.fcl new file mode 100644 index 000000000..127b740de --- /dev/null +++ b/sbndcode/JobConfigurations/dnnroi/standard_detsim_sbnd_bothrois_gpu.fcl @@ -0,0 +1,11 @@ +# File: standard_detsim_sbnd_bothrois.fcl +# Authors: Mun Jung Jung (munjung@uchicago.edu) + +# runs WC 2D signal processing during detsim using both traditional and DNN-based ROI finding +# create three Wire products: simtpc2d:gauss, simtpc2d:wiener (from traditional), simtpc2d:dnnsp (from DNN) + +#include "standard_detsim_sbnd.fcl" + +physics.producers.simtpc2d: @local::sbnd_wcls_simsp_bothrois +physics.producers.simtpc2d.wcls_main.params.wc_device: "gpu" +physics.producers.simtpc2d.wcls_main.plugins: [@sequence::sbnd_wcls_simsp_dnnroi.wcls_main.plugins, "WireCellCuda"] diff --git a/sbndcode/JobConfigurations/dnnroi/standard_detsim_sbnd_dnnroi.fcl b/sbndcode/JobConfigurations/dnnroi/standard_detsim_sbnd_dnnroi.fcl index b454e4009..b48903b74 100644 --- a/sbndcode/JobConfigurations/dnnroi/standard_detsim_sbnd_dnnroi.fcl +++ b/sbndcode/JobConfigurations/dnnroi/standard_detsim_sbnd_dnnroi.fcl @@ -7,4 +7,5 @@ #include "standard_detsim_sbnd.fcl" -physics.producers.simtpc2d: @local::sbnd_wcls_simsp_dnnroi \ No newline at end of file +physics.producers.simtpc2d: @local::sbnd_wcls_simsp_dnnroi +physics.producers.simtpc2d.wcls_main.params.wc_device: "cpu" diff --git a/sbndcode/JobConfigurations/dnnroi/standard_detsim_sbnd_dnnroi_gpu.fcl b/sbndcode/JobConfigurations/dnnroi/standard_detsim_sbnd_dnnroi_gpu.fcl new file mode 100644 index 000000000..764e8bd2d --- /dev/null +++ b/sbndcode/JobConfigurations/dnnroi/standard_detsim_sbnd_dnnroi_gpu.fcl @@ -0,0 +1,12 @@ +# File: standard_detsim_sbnd_dnnroi.fcl +# Authors: Mun Jung Jung (munjung@uchicago.edu) + +# runs WC 2D signal processing during detsim using only DNN-based ROI finding +# create one Wire product: simtpc2d:dnnsp + + +#include "standard_detsim_sbnd.fcl" + +physics.producers.simtpc2d: @local::sbnd_wcls_simsp_dnnroi +physics.producers.simtpc2d.wcls_main.params.wc_device: "gpu" +physics.producers.simtpc2d.wcls_main.plugins: [@sequence::sbnd_wcls_simsp_dnnroi.wcls_main.plugins, "WireCellCuda"] diff --git a/sbndcode/WireCell/cfg/pgrapher/experiment/sbnd/wcls-nf-sp-data.jsonnet b/sbndcode/WireCell/cfg/pgrapher/experiment/sbnd/wcls-nf-sp-data.jsonnet index ba52ceaa9..26cb87254 100644 --- a/sbndcode/WireCell/cfg/pgrapher/experiment/sbnd/wcls-nf-sp-data.jsonnet +++ b/sbndcode/WireCell/cfg/pgrapher/experiment/sbnd/wcls-nf-sp-data.jsonnet @@ -22,6 +22,7 @@ //local epoch = std.extVar('epoch'); // eg "dynamic", "after", "before", "perfect" local sigoutform = std.extVar('signal_output_form'); // eg "sparse" or "dense" +local wc_device = std.extVar('wc_device'); local raw_input_label = std.extVar('raw_input_label'); // eg "daq" local use_paramresp = std.extVar('use_paramresp'); // eg "true" or "false" local roi = std.extVar('roi'); @@ -49,9 +50,9 @@ local params = data_params { }, }; - -local tools = tools_maker(params); - +local default_tools = tools_maker(params); +local tools = if wc_device == 'gpu' then std.mergePatch(default_tools, + {dft: {type: "TorchDFT", data: {device: wc_device}}}) else default_tools; local mega_anode = { type: 'MegaAnodePlane', @@ -217,7 +218,7 @@ local ts_p0 = { tick_per_slice: tick_per_slice, data: { model: dnnroi_model_p0, - device: "cpu", + device: wc_device, concurrency: 1, }, }; @@ -228,7 +229,7 @@ local ts_p1 = { tick_per_slice: tick_per_slice, data: { model: dnnroi_model_p1, - device: "cpu", + device: wc_device, concurrency: 1, }, }; diff --git a/sbndcode/WireCell/cfg/pgrapher/experiment/sbnd/wcls-nf-sp.jsonnet b/sbndcode/WireCell/cfg/pgrapher/experiment/sbnd/wcls-nf-sp.jsonnet index 90528f446..086e2ea64 100644 --- a/sbndcode/WireCell/cfg/pgrapher/experiment/sbnd/wcls-nf-sp.jsonnet +++ b/sbndcode/WireCell/cfg/pgrapher/experiment/sbnd/wcls-nf-sp.jsonnet @@ -21,6 +21,7 @@ local epoch = std.extVar('epoch'); // eg "dynamic", "after", "before", "perfect" +local wc_device = std.extVar('wc_device'); local sigoutform = std.extVar('signal_output_form'); // eg "sparse" or "dense" local raw_input_label = std.extVar('raw_input_label'); // eg "daq" local use_dnnroi = std.extVar('use_dnnroi'); @@ -32,7 +33,9 @@ local dnnroi_model_p1 = std.extVar('dnnroi_model_p1'); local g = import 'pgraph.jsonnet'; local f = import 'pgrapher/experiment/sbnd/funcs.jsonnet'; local wc = import 'wirecell.jsonnet'; -local tools_maker = import 'pgrapher/common/tools.jsonnet'; +local default_tools = tools_maker(params); +local tools = if wc_device == 'gpu' then std.mergePatch(default_tools, + {dft: {type: "TorchDFT", data: {device: wc_device}}}) else default_tools; local simu_params = import 'simparams.jsonnet'; local params = simu_params; @@ -180,7 +183,7 @@ local ts_p0 = { tick_per_slice: tick_per_slice, data: { model: dnnroi_model_p0, - device: "cpu", + device: wc_device, concurrency: 1, }, }; @@ -191,7 +194,7 @@ local ts_p1 = { tick_per_slice: tick_per_slice, data: { model: dnnroi_model_p1, - device: "cpu", + device: wc_device, concurrency: 1, }, }; diff --git a/sbndcode/WireCell/cfg/pgrapher/experiment/sbnd/wcls-sim-drift-depoflux-nf-sp.jsonnet b/sbndcode/WireCell/cfg/pgrapher/experiment/sbnd/wcls-sim-drift-depoflux-nf-sp.jsonnet index 3535696b0..f9a2ddfe0 100644 --- a/sbndcode/WireCell/cfg/pgrapher/experiment/sbnd/wcls-sim-drift-depoflux-nf-sp.jsonnet +++ b/sbndcode/WireCell/cfg/pgrapher/experiment/sbnd/wcls-sim-drift-depoflux-nf-sp.jsonnet @@ -14,6 +14,7 @@ local dnnroi_model_p1 = std.extVar('dnnroi_model_p1'); local g = import 'pgraph.jsonnet'; local f = import 'pgrapher/common/funcs.jsonnet'; local wc = import 'wirecell.jsonnet'; +local wc_device = std.extVar('wc_device'); local io = import 'pgrapher/common/fileio.jsonnet'; local tools_maker = import 'pgrapher/common/tools.jsonnet'; @@ -39,7 +40,9 @@ local params = base { } }; -local tools = tools_maker(params); +local default_tools = tools_maker(params); +local tools = if wc_device == 'gpu' then std.mergePatch(default_tools, + {dft: {type: "TorchDFT", data: {device: wc_device}}}) else default_tools; local sim_maker = import 'pgrapher/experiment/sbnd/sim.jsonnet'; local sim = sim_maker(params, tools); local nanodes = std.length(tools.anodes); @@ -317,7 +320,7 @@ local ts_p0 = { tick_per_slice: tick_per_slice, data: { model: dnnroi_model_p0, - device: "cpu", + device: wc_device, concurrency: 1, }, }; @@ -328,7 +331,7 @@ local ts_p1 = { tick_per_slice: tick_per_slice, data: { model: dnnroi_model_p1, - device: "cpu", + device: wc_device, concurrency: 1, }, }; diff --git a/sbndcode/WireCell/wcsimsp_sbnd.fcl b/sbndcode/WireCell/wcsimsp_sbnd.fcl index 8a2e2e66c..4fe10a12e 100644 --- a/sbndcode/WireCell/wcsimsp_sbnd.fcl +++ b/sbndcode/WireCell/wcsimsp_sbnd.fcl @@ -116,6 +116,9 @@ sbnd_wcls_sp.wcls_main.params : { # This locates the input raw::RawDigit collection in the art::Event raw_input_label: "daq" + # if dnnroi is used, this device will be used + wc_device: "cpu" + # Set "data" vs. "sim". The epoch below probably should follow suit. reality: "sim" diff --git a/sbndcode/WireCell/wcsp_data_sbnd.fcl b/sbndcode/WireCell/wcsp_data_sbnd.fcl index 7f35074f5..daa40b5e8 100644 --- a/sbndcode/WireCell/wcsp_data_sbnd.fcl +++ b/sbndcode/WireCell/wcsp_data_sbnd.fcl @@ -41,6 +41,8 @@ sbnd_wcls_sp_data.wcls_main.outputers: ["wclsFrameSaver:spsaver"] sbnd_wcls_sp_data.wcls_main.params : { # This locates the input raw::RawDigit collection in the art::Event raw_input_label: "daq" + # if dnnroi is used, this device will be used + wc_device: "cpu" # if epoch is "dynamic" you MUST add # "wclsMultiChannelNoiseDB" to "inputers" and must NOT From 03aba3579b4f1ca38c913061f75db3477bb4f24c Mon Sep 17 00:00:00 2001 From: nathanielerowe Date: Mon, 21 Jul 2025 20:12:58 +0000 Subject: [PATCH 12/14] fix tools_maker --- .../cfg/pgrapher/experiment/sbnd/wcls-nf-sp.jsonnet | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/sbndcode/WireCell/cfg/pgrapher/experiment/sbnd/wcls-nf-sp.jsonnet b/sbndcode/WireCell/cfg/pgrapher/experiment/sbnd/wcls-nf-sp.jsonnet index 086e2ea64..a1bdcadb2 100644 --- a/sbndcode/WireCell/cfg/pgrapher/experiment/sbnd/wcls-nf-sp.jsonnet +++ b/sbndcode/WireCell/cfg/pgrapher/experiment/sbnd/wcls-nf-sp.jsonnet @@ -33,14 +33,15 @@ local dnnroi_model_p1 = std.extVar('dnnroi_model_p1'); local g = import 'pgraph.jsonnet'; local f = import 'pgrapher/experiment/sbnd/funcs.jsonnet'; local wc = import 'wirecell.jsonnet'; -local default_tools = tools_maker(params); -local tools = if wc_device == 'gpu' then std.mergePatch(default_tools, - {dft: {type: "TorchDFT", data: {device: wc_device}}}) else default_tools; +local tools_maker = import 'pgrapher/common/tools.jsonnet'; local simu_params = import 'simparams.jsonnet'; local params = simu_params; -local tools = tools_maker(params); +local default_tools = tools_maker(params); +local tools = if wc_device == 'gpu' then std.mergePatch(default_tools, + {dft: {type: "TorchDFT", data: {device: wc_device}}}) else default_tools; + local mega_anode = { type: 'MegaAnodePlane', From f017706ae4ef1b2783694e5404fbc4d6133ddcd6 Mon Sep 17 00:00:00 2001 From: nathanielerowe Date: Mon, 21 Jul 2025 20:53:16 +0000 Subject: [PATCH 13/14] reduce number of wc_device params --- .../dnnroi/standard_detsim_sbnd_bothrois.fcl | 1 - .../dnnroi/standard_detsim_sbnd_dnnroi.fcl | 1 - .../cfg/pgrapher/experiment/sbnd/wcls-nf-sp.jsonnet | 2 +- sbndcode/WireCell/wcsimsp_sbnd.fcl | 8 +++++--- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/sbndcode/JobConfigurations/dnnroi/standard_detsim_sbnd_bothrois.fcl b/sbndcode/JobConfigurations/dnnroi/standard_detsim_sbnd_bothrois.fcl index 94af3d0b5..8a057e9b7 100644 --- a/sbndcode/JobConfigurations/dnnroi/standard_detsim_sbnd_bothrois.fcl +++ b/sbndcode/JobConfigurations/dnnroi/standard_detsim_sbnd_bothrois.fcl @@ -7,4 +7,3 @@ #include "standard_detsim_sbnd.fcl" physics.producers.simtpc2d: @local::sbnd_wcls_simsp_bothrois -physics.producers.simtpc2d.wcls_main.params.wc_device: "cpu" diff --git a/sbndcode/JobConfigurations/dnnroi/standard_detsim_sbnd_dnnroi.fcl b/sbndcode/JobConfigurations/dnnroi/standard_detsim_sbnd_dnnroi.fcl index b48903b74..f38c18d7a 100644 --- a/sbndcode/JobConfigurations/dnnroi/standard_detsim_sbnd_dnnroi.fcl +++ b/sbndcode/JobConfigurations/dnnroi/standard_detsim_sbnd_dnnroi.fcl @@ -8,4 +8,3 @@ #include "standard_detsim_sbnd.fcl" physics.producers.simtpc2d: @local::sbnd_wcls_simsp_dnnroi -physics.producers.simtpc2d.wcls_main.params.wc_device: "cpu" diff --git a/sbndcode/WireCell/cfg/pgrapher/experiment/sbnd/wcls-nf-sp.jsonnet b/sbndcode/WireCell/cfg/pgrapher/experiment/sbnd/wcls-nf-sp.jsonnet index a1bdcadb2..467a953ea 100644 --- a/sbndcode/WireCell/cfg/pgrapher/experiment/sbnd/wcls-nf-sp.jsonnet +++ b/sbndcode/WireCell/cfg/pgrapher/experiment/sbnd/wcls-nf-sp.jsonnet @@ -33,7 +33,7 @@ local dnnroi_model_p1 = std.extVar('dnnroi_model_p1'); local g = import 'pgraph.jsonnet'; local f = import 'pgrapher/experiment/sbnd/funcs.jsonnet'; local wc = import 'wirecell.jsonnet'; -local tools_maker = import 'pgrapher/common/tools.jsonnet'; +local tools_maker = import 'pgrapher/common/tools.jsonnet';' local simu_params = import 'simparams.jsonnet'; local params = simu_params; diff --git a/sbndcode/WireCell/wcsimsp_sbnd.fcl b/sbndcode/WireCell/wcsimsp_sbnd.fcl index 4fe10a12e..d2a13060f 100644 --- a/sbndcode/WireCell/wcsimsp_sbnd.fcl +++ b/sbndcode/WireCell/wcsimsp_sbnd.fcl @@ -43,7 +43,10 @@ sbnd_wcls_simsp.wcls_main.params: { ## Set "data" vs. "sim". The epoch below probably should follow suit. reality: "sim" - + + ## if using dnnroi, use this device + wc_device: "cpu" + ## if epoch is "dynamic" you MUST add ## "wclsMultiChannelNoiseDB" to "inputers" and must NOT ## add it if not "dynamic" @@ -156,7 +159,6 @@ sbnd_wcls_samples_rec.wcls_main.params: { ## Set "data" vs. "sim". The epoch below probably should follow suit. reality: "sim" - ## if epoch is "dynamic" you MUST add ## "wclsMultiChannelNoiseDB" to "inputers" and must NOT ## add it if not "dynamic" @@ -199,7 +201,7 @@ sbnd_wcls_samples_tru.wcls_main.params: { ## Set "data" vs. "sim". The epoch below probably should follow suit. reality: "sim" - + ## if epoch is "dynamic" you MUST add ## "wclsMultiChannelNoiseDB" to "inputers" and must NOT ## add it if not "dynamic" From 7f89b3eb647938783dbe8f020e1f222158f0e62a Mon Sep 17 00:00:00 2001 From: Mun Jung Jung Date: Wed, 23 Jul 2025 21:41:27 -0500 Subject: [PATCH 14/14] update training sample making code --- ...m-drift-depoflux-nf-sp-samples_rec.jsonnet | 48 ++++++++----------- ...m-drift-depoflux-nf-sp-samples_tru.jsonnet | 29 +---------- sbndcode/WireCell/wcsimsp_sbnd.fcl | 9 ++-- 3 files changed, 27 insertions(+), 59 deletions(-) diff --git a/sbndcode/WireCell/cfg/pgrapher/experiment/sbnd/wcls-sim-drift-depoflux-nf-sp-samples_rec.jsonnet b/sbndcode/WireCell/cfg/pgrapher/experiment/sbnd/wcls-sim-drift-depoflux-nf-sp-samples_rec.jsonnet index 15895db20..c18d2c38f 100644 --- a/sbndcode/WireCell/cfg/pgrapher/experiment/sbnd/wcls-sim-drift-depoflux-nf-sp-samples_rec.jsonnet +++ b/sbndcode/WireCell/cfg/pgrapher/experiment/sbnd/wcls-sim-drift-depoflux-nf-sp-samples_rec.jsonnet @@ -157,7 +157,26 @@ local wcls_depoflux_writer = g.pnode({ }, nin=1, nout=1, uses=tools.anodes + [tools.field]); local sp_maker = import 'pgrapher/experiment/sbnd/sp.jsonnet'; -local sp = sp_maker(params, tools, { sparse: sigoutform == 'sparse' }); + +local sp_override = { + sparse: true, + use_roi_debug_mode: true, + save_negative_charge: false, // TODO: no negative charge in gauss, default is false + use_multi_plane_protection: true, + do_not_mp_protect_traditional: false, // TODO: do_not_mp_protect_traditional to make a clear ref, defualt is false + mp_tick_resolution:4, + tight_lf_tag: "", + cleanup_roi_tag: "", + break_roi_loop1_tag: "", + break_roi_loop2_tag: "", + shrink_roi_tag: "", + extend_roi_tag: "", + decon_charge_tag: "", + gauss_tag: "", + wiener_tag: "", +}; + +local sp = sp_maker(params, tools, sp_override); local sp_pipes = [sp.make_sigproc(a) for a in tools.anodes]; local magoutput = 'sbnd-data-check.root'; @@ -178,14 +197,6 @@ local chndb = [{ local nf_maker = import 'pgrapher/experiment/sbnd/nf.jsonnet'; local nf_pipes = [nf_maker(params, tools.anodes[n], chndb[n], n, name='nf%d' % n) for n in anode_iota]; -local multipass1 = [ - g.pipeline([ - sn_pipes[n], - ], - 'multipass%d' % n) - for n in anode_iota -]; - local multipass2 = [ g.pipeline([ sn_pipes[n], @@ -208,7 +219,6 @@ local multipass2 = [ local f_sp = import 'pgrapher/experiment/sbnd/funcs.jsonnet'; local outtags = ['orig%d' % n for n in anode_iota]; -local bi_manifold1 = f.fanpipe('DepoSetFanout', multipass1, 'FrameFanin', 'sn_mag_nf', outtags); local bi_manifold2 = f_sp.fanpipe('DepoSetFanout', multipass2, 'FrameFanin', 'sn_mag_nf_mod2', outtags, "true"); local retagger_sim = g.pnode({ @@ -333,21 +343,7 @@ local retagger_sp = g.pnode({ local sink_sp = g.pnode({ type: 'DumpFrames' }, nin=1, nout=0); -local graph1 = g.pipeline([ -wcls_input_sim.deposet, //sim -setdrifter, //sim -wcls_depoflux_writer, //sim -bi_manifold1, //sim -retagger_sim, //sim -wcls_output_sim.sim_digits, //sim -fanpipe, //sp -retagger_sp, //sp -wcls_output_sp.sp_signals, //sp -sink_sp //sp -]); - - -local graph2 = g.pipeline([ +local graph = g.pipeline([ wcls_input_sim.deposet, //sim setdrifter, //sim wcls_depoflux_writer, //sim @@ -359,8 +355,6 @@ sink_sp //sp local save_simdigits = std.extVar('save_simdigits'); -local graph = if save_simdigits == "true" then graph1 else graph2; - local app = { type: 'TbbFlow', data: { diff --git a/sbndcode/WireCell/cfg/pgrapher/experiment/sbnd/wcls-sim-drift-depoflux-nf-sp-samples_tru.jsonnet b/sbndcode/WireCell/cfg/pgrapher/experiment/sbnd/wcls-sim-drift-depoflux-nf-sp-samples_tru.jsonnet index cbe6c9ad6..6d73f1eb2 100644 --- a/sbndcode/WireCell/cfg/pgrapher/experiment/sbnd/wcls-sim-drift-depoflux-nf-sp-samples_tru.jsonnet +++ b/sbndcode/WireCell/cfg/pgrapher/experiment/sbnd/wcls-sim-drift-depoflux-nf-sp-samples_tru.jsonnet @@ -288,14 +288,6 @@ local hio_truth = [g.pnode({ ]; -local multipass1 = [ - g.pipeline([ - sn_pipes[n], - ], - 'multipass%d' % n) - for n in anode_iota -]; - local multipass2 = [ g.pipeline([ deposplats[n], @@ -308,9 +300,6 @@ local multipass2 = [ local f_sp = import 'pgrapher/experiment/sbnd/funcs.jsonnet'; local outtags = ['orig%d' % n for n in anode_iota]; -local bi_manifold1 = f.fanpipe('DepoFanout', multipass1, 'FrameFanin', 'sn_mag_nf', outtags); -// local bi_manifold2 = f_sp.fanpipe('DepoFanout', multipass2, 'FrameFanin', 'sn_mag_nf_mod2', outtags, "true"); -// local bi_manifold2 = f_sp.fanpipe('DepoFanout', multipass2, 'FrameFanin', 'sn_mag_nf_mod2', outtags, "true"); local bi_manifold2 = f_sp.fanpipe('DepoFanout', multipass2, 'FrameFanin'); local retagger_sim = g.pnode({ @@ -436,21 +425,7 @@ local retagger_sp = g.pnode({ local sink_sp = g.pnode({ type: 'DumpFrames' }, nin=1, nout=0); -local graph1 = g.pipeline([ -wcls_input_sim.deposet, //sim -setdrifter, //sim -wcls_deposetsimchannel_sink, //sim -bi_manifold1, //sim // sn_pipes[n] -retagger_sim, //sim -wcls_output_sim.sim_digits, //sim -fanpipe, //sp <- nf_pipes[n], sp_pipes[n] (needs channel selector to do fanout correctly) -retagger_sp, //sp -wcls_output_sp.sp_signals, //sp -sink_sp //sp -]); - - -local graph2 = g.pipeline([ +local graph = g.pipeline([ wcls_input_sim.depos, //sim drifter, //sim wcls_deposetsimchannel_sink, //sim @@ -461,8 +436,6 @@ sink_sp //sp // local save_simdigits = std.extVar('save_simdigits'); local save_simdigits = "false"; -local graph = if save_simdigits == "true" then graph1 else graph2; - local app = { type: 'TbbFlow', data: { diff --git a/sbndcode/WireCell/wcsimsp_sbnd.fcl b/sbndcode/WireCell/wcsimsp_sbnd.fcl index 8a2e2e66c..870886119 100644 --- a/sbndcode/WireCell/wcsimsp_sbnd.fcl +++ b/sbndcode/WireCell/wcsimsp_sbnd.fcl @@ -146,7 +146,8 @@ sbnd_wcls_samples_rec: @local::sbnd_wcls sbnd_wcls_samples_rec.wcls_main.plugins: ["WireCellGen", "WireCellSigProc", "WireCellRoot", "WireCellPgraph", "WireCellLarsoft", "WireCellTbb", "WireCellPytorch", "WireCellHio"] sbnd_wcls_samples_rec.wcls_main.configs: ["pgrapher/experiment/sbnd/wcls-sim-drift-depoflux-nf-sp-samples_rec.jsonnet"] sbnd_wcls_samples_rec.wcls_main.inputers: ["wclsSimDepoSetSource:"] -sbnd_wcls_samples_rec.wcls_main.outputers:[ "wclsDepoFluxWriter:postdrift", "wclsFrameSaver:simdigits"] +sbnd_wcls_samples_rec.wcls_main.outputers:["wclsDepoFluxWriter:postdrift", + "wclsFrameSaver:spsaver"] sbnd_wcls_samples_rec.wcls_main.params: { ## This locates the input SimEnergyDeposits in the art::Event inputTag: "ionandscint:" @@ -165,7 +166,7 @@ sbnd_wcls_samples_rec.wcls_main.params: { ## If save_simdigits="false", comment line with "wclsFrameSaver:simdigits" in outputers ## If save_simdigits="true", uncomment line with "wclsFrameSaver:simdigits" in outputers ## this is whether or not to save raw digits!!! - save_simdigits: "true" + save_simdigits: "false" save_track_id: "true" } @@ -189,7 +190,7 @@ sbnd_wcls_samples_tru: @local::sbnd_wcls sbnd_wcls_samples_tru.wcls_main.plugins: ["WireCellGen", "WireCellSigProc", "WireCellRoot", "WireCellPgraph", "WireCellLarsoft", "WireCellTbb", "WireCellPytorch", "WireCellHio"] sbnd_wcls_samples_tru.wcls_main.configs: ["pgrapher/experiment/sbnd/wcls-sim-drift-depoflux-nf-sp-samples_tru.jsonnet"] sbnd_wcls_samples_tru.wcls_main.inputers: ["wclsSimDepoSource:"] -sbnd_wcls_samples_tru.wcls_main.outputers:["wclsDepoFluxWriter:postdrift", "wclsFrameSaver:simdigits"] +sbnd_wcls_samples_tru.wcls_main.outputers:[] sbnd_wcls_samples_tru.wcls_main.params: { ## This locates the input SimEnergyDeposits in the art::Event inputTag: "ionandscint:" @@ -208,7 +209,7 @@ sbnd_wcls_samples_tru.wcls_main.params: { ## If save_simdigits="false", comment line with "wclsFrameSaver:simdigits" in outputers ## If save_simdigits="true", uncomment line with "wclsFrameSaver:simdigits" in outputers ## this is whether or not to save raw digits!!! - save_simdigits: "true" + save_simdigits: "false" save_track_id: "true" }