New reconstruction chain in the webApp

bye_splits/plot/display_plotly/main.py

@@ -29,7 +29,7 @@
                   dbc.Nav([dbc.DropdownMenu([
                                dbc.DropdownMenuItem('3D view', id='3D view'),
                                dbc.DropdownMenuItem('Layer view', id='Layer view')],
-                           label="Pages", nav=True)
+                           label="View", nav=True)
                           ])
               ],brand="3D trigger cells visualization",color="primary",dark=True,)]),
     html.Br(),
@@ -72,22 +72,21 @@ def update_event(particle, pu, n_click, submit_event, event, page):
     button_clicked = ctx.triggered_id
     if button_clicked == 'submit-val':
         if event is None:  raise RuntimeError("Please select manually an event or click on 'Random event'.")
-
+    
     df_dict, event = process.get_data(common.dot_dict(vars(args)), particle, pu, event)
 
     gen_info = next(reversed(df_dict.values()))
-    slider = plt.prepare_slider(next(iter(df_dict.values())), page)
+    slider = plt.prepare_slider(next(iter(df_dict['default'].values())), page)
 
-    if particle == 'photons' and pu == '200 PU':
+    if pu == '200 pileup':
         df_dict = plt.geom_selection(df_dict)
     if particle != 'pions': checkbox = ['Coarse seeding','Layer selection','Seed index']
     else: checkbox = ['Layer selection','Seed index']
 
-    df_dict = {k: v.to_json() for k, v in df_dict.items()}
-    event_selected_message = 'Event {} selected. Gen Particle (η={:.2f}, ϕ={:.2f}), pₜ={:.2f} GeV.'
-    return (event_selected_message.format(int(event), gen_info['gen_eta'].values[0], 
-            gen_info['gen_phi'].values[0], gen_info['gen_pt'].values[0]), 
-            checkbox, slider, df_dict, None)
+    df_dict = {key:val for key, val in df_dict.items() if key != 'gen'}
+    json_df_dict = {chain: {coef: df.to_json() for coef, df in v.items()} for chain, v in df_dict.items()}
+    event_selected_message = 'Event {} selected. Gen Particle (η={:.2f}, ϕ={:.2f}, p$_{{\\text{T}}}$={:.2f} GeV).'
+    return ('Event '+ event +' selected. Gen Particle η={:.2f}, ϕ={:.2f}, '.format(gen_info['gen_eta'].values[0],gen_info['gen_phi'].values[0])+ 'p$_{T}$'+'={:.2f} GeV'.format(gen_info['gen_pt'].values[0]), checkbox, slider, json_df_dict, None)
 
 @app.callback(
     [
@@ -100,13 +99,14 @@ def update_event(particle, pu, n_click, submit_event, event, page):
         Input('slider_cluster', 'value'),
         Input('mip', 'value'),
         Input('checkbox', 'value'),
+        Input('chain', 'value'),
     ],
     [State('page', 'key')]
 )
-def make_graph(data, slider_value, coef, mip, checkbox, page):
+def make_graph(data, slider_value, coef, mip, checkbox, chain, page):
     assert float(mip) >= 0.5, 'mip\u209C value out of range. Minimum value 0.5 !'
-    df_dict = {k: pd.read_json(v) for k, v in data.items()}
-    df = df_dict[str(coef)]
+    df_dict = {k: {coef: pd.read_json(df) for coef, df in v.items()} for k, v in data.items()}
+    df = df_dict[chain][str(coef)]
     df_sel = df[df.tc_mipPt >= mip]
 
     if page == '3D':
@@ -144,4 +144,4 @@ def make_graph(data, slider_value, coef, mip, checkbox, page):
     args = parsing.parser_display_plotly()
     process = processing.Processing() 
 
-    app.run_server(host=args.host, port=args.port, debug=args.debug)
+    app.run_server(host=args.host, port=args.port, debug=True)

bye_splits/plot/display_plotly/plot_event.py

@@ -14,13 +14,17 @@
 def geom_selection(df_dict):
     gen_info = next(reversed(df_dict.values()))
 
-    radius_gen = 50 # radius to select a specific region around the gen
-    for coef in list(df_dict.keys())[:-1]:
-        eta = gen_info['gen_eta'].values[0]
-        phi = gen_info['gen_phi'].values[0]
-        x_gen, y_gen = sph2cart(eta, phi)
-        df_dict[coef] = df_dict[coef][np.sqrt((x_gen-df_dict[coef].tc_x)**2+(y_gen-df_dict[coef].tc_y)**2)<radius_gen]
-
+    radius_gen = 40  # radius to select a specific region around the gen
+    for chain in df_dict.keys():
+        for coef in list(df_dict[chain].keys()):
+            eta = gen_info['gen_eta'].values[0]
+            phi = gen_info['gen_phi'].values[0]
+            x_gen, y_gen = sph2cart(eta, phi)
+
+            mask = np.sqrt((x_gen - df_dict[chain][coef]['tc_x'])**2 + \
+                           (y_gen - df_dict[chain][coef]['tc_y'])**2) < radius_gen
+            df_dict[chain][coef] = df_dict[chain][coef][mask].reset_index(drop=True)
+
     return df_dict
 
 def prepare_slider(df, page='3D'):
@@ -140,7 +144,7 @@ def produce_plot(df, opacity, plot_type, discrete):
             cmin=df.tc_mipPt.min(),
             cmax=df.tc_mipPt.max(),
             showscale=True,
-            colorbar=dict(title=dict(text="[mip\u209c]", side="right"), ticks="outside", x=1)
+            colorbar=dict(title=dict(text='Transverse mip', font=dict(size=16), side="right"), ticks="outside", x=1)
         ))
         listdata.append(datum)
     return listdata

bye_splits/plot/display_plotly/processing.py

@@ -8,6 +8,7 @@
 sys.path.insert(0, parent_dir)
 
 from dash import dcc, html
+import dash_daq as daq
 import h5py
 import re
 import plotly.express.colors as px
@@ -29,6 +30,8 @@ def __init__(self):
 
         self.reprocess = self.cfg['3Ddisplay']['reprocess']
         self.coefs = self.cfg['3Ddisplay']['coefs']
+        self.chain = {'default' : 'Default chain',
+                      'cs'      : 'Coarse seeding chain'}
         self.ds_geom = GeometryData(reprocess=False, logger=log, library='plotly').provide()
         self.filename = None
         self.list_events = None
@@ -37,7 +40,7 @@ def random_event(self, f):
         return random.choice(self.list_events)
 
     def filestore(self, particles, pu):
-        self.filename = self.cfg["clusterStudies"]["localDir"] + self.cfg["3Ddisplay"][f"PU{pu}"][particles]
+        self.filename = self.cfg["clusterStudies"]["localDir"] + self.cfg["3Ddisplay"][f"PU{pu}"][particles] + ".hdf5"
 
     def get_data(self, pars, particles, pu_str, event = ''):
         pu = re.search(r'(\d+) pileup', pu_str).group(1)
@@ -61,9 +64,11 @@ def get_data(self, pars, particles, pu_str, event = ''):
     def get_event(self, event):
         """   Load dataframes of a particular event from the hdf5 file   """
         dic = {}
-        for coef in self.coefs:
-            key = '/ev_'+str(event)+'/coef_'+str(coef)[2:]
-            dic[coef] = pd.read_hdf(self.filename, key)
+        for chain in self.chain:
+          dic[chain] = {}
+          for coef in self.coefs:
+            key = '/ev_'+str(event)+'/'+chain+'/coef_'+str(coef)[2:] # deleting 0. in the coef strings
+            dic[chain][coef] = pd.read_hdf(self.filename, key)
         dic['gen'] = pd.read_hdf(self.filename, '/ev_'+str(event)+'/gen_info')
         return dic
 
@@ -100,8 +105,9 @@ def process_event(self, dict_events, gen_info):
         self.ds_geom['sci'] = self.ds_geom['sci'].rename(columns=sci_update)
 
         for event in dict_events.keys():
-            print('Procesing event '+str(event))
-            dict_event = dict_events[event]
+          print('Procesing event '+str(event))
+          for chain in dict_events[event].keys():
+            dict_event = dict_events[event][chain]
             for coef in dict_event.keys():
                 silicon_df = pd.merge(left=dict_event[coef], right=self.ds_geom['si'], how='inner',
                                       on=['tc_layer', 'tc_wu', 'tc_wv', 'tc_cu', 'tc_cv'])
@@ -114,7 +120,7 @@ def process_event(self, dict_events, gen_info):
                 color_discrete   = common.colorscale(dict_event[coef], 'seed_idx', px.qualitative.Light24, True)
                 dict_event[coef] = dict_event[coef].assign(color_energy=color_continuous, color_clusters=color_discrete)
 
-            self.store_event('/ev_'+str(event)+'/coef_', dict_event)
+            self.store_event('/ev_'+str(event)+'/'+chain+'/coef_', dict_event)
             self.store_event('/ev_'+str(event)+'/gen_info', gen_info[gen_info.event == event])
 
 
@@ -126,21 +132,18 @@ def layout(self, **options):
                 html.Div([dbc.Checklist(options, [], inline=True, id='checkbox', switch=True)], style={"margin-left": "15px"}),
                 html.Div(id='slider-container', children=html.Div(id='out_slider', style={'width':'99%'}), style= {'display': 'block', 'width':'55%'}),
             ], style={'display': 'flex', 'flex-direction': 'row'}),
-
-            html.Div([
-                html.Div(["Threshold in [mip\u209C]: ", dcc.Input(id='mip', value=1, type='number', step=0.1)], style={'padding': 10}),
-                html.Div(["Select manually an event: ", dcc.Input(id='event', value=None, type='number')], style={'padding': 10, 'flex': 1}),
-            ], style={'display':'flex', 'flex-direction':'row'}),
-
+
             html.Div([
-                html.Div(["Select a particular clustering radius: "], style={'padding': 10}),
-                html.Div([dcc.Slider(self.coefs[0], self.coefs[-1], 0.004,value=self.coefs[-1],id='slider_cluster', included=False)], style={'width':'40%'}),
-            ], style={'display':'flex', 'flex-direction':'row'}),
-
+		html.Div([dcc.Markdown('Threshold in mip$_{\\text{T}}$: ', mathjax=True, style={'margin-left': '17px','margin-bottom': '-10px'}), dcc.Input(id='mip', value=1, type='number', step=0.1)], style={'padding': '5px', 'margin-bottom': '0px', 'margin-right': '0px', 'width': '23%'}),
+                html.Div([dcc.Markdown("Select manually an event: ", style={'margin-left': '5px','margin-bottom': '-10px'}), dcc.Input(id='event', value=None, type='number')], style={'padding': '5px', 'margin-right': '20px', 'margin-bottom': '-2px', 'width': '23%'}),
+                html.Div([dcc.Markdown("Select a reconstruction chain: ", style={'margin-left': '15px','margin-bottom': '-15px'}), dcc.Dropdown(id='chain', options=self.chain, value='default')], style={'padding': '5px', 'margin-right': '5px', 'margin-bottom': '0px', 'width': '24%'}),
+                html.Div([dcc.Markdown("Select the clustering radius: ", style={'margin-left': '25px','margin-bottom': '-10px'}), dcc.Slider(self.coefs[0], self.coefs[-1], 0.005, value=self.coefs[-1], id='slider_cluster', included=False)], style={'padding': '5px', 'margin-left': '20px', 'width': '30%'}),
+            ], style={'display':'flex','flex-direction':'row', 'marginRight': 250, 'marginLeft': 0}),
+
             html.Div([
                 dbc.Button(children='Random event', id='event-val', n_clicks=0),
                 dbc.Button(children='Submit selected event', id='submit-val', n_clicks=0, style={'display':'inline-block', "margin-left": "15px"}),
-                html.Div(id='event-display', style={'display':'inline-block', "margin-left": "15px"}), 
+                html.Div(children=[dcc.Markdown(mathjax=True, id='event-display')], style={'display':'inline-block', "margin-left": "15px"}), 
             ], style={'display':'inline-block', "margin-left": "15px"}),
 
             dcc.Loading(id="loading", children=[html.Div(dcc.Graph(id='plot'))], type="circle"),

bye_splits/scripts/run_radii_chain.py

@@ -23,44 +23,90 @@
 import argparse
 import pandas as pd
 
-def run_radii_chain(pars, particles, pu, coefs, event=None):
-    df_gen, df_cl, df_tc = get_data_reco_chain_start(nevents=30, reprocess=False, particles=particles, pu=pu, event=event)
-
-    fill_d = params.read_task_params("fill")
-    tasks.fill.fill(pars, df_gen, df_cl, df_tc, **fill_d)
-
-    smooth_d = params.read_task_params("smooth")
-    tasks.smooth.smooth(pars, **smooth_d)
-
-    seed_d = params.read_task_params("seed")
-    tasks.seed.seed(pars, **seed_d)
+with open(params.CfgPath, "r") as afile:
+    cfg = yaml.safe_load(afile)
 
-    cluster_d = params.read_task_params("cluster")
-    dict_cluster = {}
+def get_cluster_data(cluster_params, chain, coefs, pars):
+    cluster_data = {}
+
     for coef in coefs:
-        cluster_d["CoeffA"] = [coef, 0] * 50
-        nevents = tasks.cluster.cluster_default(pars, **cluster_d)
-        store_file = pd.HDFStore(common.fill_path(cluster_d["ClusterOutPlot"], **pars), mode='r')
+        cluster_params["CoeffA"] = [coef, 0] * 50
+        if chain == 'default': 
+            nevents = tasks.cluster.cluster_default(pars, **cluster_params)
+            store_file = pd.HDFStore(common.fill_path(cluster_params["ClusterOutPlot"], **pars), mode='r')
+        else:
+            nevents = tasks.cluster.cluster_cs(pars, **cluster_params)
+            store_file = pd.HDFStore(common.fill_path(cluster_params["ClusterOutPlotCS"], **pars), mode='r')
+
+        # Extract coefs and events
         df_list = []
         filtered_keys = [key for key in store_file.keys() if key.startswith('/df_')]
         events = [int(key.split('df_')[1]) for key in filtered_keys]
+
         for key in filtered_keys:
             df = store_file.get(key)
             df_list.append(df)
-        dict_cluster[str(coef)[2:]] = df_list
-        store_file.close() 
 
+        cluster_data[str(coef)[2:]] = df_list
+        store_file.close()
+
+    return cluster_data, events
+
+def merge_cluster_data_with_event(df_event_tc, cluster_data, index):
+    """ data processed from clustering step is merged with original data events
+        to get infomation about the non-clusterised TCs """
+    merged_data = {}
+
+    for coef in cluster_data.keys():
+        merged_data[coef] = pd.merge(
+            left=cluster_data[coef][index],
+            right=df_event_tc[~df_event_tc['tc_layer'].isin(cfg['selection']['disconnectedTriggerLayers'])],
+            on=['tc_wu', 'tc_wv', 'tc_cu', 'tc_cv', 'tc_layer'],
+            how='outer'
+        ).fillna(cluster_data[coef][index]['seed_idx'].max() + 1)
+
+    return merged_data
+
+def run_radii_chain(pars, particles, pu, coefs, event=None):
+    df_gen, df_cl, df_tc = get_data_reco_chain_start(nevents=30, reprocess=False, particles=particles, pu=pu, event=event)
+
+    # Default chain
+    fill_params = params.read_task_params("fill")
+    tasks.fill.fill(pars, df_gen, df_cl, df_tc, **fill_params)
+
+    smooth_params = params.read_task_params("smooth")
+    tasks.smooth.smooth(pars, **smooth_params)
+
+    seed_params = params.read_task_params("seed")
+    tasks.seed.seed(pars, **seed_params)
+
+    cluster_params = params.read_task_params("cluster")
+    cluster_data_default, _ = get_cluster_data(cluster_params, 'default', coefs, pars)
+
+    # Coarse seeding chain
+    fill_cs_params = params.read_task_params('cs')
+    tasks.coarse_seeding.coarse_seeding(pars, df_gen, df_cl, df_tc, **fill_cs_params)
+
+    seed_cs_params = params.read_task_params('seed_cs')
+    tasks.seed_cs.seed_cs(pars, **seed_cs_params)
+
+    cluster_params = params.read_task_params("cluster")
+    cluster_data_cs, events = get_cluster_data(cluster_params, 'cs', coefs, pars)
+
+    # Data unpacker
     dict_event = {}
+
     for index, ev in enumerate(events):
         dict_event[ev] = {}
-        df_event_tc = df_tc[df_tc.event == ev][['tc_mipPt','tc_eta','tc_wu','tc_wv','tc_cu','tc_cv','tc_layer']]
-        for coef in dict_cluster.keys():
-            dict_event[ev][coef] = pd.merge(left=dict_cluster[coef][index], 
-                                            right=df_event_tc[df_event_tc.tc_layer%2 != 0], 
-                                            on=['tc_wu', 'tc_wv', 'tc_cu', 'tc_cv', 'tc_layer'],
-                                            how='outer').fillna(dict_cluster[coef][index]['seed_idx'].max()+1) 
+        df_event_tc = df_tc[df_tc.event == ev][['tc_mipPt', 'tc_eta', 'tc_wu', 'tc_wv', 'tc_cu', 'tc_cv', 'tc_layer']]
+
+        # Merge cluster data with event data
+        dict_event[ev]['default'] = merge_cluster_data_with_event(df_event_tc, cluster_data_default, index)
+        dict_event[ev]['cs'] = merge_cluster_data_with_event(df_event_tc, cluster_data_cs, index)
+
     return dict_event, df_gen
 
+
 if __name__ == "__main__":
     FLAGS = parsing.parser_radii_chain()
     if FLAGS.no_valid:

config.yaml

@@ -125,15 +125,15 @@ varGeometry:
 
 3Ddisplay:
   PU0:
-    photons: 'photons_0PU_cluster.hdf5'
-    electrons: 'electrons_0PU_cluster.hdf5'
-    pions: 'pions_0PU_cluster.hdf5'
+    photons: 'photons_0PU_cluster'
+    electrons: 'electrons_0PU_cluster'
+    pions: 'pions_0PU_cluster'
   PU200:
-    photons: 'photons_200PU_cluster.hdf5'
-    electrons: 'electrons_200PU_cluster.hdf5'
-    pions: 'pions_200PU_cluster.hdf5'
+    photons: 'photons_200PU_cluster'
+    electrons: 'electrons_200PU_cluster'
+    pions: 'pions_200PU_cluster'
   reprocess: False
-  coefs: [0.006,0.010,0.014,0.018,0.022,0.026,0.030]
+  coefs: [0.010, 0.015, 0.020, 0.025]
 
 clusterStudies:
   localDir: /home/llr/cms/mchiusi/event_display/bye_splits/data/new_algos/