Merge pull request #17 from lucabaldini/metadata

Add metadata to output files

docs/api.rst

@@ -21,7 +21,7 @@ List of modules
    digi
    display
    hexagon
-   io
+   fileio
    mc
    pprint
    recon

docs/io.rst → docs/fileio.rst

@@ -1,5 +1,5 @@
-:mod:`hexsample.io` --- Input/Output
-====================================
+:mod:`hexsample.fileio` --- Input/Output
+========================================
 
 This module contains all the I/O related facilities, that is, the basic definition
 of the file format and the proper classes to create and read back data files.
@@ -19,6 +19,26 @@ of the file format and the proper classes to create and read back data files.
    the package that might see drastic changes in the future.
 
 
+Base classes
+------------
+
+The module provides base classes for input and output files, than are then
+specialized for, e.g., digitized or reconstructed data.
+
+:class:`OutputFileBase <hexsample.fileio.OutputFileBase>` is the base class for
+output files. Instantiating an object of this class causes the output file to
+be opened in write mode, and a ``header`` node to be created that can be
+used to store arbitrary user attributes---typically to keep track of the
+simulation or reconstruction settings. The file header can bu updated via the
+:meth:`update_header() <hexsample.fileio.OutputFileBase.update_header>` call, which
+loops over the keyword arguments and set the attributes one at a time.
+
+:class:`InputFileBase <hexsample.fileio.InputFileBase>` is the base class for
+input files. Instantiating an object of this class causes the input file to be
+opened in read mode and the header information to be rebuilt in the form of a
+dictionary.
+
+
 Digitized data
 --------------
 
@@ -27,7 +47,7 @@ would ordinarily be written out by the DAQ, i.e., the trigger identifier, the
 timestamp, and all the quantities that are necessary in order to uniquely identify
 the region of interest:
 
-.. literalinclude:: ../hexsample/io.py
+.. literalinclude:: ../hexsample/fileio.py
    :pyobject: DigiDescription
 
 In addition, the PHA content of the ROI (which is a variable-length array by its
@@ -37,7 +57,7 @@ group holding the digi table.
 For simulated data, digitized files contain an additional table encapsulating the
 ground truth information for the event.
 
-.. literalinclude:: ../hexsample/io.py
+.. literalinclude:: ../hexsample/fileio.py
    :pyobject: MonteCarloDescription
 
 
@@ -46,4 +66,4 @@ ground truth information for the event.
 Module documentation
 --------------------
 
-.. automodule:: hexsample.io
+.. automodule:: hexsample.fileio

docs/release.rst

@@ -5,11 +5,19 @@ Release notes
 
 * Merging https://github.com/lucabaldini/hexsample/pull/11
 * Merging https://github.com/lucabaldini/hexsample/pull/10
+* Merging https://github.com/lucabaldini/hexsample/pull/17
+* Casting the outputfile default argument to string in ArgumentParser in order
+  to avoid possible problems downstream with patlib.Path instances.
+* mc option removed from output digi and recon files.
+* Base classes for input and output files added, and machinery for adding
+  and retrieving metadata information to/from file headers added.
+* io module renamed as fileio
 * Added protection against mistyped parameter names in pipeline calls.
 * uncertainties added as a requirement.
 * PlotCard class completely refactored.
 * Updating the hxview script.
 * Issue(s) closed:
+      * https://github.com/lucabaldini/hexsample/issues/14
       * https://github.com/lucabaldini/hexsample/issues/15
 
 

hexsample/app.py

@@ -100,9 +100,14 @@ def add_numevents(self, default : int) -> None:
 
     def add_outfile(self, default : str) -> None:
         """Add an option for the output file.
+
+        Note that we cast the default to a string---this prevents having
+        pathlib.Path instances around, which would then needed to be handled
+        properly in specific places (such as adding metadata to the output HDF5
+        file headers).
         """
         help = 'path to the output file'
-        self.add_argument('--outfile', '-o', type=str, default=default, help=help)
+        self.add_argument('--outfile', '-o', type=str, default=str(default), help=help)
 
     def add_suffix(self, default : str) -> None:
         """Add an option for the output suffix.

hexsample/bin/hxdisplay.py

@@ -25,7 +25,7 @@
 from hexsample.app import ArgumentParser
 from hexsample.digi import Xpol3
 from hexsample.display import HexagonalGridDisplay
-from hexsample.io import DigiInputFile
+from hexsample.fileio import DigiInputFile
 from hexsample.plot import plt
 
 

hexsample/bin/hxrecon.py

@@ -27,7 +27,7 @@
 from hexsample.app import ArgumentParser, check_required_args
 from hexsample.clustering import ClusteringNN
 from hexsample.digi import Xpol3
-from hexsample.io import DigiInputFile, ReconOutputFile
+from hexsample.fileio import DigiInputFile, ReconOutputFile
 from hexsample.recon import ReconEvent
 
 
@@ -54,7 +54,8 @@ def hxrecon(**kwargs):
     input_file = DigiInputFile(input_file_path)
     suffix = kwargs['suffix']
     output_file_path = input_file_path.replace('.h5', f'_{suffix}.h5')
-    output_file = ReconOutputFile(output_file_path, mc=True)
+    output_file = ReconOutputFile(output_file_path)
+    output_file.update_header(**kwargs)
     for i, event in tqdm(enumerate(input_file)):
         cluster = clustering.run(event)
         args = event.trigger_id, event.timestamp(), event.livetime, event.roi.size, cluster

hexsample/bin/hxsim.py

@@ -28,7 +28,7 @@
 from hexsample import HEXSAMPLE_DATA
 from hexsample.app import ArgumentParser
 from hexsample.digi import Xpol3
-from hexsample.io import DigiOutputFile
+from hexsample.fileio import DigiOutputFile
 from hexsample.mc import PhotonList
 from hexsample.roi import Padding
 from hexsample.source import LineForest, GaussianBeam, Source
@@ -60,7 +60,8 @@ def hxsim(**kwargs):
     photon_list = PhotonList(source, sensor, kwargs['numevents'])
     readout = Xpol3(kwargs['noise'], kwargs['gain'])
     output_file_path = kwargs.get('outfile')
-    output_file = DigiOutputFile(output_file_path, mc=True)
+    output_file = DigiOutputFile(output_file_path)
+    output_file.update_header(**kwargs)
     padding = Padding(*kwargs['padding'])
     readout_args = kwargs['trgthreshold'], padding, kwargs['zsupthreshold'], kwargs['offset']
     logger.info('Starting the event loop...')
@@ -71,8 +72,7 @@ def hxsim(**kwargs):
     logger.info('Done!')
     output_file.flush()
     output_file.close()
-    # Cast the output_file_path to string, in case it happens to be a pathlib.Path object.
-    return str(output_file_path)
+    return output_file_path
 
 
 

hexsample/bin/hxview.py

@@ -27,7 +27,7 @@
 from hexsample.app import ArgumentParser
 from hexsample.fitting import fit_histogram
 from hexsample.hist import Histogram1d
-from hexsample.io import ReconInputFile
+from hexsample.fileio import ReconInputFile
 from hexsample.modeling import Gaussian
 from hexsample.plot import plt