Merge pull request #165 from zhujun98/move_definition_of_meta_source

Move definition of meta source from config to SourceCatalog

extra_foam/config.py

@@ -389,13 +389,6 @@ def __init__(self):
         self.control_sources = dict()
         # pipeline data sources listed in the DataSourceWidget
         self.pipeline_sources = dict()
-        # meta data sources which is not listed in the DataSourceWidget but
-        # is used in statistics analysis
-        self.meta_sources = {
-            "Metadata": {
-                "META": ["timestamp.tid"],
-            }
-        }
 
         self.appendix_streamers = []
 
@@ -554,10 +547,6 @@ def control_sources(self):
     def pipeline_sources(self):
         return self._data.pipeline_sources
 
-    @property
-    def meta_sources(self):
-        return self._data.meta_sources
-
     @property
     def appendix_streamers(self):
         return self._data.appendix_streamers

extra_foam/database/data_source.py

@@ -29,6 +29,10 @@ class SourceCatalog(abc.Collection):
 
     Served as a catalog for searching data sources.
     """
+
+    TRAIN_ID = "META timestamp.tid"
+    _meta = (TRAIN_ID,)
+
     def __init__(self):
         super().__init__()
 
@@ -40,9 +44,9 @@ def __init__(self):
         self._main_detector_category = config["DETECTOR"]
         self._main_detector = ''
 
-    def __contains__(self, item):
+    def __contains__(self, src):
         """Override."""
-        return self._items.__contains__(item)
+        return self._items.__contains__(src) or self._meta.__contains__(src)
 
     def __len__(self):
         """Override."""

extra_foam/database/tests/test_datasource.py

@@ -14,6 +14,8 @@ def testGeneral(self):
         catalog = SourceCatalog()
         item = SourceItem('DSSC', 'dssc_device_id', [], 'image.data', None, None)
 
+        self.assertIn("META timestamp.tid", catalog)
+
         src = f"{item.name} {item.property}"
         catalog.add_item(item)
         self.assertEqual(src, catalog.main_detector)

extra_foam/gui/ctrl_widgets/bin_ctrl_widget.py

@@ -21,6 +21,7 @@
 from ..gui_helpers import invert_dict
 from ...config import AnalysisType, BinMode, config
 from ...database import Metadata as mt
+from ...database import SourceCatalog
 
 
 _N_PARAMS = 2
@@ -50,6 +51,7 @@ class BinCtrlWidget(_AbstractGroupBoxCtrlWidget):
     _user_defined_key = config["SOURCE_USER_DEFINED_CATEGORY"]
 
     _UNDEFINED_CATEGORY = ''
+    _META_CATEGORY = 'Metadata'
 
     def __init__(self, *args, **kwargs):
         super().__init__("Binning setup", *args, **kwargs)
@@ -67,7 +69,12 @@ def __init__(self, *args, **kwargs):
         self._auto_level_btn = QPushButton("Auto level")
 
         self._src_instrument = config.control_sources
-        self._src_metadata = config.meta_sources
+        tid_key_split = SourceCatalog.TRAIN_ID.split(" ")
+        self._src_metadata = {
+            self._META_CATEGORY: {
+                tid_key_split[0]: [tid_key_split[1]],
+            }
+        }
 
         self.initUI()
         self.initConnections()
@@ -199,7 +206,7 @@ def onCategoryChange(self, i_row, category):
 
             self.onBinParamChangeLe(i_row)
         else:
-            srcs = self._src_metadata if category in config.meta_sources \
+            srcs = self._src_metadata if category in self._src_metadata \
                 else self._src_instrument
             category_srcs = srcs.get(category, dict())
             device_id_cb = QComboBox()

extra_foam/gui/ctrl_widgets/correlation_ctrl_widget.py

@@ -22,6 +22,7 @@
 from ..gui_helpers import invert_dict
 from ...config import AnalysisType, config
 from ...database import Metadata as mt
+from ...database import SourceCatalog
 
 _N_PARAMS = 2  # maximum number of correlated parameters
 _DEFAULT_RESOLUTION = 0.0
@@ -42,6 +43,7 @@ class CorrelationCtrlWidget(_AbstractGroupBoxCtrlWidget):
     _user_defined_key = config["SOURCE_USER_DEFINED_CATEGORY"]
 
     _UNDEFINED_CATEGORY = ''
+    _META_CATEGORY = 'Metadata'
 
     def __init__(self, *args, **kwargs):
         super().__init__("Correlation setup", *args, **kwargs)
@@ -55,8 +57,12 @@ def __init__(self, *args, **kwargs):
         self._table = QTableWidget()
 
         self._src_instrument = config.control_sources
-        self._src_metadata = config.meta_sources
-
+        tid_key_split = SourceCatalog.TRAIN_ID.split(" ")
+        self._src_metadata = {
+            self._META_CATEGORY: {
+                tid_key_split[0]: [tid_key_split[1]],
+            }
+        }
         self.initUI()
         self.initConnections()
 
@@ -165,7 +171,7 @@ def onCategoryChange(self, i_row, category):
 
             self.onCorrelationParamChangeLe(i_row)
         else:
-            srcs = self._src_metadata if category in config.meta_sources \
+            srcs = self._src_metadata if category in self._src_metadata \
                 else self._src_instrument
             category_srcs = srcs.get(category, dict())
             device_id_cb = QComboBox()

extra_foam/pipeline/processors/image_assembler.py

@@ -17,12 +17,14 @@
 from .base_processor import _RedisParserMixin
 from ..exceptions import AssemblingError
 from ...config import config, GeomAssembler, DataSource
+from ...database import SourceCatalog
 from ...geometries import load_geometry
 from ...ipc import process_logger as logger
 
 
 _IMAGE_DTYPE = config['SOURCE_PROC_IMAGE_DTYPE']
 _RAW_IMAGE_DTYPE = config['SOURCE_RAW_IMAGE_DTYPE']
+_TRAIN_ID = SourceCatalog.TRAIN_ID
 
 
 class ImageAssemblerFactory(ABC):
@@ -238,7 +240,7 @@ def process(self, data):
             }
             # Assign the global train ID once the main detector was
             # successfully assembled.
-            raw["META timestamp.tid"] = meta[src]["tid"]
+            raw[_TRAIN_ID] = meta[src]["tid"]
 
     class AgipdImageAssembler(BaseAssembler):
         def _get_modules_bridge(self, data, src):