DM-39852: Have TMA events know their block numbers

python/lsst/summit/utils/tmaUtils.py

@@ -33,7 +33,8 @@
 import matplotlib.pyplot as plt
 from lsst.utils.iteration import ensure_iterable
 
-from .enums import ScriptState, AxisMotionState, PowerState
+from .enums import AxisMotionState, PowerState
+from .blockUtils import BlockParser
 from .utils import getCurrentDayObs_int, dayObsIntToString
 from .efdUtils import (getEfdData,
                        makeEfdClient,
@@ -42,7 +43,6 @@
                        getDayObsForTime,
                        getDayObsStartTime,
                        getDayObsEndTime,
-                       clipDataToEvent,
                        )
 
 __all__ = (
@@ -382,77 +382,7 @@ def _initializeTma(tma):
     tma._parts['elevationSystemState'] = PowerState.ON
 
 
-@dataclass(slots=True, kw_only=True, frozen=True)
-class BlockInfo:
-    """The block info relating to a TMAEvent.
-
-    Parameters
-    ----------
-    blockNumber : `int`
-        The block number, as an integer.
-    blockId : `str`
-        The block ID, as a string.
-    salIndices : `list` of `int`
-        One or more SAL indices, relating to the block.
-    tickets : `list` of `str`
-        One or more SITCOM tickets, relating to the block.
-    states : `list` of `ScriptStatePoint`
-        The states of the script during the block. Each element is a
-        ``ScriptStatePoint`` which contains:
-            - the time, as an astropy.time.Time
-            - the state, as a ``ScriptState`` enum
-            - the reason for state change, as a string
-    """
-    blockNumber: int
-    blockId: str
-    salIndices: int
-    tickets: list
-    states: list
-
-    def __repr__(self):
-        return (
-            f"BlockInfo(blockNumber={self.blockNumber}, blockId={self.blockId}, salIndices={self.salIndices},"
-            f" tickets={self.tickets}, states={self.states!r}"
-        )
-
-    def _ipython_display_(self):
-        print(self.__str__())
-
-    def __str__(self):
-        # You can't put the characters '\n' directly into the evaluated part of
-        # an f-string i.e. inside the {} part, until py 3.12, so this must go
-        # in via a variable until then.
-        newline = '  \n'
-
-        return (
-            f"blockNumber: {self.blockNumber}\n"
-            f"blockId: {self.blockId}\n"
-            f"salIndices: {self.salIndices}\n"
-            f"tickets: {self.tickets}\n"
-            f"states: \n{newline.join([str(state) for state in self.states])}"
-        )
-
-
-@dataclass(slots=True, kw_only=True, frozen=True)
-class ScriptStatePoint:
-    time: Time
-    state: ScriptState
-    reason: str
-
-    def __repr__(self):
-        return (
-            f"ScriptStatePoint(time={self.time!r}, state={self.state!r}, reason={self.reason!r})"
-        )
-
-    def _ipython_display_(self):
-        print(self.__str__())
-
-    def __str__(self):
-        reasonStr = f" - {self.reason}" if self.reason else ""
-        return (f"{self.state.name:>10} @ {self.time.isot}{reasonStr}")
-
-
-@dataclass(slots=True, kw_only=True, frozen=True)
+@dataclass(kw_only=True, frozen=True)
 class TMAEvent:
     """A movement event for the TMA.
 
@@ -497,8 +427,9 @@ class TMAEvent:
         The time the event began.
     end : `astropy.time.Time`
         The time the event ended.
-    blockInfo : `lsst.summit.utils.tmaUtils.BlockInfo`
-        The block info relating to the event.
+    blockInfos : `list` of `lsst.summit.utils.tmaUtils.BlockInfo`, or `None`
+        The block infomation, if any, relating to the event. Could be `None`,
+        or one or more block informations.
     version : `int`
         The version of the TMAEvent class. Equality between events is only
         valid for a given version of the class. If the class definition
@@ -516,7 +447,7 @@ class TMAEvent:
     duration: float  # seconds
     begin: Time
     end: Time
-    blockInfo: BlockInfo = None
+    blockInfos: list = None
     version: int = 0  # update this number any time a code change which could change event definitions is made
     _startRow: int
     _endRow: int
@@ -543,10 +474,22 @@ def _ipython_display_(self):
         print(self.__str__())
 
     def __str__(self):
+        def indent(string):
+            return '\n' + '\n'.join(['    ' + s for s in string.splitlines()])
+
+        blockInfoStr = 'None'
+        if self.blockInfos is not None:
+            blockInfoStr = ''.join(indent(str(i)) for i in self.blockInfos)
+
         return (
-            f"dayObs: {self.dayObs}\nseqNum: {self.seqNum}\ntype: {self.type.name}"
-            f"\nendReason: {self.endReason.name}\nduration: {self.duration}\nbegin: {self.begin!r},"
-            f"\nend: {self.end!r}"
+            f"dayObs: {self.dayObs}\n"
+            f"seqNum: {self.seqNum}\n"
+            f"type: {self.type.name}\n"
+            f"endReason: {self.endReason.name}\n"
+            f"duration: {self.duration}\n"
+            f"begin: {self.begin!r}\n"
+            f"end: {self.end!r}\n"
+            f"blockInfos: {blockInfoStr}"
         )
 
 
@@ -1047,6 +990,16 @@ def _mergeData(self, data):
         if len(merged) != originalRowCounter:
             self.log.warning("Merged data has a different number of rows to the original data, some"
                              " timestamps (rows) will contain more than one piece of actual information.")
+
+        # if the index is still a DatetimeIndex here then we didn't actually
+        # merge any data, so there is only data from a single component.
+        # This is likely to result in no events, but not necessarily, and for
+        # generality, instead we convert to a range index to ensure consistency
+        # in the returned data, and allow processing to continue.
+        if isinstance(merged.index, pd.DatetimeIndex):
+            self.log.warning("Data was only found for a single component in the EFD.")
+            merged.reset_index(drop=True, inplace=True)
+
         return merged
 
     def getEvents(self, dayObs):
@@ -1214,7 +1167,7 @@ def _calculateEventsFromMergedData(self, data, dayObs, dataIsForCurrentDay):
 
         stateTuples = self._statesToEventTuples(tmaStates, dataIsForCurrentDay)
         events = self._makeEventsFromStateTuples(stateTuples, dayObs, data)
-        self.addBlockDataToEvents(events)
+        self.addBlockDataToEvents(dayObs, events)
         return events
 
     def _statesToEventTuples(self, states, dataIsForCurrentDay):
@@ -1318,7 +1271,7 @@ def _statesToEventTuples(self, states, dataIsForCurrentDay):
 
         return parsedStates
 
-    def addBlockDataToEvents(self, events):
+    def addBlockDataToEvents(self, dayObs, events):
         """Find all the block data in the EFD for the specified events.
 
         Finds all the block data in the EFD relating to the events, parses it,
@@ -1330,104 +1283,29 @@ def addBlockDataToEvents(self, events):
                  `list` of `lsst.summit.utils.tmaUtils.TMAEvent`
             One or more events to get the block data for.
         """
-        events = ensure_iterable(events)
-        events = sorted(events)
-
-        # Get all the data in one go and then clip to the events in the loop.
-        # This is orders of magnitude faster than querying individually.
-        allData = getEfdData(self.client,
-                             "lsst.sal.Script.logevent_state",
-                             begin=events[0].begin,  # time ordered, so this is the start of the window
-                             end=events[-1].end,  # and this is the end
-                             warn=False)
-        if allData.empty:
-            self.log.info('No block data found for the specified events')
-            return {}
-
-        blockPattern = r"BLOCK-(\d+)"
-        blockIdPattern = r"BL\d+(?:_\w+)+"
-        sitcomPattern = r"SITCOM-(\d+)"
-
-        for event in events:
-            eventData = clipDataToEvent(allData, event)
-            if eventData.empty:
-                continue
-
-            blockNums = set()
-            blockIds = set()
-            tickets = set()
-            salIndices = set()
-            stateList = []
-
-            # for each for in the data which corresponds to the event, extract
-            # the block number, block id, sitcom tickets, sal index and state
-            # some may have multiple values, some may be None, so collect in
-            # sets and then remove None, and validate on ones which must not
-            # contain duplicate values
-            for rowNum, row in eventData.iterrows():
-                # the lastCheckpoint column contains the block number, blockId,
-                # and any sitcom tickets.
-                rowStr = row['lastCheckpoint']
-
-                blockMatch = re.search(blockPattern, rowStr)
-                blockNumber = int(blockMatch.group(1)) if blockMatch else None
-                blockNums.add(blockNumber)
-
-                blockIdMatch = re.search(blockIdPattern, rowStr)
-                blockId = blockIdMatch.group(0) if blockIdMatch else None
-                blockIds.add(blockId)
-
-                sitcomMatches = re.findall(sitcomPattern, rowStr)
-                sitcomTicketNumbers = [int(match) for match in sitcomMatches]
-                tickets.update(sitcomTicketNumbers)
-
-                salIndices.add(row['salIndex'])
-
-                state = row['state']
-                state = ScriptState(state)  # cast this back to its native enum
-                stateReason = row['reason']  # might be empty, might contain useful error messages
-                stateTimestamp = efdTimestampToAstropy(row['private_efdStamp'])
-                scriptStatePoint = ScriptStatePoint(time=stateTimestamp,
-                                                    state=state,
-                                                    reason=stateReason)
-                stateList.append(scriptStatePoint)
-
-            # remove all the Nones from the sets, and then check the lengths
-            for fieldSet in (blockNums, blockIds, salIndices):
-                if None in fieldSet:
-                    fieldSet.remove(None)
-
-            # if we didn't find any block numbers at all, that is fine, just
-            # continue as this event doesn't relate to a BLOCK
-            if not blockNums:
-                continue
-
-            # but if it does related to a BLOCK then it must not have more than
-            # one. If this is the case something is wrong with a SAL script, so
-            # raise here to indicate the something needs debugging in the
-            # scriptQueue or something like that.
-            if len(blockNums) > 1:
-                raise RuntimeError(f"Found multiple BLOCK values ({blockNums}) for {event}")
-            blockNumber = blockNums.pop()
-
-            # likewise for the blockIds
-            if len(blockIds) > 1:
-                raise RuntimeError(f"Found multiple blockIds ({blockIds}) for {event}")
-            blockId = blockIds.pop()
-
-            blockInfo = BlockInfo(
-                blockNumber=blockNumber,
-                blockId=blockId,
-                salIndices=sorted([i for i in salIndices]),
-                tickets=[f'SITCOM-{ticket}' for ticket in tickets],
-                states=stateList,
-            )
-
-            # Add the blockInfo to the TMAEvent. Because this is a frozen
-            # dataclass, use object.__setattr__ to set the attribute. This is
-            # the correct way to set a frozen dataclass attribute after
-            # creation.
-            object.__setattr__(event, 'blockInfo', blockInfo)
+        blockParser = BlockParser(dayObs, client=self.client)
+        blocks = blockParser.getBlockNums()
+        blockDict = {}
+        for block in blocks:
+            blockDict[block] = blockParser.getSeqNums(block)
+
+        for block, seqNums in blockDict.items():
+            for seqNum in seqNums:
+                blockInfo = blockParser.getBlockInfo(block=block, seqNum=seqNum)
+
+                relatedEvents = blockParser.getEventsForBlock(events, block=block, seqNum=seqNum)
+                for event in relatedEvents:
+                    toSet = [blockInfo]
+                    if event.blockInfos is not None:
+                        existingInfo = event.blockInfos
+                        existingInfo.append(blockInfo)
+                        toSet = existingInfo
+
+                    # Add the blockInfo to the TMAEvent. Because this is a
+                    # frozen dataclass, use object.__setattr__ to set the
+                    # attribute. This is the correct way to set a frozen
+                    # dataclass attribute after creation.
+                    object.__setattr__(event, 'blockInfos', toSet)
 
     def _makeEventsFromStateTuples(self, states, dayObs, data):
         """For the list of state-tuples, create a list of ``TMAEvent`` objects.
@@ -1466,7 +1344,7 @@ def _makeEventsFromStateTuples(self, states, dayObs, data):
                 duration=duration,
                 begin=beginAstropy,
                 end=endAstropy,
-                blockInfo=None,  # this is added later
+                blockInfos=None,  # this is added later
                 _startRow=parsedState.eventStart,
                 _endRow=parsedState.eventEnd,
             )

tests/data/blockInfoData.json

@@ -0,0 +1 @@
+{"6||1": "BL6_O_20230615_000001||1686872864.4860544||1686873078.9402378||[100017]||['SITCOM-906']||6", "6||2": "BL6_O_20230615_000002||1686873218.2808762||1686873338.3378706||[100018]||['SITCOM-906']||5", "6||3": "BL6_O_20230615_000003||1686873973.9807594||1686876507.2946963||[100019]||['SITCOM-906']||7", "7||1": "BL7_O_20230615_000001||1686877986.0514321||1686879969.7464483||[100020]||['SITCOM-901', 'SITCOM-903', 'SITCOM-904']||6", "7||2": "BL7_O_20230615_000002||1686880781.7232163||1686884505.9864519||[100022]||['SITCOM-901', 'SITCOM-903', 'SITCOM-904']||6", "8||1": "BL8_O_20230615_000001||1686891087.1351101||1686897877.212973||[100025]||['SITCOM-903', 'SITCOM-904']||15", "8||2": "BL8_O_20230615_000002||1686900229.4312751||1686906978.6924145||[100026]||['SITCOM-903', 'SITCOM-904']||15", "8||3": "BL8_O_20230615_000003||1686910489.00772||1686914113.4434297||[100029]||['SITCOM-903', 'SITCOM-904']||11", "9||1": "BL9_O_20230615_000001||1686907100.0435104||1686908617.485946||[100027]||['SITCOM-903']||14", "9||2": "BL9_O_20230615_000002||1686908659.632222||1686910277.8725371||[100028]||['SITCOM-903']||14", "11||1": "BL11_O_20230615_000001||1686887770.7858129||1686890795.5148818||[100024]||['SITCOM-904']||7", "12||1": "BL12_O_20230615_000001||1686884595.1160328||1686887680.5445957||[100023]||['SITCOM-904', 'SITCOM-905']||7"}