Adding support for scoreboard channel specific timeouts (#21)

docs/components.md

@@ -217,6 +217,91 @@ If, for any reason, you do not want a monitor to be attached to the scoreboard
 then you may provide the argument `scoreboard=False` to the `self.register(...)`
 call.
 
+### Scoreboard Channel Types
+
+When `self.register(...)` is called it will register the monitor against the
+scoreboard and queue captured transactions into a dedicated channel (unless
+`scoreboard=False` is set). If no other parameters are provided then the
+scoreboard will create a simpple channel.
+
+A simple channel will expect all transactions submitted to the reference queue
+to appear in the same order in the monitor's queue, and whenever a mismatch is
+detected it will flag an error.
+
+```python title="tb/testbench.py"
+from forastero import BaseBench, IORole
+from .stream import StreamIO, StreamMonitor
+
+class Testbench(BaseBench):
+    def __init__(self, dut) -> None:
+        super().__init__(dut, clk=dut.i_clk, rst=dut.i_rst)
+        stream_io = StreamIO(dut, "stream", IORole.INITIATOR)
+        self.register("stream_mon",
+                      StreamMonitor(self, stream_io, self.clk, self.rst))
+
+    def model_stream(self):
+        self.scoreboard.channels["stream_mon"].push_reference(StreamTransaction(...))
+```
+
+Another option is to register the monitor with multiple named scoreboard queues,
+thus creating a "funnel" channel. In this case each named queue of the channel
+must maintain order, but the scoreboard can pop entries from different queus in
+any order - this is great for blackbox verification of components like arbiters
+where the decision on which transaction is going to be taken may be influenced
+by many factors internal to the device.
+
+```python title="tb/testbench.py"
+from forastero import BaseBench, IORole
+from .stream import StreamIO, StreamMonitor
+
+class Testbench(BaseBench):
+    def __init__(self, dut) -> None:
+        super().__init__(dut, clk=dut.i_clk, rst=dut.i_rst)
+        stream_io = StreamIO(dut, "stream", IORole.INITIATOR)
+        self.register("arb_output_mon",
+                      StreamMonitor(self, stream_io, self.clk, self.rst),
+                      scoreboard_queues=["a", "b", "c"])
+
+    def model_arbiter_src_b(self):
+        self.scoreboard.channels["arb_output_mon"].push_reference(
+            "b", StreamTransaction(...)
+        )
+```
+
+### Scoreboard Channel Timeouts
+
+While each registered testcase can provide a timeout, this in many cases may be
+set at a very high time to cope with complex, long running testcases. To provide
+a finer granularity of timeout control, timeouts may be configured specifically
+to scoreboard channels that specify how long it is acceptable for a transaction
+to sit at the front of the monitor's queue before the scoreboard matches it to
+a reference transaction. There are two key parameters to `self.register(...)`:
+
+ * `scoreboard_timeout_ns` - the maximum acceptable age a transaction may be at
+   the front of the monitor's channel. The age is determined by substracting the
+   transaction's `timestamp` field (a default property of `BaseTransaction`)
+   from the current simulation time.
+
+ * `scoreboard_polling_ns` - the frequency with which to check the front of the
+   scoreboard's monitor queue, this defaults to `100 ns`.
+
+Due to the interaction of the polling timeout and polling period, transactions
+may live longer than the timeout in certain cases but this is bounded by a
+maximum delay of one polling interval.
+
+```python title="tb/testbench.py"
+from forastero import BaseBench, IORole
+from .stream import StreamIO, StreamMonitor
+
+class Testbench(BaseBench):
+    def __init__(self, dut) -> None:
+        super().__init__(dut, clk=dut.i_clk, rst=dut.i_rst)
+        stream_io = StreamIO(dut, "stream", IORole.INITIATOR)
+        self.register("stream_mon",
+                      StreamMonitor(self, stream_io, self.clk, self.rst),
+                      scoreboard_timeout_ns=10)
+```
+
 ## Logging
 
 Both `BaseDriver` and `BaseMonitor` inherit from `Component`, and this root class

forastero/bench.py

@@ -206,6 +206,8 @@ def register(
         scoreboard_verbose: bool = False,
         scoreboard_queues: list[str] | None = None,
         scoreboard_filter: Callable | None = None,
+        scoreboard_timeout_ns: int | None = None,
+        scoreboard_polling_ns: int = 100,
     ) -> Component | Coroutine:
         """
         Register a driver, monitor, or coroutine with the testbench. Drivers and
@@ -214,18 +216,25 @@ def register(
         the scoreboard unless explicitly requested. Coroutines must also be named
         and are required to complete before the test will shutdown.
 
-        :param name:               Name of the component or coroutine
-        :param comp_or_coro:       Component instance or coroutine
-        :param scoreboard:         Only applies to monitors, controls whether it
-                                   is registered with the scoreboard
-        :param scoreboard_verbose: Only applies to scoreboarded monitors,
-                                   controls whether to log each transaction,
-                                   even when they don't mismatch
-        :param scoreboard_queues:  A list of named queues used when a funnel
-                                   type scoreboard channel is required
-        :param scoreboard_filter:  A function that can filter or modify items
-                                   recorded by the monitor before they are passed
-                                   to the scoreboard
+        :param name:                  Name of the component or coroutine
+        :param comp_or_coro:          Component instance or coroutine
+        :param scoreboard:            Only applies to monitors, controls whether
+                                      it is registered with the scoreboard
+        :param scoreboard_verbose:    Only applies to scoreboarded monitors,
+                                      controls whether to log each transaction,
+                                      even when they don't mismatch
+        :param scoreboard_queues:     A list of named queues used when a funnel
+                                      type scoreboard channel is required
+        :param scoreboard_filter:     A function that can filter or modify items
+                                      recorded by the monitor before they are
+                                      passed to the scoreboard
+        :param scoreboard_timeout_ns: Optional timeout to allow for a object sat
+                                      at the front of the monitor queue to remain
+                                      unmatched (in nanoseconds, a value of None
+                                      disables the timeout mechanism)
+        :param scoreboard_polling_ns: How frequently to poll to check for unmatched
+                                      items stuck in the monitor queue in nanoseconds
+                                      (defaults to 100 ns)
         """
         assert isinstance(name, str), f"Name must be a string '{name}'"
         if asyncio.iscoroutine(comp_or_coro):
@@ -244,6 +253,8 @@ def register(
                     verbose=scoreboard_verbose,
                     filter_fn=scoreboard_filter,
                     queues=scoreboard_queues,
+                    timeout_ns=scoreboard_timeout_ns,
+                    polling_ns=scoreboard_polling_ns,
                 )
         else:
             raise TypeError(f"Unsupported object: {comp_or_coro}")

forastero/scoreboard.py

@@ -17,7 +17,8 @@
 
 import cocotb
 from cocotb.log import SimLog
-from cocotb.triggers import Event, First, Lock, RisingEdge
+from cocotb.triggers import Event, First, Lock, RisingEdge, Timer
+from cocotb.utils import get_sim_time
 
 from .monitor import BaseMonitor, MonitorEvent
 from .transaction import BaseTransaction
@@ -27,6 +28,10 @@ class QueueEmptyError(Exception):
     pass
 
 
+class ChannelTimeoutError(AssertionError):
+    pass
+
+
 class Queue:
     """
     A custom queue implementation that allows peeking onto the head of the queue,
@@ -92,19 +97,32 @@ class Channel:
     both queues and tested for equivalence. Any mismatches are reported to the
     scoreboard.
 
-    :param name:      Name of this scoreboard channel
-    :param monitor:   Handle to the monitor capturing traffic from the DUT
-    :param log:       Handle to the scoreboard log
-    :param filter_fn: Function to filter or modify captured transactions
+    :param name:       Name of this scoreboard channel
+    :param monitor:    Handle to the monitor capturing traffic from the DUT
+    :param log:        Handle to the scoreboard log
+    :param filter_fn:  Function to filter or modify captured transactions
+    :param timeout_ns: Optional timeout to allow for a object sat at the front
+                       of the monitor queue to remain unmatched (in nanoseconds,
+                       a value of None disables the timeout mechanism)
+    :param polling_ns: How frequently to poll to check for unmatched items stuck
+                       in the monitor queue in nanoseconds (defaults to 100 ns)
     """
 
     def __init__(
-        self, name: str, monitor: BaseMonitor, log: SimLog, filter_fn: Callable | None
+        self,
+        name: str,
+        monitor: BaseMonitor,
+        log: SimLog,
+        filter_fn: Callable | None,
+        timeout_ns: int | None = None,
+        polling_ns: int = 100,
     ) -> None:
         self.name = name
         self.monitor = monitor
         self.log = log
         self.filter_fn = filter_fn
+        self.timeout_ns = timeout_ns
+        self.polling_ns = polling_ns
         self._q_mon = Queue()
         self._q_ref = Queue()
         self._lock = Lock()
@@ -121,6 +139,7 @@ def _sample(mon: BaseMonitor, evt: MonitorEvent, obj: BaseTransaction) -> None:
                     self.push_monitor(obj)
 
         self.monitor.subscribe(MonitorEvent.CAPTURE, _sample)
+        cocotb.start_soon(self._polling())
 
     @property
     def monitor_depth(self) -> int:
@@ -185,6 +204,31 @@ async def _dequeue(self) -> tuple[BaseTransaction, BaseTransaction]:
         # Return monitor-reference pair (don't release lock yet)
         return next_mon, next_ref
 
+    async def _polling(self) -> None:
+        """
+        Polling loop that checks for items getting stuck in the channel's
+        monitor queue
+        """
+        while True:
+            # Wait for polling delay
+            await Timer(self.polling_ns, units="ns")
+            # Check for object at the front of the monitor queue
+            if (
+                (self.timeout_ns is not None)
+                and (self._q_mon.level > 0)
+                and (
+                    (age := (get_sim_time(units="ns") - self._q_mon.peek().timestamp))
+                    > self.timeout_ns
+                )
+            ):
+                self.log.error(
+                    f"Object at the front of the of the {self.name} monitor "
+                    f"queue has been stalled for {age} ns which exceeds the "
+                    f"configured timeout of {self.timeout_ns} ns"
+                )
+                self.report()
+                raise ChannelTimeoutError(f"Channel {self.name} timed out")
+
     async def loop(self, mismatch: Callable, match: Callable | None = None) -> None:
         """
         Continuously dequeue pairs of transactions from the monitor and reference
@@ -256,6 +300,11 @@ class FunnelChannel(Channel):
     :param log:        Handle to the scoreboard log
     :param filter_fn:  Function to filter or modify captured transactions
     :param ref_queues: List of reference queue names
+    :param timeout_ns: Optional timeout to allow for a object sat at the front
+                       of the monitor queue to remain unmatched (in nanoseconds,
+                       a value of None disables the timeout mechanism)
+    :param polling_ns: How frequently to poll to check for unmatched items stuck
+                       in the monitor queue in nanoseconds (defaults to 100 ns)
     """
 
     def __init__(
@@ -265,8 +314,12 @@ def __init__(
         log: SimLog,
         filter_fn: Callable | None,
         ref_queues: list[str],
+        timeout_ns: int | None = None,
+        polling_ns: int = 100,
     ) -> None:
-        super().__init__(name, monitor, log, filter_fn)
+        super().__init__(
+            name, monitor, log, filter_fn, timeout_ns=timeout_ns, polling_ns=polling_ns
+        )
         self._q_ref = {x: Queue() for x in ref_queues}
 
     @property
@@ -387,17 +440,24 @@ def attach(
         verbose=False,
         filter_fn: Callable | None = None,
         queues: list[str] | None = None,
+        timeout_ns: int | None = None,
+        polling_ns: int = 100,
     ) -> None:
         """
         Attach a monitor to the scoreboard, creating and scheduling a new
         channel in the process.
 
-        :param monitor:   The monitor to attach
-        :param verbose:   Whether to tabulate matches as well as mismatches
-        :param queues:    List of reference queue names, this causes a funnel
-                          type scoreboard channel to be used
-        :param filter_fn: A filter function that can either drop or modify items
-                          recorded by the monitor prior to scoreboarding
+        :param monitor:    The monitor to attach
+        :param verbose:    Whether to tabulate matches as well as mismatches
+        :param queues:     List of reference queue names, this causes a funnel
+                           type scoreboard channel to be used
+        :param filter_fn:  A filter function that can either drop or modify items
+                           recorded by the monitor prior to scoreboarding
+        :param timeout_ns: Optional timeout to allow for a object sat at the front
+                           of the monitor queue to remain unmatched (in nanoseconds,
+                           a value of None disables the timeout mechanism)
+        :param polling_ns: How frequently to poll to check for unmatched items stuck
+                           in the monitor queue in nanoseconds (defaults to 100 ns)
         """
         assert monitor.name not in self.channels, f"Monitor known for '{monitor.name}'"
         if isinstance(queues, list) and len(queues) > 0:
@@ -407,13 +467,17 @@ def attach(
                 self.tb.fork_log("channel", monitor.name),
                 filter_fn,
                 queues,
+                timeout_ns=timeout_ns,
+                polling_ns=polling_ns,
             )
         else:
             channel = Channel(
                 monitor.name,
                 monitor,
                 self.tb.fork_log("channel", monitor.name),
                 filter_fn,
+                timeout_ns=timeout_ns,
+                polling_ns=polling_ns,
             )
         self.channels[channel.name] = channel
         if verbose: