Skip to content

Commit

Permalink
Improve memory requirements of mempool_monitor.py
Browse files Browse the repository at this point in the history
Because the demo script was never intended to be run for long periods of
time, the script simply collected *all* event timestamps in lists for
the calculation of the event count and rate metrics. This results in
increasing memory and CPU usage over time.

The problem is resolved by explicitly keeping count of events older than
then minutes and removing the corresponding timestamps from lists.
  • Loading branch information
virtu committed Mar 20, 2023
1 parent 4684aa3 commit 129d70e
Showing 1 changed file with 11 additions and 2 deletions.
13 changes: 11 additions & 2 deletions contrib/tracing/mempool_monitor.py
Original file line number Diff line number Diff line change
Expand Up @@ -182,6 +182,7 @@ def __init__(self, screen):
self._screen = screen
self._time_started = datetime.now(timezone.utc)
self._timestamps = {"added": [], "removed": [], "rejected": [], "replaced": []}
self._event_history = {"added": 0, "removed": 0, "rejected": 0, "replaced": 0}
self._init_windows()

def _init_windows(self):
Expand Down Expand Up @@ -263,10 +264,18 @@ def calculate_metrics(self, events):
for event_ts, event_type, event_data in events:
self._timestamps[event_type].append(event_ts)
for event_type, ts in self._timestamps.items():
# remove timestamps older than ten minutes but keep track of their
# count for the 'total' metric
#
self._event_history[event_type] += len(
[t for t in ts if Dashboard.timestamp_age(t) >= 600]
)
ts = [t for t in ts if Dashboard.timestamp_age(t) < 600]
self._timestamps[event_type] = ts
# count metric
count_1m = len([t for t in ts if Dashboard.timestamp_age(t) < 60])
count_10m = len([t for t in ts if Dashboard.timestamp_age(t) < 600])
count_total = len(ts)
count_10m = len(ts)
count_total = self._event_history[event_type] + len(ts)
count[event_type] = (count_total, count_1m, count_10m)
# rate metric
runtime = Dashboard.timestamp_age(self._time_started)
Expand Down

0 comments on commit 129d70e

Please sign in to comment.