diff --git a/src/middlewared/middlewared/plugins/alert.py b/src/middlewared/middlewared/plugins/alert.py index 368a10cc3f4e..b4104a37203b 100644 --- a/src/middlewared/middlewared/plugins/alert.py +++ b/src/middlewared/middlewared/plugins/alert.py @@ -1,7 +1,8 @@ -from collections import defaultdict +from collections import defaultdict, namedtuple from datetime import datetime import errno import os +import time import traceback import uuid @@ -34,6 +35,8 @@ ALERT_SOURCES = {} ALERT_SERVICES_FACTORIES = {} +AlertSourceLock = namedtuple("AlertSourceLock", ["source_name", "expires_at"]) + class AlertSourceRunFailedAlertClass(AlertClass): category = AlertCategory.SYSTEM @@ -401,6 +404,10 @@ async def __run_alerts(self): if remote_failover_status == "BACKUP": run_on_backup_node = True + for k, source_lock in list(self.sources_locks.items()): + if source_lock.expires_at <= time.monotonic(): + await self.unblock_source(k) + for alert_source in ALERT_SOURCES.values(): if not alert_source.schedule.should_run(datetime.utcnow(), self.alert_source_last_run[alert_source.name]): continue @@ -496,19 +503,19 @@ async def run_source(self, source_name): raise CallError("This alert checker is unavailable", CallError.EALERTCHECKERUNAVAILABLE) @private - async def block_source(self, source_name): + async def block_source(self, source_name, timeout=3600): if source_name not in ALERT_SOURCES: raise CallError("Invalid alert source") lock = str(uuid.uuid4()) self.blocked_sources[source_name].add(lock) - self.sources_locks[lock] = source_name + self.sources_locks[lock] = AlertSourceLock(source_name, time.monotonic() + timeout) return lock @private async def unblock_source(self, lock): - source_name = self.sources_locks.pop(lock) - self.blocked_sources[source_name].remove(lock) + source_lock = self.sources_locks.pop(lock) + self.blocked_sources[source_lock.source_name].remove(lock) async def __run_source(self, source_name): alert_source = ALERT_SOURCES[source_name]