kv: Attempt to notify watchers when disconnected

Use the local cache to notify anything locally if possible.
This allows us to use a watch early in the process.

Signed-off-by: Joshua Hesketh <jhesketh@suse.com>

src/gravel/controllers/kv.py

@@ -251,6 +251,9 @@ def _config_notify(
         logger.debug(
             f"Got notify on config object {notify_id} {notifier_id} {watch_id} {key}"
         )
+        self._notify_watches(key)
+
+    def _notify_watches(self, key: str):
         if key not in self._watches:
             return
 
@@ -322,6 +325,10 @@ async def put(self, key: str, value: str) -> None:
             except Exception as e:
                 # e.g. RADOS state (You cannot perform that operation on a Rados object in state configuring.)
                 logger.exception(str(e))
+        else:
+            # When we're not connected to ioctx we should try to notify any
+            # watchers that we may be aware of via the cache
+            self._notify_watches(key)
 
         logger.debug(f"Writing {key}: {value} to local cache")
         self._db[key] = bvalue