Make system monitor interval configurable (#4447)

Sometimes we want to reduce this to improve performance

distributed/core.py

@@ -199,7 +199,13 @@ def stop():
 
         self.periodic_callbacks = dict()
 
-        pc = PeriodicCallback(self.monitor.update, 500)
+        pc = PeriodicCallback(
+            self.monitor.update,
+            parse_timedelta(
+                dask.config.get("distributed.admin.system-monitor.interval")
+            )
+            * 1000,
+        )
         self.periodic_callbacks["monitor"] = pc
 
         self._last_tick = time()

distributed/distributed-schema.yaml

@@ -771,6 +771,15 @@ properties:
             type: boolean
             description: Enter Python Debugger on scheduling error
 
+          system-monitor:
+            type: object
+            description: |
+              Options for the periodic system monitor
+            properties:
+              interval:
+                type: string
+                description: Polling time to query cpu/memory statistics default 500ms
+
   rmm:
     type: object
     description: |

distributed/distributed.yaml

@@ -174,6 +174,8 @@ distributed:
     log-length: 10000  # default length of logs to keep in memory
     log-format: '%(name)s - %(levelname)s - %(message)s'
     pdb-on-err: False       # enter debug mode on scheduling error
+    system-monitor:
+      interval: 500ms
     event-loop: tornado
 rmm:
   pool-size: null