Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

MINOR: Enable GIL monitoring when gilknocker installed #7730

Merged
merged 7 commits into from Apr 26, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
2 changes: 1 addition & 1 deletion distributed/distributed.yaml
Expand Up @@ -312,7 +312,7 @@ distributed:
disk: true # Monitor host-wide disk I/O
host-cpu: false # Monitor host-wide CPU usage, with very granular breakdown
gil:
enabled: false # Monitor GIL contention
enabled: true # Monitor GIL contention
interval: "1ms" # Frequency to poll GIL
event-loop: tornado
rmm:
Expand Down
1 change: 1 addition & 0 deletions distributed/http/scheduler/tests/test_scheduler_http.py
Expand Up @@ -107,6 +107,7 @@ async def test_prometheus(c, s, a, b):
expected_metrics = {
"dask_scheduler_clients",
"dask_scheduler_desired_workers",
"dask_scheduler_gil_contention",
"dask_scheduler_workers",
"dask_scheduler_last_time",
"dask_scheduler_tasks",
Expand Down
1 change: 1 addition & 0 deletions distributed/http/worker/tests/test_worker_http.py
Expand Up @@ -32,6 +32,7 @@ async def test_prometheus(c, s, a):
)
expected_metrics = {
"dask_worker_concurrent_fetch_requests",
"dask_worker_gil_contention_total",
"dask_worker_latency_seconds",
"dask_worker_memory_bytes",
"dask_worker_spill_bytes_total",
Expand Down
5 changes: 1 addition & 4 deletions distributed/system_monitor.py
Expand Up @@ -233,7 +233,4 @@ def range_query(self, start: int) -> dict[str, list]:

def close(self) -> None:
if self.monitor_gil_contention:
try:
self._gilknocker.stop()
except ValueError: # Wasn't started or already stopped
pass
self._gilknocker.stop()
20 changes: 11 additions & 9 deletions distributed/tests/test_system_monitor.py
Expand Up @@ -101,20 +101,22 @@ def test_host_cpu():
def test_gil_contention():
pytest.importorskip("gilknocker")

# Default enabled if gilknocker installed
sm = SystemMonitor()
a = sm.update()
assert "gil_contention" not in a

sm = SystemMonitor(monitor_gil_contention=True)
a = sm.update()
assert "gil_contention" in a

with dask.config.set({"distributed.admin.system-monitor.gil.enabled": True}):
sm = SystemMonitor()
a = sm.update()
assert "gil_contention" in a

assert sm._gilknocker.is_running
sm.close()
sm.close() # Idempotent
assert not sm._gilknocker.is_running

sm = SystemMonitor(monitor_gil_contention=False)
a = sm.update()
assert "gil_contention" not in a

assert dask.config.get("distributed.admin.system-monitor.gil.enabled")
with dask.config.set({"distributed.admin.system-monitor.gil.enabled": False}):
sm = SystemMonitor()
a = sm.update()
assert "gil_contention" not in a