This repository has been archived by the owner on Jul 1, 2023. It is now read-only.
forked from kubernetes/kubernetes
-
Notifications
You must be signed in to change notification settings - Fork 1
/
deadlock-detector.go
120 lines (102 loc) · 2.92 KB
/
deadlock-detector.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
/*
Copyright 2015 The Kubernetes Authors All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package threading
import (
"os"
"sync"
"time"
"github.com/golang/glog"
)
type rwMutexToLockableAdapter struct {
rw *sync.RWMutex
}
func (r *rwMutexToLockableAdapter) Lock() {
r.rw.RLock()
}
func (r *rwMutexToLockableAdapter) Unlock() {
r.rw.RUnlock()
}
type deadlockDetector struct {
name string
lock sync.Locker
maxLockPeriod time.Duration
exiter exiter
exitChannelFn func() <-chan time.Time
// Really only useful for testing
stopChannel <-chan bool
}
// DeadlockWatchdogReadLock creates a watchdog on read/write mutex. If the mutex can not be acquired
// for read access within 'maxLockPeriod', the program exits via glog.Exitf() or os.Exit() if that fails
// 'name' is a semantic name that is useful for the user and is printed on exit.
func DeadlockWatchdogReadLock(lock *sync.RWMutex, name string, maxLockPeriod time.Duration) {
DeadlockWatchdog(&rwMutexToLockableAdapter{lock}, name, maxLockPeriod)
}
func DeadlockWatchdog(lock sync.Locker, name string, maxLockPeriod time.Duration) {
if maxLockPeriod <= 0 {
panic("maxLockPeriod is <= 0, that can't be what you wanted")
}
detector := &deadlockDetector{
lock: lock,
name: name,
maxLockPeriod: maxLockPeriod,
exitChannelFn: func() <-chan time.Time { return time.After(maxLockPeriod) },
stopChannel: make(chan bool),
}
go detector.run()
}
// Useful for injecting tests
type exiter interface {
Exitf(format string, args ...interface{})
}
type realExiter struct{}
func (realExiter) Exitf(format string, args ...interface{}) {
func() {
defer func() {
// Let's just be extra sure we die, even if Exitf panics
if r := recover(); r != nil {
glog.Errorf(format, args...)
os.Exit(2)
}
}()
glog.Exitf(format, args...)
}()
}
func (d *deadlockDetector) run() {
for {
if !d.runOnce() {
return
}
time.Sleep(d.maxLockPeriod / 2)
}
}
func (d *deadlockDetector) runOnce() bool {
ch := make(chan bool, 1)
go func() {
d.lock.Lock()
d.lock.Unlock()
ch <- true
}()
exitCh := d.exitChannelFn()
select {
case <-exitCh:
d.exiter.Exitf("Deadlock on %s, exiting", d.name)
// return is here for when we use a fake exiter in testing
return false
case <-ch:
glog.V(6).Infof("%s is not deadlocked", d.name)
case <-d.stopChannel:
glog.V(4).Infof("Stopping deadlock detector for %s", d.name)
return false
}
return true
}