/
gpumanager.go
101 lines (85 loc) · 2.44 KB
/
gpumanager.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
package nvidia
import (
"fmt"
"syscall"
"time"
"github.com/NVIDIA/gpu-monitoring-tools/bindings/go/nvml"
"github.com/fsnotify/fsnotify"
log "github.com/golang/glog"
pluginapi "k8s.io/kubernetes/pkg/kubelet/apis/deviceplugin/v1beta1"
)
type sharedGPUManager struct {
enableMPS bool
healthCheck bool
}
func NewSharedGPUManager(enableMPS, healthCheck bool, bp MemoryUnit) *sharedGPUManager {
metric = bp
return &sharedGPUManager{
enableMPS: enableMPS,
healthCheck: healthCheck,
}
}
func (ngm *sharedGPUManager) Run() error {
log.V(1).Infoln("Loading NVML")
if err := nvml.Init(); err != nil {
log.V(1).Infof("Failed to initialize NVML: %s.", err)
log.V(1).Infof("If this is a GPU node, did you set the docker default runtime to `nvidia`?")
select {}
}
defer func() { log.V(1).Infoln("Shutdown of NVML returned:", nvml.Shutdown()) }()
log.V(1).Infoln("Fetching devices.")
if getDeviceCount() == uint(0) {
log.V(1).Infoln("No devices found. Waiting indefinitely.")
select {}
}
log.V(1).Infoln("Starting FS watcher.")
watcher, err := newFSWatcher(pluginapi.DevicePluginPath)
if err != nil {
log.V(1).Infoln("Failed to created FS watcher.")
return err
}
defer watcher.Close()
log.V(1).Infoln("Starting OS watcher.")
sigs := newOSWatcher(syscall.SIGHUP, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT)
restart := true
var devicePlugin *NvidiaDevicePlugin
L:
for {
if restart {
if devicePlugin != nil {
devicePlugin.Stop()
}
devicePlugin = NewNvidiaDevicePlugin(ngm.enableMPS, ngm.healthCheck)
if err := devicePlugin.Serve(); err != nil {
log.Warningf("Failed to start device plugin due to %v", err)
} else {
restart = false
}
}
select {
case event := <-watcher.Events:
if event.Name == pluginapi.KubeletSocket && event.Op&fsnotify.Create == fsnotify.Create {
log.V(1).Infof("inotify: %s created, restarting.", pluginapi.KubeletSocket)
restart = true
}
case err := <-watcher.Errors:
log.Warningf("inotify: %s", err)
case s := <-sigs:
switch s {
case syscall.SIGHUP:
log.V(1).Infoln("Received SIGHUP, restarting.")
restart = true
case syscall.SIGQUIT:
t := time.Now()
timestamp := fmt.Sprint(t.Format("20060102150405"))
log.Infoln("generate core dump")
coredump("/etc/kubernetes/go_" + timestamp + ".txt")
default:
log.V(1).Infof("Received signal \"%v\", shutting down.", s)
devicePlugin.Stop()
break L
}
}
}
return nil
}