forked from google/cadvisor
/
factory.go
396 lines (322 loc) · 11.6 KB
/
factory.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
// Copyright 2014 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package docker
import (
"flag"
"fmt"
"path"
"regexp"
"strconv"
"strings"
"sync"
"time"
"github.com/blang/semver"
dockertypes "github.com/docker/docker/api/types"
"github.com/google/cadvisor/container"
dockerutil "github.com/google/cadvisor/container/docker/utils"
"github.com/google/cadvisor/container/libcontainer"
"github.com/google/cadvisor/devicemapper"
"github.com/google/cadvisor/fs"
info "github.com/google/cadvisor/info/v1"
"github.com/google/cadvisor/machine"
"github.com/google/cadvisor/watcher"
"github.com/google/cadvisor/zfs"
docker "github.com/docker/docker/client"
"golang.org/x/net/context"
"k8s.io/klog/v2"
)
var ArgDockerEndpoint = flag.String("docker", "unix:///var/run/docker.sock", "docker endpoint")
var ArgDockerTLS = flag.Bool("docker-tls", false, "use TLS to connect to docker")
var ArgDockerCert = flag.String("docker-tls-cert", "cert.pem", "path to client certificate")
var ArgDockerKey = flag.String("docker-tls-key", "key.pem", "path to private key")
var ArgDockerCA = flag.String("docker-tls-ca", "ca.pem", "path to trusted CA")
var dockerEnvMetadataWhiteList = flag.String("docker_env_metadata_whitelist", "", "DEPRECATED: this flag will be removed, please use `env_metadata_whitelist`. A comma-separated list of environment variable keys matched with specified prefix that needs to be collected for docker containers")
// The namespace under which Docker aliases are unique.
const DockerNamespace = "docker"
// The retry times for getting docker root dir
const rootDirRetries = 5
// The retry period for getting docker root dir, Millisecond
const rootDirRetryPeriod time.Duration = 1000 * time.Millisecond
// Regexp that identifies docker cgroups, containers started with
// --cgroup-parent have another prefix than 'docker'
var dockerCgroupRegexp = regexp.MustCompile(`([a-z0-9]{64})`)
var (
// Basepath to all container specific information that libcontainer stores.
dockerRootDir string
dockerRootDirFlag = flag.String("docker_root", "/var/lib/docker", "DEPRECATED: docker root is read from docker info (this is a fallback, default: /var/lib/docker)")
dockerRootDirOnce sync.Once
// flag that controls globally disabling thin_ls pending future enhancements.
// in production, it has been found that thin_ls makes excessive use of iops.
// in an iops restricted environment, usage of thin_ls must be controlled via blkio.
// pending that enhancement, disable its usage.
disableThinLs = true
)
func RootDir() string {
dockerRootDirOnce.Do(func() {
for i := 0; i < rootDirRetries; i++ {
status, err := Status()
if err == nil && status.RootDir != "" {
dockerRootDir = status.RootDir
break
} else {
time.Sleep(rootDirRetryPeriod)
}
}
if dockerRootDir == "" {
dockerRootDir = *dockerRootDirFlag
}
})
return dockerRootDir
}
type storageDriver string
const (
devicemapperStorageDriver storageDriver = "devicemapper"
aufsStorageDriver storageDriver = "aufs"
overlayStorageDriver storageDriver = "overlay"
overlay2StorageDriver storageDriver = "overlay2"
zfsStorageDriver storageDriver = "zfs"
vfsStorageDriver storageDriver = "vfs"
)
type dockerFactory struct {
machineInfoFactory info.MachineInfoFactory
storageDriver storageDriver
storageDir string
client *docker.Client
// Information about the mounted cgroup subsystems.
cgroupSubsystems map[string]string
// Information about mounted filesystems.
fsInfo fs.FsInfo
dockerVersion []int
dockerAPIVersion []int
includedMetrics container.MetricSet
thinPoolName string
thinPoolWatcher *devicemapper.ThinPoolWatcher
zfsWatcher *zfs.ZfsWatcher
}
func (f *dockerFactory) String() string {
return DockerNamespace
}
func (f *dockerFactory) NewContainerHandler(name string, metadataEnvAllowList []string, inHostNamespace bool) (handler container.ContainerHandler, err error) {
client, err := Client()
if err != nil {
return
}
dockerMetadataEnvAllowList := strings.Split(*dockerEnvMetadataWhiteList, ",")
// prefer using the unified metadataEnvAllowList
if len(metadataEnvAllowList) != 0 {
dockerMetadataEnvAllowList = metadataEnvAllowList
}
handler, err = newDockerContainerHandler(
client,
name,
f.machineInfoFactory,
f.fsInfo,
f.storageDriver,
f.storageDir,
f.cgroupSubsystems,
inHostNamespace,
dockerMetadataEnvAllowList,
f.dockerVersion,
f.includedMetrics,
f.thinPoolName,
f.thinPoolWatcher,
f.zfsWatcher,
)
return
}
// Returns the Docker ID from the full container name.
func ContainerNameToDockerId(name string) string {
id := path.Base(name)
if matches := dockerCgroupRegexp.FindStringSubmatch(id); matches != nil {
return matches[1]
}
return id
}
// isContainerName returns true if the cgroup with associated name
// corresponds to a docker container.
func isContainerName(name string) bool {
// always ignore .mount cgroup even if associated with docker and delegate to systemd
if strings.HasSuffix(name, ".mount") {
return false
}
return dockerCgroupRegexp.MatchString(path.Base(name))
}
// Docker handles all containers under /docker
func (f *dockerFactory) CanHandleAndAccept(name string) (bool, bool, error) {
// if the container is not associated with docker, we can't handle it or accept it.
if !isContainerName(name) {
return false, false, nil
}
// Check if the container is known to docker and it is active.
id := ContainerNameToDockerId(name)
// We assume that if Inspect fails then the container is not known to docker.
ctnr, err := f.client.ContainerInspect(context.Background(), id)
if err != nil || !ctnr.State.Running {
return false, true, fmt.Errorf("error inspecting container: %v", err)
}
return true, true, nil
}
func (f *dockerFactory) DebugInfo() map[string][]string {
return map[string][]string{}
}
var (
versionRegexpString = `(\d+)\.(\d+)\.(\d+)`
versionRe = regexp.MustCompile(versionRegexpString)
apiVersionRegexpString = `(\d+)\.(\d+)`
apiVersionRe = regexp.MustCompile(apiVersionRegexpString)
)
func startThinPoolWatcher(dockerInfo *dockertypes.Info) (*devicemapper.ThinPoolWatcher, error) {
_, err := devicemapper.ThinLsBinaryPresent()
if err != nil {
return nil, err
}
if err := ensureThinLsKernelVersion(machine.KernelVersion()); err != nil {
return nil, err
}
if disableThinLs {
return nil, fmt.Errorf("usage of thin_ls is disabled to preserve iops")
}
dockerThinPoolName, err := dockerutil.DockerThinPoolName(*dockerInfo)
if err != nil {
return nil, err
}
dockerMetadataDevice, err := dockerutil.DockerMetadataDevice(*dockerInfo)
if err != nil {
return nil, err
}
thinPoolWatcher, err := devicemapper.NewThinPoolWatcher(dockerThinPoolName, dockerMetadataDevice)
if err != nil {
return nil, err
}
go thinPoolWatcher.Start()
return thinPoolWatcher, nil
}
func startZfsWatcher(dockerInfo *dockertypes.Info) (*zfs.ZfsWatcher, error) {
filesystem, err := dockerutil.DockerZfsFilesystem(*dockerInfo)
if err != nil {
return nil, err
}
zfsWatcher, err := zfs.NewZfsWatcher(filesystem)
if err != nil {
return nil, err
}
go zfsWatcher.Start()
return zfsWatcher, nil
}
func ensureThinLsKernelVersion(kernelVersion string) error {
// kernel 4.4.0 has the proper bug fixes to allow thin_ls to work without corrupting the thin pool
minKernelVersion := semver.MustParse("4.4.0")
// RHEL 7 kernel 3.10.0 release >= 366 has the proper bug fixes backported from 4.4.0 to allow
// thin_ls to work without corrupting the thin pool
minRhel7KernelVersion := semver.MustParse("3.10.0")
matches := versionRe.FindStringSubmatch(kernelVersion)
if len(matches) < 4 {
return fmt.Errorf("error parsing kernel version: %q is not a semver", kernelVersion)
}
sem, err := semver.Make(matches[0])
if err != nil {
return err
}
if sem.GTE(minKernelVersion) {
// kernel 4.4+ - good
return nil
}
// Certain RHEL/Centos 7.x kernels have a backport to fix the corruption bug
if !strings.Contains(kernelVersion, ".el7") {
// not a RHEL 7.x kernel - won't work
return fmt.Errorf("kernel version 4.4.0 or later is required to use thin_ls - you have %q", kernelVersion)
}
// RHEL/Centos 7.x from here on
if sem.Major != 3 {
// only 3.x kernels *may* work correctly
return fmt.Errorf("RHEL/Centos 7.x kernel version 3.10.0-366 or later is required to use thin_ls - you have %q", kernelVersion)
}
if sem.GT(minRhel7KernelVersion) {
// 3.10.1+ - good
return nil
}
if sem.EQ(minRhel7KernelVersion) {
// need to check release
releaseRE := regexp.MustCompile(`^[^-]+-([0-9]+)\.`)
releaseMatches := releaseRE.FindStringSubmatch(kernelVersion)
if len(releaseMatches) != 2 {
return fmt.Errorf("unable to determine RHEL/Centos 7.x kernel release from %q", kernelVersion)
}
release, err := strconv.Atoi(releaseMatches[1])
if err != nil {
return fmt.Errorf("error parsing release %q: %v", releaseMatches[1], err)
}
if release >= 366 {
return nil
}
}
return fmt.Errorf("RHEL/Centos 7.x kernel version 3.10.0-366 or later is required to use thin_ls - you have %q", kernelVersion)
}
// Register root container before running this function!
func Register(factory info.MachineInfoFactory, fsInfo fs.FsInfo, includedMetrics container.MetricSet) error {
client, err := Client()
if err != nil {
return fmt.Errorf("unable to communicate with docker daemon: %v", err)
}
dockerInfo, err := ValidateInfo()
if err != nil {
return fmt.Errorf("failed to validate Docker info: %v", err)
}
// Version already validated above, assume no error here.
dockerVersion, _ := parseVersion(dockerInfo.ServerVersion, versionRe, 3)
dockerAPIVersion, _ := APIVersion()
cgroupSubsystems, err := libcontainer.GetCgroupSubsystems(includedMetrics)
if err != nil {
return fmt.Errorf("failed to get cgroup subsystems: %v", err)
}
var (
thinPoolWatcher *devicemapper.ThinPoolWatcher
thinPoolName string
zfsWatcher *zfs.ZfsWatcher
)
if includedMetrics.Has(container.DiskUsageMetrics) {
if storageDriver(dockerInfo.Driver) == devicemapperStorageDriver {
thinPoolWatcher, err = startThinPoolWatcher(dockerInfo)
if err != nil {
klog.Errorf("devicemapper filesystem stats will not be reported: %v", err)
}
// Safe to ignore error - driver status should always be populated.
status, _ := StatusFromDockerInfo(*dockerInfo)
thinPoolName = status.DriverStatus[dockerutil.DriverStatusPoolName]
}
if storageDriver(dockerInfo.Driver) == zfsStorageDriver {
zfsWatcher, err = startZfsWatcher(dockerInfo)
if err != nil {
klog.Errorf("zfs filesystem stats will not be reported: %v", err)
}
}
}
klog.V(1).Infof("Registering Docker factory")
f := &dockerFactory{
cgroupSubsystems: cgroupSubsystems,
client: client,
dockerVersion: dockerVersion,
dockerAPIVersion: dockerAPIVersion,
fsInfo: fsInfo,
machineInfoFactory: factory,
storageDriver: storageDriver(dockerInfo.Driver),
storageDir: RootDir(),
includedMetrics: includedMetrics,
thinPoolName: thinPoolName,
thinPoolWatcher: thinPoolWatcher,
zfsWatcher: zfsWatcher,
}
container.RegisterContainerHandlerFactory(f, []watcher.ContainerWatchSource{watcher.Raw})
return nil
}