Skip to content

Commit

Permalink
Add devicemapper support for docker containers
Browse files Browse the repository at this point in the history
  • Loading branch information
pmorie committed May 18, 2016
1 parent 8cf6ed3 commit 647224c
Show file tree
Hide file tree
Showing 14 changed files with 888 additions and 43 deletions.
32 changes: 30 additions & 2 deletions container/docker/factory.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,9 +24,11 @@ import (

"github.com/google/cadvisor/container"
"github.com/google/cadvisor/container/libcontainer"
"github.com/google/cadvisor/devicemapper"
"github.com/google/cadvisor/fs"
info "github.com/google/cadvisor/info/v1"
"github.com/google/cadvisor/manager/watcher"
dockerutil "github.com/google/cadvisor/utils/docker"

docker "github.com/docker/engine-api/client"
"github.com/golang/glog"
Expand Down Expand Up @@ -68,7 +70,6 @@ func RootDir() string {
type storageDriver string

const (
// TODO: Add support for devicemapper storage usage.
devicemapperStorageDriver storageDriver = "devicemapper"
aufsStorageDriver storageDriver = "aufs"
overlayStorageDriver storageDriver = "overlay"
Expand All @@ -92,6 +93,8 @@ type dockerFactory struct {
dockerVersion []int

ignoreMetrics container.MetricSet

thinPoolWatcher *devicemapper.ThinPoolWatcher
}

func (self *dockerFactory) String() string {
Expand All @@ -118,6 +121,7 @@ func (self *dockerFactory) NewContainerHandler(name string, inHostNamespace bool
metadataEnvs,
self.dockerVersion,
self.ignoreMetrics,
self.thinPoolWatcher,
)
return
}
Expand Down Expand Up @@ -187,7 +191,30 @@ func Register(factory info.MachineInfoFactory, fsInfo fs.FsInfo, ignoreMetrics c
return fmt.Errorf("failed to get cgroup subsystems: %v", err)
}

glog.Infof("Registering Docker factory")
var (
dockerStorageDriver = storageDriver(dockerInfo.Driver)
thinPoolWatcher *devicemapper.ThinPoolWatcher = nil
)

if dockerStorageDriver == devicemapperStorageDriver {
// If the storage drive is devicemapper, create and start a
// ThinPoolWatcher to monitor the size of container CoW layers with
// thin_ls.
dockerThinPoolName, err := dockerutil.DockerThinPoolName(*dockerInfo)
if err != nil {
return fmt.Errorf("couldn't find device mapper thin pool name: %v", err)
}

dockerMetadataDevice, err := dockerutil.DockerMetadataDevice(*dockerInfo)
if err != nil {
return fmt.Errorf("couldn't determine devicemapper metadata device")
}

thinPoolWatcher = devicemapper.NewThinPoolWatcher(dockerThinPoolName, dockerMetadataDevice)
go thinPoolWatcher.Start()
}

glog.Infof("registering Docker factory")
f := &dockerFactory{
cgroupSubsystems: cgroupSubsystems,
client: client,
Expand All @@ -197,6 +224,7 @@ func Register(factory info.MachineInfoFactory, fsInfo fs.FsInfo, ignoreMetrics c
storageDriver: storageDriver(dockerInfo.Driver),
storageDir: RootDir(),
ignoreMetrics: ignoreMetrics,
thinPoolWatcher: thinPoolWatcher,
}

container.RegisterContainerHandlerFactory(f, []watcher.ContainerWatchSource{watcher.Raw})
Expand Down
111 changes: 98 additions & 13 deletions container/docker/handler.go
Original file line number Diff line number Diff line change
Expand Up @@ -25,11 +25,14 @@ import (
"github.com/google/cadvisor/container"
"github.com/google/cadvisor/container/common"
containerlibcontainer "github.com/google/cadvisor/container/libcontainer"
"github.com/google/cadvisor/devicemapper"
"github.com/google/cadvisor/fs"
info "github.com/google/cadvisor/info/v1"
dockerutil "github.com/google/cadvisor/utils/docker"

docker "github.com/docker/engine-api/client"
dockercontainer "github.com/docker/engine-api/types/container"
"github.com/golang/glog"
"github.com/opencontainers/runc/libcontainer/cgroups"
cgroupfs "github.com/opencontainers/runc/libcontainer/cgroups/fs"
libcontainerconfigs "github.com/opencontainers/runc/libcontainer/configs"
Expand Down Expand Up @@ -57,10 +60,18 @@ type dockerContainerHandler struct {
// Manager of this container's cgroups.
cgroupManager cgroups.Manager

// the docker storage driver
storageDriver storageDriver
fsInfo fs.FsInfo
rootfsStorageDir string

// devicemapper state

// the devicemapper poolname
poolName string
// the devicemapper device id for the container
deviceID string

// Time at which this container was created.
creationTime time.Time

Expand All @@ -84,8 +95,13 @@ type dockerContainerHandler struct {
fsHandler common.FsHandler

ignoreMetrics container.MetricSet

// thin pool watcher
thinPoolWatcher *devicemapper.ThinPoolWatcher
}

var _ container.ContainerHandler = &dockerContainerHandler{}

func getRwLayerID(containerID, storageDir string, sd storageDriver, dockerVersion []int) (string, error) {
const (
// Docker version >=1.10.0 have a randomized ID for the root fs of a container.
Expand All @@ -103,6 +119,7 @@ func getRwLayerID(containerID, storageDir string, sd storageDriver, dockerVersio
return string(bytes), err
}

// newDockerContainerHandler returns a new container.ContainerHandler
func newDockerContainerHandler(
client *docker.Client,
name string,
Expand All @@ -115,6 +132,7 @@ func newDockerContainerHandler(
metadataEnvs []string,
dockerVersion []int,
ignoreMetrics container.MetricSet,
thinPoolWatcher *devicemapper.ThinPoolWatcher,
) (container.ContainerHandler, error) {
// Create the cgroup paths.
cgroupPaths := make(map[string]string, len(cgroupSubsystems.MountPoints))
Expand Down Expand Up @@ -146,14 +164,27 @@ func newDockerContainerHandler(
if err != nil {
return nil, err
}
var rootfsStorageDir string

// Determine the rootfs storage dir OR the pool name to determine the device
var (
rootfsStorageDir string
poolName string
)
switch storageDriver {
case aufsStorageDriver:
rootfsStorageDir = path.Join(storageDir, string(aufsStorageDriver), aufsRWLayer, rwLayerID)
case overlayStorageDriver:
rootfsStorageDir = path.Join(storageDir, string(overlayStorageDriver), rwLayerID)
case devicemapperStorageDriver:
status, err := Status()
if err != nil {
return nil, fmt.Errorf("unable to determine docker status: %v", err)
}

poolName = status.DriverStatus[dockerutil.DriverStatusPoolName]
}

// TODO: extract object mother method
handler := &dockerContainerHandler{
id: id,
client: client,
Expand All @@ -164,13 +195,11 @@ func newDockerContainerHandler(
storageDriver: storageDriver,
fsInfo: fsInfo,
rootFs: rootFs,
poolName: poolName,
rootfsStorageDir: rootfsStorageDir,
envs: make(map[string]string),
ignoreMetrics: ignoreMetrics,
}

if !ignoreMetrics.Has(container.DiskUsageMetrics) {
handler.fsHandler = common.NewFsHandler(time.Minute, rootfsStorageDir, otherStorageDir, fsInfo)
thinPoolWatcher: thinPoolWatcher,
}

// We assume that if Inspect fails then the container is not known to docker.
Expand All @@ -191,6 +220,15 @@ func newDockerContainerHandler(
handler.labels = ctnr.Config.Labels
handler.image = ctnr.Config.Image
handler.networkMode = ctnr.HostConfig.NetworkMode
handler.deviceID = ctnr.GraphDriver.Data["DeviceId"]

if !ignoreMetrics.Has(container.DiskUsageMetrics) {
handler.fsHandler = &dockerFsHandler{
fsHandler: common.NewFsHandler(time.Minute, rootfsStorageDir, otherStorageDir, fsInfo),
thinPoolWatcher: thinPoolWatcher,
deviceID: handler.deviceID,
}
}

// split env vars to get metadata map.
for _, exposedEnv := range metadataEnvs {
Expand All @@ -205,6 +243,48 @@ func newDockerContainerHandler(
return handler, nil
}

// dockerFsHandler is a composite FsHandler implementation the incorporates
// the common fs handler and a devicemapper ThinPoolWatcher.
type dockerFsHandler struct {
fsHandler common.FsHandler

// thinPoolWatcher is the devicemapper thin pool watcher
thinPoolWatcher *devicemapper.ThinPoolWatcher
// deviceID is the id of the container's fs device
deviceID string
}

var _ common.FsHandler = &dockerFsHandler{}

func (h *dockerFsHandler) Start() {
h.fsHandler.Start()
}

func (h *dockerFsHandler) Stop() {
h.fsHandler.Stop()
}

func (h *dockerFsHandler) Usage() (uint64, uint64) {
baseUsage, usage := h.fsHandler.Usage()

// When devicemapper is the storage driver, the base usage of the container comes from the thin pool.
// We still need the result of the fsHandler for any extra storage associated with the container.
// To correctly factor in the thin pool usage, we should:
// * Usage the thin pool usage as the base usage
// * Calculate the overall usage by adding the overall usage from the fs handler to the thin pool usage
if h.thinPoolWatcher != nil {
thinPoolUsage, err := h.thinPoolWatcher.GetUsage(h.deviceID)
if err != nil {
glog.Errorf("unable to get fs usage from thin pool for device %v: %v", h.deviceID, err)
} else {
baseUsage = thinPoolUsage
usage += thinPoolUsage
}
}

return baseUsage, usage
}

func (self *dockerContainerHandler) Start() {
if self.fsHandler != nil {
self.fsHandler.Start()
Expand Down Expand Up @@ -249,17 +329,22 @@ func (self *dockerContainerHandler) getFsStats(stats *info.ContainerStats) error
if self.ignoreMetrics.Has(container.DiskUsageMetrics) {
return nil
}
var device string
switch self.storageDriver {
case devicemapperStorageDriver:
// Device has to be the pool name to correlate with the device name as
// set in the machine info filesystems.
device = self.poolName
case aufsStorageDriver, overlayStorageDriver, zfsStorageDriver:
deviceInfo, err := self.fsInfo.GetDirFsDevice(self.rootfsStorageDir)
if err != nil {
return fmt.Errorf("unable to determine device info for dir: %v: %v", self.rootfsStorageDir, err)
}
device = deviceInfo.Device
default:
return nil
}

deviceInfo, err := self.fsInfo.GetDirFsDevice(self.rootfsStorageDir)
if err != nil {
return err
}

mi, err := self.machineInfoFactory.GetMachineInfo()
if err != nil {
return err
Expand All @@ -272,16 +357,16 @@ func (self *dockerContainerHandler) getFsStats(stats *info.ContainerStats) error

// Docker does not impose any filesystem limits for containers. So use capacity as limit.
for _, fs := range mi.Filesystems {
if fs.Device == deviceInfo.Device {
if fs.Device == device {
limit = fs.Capacity
fsType = fs.Type
break
}
}

fsStat := info.FsStats{Device: deviceInfo.Device, Type: fsType, Limit: limit}

fsStat := info.FsStats{Device: device, Type: fsType, Limit: limit}
fsStat.BaseUsage, fsStat.Usage = self.fsHandler.Usage()

stats.Filesystem = append(stats.Filesystem, fsStat)

return nil
Expand Down
56 changes: 56 additions & 0 deletions devicemapper/dmsetup_client.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,56 @@
// Copyright 2016 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package devicemapper

import (
"os/exec"
"strconv"
"strings"

"github.com/golang/glog"
)

// DmsetupClient is a low-level client for interacting with devicemapper via
// the dmsetup utility.
type DmsetupClient interface {
Table(deviceName string) ([]byte, error)
Message(deviceName string, sector int, message string) ([]byte, error)
Status(deviceName string) ([]byte, error)
}

func NewDmsetupClient() DmsetupClient {
return &defaultDmsetupClient{}
}

// defaultDmsetupClient implements the standard behavior for interacting with dmsetup.
type defaultDmsetupClient struct{}

var _ DmsetupClient = &defaultDmsetupClient{}

func (c *defaultDmsetupClient) Table(deviceName string) ([]byte, error) {
return c.dmsetup("table", deviceName)
}

func (c *defaultDmsetupClient) Message(deviceName string, sector int, message string) ([]byte, error) {
return c.dmsetup("message", deviceName, strconv.Itoa(sector), message)
}

func (c *defaultDmsetupClient) Status(deviceName string) ([]byte, error) {
return c.dmsetup("status", deviceName)
}

func (*defaultDmsetupClient) dmsetup(args ...string) ([]byte, error) {
glog.V(5).Infof("running dmsetup %v", strings.Join(args, " "))
return exec.Command("dmsetup", args...).Output()
}
16 changes: 16 additions & 0 deletions devicemapper/doc.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
// Copyright 2016 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package devicemapper contains code for working with devicemapper
package devicemapper
Loading

0 comments on commit 647224c

Please sign in to comment.