cgroups: implement a devices cgroupv1 emulator

Okay, this requires a bit of explanation.

The reason for this emulation is to allow us to have seamless updates of
the devices cgroup for running containers. This was triggered by several
users having issues where our initial writing of a deny-all rule (in all
cases) results in spurrious errors.

The obvious solution would be to just remove the deny-all rule, right?
Well, it turns out that runc doesn't actually control the deny-all rule
because all users of runc have explicitly specified their own deny-all
rule for many years. This appears to have been done to work around a bug
in runc (which this series has fixed in [1]) where we would actually act
as a black-list despite this being a violation of the OCI spec.

This means that not adding our own deny-all rule in the case of updates
won't solve the issue. However, it will also not solve the issue in
several other cases (the most notable being where a container is being
switched between default-permission modes).

So in order to handle all of these cases, a way of tracking the relevant
internal cgroup state (given a certain state of "cgroups.list" and a set
of rules to apply) is necessary. That is the purpose of DevicesEmulator.
Reading "devices.list" is quite important because that's the only way we
can tell if it's safe to skip the troublesome deny-all rules without
making potentially-dangerous assumptions about the container.

We also are currently bug-compatible with the devices cgroup (namely,
removing rules that don't exist or having superfluous rules all works as
with the in-kernel implementation). The only exception to this is that
we give an error if a user requests to revoke part of a wildcard
exception, because allowing such configurations could result in security
holes (cgroupv1 silently ignores such rules, meaning in white-list mode
that the access is still permitted).

[1]: b2bec98 ("cgroup: devices: eradicate the Allow/Deny lists")

Signed-off-by: Aleksa Sarai <asarai@suse.de>

libcontainer/cgroups/devices/devices_emulator.go

@@ -0,0 +1,351 @@
+// +build linux
+
+package devices
+
+import (
+	"bufio"
+	"io"
+	"regexp"
+	"sort"
+	"strconv"
+
+	"github.com/opencontainers/runc/libcontainer/configs"
+
+	"github.com/pkg/errors"
+)
+
+// deviceMeta is a DeviceRule without the Allow or Permissions fields, and no
+// wildcard-type support. It's effectively the "match" portion of a metadata
+// rule, for the purposes of our emulation.
+type deviceMeta struct {
+	node  configs.DeviceType
+	major int64
+	minor int64
+}
+
+// deviceRule is effectively the tuple (deviceMeta, DevicePermissions).
+type deviceRule struct {
+	meta  deviceMeta
+	perms configs.DevicePermissions
+}
+
+// deviceRules is a mapping of device metadata rules to the associated
+// permissions in the ruleset.
+type deviceRules map[deviceMeta]configs.DevicePermissions
+
+func (r deviceRules) orderedEntries() []deviceRule {
+	var rules []deviceRule
+	for meta, perms := range r {
+		rules = append(rules, deviceRule{meta: meta, perms: perms})
+	}
+	sort.Slice(rules, func(i, j int) bool {
+		// Sort by (major, minor, type).
+		a, b := rules[i].meta, rules[j].meta
+		return a.major < b.major ||
+			(a.major == b.major && a.minor < b.minor) ||
+			(a.major == b.major && a.minor == b.minor && a.node < b.node)
+	})
+	return rules
+}
+
+type Emulator struct {
+	defaultAllow bool
+	rules        deviceRules
+}
+
+func (e *Emulator) IsBlacklist() bool {
+	return e.defaultAllow
+}
+
+var devicesListRegexp = regexp.MustCompile(`^([abc])\s+(\d+|\*):(\d+|\*)\s+([rwm]+)$`)
+
+func parseLine(line string) (*deviceRule, error) {
+	matches := devicesListRegexp.FindStringSubmatch(line)
+	if matches == nil {
+		return nil, errors.Errorf("line doesn't match devices.list format")
+	}
+	var (
+		rule  deviceRule
+		node  = matches[1]
+		major = matches[2]
+		minor = matches[3]
+		perms = matches[4]
+	)
+
+	// Parse the node type.
+	switch node {
+	case "a":
+		// Super-special case -- "a" always means every device with every
+		// access mode. In fact, for devices.list this actually indicates that
+		// the cgroup is in black-list mode.
+		// TODO: Double-check that the entire file is "a *:* rwm".
+		return nil, nil
+	case "b":
+		rule.meta.node = configs.BlockDevice
+	case "c":
+		rule.meta.node = configs.CharDevice
+	default:
+		// Should never happen!
+		return nil, errors.Errorf("unknown device type %q", node)
+	}
+
+	// Parse the major number.
+	if major == "*" {
+		rule.meta.major = configs.Wildcard
+	} else {
+		val, err := strconv.ParseUint(major, 10, 32)
+		if err != nil {
+			return nil, errors.Wrap(err, "parse major number")
+		}
+		rule.meta.major = int64(val)
+	}
+
+	// Parse the minor number.
+	if minor == "*" {
+		rule.meta.minor = configs.Wildcard
+	} else {
+		val, err := strconv.ParseUint(minor, 10, 32)
+		if err != nil {
+			return nil, errors.Wrap(err, "parse minor number")
+		}
+		rule.meta.minor = int64(val)
+	}
+
+	// Parse the access permissions.
+	rule.perms = configs.DevicePermissions(perms)
+	if !rule.perms.IsValid() || rule.perms.IsEmpty() {
+		// Should never happen!
+		return nil, errors.Errorf("parse access mode: contained unknown modes or is empty: %q", perms)
+	}
+	return &rule, nil
+}
+
+func (e *Emulator) addRule(rule deviceRule) error {
+	if e.rules == nil {
+		e.rules = make(map[deviceMeta]configs.DevicePermissions)
+	}
+
+	// Merge with any pre-existing permissions.
+	oldPerms := e.rules[rule.meta]
+	newPerms := rule.perms.Union(oldPerms)
+	e.rules[rule.meta] = newPerms
+	return nil
+}
+
+func (e *Emulator) rmRule(rule deviceRule) error {
+	// Give an error if any of the permissions requested to be removed are
+	// present in a partially-matching wildcard rule, because such rules will
+	// be ignored by cgroupv1.
+	//
+	// This is a diversion from cgroupv1, but is necessary to avoid leading
+	// users into a false sense of security. cgroupv1 will silently(!) ignore
+	// requests to remove partial exceptions, but we really shouldn't do that.
+	//
+	// It may seem like we could just "split" wildcard rules which hit this
+	// issue, but unfortunately there are 2^32 possible major and minor
+	// numbers, which would exhaust kernel memory quickly if we did this. Not
+	// to mention it'd be really slow (the kernel side is implemented as a
+	// linked-list of exceptions).
+	for _, partialMeta := range []deviceMeta{
+		{node: rule.meta.node, major: configs.Wildcard, minor: rule.meta.minor},
+		{node: rule.meta.node, major: rule.meta.major, minor: configs.Wildcard},
+		{node: rule.meta.node, major: configs.Wildcard, minor: configs.Wildcard},
+	} {
+		// This wildcard rule is equivalent to the requested rule, so skip it.
+		if rule.meta == partialMeta {
+			continue
+		}
+		// Only give an error if the set of permissions overlap.
+		partialPerms := e.rules[partialMeta]
+		if !partialPerms.Intersection(rule.perms).IsEmpty() {
+			return errors.Errorf("requested rule [%v %v] not supported by devices cgroupv1 (cannot punch hole in existing wildcard rule [%v %v])", rule.meta, rule.perms, partialMeta, partialPerms)
+		}
+	}
+
+	// Subtract all of the permissions listed from the full match rule. If the
+	// rule didn't exist, all of this is a no-op.
+	newPerms := e.rules[rule.meta].Difference(rule.perms)
+	if newPerms.IsEmpty() {
+		delete(e.rules, rule.meta)
+	} else {
+		e.rules[rule.meta] = newPerms
+	}
+	// TODO: The actual cgroup code doesn't care if an exception didn't exist
+	//       during removal, so not erroring out here is /accurate/ but quite
+	//       worrying. Maybe we should do additional validation, but again we
+	//       have to worry about backwards-compatibility.
+	return nil
+}
+
+func (e *Emulator) allow(rule *deviceRule) error {
+	// This cgroup is configured as a black-list. Reset the entire emulator,
+	// and put is into black-list mode.
+	if rule == nil || rule.meta.node == configs.WildcardDevice {
+		*e = Emulator{
+			defaultAllow: true,
+			rules:        nil,
+		}
+		return nil
+	}
+
+	var err error
+	if e.defaultAllow {
+		err = errors.Wrap(e.rmRule(*rule), "remove 'deny' exception")
+	} else {
+		err = errors.Wrap(e.addRule(*rule), "add 'allow' exception")
+	}
+	return err
+}
+
+func (e *Emulator) deny(rule *deviceRule) error {
+	// This cgroup is configured as a white-list. Reset the entire emulator,
+	// and put is into white-list mode.
+	if rule == nil || rule.meta.node == configs.WildcardDevice {
+		*e = Emulator{
+			defaultAllow: false,
+			rules:        nil,
+		}
+		return nil
+	}
+
+	var err error
+	if e.defaultAllow {
+		err = errors.Wrap(e.addRule(*rule), "add 'deny' exception")
+	} else {
+		err = errors.Wrap(e.rmRule(*rule), "remove 'allow' exception")
+	}
+	return err
+}
+
+func (e *Emulator) Apply(rule configs.DeviceRule) error {
+	if !rule.Type.CanCgroup() {
+		return errors.Errorf("cannot add rule [%#v] with non-cgroup type %q", rule, rule.Type)
+	}
+
+	innerRule := &deviceRule{
+		meta: deviceMeta{
+			node:  rule.Type,
+			major: rule.Major,
+			minor: rule.Minor,
+		},
+		perms: rule.Permissions,
+	}
+	if innerRule.meta.node == configs.WildcardDevice {
+		innerRule = nil
+	}
+
+	if rule.Allow {
+		return e.allow(innerRule)
+	} else {
+		return e.deny(innerRule)
+	}
+}
+
+// EmulatorFromList takes a reader to a "devices.list"-like source, and returns
+// a new Emulator that represents the state of the devices cgroup. Note that
+// black-list devices cgroups cannot be fully reconstructed, due to limitations
+// in the devices cgroup API. Instead, such cgroups are always treated as
+// "allow all" cgroups.
+func EmulatorFromList(list io.Reader) (*Emulator, error) {
+	// Normally cgroups are in black-list mode by default, but the way we
+	// figure out the current mode is whether or not devices.list has an
+	// allow-all rule. So we default to a white-list, and the existence of an
+	// "a *:* rwm" entry will tell us otherwise.
+	e := &Emulator{
+		defaultAllow: false,
+	}
+
+	// Parse the "devices.list".
+	s := bufio.NewScanner(list)
+	for s.Scan() {
+		line := s.Text()
+		deviceRule, err := parseLine(line)
+		if err != nil {
+			return nil, errors.Wrapf(err, "parsing line %q", line)
+		}
+		// "devices.list" is an allow list. Note that this means that in
+		// black-list mode, we have no idea what rules are in play. As a
+		// result, we need to be very careful in Transition().
+		if err := e.allow(deviceRule); err != nil {
+			return nil, errors.Wrapf(err, "adding devices.list rule")
+		}
+	}
+	if err := s.Err(); err != nil {
+		return nil, errors.Wrap(err, "reading devices.list lines")
+	}
+	return e, nil
+}
+
+// Transition calculates what is the minimally-disruptive set of rules need to
+// be applied to a devices cgroup in order to transition to the given target.
+// This means that any already-existing rules will not be applied, and
+// disruptive rules (like denying all device access) will only be applied if
+// necessary.
+//
+// This function is the sole reason for all of Emulator -- to allow us
+// to figure out how to update a containers' cgroups without causing spurrious
+// device errors (if possible).
+func (source *Emulator) Transition(target *Emulator) ([]*configs.DeviceRule, error) {
+	var transitionRules []*configs.DeviceRule
+	oldRules := source.rules
+
+	// If the default policy doesn't match, we need to include a "disruptive"
+	// rule (either allow-all or deny-all) in order to switch the cgroup to the
+	// correct default policy.
+	//
+	// However, due to a limitation in "devices.list" we cannot be sure what
+	// deny rules are in place in a black-list cgroup. Thus if the source is a
+	// black-list we also have to include a disruptive rule.
+	if source.IsBlacklist() || source.defaultAllow != target.defaultAllow {
+		transitionRules = append(transitionRules, &configs.DeviceRule{
+			Type:        'a',
+			Major:       -1,
+			Minor:       -1,
+			Permissions: configs.DevicePermissions("rwm"),
+			Allow:       target.defaultAllow,
+		})
+		// The old rules are only relevant if we aren't starting out with a
+		// disruptive rule.
+		oldRules = nil
+	}
+
+	// NOTE: We traverse through the rules in a sorted order so we always write
+	//       the same set of rules (this is to aid testing).
+
+	// First, we create inverse rules for any old rules not in the new set.
+	// This includes partial-inverse rules for specific permissions. This is a
+	// no-op if we added a disruptive rule, since oldRules will be empty.
+	for _, rule := range oldRules.orderedEntries() {
+		meta, oldPerms := rule.meta, rule.perms
+		newPerms := target.rules[meta]
+		droppedPerms := oldPerms.Difference(newPerms)
+		if !droppedPerms.IsEmpty() {
+			transitionRules = append(transitionRules, &configs.DeviceRule{
+				Type:        meta.node,
+				Major:       meta.major,
+				Minor:       meta.minor,
+				Permissions: droppedPerms,
+				Allow:       target.defaultAllow,
+			})
+		}
+	}
+
+	// Add any additional rules which weren't in the old set. We happen to
+	// filter out rules which are present in both sets, though this isn't
+	// strictly necessary.
+	for _, rule := range target.rules.orderedEntries() {
+		meta, newPerms := rule.meta, rule.perms
+		oldPerms := oldRules[meta]
+		gainedPerms := newPerms.Difference(oldPerms)
+		if !gainedPerms.IsEmpty() {
+			transitionRules = append(transitionRules, &configs.DeviceRule{
+				Type:        meta.node,
+				Major:       meta.major,
+				Minor:       meta.minor,
+				Permissions: gainedPerms,
+				Allow:       !target.defaultAllow,
+			})
+		}
+	}
+	return transitionRules, nil
+}