Merge pull request #4109 from pmores/drop-in-cfg-files-support

Drop in cfg files support

src/runtime/README.md

@@ -87,6 +87,27 @@ following locations (in order):
 
 > **Note:** For both binaries, the first path that exists will be used.
 
+#### Drop-in configuration file fragments
+
+To enable changing configuration without changing the configuration file
+itself, drop-in configuration file fragments are supported.  Once a
+configuration file is parsed, if there is a subdirectory called `config.d` in
+the same directory as the configuration file its contents will be loaded
+in alphabetical order and each item will be parsed as a config file.  Settings
+loaded from these configuration file fragments override settings loaded from
+the main configuration file and earlier fragments.  Users are encouraged to use
+familiar naming conventions to order the fragments (e.g. `config.d/10-this`,
+`config.d/20-that` etc.).
+
+Non-existent or empty `config.d` directory is not an error (in other words, not
+using configuration file fragments is fine).  On the other hand, if fragments
+are used, they must be valid - any errors while parsing fragments (unreadable
+fragment files, contents not valid TOML) are treated the same as errors
+while parsing the main configuration file.  A `config.d` subdirectory affects
+only the `configuration.toml` _in the same directory_.  For fragments in
+`config.d` to be parsed, there has to be a valid main configuration file _in
+that location_ (it can be empty though).
+
 ### Hypervisor specific configuration
 
 Kata Containers supports multiple hypervisors so your `configuration.toml`

src/runtime/pkg/katautils/config.go

@@ -10,8 +10,10 @@ package katautils
 import (
 	"errors"
 	"fmt"
+	"io/ioutil"
 	"os"
 	"path/filepath"
+	"reflect"
 	goruntime "runtime"
 	"strings"
 
@@ -178,6 +180,20 @@ type agent struct {
 	DialTimeout         uint32   `toml:"dial_timeout"`
 }
 
+func (orig *tomlConfig) Clone() tomlConfig {
+	clone := *orig
+	clone.Hypervisor = make(map[string]hypervisor)
+	clone.Agent = make(map[string]agent)
+
+	for key, value := range orig.Hypervisor {
+		clone.Hypervisor[key] = value
+	}
+	for key, value := range orig.Agent {
+		clone.Agent[key] = value
+	}
+	return clone
+}
+
 func (h hypervisor) path() (string, error) {
 	p := h.Path
 
@@ -1322,9 +1338,179 @@ func decodeConfig(configPath string) (tomlConfig, string, error) {
 		return tomlConf, resolved, err
 	}
 
+	err = decodeDropIns(resolved, &tomlConf)
+	if err != nil {
+		return tomlConf, resolved, err
+	}
+
 	return tomlConf, resolved, nil
 }
 
+func decodeDropIns(mainConfigPath string, tomlConf *tomlConfig) error {
+	configDir := filepath.Dir(mainConfigPath)
+	dropInDir := filepath.Join(configDir, "config.d")
+
+	files, err := ioutil.ReadDir(dropInDir)
+	if err != nil {
+		if !os.IsNotExist(err) {
+			return fmt.Errorf("error reading %q directory: %s", dropInDir, err)
+		} else {
+			return nil
+		}
+	}
+
+	for _, file := range files {
+		dropInFpath := filepath.Join(dropInDir, file.Name())
+
+		err = updateFromDropIn(dropInFpath, tomlConf)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func updateFromDropIn(dropInFpath string, tomlConf *tomlConfig) error {
+	configData, err := os.ReadFile(dropInFpath)
+	if err != nil {
+		return fmt.Errorf("error reading file %q: %s", dropInFpath, err)
+	}
+
+	// Ordinarily, BurntSushi only updates fields of tomlConfig that are
+	// changed by the file and leaves the rest alone.  This doesn't apply
+	// though to tomlConfig substructures that are stored in maps.  Their
+	// previous contents are erased by toml.Decode() and only fields changed by
+	// the file are set.  To work around this, a bit of juggling is needed to
+	// preserve the previous contents and merge them manually with the incoming
+	// changes afterwards, using reflection.
+	tomlConfOrig := tomlConf.Clone()
+
+	var md toml.MetaData
+	md, err = toml.Decode(string(configData), &tomlConf)
+
+	if err != nil {
+		return fmt.Errorf("error decoding file %q: %s", dropInFpath, err)
+	}
+
+	if len(md.Undecoded()) > 0 {
+		msg := fmt.Sprintf("warning: undecoded keys in %q: %+v", dropInFpath, md.Undecoded())
+		kataUtilsLogger.Warn(msg)
+	}
+
+	for _, key := range md.Keys() {
+		err = applyKey(*tomlConf, key, &tomlConfOrig)
+		if err != nil {
+			return fmt.Errorf("error applying key '%+v' from drop-in file %q: %s", key, dropInFpath, err)
+		}
+	}
+
+	tomlConf.Hypervisor = tomlConfOrig.Hypervisor
+	tomlConf.Agent = tomlConfOrig.Agent
+
+	return nil
+}
+
+func applyKey(sourceConf tomlConfig, key []string, targetConf *tomlConfig) error {
+	// Any key that might need treatment provided by this function has to have
+	// (at least) three components: [ map_name map_key_name field_toml_tag ],
+	// e.g. [agent kata enable_tracing] or [hypervisor qemu confidential_guest].
+	if len(key) < 3 {
+		return nil
+	}
+	switch key[0] {
+	case "agent":
+		return applyAgentKey(sourceConf, key[1:], targetConf)
+	case "hypervisor":
+		return applyHypervisorKey(sourceConf, key[1:], targetConf)
+		// The table the 'key' is in is not stored in a map so no special handling
+		// is needed.
+	}
+	return nil
+}
+
+// Both of the following functions copy the value of a 'sourceConf' field
+// identified by the TOML tag in 'key' into the corresponding field in
+// 'targetConf'.
+func applyAgentKey(sourceConf tomlConfig, key []string, targetConf *tomlConfig) error {
+	agentName := key[0]
+	tomlKeyName := key[1]
+
+	sourceAgentConf := sourceConf.Agent[agentName]
+	targetAgentConf := targetConf.Agent[agentName]
+
+	err := copyFieldValue(reflect.ValueOf(&sourceAgentConf).Elem(), tomlKeyName, reflect.ValueOf(&targetAgentConf).Elem())
+	if err != nil {
+		return err
+	}
+
+	targetConf.Agent[agentName] = targetAgentConf
+	return nil
+}
+
+func applyHypervisorKey(sourceConf tomlConfig, key []string, targetConf *tomlConfig) error {
+	hypervisorName := key[0]
+	tomlKeyName := key[1]
+
+	sourceHypervisorConf := sourceConf.Hypervisor[hypervisorName]
+	targetHypervisorConf := targetConf.Hypervisor[hypervisorName]
+
+	err := copyFieldValue(reflect.ValueOf(&sourceHypervisorConf).Elem(), tomlKeyName, reflect.ValueOf(&targetHypervisorConf).Elem())
+	if err != nil {
+		return err
+	}
+
+	targetConf.Hypervisor[hypervisorName] = targetHypervisorConf
+	return nil
+}
+
+// Copies a TOML value of the source field identified by its TOML key to the
+// corresponding field of the target.  Basically
+// 'target[tomlKeyName] = source[tomlKeyNmae]'.
+func copyFieldValue(source reflect.Value, tomlKeyName string, target reflect.Value) error {
+	val, err := getValue(source, tomlKeyName)
+	if err != nil {
+		return fmt.Errorf("error getting key %q from a decoded drop-in conf file: %s", tomlKeyName, err)
+	}
+	err = setValue(target, tomlKeyName, val)
+	if err != nil {
+		return fmt.Errorf("error setting key %q to a new value '%v': %s", tomlKeyName, val.Interface(), err)
+	}
+	return nil
+}
+
+// The first argument is expected to be a reflect.Value of a tomlConfig
+// substructure (hypervisor, agent), the second argument is a TOML key
+// corresponding to the substructure field whose TOML value is queried.
+// Return value corresponds to 'tomlConfStruct[tomlKey]'.
+func getValue(tomlConfStruct reflect.Value, tomlKey string) (reflect.Value, error) {
+	tomlConfStructType := tomlConfStruct.Type()
+	for j := 0; j < tomlConfStruct.NumField(); j++ {
+		fieldTomlTag := tomlConfStructType.Field(j).Tag.Get("toml")
+		if fieldTomlTag == tomlKey {
+			return tomlConfStruct.Field(j), nil
+		}
+	}
+	return reflect.Value{}, fmt.Errorf("key %q not found", tomlKey)
+}
+
+// The first argument is expected to be a reflect.Value of a tomlConfig
+// substructure (hypervisor, agent), the second argument is a TOML key
+// corresponding to the substructure field whose TOML value is to be changed,
+// the third argument is a reflect.Value representing the new TOML value.
+// An equivalent of 'tomlConfStruct[tomlKey] = newVal'.
+func setValue(tomlConfStruct reflect.Value, tomlKey string, newVal reflect.Value) error {
+	tomlConfStructType := tomlConfStruct.Type()
+	for j := 0; j < tomlConfStruct.NumField(); j++ {
+		fieldTomlTag := tomlConfStructType.Field(j).Tag.Get("toml")
+		if fieldTomlTag == tomlKey {
+			tomlConfStruct.Field(j).Set(newVal)
+			return nil
+		}
+	}
+	return fmt.Errorf("key %q not found", tomlKey)
+}
+
 // checkConfig checks the validity of the specified config.
 func checkConfig(config oci.RuntimeConfig) error {
 	if err := checkNetNsConfig(config); err != nil {

src/runtime/pkg/katautils/config_test.go

@@ -1658,3 +1658,69 @@ func TestValidateBindMounts(t *testing.T) {
 		}
 	}
 }
+
+func TestLoadDropInConfiguration(t *testing.T) {
+	tmpdir := t.TempDir()
+
+	// Test Runtime and Hypervisor to represent structures stored directly and
+	// in maps, respectively.  For each of them, test
+	// - a key that's only set in the base config file
+	// - a key that's only set in a drop-in
+	// - a key that's set in the base config file and then changed by a drop-in
+	// - a key that's set in a drop-in and then overridden by another drop-in
+	// Avoid default values to reduce the risk of mistaking a result of
+	// something having gone wrong with the expected value.
+
+	runtimeConfigFileData := `
+[hypervisor.qemu]
+path = "/usr/bin/qemu-kvm"
+default_bridges = 3
+[runtime]
+enable_debug = true
+internetworking_model="tcfilter"
+`
+	dropInData := `
+[hypervisor.qemu]
+default_vcpus = 2
+default_bridges = 4
+shared_fs = "virtio-fs"
+[runtime]
+sandbox_cgroup_only=true
+internetworking_model="macvtap"
+vfio_mode="guest-kernel"
+`
+	dropInOverrideData := `
+[hypervisor.qemu]
+shared_fs = "virtio-9p"
+[runtime]
+vfio_mode="vfio"
+`
+
+	configPath := path.Join(tmpdir, "runtime.toml")
+	err := createConfig(configPath, runtimeConfigFileData)
+	assert.NoError(t, err)
+
+	dropInDir := path.Join(tmpdir, "config.d")
+	err = os.Mkdir(dropInDir, os.FileMode(0777))
+	assert.NoError(t, err)
+
+	dropInPath := path.Join(dropInDir, "10-base")
+	err = createConfig(dropInPath, dropInData)
+	assert.NoError(t, err)
+
+	dropInOverridePath := path.Join(dropInDir, "10-override")
+	err = createConfig(dropInOverridePath, dropInOverrideData)
+	assert.NoError(t, err)
+
+	config, _, err := decodeConfig(configPath)
+	assert.NoError(t, err)
+
+	assert.Equal(t, config.Hypervisor["qemu"].Path, "/usr/bin/qemu-kvm")
+	assert.Equal(t, config.Hypervisor["qemu"].NumVCPUs, int32(2))
+	assert.Equal(t, config.Hypervisor["qemu"].DefaultBridges, uint32(4))
+	assert.Equal(t, config.Hypervisor["qemu"].SharedFS, "virtio-9p")
+	assert.Equal(t, config.Runtime.Debug, true)
+	assert.Equal(t, config.Runtime.SandboxCgroupOnly, true)
+	assert.Equal(t, config.Runtime.InterNetworkModel, "macvtap")
+	assert.Equal(t, config.Runtime.VfioMode, "vfio")
+}