sysconfig/cloudinit.go: measure (but don't use) gadget cloud-init datasource

sysconfig/cloudinit.go

@@ -27,6 +27,10 @@ import (
 	"os/exec"
 	"path/filepath"
 	"regexp"
+	"sort"
+	"strings"
+
+	yaml "gopkg.in/yaml.v2"
 
 	"github.com/snapcore/snapd/asserts"
 	"github.com/snapcore/snapd/dirs"
@@ -63,6 +67,108 @@ func DisableCloudInit(rootDir string) error {
 	return nil
 }
 
+// supportedFilteredCloudConfig is a struct of the supported values for
+// cloud-init configuration file.
+type supportedFilteredCloudConfig struct {
+	Datasource map[string]supportedFilteredDatasource `yaml:"datasource,omitempty"`
+	Network    map[string]interface{}                 `yaml:"network,omitempty"`
+	// DatasourceList is a pointer so we can distinguish between:
+	// datasource_list: []
+	// and not setting the datasource at all
+	// for example there might be gadgets which don't want to use any
+	// datasources, but still wants to set some networking config
+	DatasourceList *[]string                             `yaml:"datasource_list,omitempty"`
+	Reporting      map[string]supportedFilteredReporting `yaml:"reporting,omitempty"`
+}
+
+type supportedFilteredDatasource struct {
+	// these are for MAAS
+	ConsumerKey string `yaml:"consumer_key"`
+	MetadataURL string `yaml:"metadata_url"`
+	TokenKey    string `yaml:"token_key"`
+	TokenSecret string `yaml:"token_secret"`
+}
+
+type supportedFilteredReporting struct {
+	Type        string `yaml:"type"`
+	Endpoint    string `yaml:"endpoint"`
+	ConsumerKey string `yaml:"consumer_key"`
+	TokenKey    string `yaml:"token_key"`
+	TokenSecret string `yaml:"token_secret"`
+}
+
+type cloudDatasourcesInUseResult struct {
+	// ExplicitlyAllowed is the value of datasource_list. If this is empty,
+	// consult ExplicitlyNoneAllowed to tell if it was specified as empty in the
+	// config or if it was just absent from the config
+	ExplicitlyAllowed []string
+	// ExplicitlyNoneAllowed is true when datasource_list was set to
+	// specifically the empty list, thus disallowing use of any datasource
+	ExplicitlyNoneAllowed bool
+	// Mentioned is the full set of datasources mentioned in the yaml config.
+	Mentioned []string
+}
+
+// cloudDatasourcesInUse returns the datasources in use by the specified config
+// file. All datasource names are made upper case to be comparable. This is an
+// arbitrary choice between making them upper case or making them lower case,
+// but cloud-init treats "maas" the same as "MAAS", so we need to treat them the
+// same too.
+func cloudDatasourcesInUse(configFile string) (*cloudDatasourcesInUseResult, error) {
+	// TODO: are there other keys in addition to those that we support in
+	// filtering that might mention datasources ?
+
+	b, err := ioutil.ReadFile(configFile)
+	if err != nil {
+		return nil, err
+	}
+
+	var cfg supportedFilteredCloudConfig
+	if err := yaml.Unmarshal(b, &cfg); err != nil {
+		return nil, err
+	}
+
+	res := &cloudDatasourcesInUseResult{}
+
+	sourcesMentionedInCfg := map[string]bool{}
+
+	// datasource key is a map with the datasource name as a key
+	for ds := range cfg.Datasource {
+		sourcesMentionedInCfg[strings.ToUpper(ds)] = true
+	}
+
+	// same for reporting
+	for ds := range cfg.Reporting {
+		sourcesMentionedInCfg[strings.ToUpper(ds)] = true
+	}
+
+	// we can also have datasources mentioned in the datasource list config
+	if cfg.DatasourceList != nil {
+		if len(*cfg.DatasourceList) == 0 {
+			res.ExplicitlyNoneAllowed = true
+		} else {
+			explicitlyAllowed := map[string]bool{}
+			for _, ds := range *cfg.DatasourceList {
+				dsName := strings.ToUpper(ds)
+				sourcesMentionedInCfg[dsName] = true
+				explicitlyAllowed[dsName] = true
+			}
+			res.ExplicitlyAllowed = make([]string, 0, len(explicitlyAllowed))
+			for ds := range explicitlyAllowed {
+				res.ExplicitlyAllowed = append(res.ExplicitlyAllowed, ds)
+			}
+			sort.Strings(res.ExplicitlyAllowed)
+		}
+	}
+
+	for ds := range sourcesMentionedInCfg {
+		res.Mentioned = append(res.Mentioned, strings.ToUpper(ds))
+	}
+	sort.Strings(res.Mentioned)
+
+	return res, nil
+}
+
 type cloudInitConfigInstallOptions struct {
 	// Prefix is the prefix to add to files when installing them.
 	Prefix string
@@ -79,7 +185,8 @@ type cloudInitConfigInstallOptions struct {
 }
 
 // installCloudInitCfgDir installs glob cfg files from the source directory to
-// the cloud config dir.
+// the cloud config dir, optionally filtering the files for safe and supported
+// keys in the configuration before installing them.
 func installCloudInitCfgDir(src, targetdir string, opts *cloudInitConfigInstallOptions) error {
 	if opts == nil {
 		opts = &cloudInitConfigInstallOptions{}
@@ -94,9 +201,6 @@ func installCloudInitCfgDir(src, targetdir string, opts *cloudInitConfigInstallO
 		return nil
 	}
 
-	// TODO: handle opts.Filter and opts.AllowedDatasources when installing
-	// config
-
 	ubuntuDataCloudCfgDir := filepath.Join(ubuntuDataCloudDir(targetdir), "cloud.cfg.d/")
 	if err := os.MkdirAll(ubuntuDataCloudCfgDir, 0755); err != nil {
 		return fmt.Errorf("cannot make cloud config dir: %v", err)
@@ -114,18 +218,22 @@ func installCloudInitCfgDir(src, targetdir string, opts *cloudInitConfigInstallO
 // gadget snap to the /etc/cloud config dir as "80_device_gadget.cfg". It also
 // parses and returns what datasources are detected to be in use for the gadget
 // cloud-config.
-// TODO: return the detected datasources in use from the gadget config, if any.
-func installGadgetCloudInitCfg(src, targetdir string) error {
+func installGadgetCloudInitCfg(src, targetdir string) (*cloudDatasourcesInUseResult, error) {
 	ubuntuDataCloudCfgDir := filepath.Join(ubuntuDataCloudDir(targetdir), "cloud.cfg.d/")
 	if err := os.MkdirAll(ubuntuDataCloudCfgDir, 0755); err != nil {
-		return fmt.Errorf("cannot make cloud config dir: %v", err)
+		return nil, fmt.Errorf("cannot make cloud config dir: %v", err)
+	}
+
+	datasourcesRes, err := cloudDatasourcesInUse(src)
+	if err != nil {
+		return nil, err
 	}
 
 	configFile := filepath.Join(ubuntuDataCloudCfgDir, "80_device_gadget.cfg")
 	if err := osutil.CopyFile(src, configFile, 0); err != nil {
-		return err
+		return nil, err
 	}
-	return nil
+	return datasourcesRes, nil
 }
 
 func configureCloudInit(model *asserts.Model, opts *Options) (err error) {
@@ -154,9 +262,9 @@ func configureCloudInit(model *asserts.Model, opts *Options) (err error) {
 		// then copy / install the gadget config first
 		gadgetCloudConf := filepath.Join(opts.GadgetDir, "cloud.conf")
 
-		// TODO: this should also return what datasources the gadget allows from
-		// it's cloud-init config
-		if err := installGadgetCloudInitCfg(gadgetCloudConf, WritableDefaultsDir(opts.TargetRootDir)); err != nil {
+		// TODO: save the gadget datasource and use it below in deciding what to
+		// allow through for grade: signed
+		if _, err := installGadgetCloudInitCfg(gadgetCloudConf, WritableDefaultsDir(opts.TargetRootDir)); err != nil {
 			return err
 		}
 

sysconfig/cloudinit_test.go

@@ -65,7 +65,7 @@ func (s *sysconfigSuite) makeCloudCfgSrcDirFiles(c *C) string {
 func (s *sysconfigSuite) makeGadgetCloudConfFile(c *C) string {
 	gadgetDir := c.MkDir()
 	gadgetCloudConf := filepath.Join(gadgetDir, "cloud.conf")
-	err := ioutil.WriteFile(gadgetCloudConf, []byte("gadget cloud config"), 0644)
+	err := ioutil.WriteFile(gadgetCloudConf, []byte("#cloud-config gadget cloud config"), 0644)
 	c.Assert(err, IsNil)
 
 	return gadgetDir
@@ -232,7 +232,7 @@ func (s *sysconfigSuite) TestInstallModeCloudInitInstallsOntoHostRunModeWithGadg
 
 	// and did copy the gadget cloud-init file
 	ubuntuDataCloudCfg := filepath.Join(boot.InstallHostWritableDir, "_writable_defaults/etc/cloud/cloud.cfg.d/")
-	c.Check(filepath.Join(ubuntuDataCloudCfg, "80_device_gadget.cfg"), testutil.FileEquals, "gadget cloud config")
+	c.Check(filepath.Join(ubuntuDataCloudCfg, "80_device_gadget.cfg"), testutil.FileEquals, "#cloud-config gadget cloud config")
 }
 
 func (s *sysconfigSuite) TestInstallModeCloudInitInstallsOntoHostRunModeWithGadgetCloudConfAlsoInstallsUbuntuSeedConfig(c *C) {
@@ -249,7 +249,7 @@ func (s *sysconfigSuite) TestInstallModeCloudInitInstallsOntoHostRunModeWithGadg
 
 	// we did copy the gadget cloud-init file
 	ubuntuDataCloudCfg := filepath.Join(boot.InstallHostWritableDir, "_writable_defaults/etc/cloud/cloud.cfg.d/")
-	c.Check(filepath.Join(ubuntuDataCloudCfg, "80_device_gadget.cfg"), testutil.FileEquals, "gadget cloud config")
+	c.Check(filepath.Join(ubuntuDataCloudCfg, "80_device_gadget.cfg"), testutil.FileEquals, "#cloud-config gadget cloud config")
 
 	// and we also copied the ubuntu-seed files with a new prefix such that they
 	// take precedence over the gadget file by being ordered lexically after the
@@ -655,3 +655,150 @@ func (s *sysconfigSuite) TestRestrictCloudInit(c *C) {
 		}
 	}
 }
+
+const maasGadgetCloudInitImplictYAML = `
+datasource:
+  MAAS:
+    foo: bar
+`
+
+const maasGadgetCloudInitImplictLowerCaseYAML = `
+datasource:
+  maas:
+    foo: bar
+`
+
+const explicitlyNoDatasourceYAML = `datasource_list: []`
+
+const explicitlyNoDatasourceButAlsoImplicitlyAnotherYAML = `
+datasource_list: []
+reporting:
+  NoCloud:
+    foo: bar
+`
+
+const explicitlyMultipleMixedCaseMentioned = `
+reporting:
+  NoCloud:
+    foo: bar
+  maas:
+    foo: bar
+datasource:
+  MAAS:
+    foo: bar
+  NOCLOUD:
+    foo: bar
+`
+
+func (s *sysconfigSuite) TestCloudDatasourcesInUse(c *C) {
+	tt := []struct {
+		configFileContent string
+		expError          string
+		expRes            *sysconfig.CloudDatasourcesInUseResult
+		comment           string
+	}{
+		{
+			configFileContent: `datasource_list: [MAAS]`,
+			expRes: &sysconfig.CloudDatasourcesInUseResult{
+				ExplicitlyAllowed: []string{"MAAS"},
+				Mentioned:         []string{"MAAS"},
+			},
+			comment: "explicitly allowed via datasource_list in upper case",
+		},
+		{
+			configFileContent: `datasource_list: [maas]`,
+			expRes: &sysconfig.CloudDatasourcesInUseResult{
+				ExplicitlyAllowed: []string{"MAAS"},
+				Mentioned:         []string{"MAAS"},
+			},
+			comment: "explicitly allowed via datasource_list in lower case",
+		},
+		{
+			configFileContent: `datasource_list: [mAaS]`,
+			expRes: &sysconfig.CloudDatasourcesInUseResult{
+				ExplicitlyAllowed: []string{"MAAS"},
+				Mentioned:         []string{"MAAS"},
+			},
+			comment: "explicitly allowed via datasource_list in random case",
+		},
+		{
+			configFileContent: `datasource_list: [maas, maas]`,
+			expRes: &sysconfig.CloudDatasourcesInUseResult{
+				ExplicitlyAllowed: []string{"MAAS"},
+				Mentioned:         []string{"MAAS"},
+			},
+			comment: "duplicated datasource in datasource_list",
+		},
+		{
+			configFileContent: `datasource_list: [maas, MAAS]`,
+			expRes: &sysconfig.CloudDatasourcesInUseResult{
+				ExplicitlyAllowed: []string{"MAAS"},
+				Mentioned:         []string{"MAAS"},
+			},
+			comment: "duplicated datasource in datasource_list with different cases",
+		},
+		{
+			configFileContent: `datasource_list: [maas, GCE]`,
+			expRes: &sysconfig.CloudDatasourcesInUseResult{
+				ExplicitlyAllowed: []string{"GCE", "MAAS"},
+				Mentioned:         []string{"GCE", "MAAS"},
+			},
+			comment: "multiple datasources in datasource list",
+		},
+		{
+			configFileContent: maasGadgetCloudInitImplictYAML,
+			expRes: &sysconfig.CloudDatasourcesInUseResult{
+				Mentioned: []string{"MAAS"},
+			},
+			comment: "implicitly mentioned datasource",
+		},
+		{
+			configFileContent: maasGadgetCloudInitImplictLowerCaseYAML,
+			expRes: &sysconfig.CloudDatasourcesInUseResult{
+				Mentioned: []string{"MAAS"},
+			},
+			comment: "implicitly mentioned datasource in lower case",
+		},
+		{
+			configFileContent: explicitlyNoDatasourceYAML,
+			expRes: &sysconfig.CloudDatasourcesInUseResult{
+				ExplicitlyNoneAllowed: true,
+			},
+			comment: "no datasources allowed at all",
+		},
+		{
+			configFileContent: explicitlyNoDatasourceButAlsoImplicitlyAnotherYAML,
+			expRes: &sysconfig.CloudDatasourcesInUseResult{
+				ExplicitlyNoneAllowed: true,
+				Mentioned:             []string{"NOCLOUD"},
+			},
+			comment: "explicitly no datasources allowed, but still some mentioned",
+		},
+		{
+			configFileContent: explicitlyMultipleMixedCaseMentioned,
+			expRes: &sysconfig.CloudDatasourcesInUseResult{
+				Mentioned: []string{"MAAS", "NOCLOUD"},
+			},
+			comment: "multiple of same datasources mentioned in different cases",
+		},
+		{
+			configFileContent: "i'm not yaml",
+			expError:          "yaml: unmarshal errors.*\n.*cannot unmarshal.*",
+			comment:           "invalid yaml",
+		},
+	}
+
+	for _, t := range tt {
+		comment := Commentf(t.comment)
+		configFile := filepath.Join(c.MkDir(), "cloud.conf")
+		err := ioutil.WriteFile(configFile, []byte(t.configFileContent), 0644)
+		c.Assert(err, IsNil, comment)
+		res, err := sysconfig.CloudDatasourcesInUse(configFile)
+		if t.expError != "" {
+			c.Assert(err, ErrorMatches, t.expError, comment)
+			continue
+		}
+
+		c.Assert(res, DeepEquals, t.expRes, comment)
+	}
+}

sysconfig/export_test.go

@@ -0,0 +1,30 @@
+// -*- Mode: Go; indent-tabs-mode: t -*-
+
+/*
+ * Copyright (C) 2021 Canonical Ltd
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 3 as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+package sysconfig
+
+func CloudDatasourcesInUse(configFile string) (*CloudDatasourcesInUseResult, error) {
+	res, err := cloudDatasourcesInUse(configFile)
+	if err != nil {
+		return nil, err
+	}
+	return (*CloudDatasourcesInUseResult)(res), err
+}
+
+type CloudDatasourcesInUseResult = cloudDatasourcesInUseResult