instance-data: add cloud-init merged_cfg and sys_info keys to json (c…

…anonical#214)

Cloud-config userdata provided as jinja templates are now distro,
platform and merged cloud config aware. The cloud-init query command
will also surface this config data.

Now users can selectively render portions of cloud-config based on:
* distro name, version, release
* python version
* merged cloud config values
* machine platform
* kernel

To support template handling of this config, add new top-level
keys to /run/cloud-init/instance-data.json.

The new 'merged_cfg' key represents merged cloud config from
/etc/cloud/cloud.cfg and /etc/cloud/cloud.cfg.d/*.

The new 'sys_info' key which captures distro and platform
info from cloudinit.util.system_info.

Cloud config userdata templates can render conditional content
based on these additional environmental checks such as the following
simple example:

```
  ## template: jinja
  #cloud-config
   runcmd:
  {% if distro == 'opensuse' %}
    - sh /custom-setup-sles
  {% elif distro == 'centos' %}
    - sh /custom-setup-centos
  {% elif distro == 'debian' %}
    - sh /custom-setup-debian
  {% endif %}
```

To see all values: sudo cloud-init query --all

Any keys added to the standardized v1 keys are guaranteed to not
change or drop on future released of cloud-init. 'v1' keys will be retained
for backward-compatibility even if a new standardized 'v2' set of keys
are introduced

The following standardized v1 keys are added:
* distro, distro_release, distro_version, kernel_version, machine,
python_version, system_platform, variant

LP: #1865969

cloudinit/sources/__init__.py

@@ -89,26 +89,26 @@ def process_instance_metadata(metadata, key_path='', sensitive_keys=()):
     @return Dict copy of processed metadata.
     """
     md_copy = copy.deepcopy(metadata)
-    md_copy['base64_encoded_keys'] = []
-    md_copy['sensitive_keys'] = []
+    base64_encoded_keys = []
+    sens_keys = []
     for key, val in metadata.items():
         if key_path:
             sub_key_path = key_path + '/' + key
         else:
             sub_key_path = key
         if key in sensitive_keys or sub_key_path in sensitive_keys:
-            md_copy['sensitive_keys'].append(sub_key_path)
+            sens_keys.append(sub_key_path)
         if isinstance(val, str) and val.startswith('ci-b64:'):
-            md_copy['base64_encoded_keys'].append(sub_key_path)
+            base64_encoded_keys.append(sub_key_path)
             md_copy[key] = val.replace('ci-b64:', '')
         if isinstance(val, dict):
             return_val = process_instance_metadata(
                 val, sub_key_path, sensitive_keys)
-            md_copy['base64_encoded_keys'].extend(
-                return_val.pop('base64_encoded_keys'))
-            md_copy['sensitive_keys'].extend(
-                return_val.pop('sensitive_keys'))
+            base64_encoded_keys.extend(return_val.pop('base64_encoded_keys'))
+            sens_keys.extend(return_val.pop('sensitive_keys'))
             md_copy[key] = return_val
+    md_copy['base64_encoded_keys'] = sorted(base64_encoded_keys)
+    md_copy['sensitive_keys'] = sorted(sens_keys)
     return md_copy
 
 
@@ -193,7 +193,7 @@ class DataSource(metaclass=abc.ABCMeta):
 
     # N-tuple of keypaths or keynames redact from instance-data.json for
     # non-root users
-    sensitive_metadata_keys = ('security-credentials',)
+    sensitive_metadata_keys = ('merged_cfg', 'security-credentials',)
 
     def __init__(self, sys_cfg, distro, paths, ud_proc=None):
         self.sys_cfg = sys_cfg
@@ -218,29 +218,38 @@ def __init__(self, sys_cfg, distro, paths, ud_proc=None):
     def __str__(self):
         return type_utils.obj_name(self)
 
-    def _get_standardized_metadata(self):
+    def _get_standardized_metadata(self, instance_data):
         """Return a dictionary of standardized metadata keys."""
         local_hostname = self.get_hostname()
         instance_id = self.get_instance_id()
         availability_zone = self.availability_zone
         # In the event of upgrade from existing cloudinit, pickled datasource
         # will not contain these new class attributes. So we need to recrawl
-        # metadata to discover that content.
+        # metadata to discover that content
+        sysinfo = instance_data["sys_info"]
         return {
             'v1': {
                 '_beta_keys': ['subplatform'],
                 'availability-zone': availability_zone,
                 'availability_zone': availability_zone,
                 'cloud-name': self.cloud_name,
                 'cloud_name': self.cloud_name,
+                'distro': sysinfo["dist"][0],
+                'distro_version': sysinfo["dist"][1],
+                'distro_release': sysinfo["dist"][2],
                 'platform': self.platform_type,
                 'public_ssh_keys': self.get_public_ssh_keys(),
+                'python_version': sysinfo["python"],
                 'instance-id': instance_id,
                 'instance_id': instance_id,
+                'kernel_release': sysinfo["uname"][2],
                 'local-hostname': local_hostname,
                 'local_hostname': local_hostname,
+                'machine': sysinfo["uname"][4],
                 'region': self.region,
-                'subplatform': self.subplatform}}
+                'subplatform': self.subplatform,
+                'system_platform': sysinfo["platform"],
+                'variant': sysinfo["variant"]}}
 
     def clear_cached_attrs(self, attr_defaults=()):
         """Reset any cached metadata attributes to datasource defaults.
@@ -299,9 +308,15 @@ def persist_instance_data(self):
                 ec2_metadata = getattr(self, 'ec2_metadata')
                 if ec2_metadata != UNSET:
                     instance_data['ds']['ec2_metadata'] = ec2_metadata
-        instance_data.update(
-            self._get_standardized_metadata())
         instance_data['ds']['_doc'] = EXPERIMENTAL_TEXT
+        # Add merged cloud.cfg and sys info for jinja templates and cli query
+        instance_data['merged_cfg'] = copy.deepcopy(self.sys_cfg)
+        instance_data['merged_cfg']['_doc'] = (
+            'Merged cloud-init system config from /etc/cloud/cloud.cfg and'
+            ' /etc/cloud/cloud.cfg.d/')
+        instance_data['sys_info'] = util.system_info()
+        instance_data.update(
+            self._get_standardized_metadata(instance_data))
         try:
             # Process content base64encoding unserializable values
             content = util.json_dumps(instance_data)

cloudinit/sources/tests/test_init.py

@@ -55,6 +55,7 @@ class InvalidDataSourceTestSubclassNet(DataSource):
 class TestDataSource(CiTestCase):
 
     with_logs = True
+    maxDiff = None
 
     def setUp(self):
         super(TestDataSource, self).setUp()
@@ -288,27 +289,47 @@ def test_get_data_writes_json_instance_data_on_success(self):
         tmp = self.tmp_dir()
         datasource = DataSourceTestSubclassNet(
             self.sys_cfg, self.distro, Paths({'run_dir': tmp}))
-        datasource.get_data()
+        sys_info = {
+            "python": "3.7",
+            "platform":
+                "Linux-5.4.0-24-generic-x86_64-with-Ubuntu-20.04-focal",
+            "uname": ["Linux", "myhost", "5.4.0-24-generic", "SMP blah",
+                      "x86_64"],
+            "variant": "ubuntu", "dist": ["ubuntu", "20.04", "focal"]}
+        with mock.patch("cloudinit.util.system_info", return_value=sys_info):
+            datasource.get_data()
         json_file = self.tmp_path(INSTANCE_JSON_FILE, tmp)
         content = util.load_file(json_file)
         expected = {
             'base64_encoded_keys': [],
-            'sensitive_keys': [],
+            'merged_cfg': REDACT_SENSITIVE_VALUE,
+            'sensitive_keys': ['merged_cfg'],
+            'sys_info': sys_info,
             'v1': {
                 '_beta_keys': ['subplatform'],
                 'availability-zone': 'myaz',
                 'availability_zone': 'myaz',
                 'cloud-name': 'subclasscloudname',
                 'cloud_name': 'subclasscloudname',
+                'distro': 'ubuntu',
+                'distro_release': 'focal',
+                'distro_version': '20.04',
                 'instance-id': 'iid-datasource',
                 'instance_id': 'iid-datasource',
                 'local-hostname': 'test-subclass-hostname',
                 'local_hostname': 'test-subclass-hostname',
+                'kernel_release': '5.4.0-24-generic',
+                'machine': 'x86_64',
                 'platform': 'mytestsubclass',
                 'public_ssh_keys': [],
+                'python_version': '3.7',
                 'region': 'myregion',
-                'subplatform': 'unknown'},
+                'system_platform':
+                    'Linux-5.4.0-24-generic-x86_64-with-Ubuntu-20.04-focal',
+                'subplatform': 'unknown',
+                'variant': 'ubuntu'},
             'ds': {
+
                 '_doc': EXPERIMENTAL_TEXT,
                 'meta_data': {'availability_zone': 'myaz',
                               'local-hostname': 'test-subclass-hostname',
@@ -329,28 +350,49 @@ def test_get_data_writes_redacted_public_json_instance_data(self):
                 'region': 'myregion',
                 'some': {'security-credentials': {
                     'cred1': 'sekret', 'cred2': 'othersekret'}}})
-        self.assertEqual(
-            ('security-credentials',), datasource.sensitive_metadata_keys)
-        datasource.get_data()
+        self.assertItemsEqual(
+            ('merged_cfg', 'security-credentials',),
+            datasource.sensitive_metadata_keys)
+        sys_info = {
+            "python": "3.7",
+            "platform":
+                "Linux-5.4.0-24-generic-x86_64-with-Ubuntu-20.04-focal",
+            "uname": ["Linux", "myhost", "5.4.0-24-generic", "SMP blah",
+                      "x86_64"],
+            "variant": "ubuntu", "dist": ["ubuntu", "20.04", "focal"]}
+        with mock.patch("cloudinit.util.system_info", return_value=sys_info):
+            datasource.get_data()
         json_file = self.tmp_path(INSTANCE_JSON_FILE, tmp)
         redacted = util.load_json(util.load_file(json_file))
         expected = {
             'base64_encoded_keys': [],
-            'sensitive_keys': ['ds/meta_data/some/security-credentials'],
+            'merged_cfg': REDACT_SENSITIVE_VALUE,
+            'sensitive_keys': [
+                'ds/meta_data/some/security-credentials', 'merged_cfg'],
+            'sys_info': sys_info,
             'v1': {
                 '_beta_keys': ['subplatform'],
                 'availability-zone': 'myaz',
                 'availability_zone': 'myaz',
                 'cloud-name': 'subclasscloudname',
                 'cloud_name': 'subclasscloudname',
+                'distro': 'ubuntu',
+                'distro_release': 'focal',
+                'distro_version': '20.04',
                 'instance-id': 'iid-datasource',
                 'instance_id': 'iid-datasource',
                 'local-hostname': 'test-subclass-hostname',
                 'local_hostname': 'test-subclass-hostname',
+                'kernel_release': '5.4.0-24-generic',
+                'machine': 'x86_64',
                 'platform': 'mytestsubclass',
                 'public_ssh_keys': [],
+                'python_version': '3.7',
                 'region': 'myregion',
-                'subplatform': 'unknown'},
+                'system_platform':
+                    'Linux-5.4.0-24-generic-x86_64-with-Ubuntu-20.04-focal',
+                'subplatform': 'unknown',
+                'variant': 'ubuntu'},
             'ds': {
                 '_doc': EXPERIMENTAL_TEXT,
                 'meta_data': {
@@ -359,7 +401,7 @@ def test_get_data_writes_redacted_public_json_instance_data(self):
                     'region': 'myregion',
                     'some': {'security-credentials': REDACT_SENSITIVE_VALUE}}}
         }
-        self.assertEqual(expected, redacted)
+        self.assertItemsEqual(expected, redacted)
         file_stat = os.stat(json_file)
         self.assertEqual(0o644, stat.S_IMODE(file_stat.st_mode))
 
@@ -376,28 +418,54 @@ def test_get_data_writes_json_instance_data_sensitive(self):
                 'region': 'myregion',
                 'some': {'security-credentials': {
                     'cred1': 'sekret', 'cred2': 'othersekret'}}})
-        self.assertEqual(
-            ('security-credentials',), datasource.sensitive_metadata_keys)
-        datasource.get_data()
+        sys_info = {
+            "python": "3.7",
+            "platform":
+                "Linux-5.4.0-24-generic-x86_64-with-Ubuntu-20.04-focal",
+            "uname": ["Linux", "myhost", "5.4.0-24-generic", "SMP blah",
+                      "x86_64"],
+            "variant": "ubuntu", "dist": ["ubuntu", "20.04", "focal"]}
+
+        self.assertItemsEqual(
+            ('merged_cfg', 'security-credentials',),
+            datasource.sensitive_metadata_keys)
+        with mock.patch("cloudinit.util.system_info", return_value=sys_info):
+            datasource.get_data()
         sensitive_json_file = self.tmp_path(INSTANCE_JSON_SENSITIVE_FILE, tmp)
         content = util.load_file(sensitive_json_file)
         expected = {
             'base64_encoded_keys': [],
-            'sensitive_keys': ['ds/meta_data/some/security-credentials'],
+            'merged_cfg': {
+                 '_doc': (
+                     'Merged cloud-init system config from '
+                     '/etc/cloud/cloud.cfg and /etc/cloud/cloud.cfg.d/'),
+                 'datasource': {'_undef': {'key1': False}}},
+            'sensitive_keys': [
+                'ds/meta_data/some/security-credentials', 'merged_cfg'],
+            'sys_info': sys_info,
             'v1': {
                 '_beta_keys': ['subplatform'],
                 'availability-zone': 'myaz',
                 'availability_zone': 'myaz',
                 'cloud-name': 'subclasscloudname',
                 'cloud_name': 'subclasscloudname',
+                'distro': 'ubuntu',
+                'distro_release': 'focal',
+                'distro_version': '20.04',
                 'instance-id': 'iid-datasource',
                 'instance_id': 'iid-datasource',
+                'kernel_release': '5.4.0-24-generic',
                 'local-hostname': 'test-subclass-hostname',
                 'local_hostname': 'test-subclass-hostname',
+                'machine': 'x86_64',
                 'platform': 'mytestsubclass',
                 'public_ssh_keys': [],
+                'python_version': '3.7',
                 'region': 'myregion',
-                'subplatform': 'unknown'},
+                'subplatform': 'unknown',
+                'system_platform':
+                    'Linux-5.4.0-24-generic-x86_64-with-Ubuntu-20.04-focal',
+                'variant': 'ubuntu'},
             'ds': {
                 '_doc': EXPERIMENTAL_TEXT,
                 'meta_data': {
@@ -408,7 +476,7 @@ def test_get_data_writes_json_instance_data_sensitive(self):
                         'security-credentials':
                             {'cred1': 'sekret', 'cred2': 'othersekret'}}}}
         }
-        self.assertEqual(expected, util.load_json(content))
+        self.assertItemsEqual(expected, util.load_json(content))
         file_stat = os.stat(sensitive_json_file)
         self.assertEqual(0o600, stat.S_IMODE(file_stat.st_mode))
         self.assertEqual(expected, util.load_json(content))

cloudinit/util.py

@@ -656,7 +656,7 @@ def system_info():
         'system': platform.system(),
         'release': platform.release(),
         'python': platform.python_version(),
-        'uname': platform.uname(),
+        'uname': list(platform.uname()),
         'dist': get_linux_distro()
     }
     system = info['system'].lower()