fix cloud-config-archive handling

handling of cloud-config-archive input would fail in fully_decoded_payload.
part.get_charset() would return a Charset object, but
get_charset.input_codec is a string suitable for passing to decode.

This handles that correctly, and is more careful about binary data inside
input.

The test added verifies that cloud-config inside a cloud-config-archive
is handled correctly and also that binary data there is ignored without
exceptions raised.

LP: #1445143

cloudinit/handlers/__init__.py

@@ -263,7 +263,10 @@ def fixup_handler(mod, def_freq=PER_INSTANCE):
 
 
 def type_from_starts_with(payload, default=None):
-    payload_lc = payload.lower()
+    try:
+        payload_lc = util.decode_binary(payload).lower()
+    except UnicodeDecodeError:
+        return default
     payload_lc = payload_lc.lstrip()
     for text in INCLUSION_SRCH:
         if payload_lc.startswith(text):

cloudinit/user_data.py

@@ -49,6 +49,7 @@
 ARCHIVE_TYPES = ["text/cloud-config-archive"]
 UNDEF_TYPE = "text/plain"
 ARCHIVE_UNDEF_TYPE = "text/cloud-config"
+ARCHIVE_UNDEF_BINARY_TYPE = "application/octet-stream"
 
 # This seems to hit most of the gzip possible content types.
 DECOMP_TYPES = [
@@ -265,11 +266,15 @@ def _explode_archive(self, archive, append_msg):
             content = ent.get('content', '')
             mtype = ent.get('type')
             if not mtype:
-                mtype = handlers.type_from_starts_with(content,
-                                                       ARCHIVE_UNDEF_TYPE)
+                default = ARCHIVE_UNDEF_TYPE
+                if isinstance(content, six.binary_type):
+                    default = ARCHIVE_UNDEF_BINARY_TYPE
+                mtype = handlers.type_from_starts_with(content, default)
 
             maintype, subtype = mtype.split('/', 1)
             if maintype == "text":
+                if isinstance(content, six.binary_type):
+                    content = content.decode()
                 msg = MIMEText(content, _subtype=subtype)
             else:
                 msg = MIMEBase(maintype, subtype)

cloudinit/util.py

@@ -121,8 +121,12 @@ def fully_decoded_payload(part):
     if (six.PY3 and
             part.get_content_maintype() == 'text' and
             isinstance(cte_payload, bytes)):
-        charset = part.get_charset() or 'utf-8'
-        return cte_payload.decode(charset, errors='surrogateescape')
+        charset = part.get_charset()
+        if charset and charset.input_codec:
+            encoding = charset.input_codec
+        else:
+            encoding = 'utf-8'
+        return cte_payload.decode(encoding, errors='surrogateescape')
     return cte_payload
 
 

tests/unittests/test_data.py

@@ -512,6 +512,33 @@ def test_mime_application_octet_stream(self):
             ci.paths.get_ipath("cloud_config"), "", 0o600)
 
 
+    def test_cloud_config_archive(self):
+        non_decodable = b'\x11\xc9\xb4gTH\xee\x12'
+        data = [{'content': '#cloud-config\npassword: gocubs\n'},
+                {'content': '#cloud-config\nlocale: chicago\n'},
+                {'content': non_decodable}]
+        message = b'#cloud-config-archive\n' + util.yaml_dumps(data).encode()
+
+        ci = stages.Init()
+        ci.datasource = FakeDataSource(message)
+
+        fs = {}
+
+        def fsstore(filename, content, mode=0o0644, omode="wb"):
+            fs[filename] = content
+
+        # consuming the user-data provided should write 'cloud_config' file
+        # which will have our yaml in it.
+        with mock.patch('cloudinit.util.write_file') as mockobj:
+            mockobj.side_effect = fsstore
+            ci.fetch()
+            ci.consume_data()
+
+        cfg = util.load_yaml(fs[ci.paths.get_ipath("cloud_config")])
+        self.assertEqual(cfg.get('password'), 'gocubs')
+        self.assertEqual(cfg.get('locale'), 'chicago')
+
+
 class TestUDProcess(helpers.ResourceUsingTestCase):
 
     def test_bytes_in_userdata(self):