chore(safeyaml): Remove unicode helper for Python2 (#5142)

PyYAML has built-in unicode support in Python3+.

The original code[1] was added as a helper to add
support for unicode to `yaml.safe_load()`. We
don't need this anymore, and can jettison it and
prefer `yaml.safe_load()`.

[1] a7a9de1

cloudinit/cmd/devel/net_convert.py

@@ -115,7 +115,7 @@ def handle_args(name, args):
     if args.kind == "eni":
         pre_ns = eni.convert_eni_data(net_data)
     elif args.kind == "yaml":
-        pre_ns = safeyaml.load(net_data)
+        pre_ns = yaml.safe_load(net_data)
         if "network" in pre_ns:
             pre_ns = pre_ns.get("network")
         if args.debug:

cloudinit/cmd/main.py

@@ -19,6 +19,7 @@
 import time
 import traceback
 import logging
+import yaml
 from typing import Tuple
 
 from cloudinit import netinfo
@@ -37,7 +38,6 @@
 from cloudinit.config.schema import validate_cloudconfig_schema
 from cloudinit import log
 from cloudinit.reporting import events
-from cloudinit.safeyaml import load
 from cloudinit.settings import PER_INSTANCE, PER_ALWAYS, PER_ONCE, CLOUD_CONFIG
 
 # Welcome message template
@@ -481,7 +481,7 @@ def main_init(name, args):
     cloud_cfg_path = init.paths.get_ipath_cur("cloud_config")
     if os.path.exists(cloud_cfg_path) and os.stat(cloud_cfg_path).st_size != 0:
         validate_cloudconfig_schema(
-            config=load(util.load_text_file(cloud_cfg_path)),
+            config=yaml.safe_load(util.load_text_file(cloud_cfg_path)),
             strict=False,
             log_details=False,
             log_deprecations=True,

cloudinit/config/cc_lxd.py

@@ -11,7 +11,9 @@
 from textwrap import dedent
 from typing import List, Tuple
 
-from cloudinit import safeyaml, subp, util
+import yaml
+
+from cloudinit import subp, util
 from cloudinit.cloud import Cloud
 from cloudinit.config import Config
 from cloudinit.config.schema import MetaSchema, get_meta_doc
@@ -512,8 +514,8 @@ def get_required_packages(init_cfg: dict, preseed_str: str) -> List[str]:
     if preseed_str and "storage_pools" in preseed_str:
         # Assume correct YAML preseed format
         try:
-            preseed_cfg = safeyaml.load(preseed_str)
-        except (safeyaml.YAMLError, TypeError, ValueError):
+            preseed_cfg = yaml.safe_load(preseed_str)
+        except (yaml.YAMLError, TypeError, ValueError):
             LOG.warning(
                 "lxd.preseed string value is not YAML. "
                 " Unable to determine required storage driver packages to"

cloudinit/config/schema.py

@@ -1083,7 +1083,7 @@ def validate_cloudconfig_file(
         if annotate:
             cloudconfig, marks = safeyaml.load_with_marks(content)
         else:
-            cloudconfig = safeyaml.load(content)
+            cloudconfig = yaml.safe_load(content)
             marks = {}
     except yaml.YAMLError as e:
         line = column = 1

cloudinit/safeyaml.py

@@ -10,7 +10,6 @@
 
 import yaml
 
-YAMLError = yaml.YAMLError
 
 # SchemaPathMarks track the path to an element within a loaded YAML file.
 # The start_mark and end_mark contain the row and column indicators
@@ -49,11 +48,6 @@ def __eq__(self, other):
         )
 
 
-class _CustomSafeLoader(yaml.SafeLoader):
-    def construct_python_unicode(self, node):
-        return super().construct_scalar(node)
-
-
 def _find_closest_parent(child_mark, marks):
     for mark in marks[::-1]:
         if child_mark in mark and not child_mark == mark:
@@ -236,12 +230,6 @@ def get_single_data(self):
         return data
 
 
-_CustomSafeLoader.add_constructor(
-    "tag:yaml.org,2002:python/unicode",
-    _CustomSafeLoader.construct_python_unicode,
-)
-
-
 class NoAliasSafeDumper(yaml.dumper.SafeDumper):
     """A class which avoids constructing anchors/aliases on yaml dump"""
 
@@ -270,10 +258,6 @@ def load_with_marks(blob) -> Tuple[Any, Dict[str, int]]:
     return result, schemamarks
 
 
-def load(blob):
-    return yaml.load(blob, Loader=_CustomSafeLoader)
-
-
 def dumps(obj, explicit_start=True, explicit_end=True, noalias=False):
     """Return data in nicely formatted yaml."""
 

cloudinit/sources/DataSourceOVF.py

@@ -19,7 +19,9 @@
 import re
 from xml.dom import minidom  # nosec B408
 
-from cloudinit import safeyaml, sources, subp, util
+import yaml
+
+from cloudinit import sources, subp, util
 
 LOG = logging.getLogger(__name__)
 
@@ -408,4 +410,4 @@ def safeload_yaml_or_dict(data):
     """
     if not data:
         return {}
-    return safeyaml.load(data)
+    return yaml.safe_load(data)

cloudinit/sources/helpers/vmware/imc/guestcust_util.py

@@ -11,7 +11,9 @@
 import re
 import time
 
-from cloudinit import safeyaml, subp, util
+import yaml
+
+from cloudinit import subp, util
 
 from .config import Config
 from .config_custom_script import PostCustomScript, PreCustomScript
@@ -263,8 +265,8 @@ def get_data_from_imc_raw_data_cust_cfg(cust_cfg):
 
         try:
             logger.debug("Validating if meta data is valid or not")
-            md = safeyaml.load(md)
-        except safeyaml.YAMLError as e:
+            md = yaml.safe_load(md)
+        except yaml.YAMLError as e:
             set_cust_error_status(
                 "Error parsing the cloud-init meta data",
                 str(e),

cloudinit/util.py

@@ -56,12 +56,13 @@
 )
 from urllib import parse
 
+import yaml
+
 from cloudinit import (
     features,
     importer,
     mergers,
     net,
-    safeyaml,
     settings,
     subp,
     temp_utils,
@@ -1009,7 +1010,7 @@ def load_yaml(blob, default=None, allowed=(dict,)):
             len(blob),
             allowed,
         )
-        converted = safeyaml.load(blob)
+        converted = yaml.safe_load(blob)
         if converted is None:
             LOG.debug("loaded blob returned None, returning default.")
             converted = default
@@ -1020,7 +1021,7 @@ def load_yaml(blob, default=None, allowed=(dict,)):
                 % (allowed, type_utils.obj_name(converted))
             )
         loaded = converted
-    except (safeyaml.YAMLError, TypeError, ValueError) as e:
+    except (yaml.YAMLError, TypeError, ValueError) as e:
         msg = "Failed loading yaml blob"
         mark = None
         if hasattr(e, "context_mark") and getattr(e, "context_mark"):

tests/integration_tests/datasources/test_lxd_hotplug.py

@@ -1,8 +1,8 @@
 import json
 
 import pytest
+import yaml
 
-from cloudinit import safeyaml
 from cloudinit.subp import subp
 from cloudinit.util import is_true
 from tests.integration_tests.decorators import retry
@@ -142,10 +142,10 @@ def test_network_config_applied(self, class_client: IntegrationInstance):
             f"nictype=bridged parent=ci-test-br-eth2".split()
         )
         ensure_hotplug_exited(client)
-        post_netplan = safeyaml.load(
+        post_netplan = yaml.safe_load(
             client.read_from_file("/etc/netplan/50-cloud-init.yaml")
         )
-        expected_netplan = safeyaml.load(UPDATED_NETWORK_CONFIG)
+        expected_netplan = yaml.safe_load(UPDATED_NETWORK_CONFIG)
         expected_netplan = {"network": expected_netplan}
         assert post_netplan == expected_netplan, client.read_from_file(
             "/var/log/cloud-init.log"

tests/unittests/cmd/devel/test_net_convert.py

@@ -3,8 +3,8 @@
 import itertools
 
 import pytest
+import yaml
 
-from cloudinit import safeyaml as yaml
 from cloudinit.cmd.devel import net_convert
 from cloudinit.distros.debian import NETWORK_FILE_HEADER
 from tests.unittests.helpers import mock
@@ -243,4 +243,4 @@ def test_convert_netplan_passthrough(
         with mock.patch("cloudinit.util.chownbyname"):
             net_convert.handle_args("somename", args)
         outfile = tmpdir.join("etc/netplan/50-cloud-init.yaml")
-        assert yaml.load(content) == yaml.load(outfile.read())
+        assert yaml.safe_load(content) == yaml.safe_load(outfile.read())

tests/unittests/config/test_schema.py

@@ -18,6 +18,7 @@
 from typing import List, Optional, Sequence, Set
 
 import pytest
+import yaml
 
 from cloudinit.config.schema import (
     VERSIONED_USERDATA_SCHEMA_FILE,
@@ -39,7 +40,7 @@
     validate_cloudconfig_schema,
 )
 from cloudinit.distros import OSFAMILIES
-from cloudinit.safeyaml import load, load_with_marks
+from cloudinit.safeyaml import load_with_marks
 from cloudinit.settings import FREQUENCIES
 from cloudinit.sources import DataSourceNotFoundException
 from cloudinit.templater import JinjaSyntaxParsingException
@@ -777,7 +778,7 @@ def test_validateconfig_schema_of_example(self, schema_id, example):
         according to the unified schema of all config modules
         """
         schema = get_schema()
-        config_load = load(example)
+        config_load = yaml.safe_load(example)
         # cloud-init-schema-v1 is permissive of additionalProperties at the
         # top-level.
         # To validate specific schemas against known documented examples
@@ -2176,7 +2177,7 @@ def test_cloud_config_schema_doc_examples(self, example_path):
     @skipUnlessJsonSchema()
     def test_network_config_schema_v1_doc_examples(self, example_path):
         validate_cloudconfig_schema(
-            config=load(open(example_path)),
+            config=yaml.safe_load(open(example_path)),
             schema=self.net_schema,
             strict=True,
         )

tests/unittests/distros/test_netconfig.py

@@ -7,15 +7,9 @@
 from textwrap import dedent
 from unittest import mock
 
-from cloudinit import (
-    distros,
-    features,
-    helpers,
-    safeyaml,
-    settings,
-    subp,
-    util,
-)
+import yaml
+
+from cloudinit import distros, features, helpers, settings, subp, util
 from cloudinit.distros.parsers.sys_conf import SysConf
 from cloudinit.net.activators import IfUpDownActivator
 from tests.unittests.helpers import (
@@ -1148,7 +1142,7 @@ def test_photon_network_config_v1_with_duplicates(self):
         [Address]
         Address=192.168.0.102/24"""
 
-        net_cfg = safeyaml.load(V1_NET_CFG_WITH_DUPS)
+        net_cfg = yaml.safe_load(V1_NET_CFG_WITH_DUPS)
 
         expected = self.create_conf_dict(expected.splitlines())
         expected_cfgs = {
@@ -1273,7 +1267,7 @@ def test_mariner_network_config_v1_with_duplicates(self):
         [Address]
         Address=192.168.0.102/24"""
 
-        net_cfg = safeyaml.load(V1_NET_CFG_WITH_DUPS)
+        net_cfg = yaml.safe_load(V1_NET_CFG_WITH_DUPS)
 
         expected = self.create_conf_dict(expected.splitlines())
         expected_cfgs = {
@@ -1398,7 +1392,7 @@ def test_azurelinux_network_config_v1_with_duplicates(self):
         [Address]
         Address=192.168.0.102/24"""
 
-        net_cfg = safeyaml.load(V1_NET_CFG_WITH_DUPS)
+        net_cfg = yaml.safe_load(V1_NET_CFG_WITH_DUPS)
 
         expected = self.create_conf_dict(expected.splitlines())
         expected_cfgs = {

tests/unittests/distros/test_networking.py

@@ -5,9 +5,9 @@
 from unittest import mock
 
 import pytest
+import yaml
 
 from cloudinit import net
-from cloudinit import safeyaml as yaml
 from cloudinit.distros.networking import (
     BSDNetworking,
     LinuxNetworking,
@@ -356,7 +356,7 @@ def test_apply_renames(
         networking = LinuxNetworking()
         m_device_driver.return_value = "virtio_net"
         m_device_devid.return_value = "0x15d8"
-        netcfg = yaml.load(getattr(self, config_attr))
+        netcfg = yaml.safe_load(getattr(self, config_attr))
 
         with mock.patch.object(
             networking, "_rename_interfaces"
@@ -381,7 +381,7 @@ def test_apply_v2_renames_skips_without_setname_or_mac(
         self, config_attr: str
     ):
         networking = LinuxNetworking()
-        netcfg = yaml.load(getattr(self, config_attr))
+        netcfg = yaml.safe_load(getattr(self, config_attr))
         with mock.patch.object(
             networking, "_rename_interfaces"
         ) as m_rename_interfaces:
@@ -391,4 +391,4 @@ def test_apply_v2_renames_skips_without_setname_or_mac(
     def test_apply_v2_renames_raises_runtime_error_on_unknown_version(self):
         networking = LinuxNetworking()
         with pytest.raises(RuntimeError):
-            networking.apply_network_config_names(yaml.load("version: 3"))
+            networking.apply_network_config_names(yaml.safe_load("version: 3"))

tests/unittests/net/test_dns.py

@@ -2,7 +2,8 @@
 
 from unittest import mock
 
-from cloudinit import safeyaml
+import yaml
+
 from cloudinit.net import network_state
 
 
@@ -15,7 +16,7 @@ def test_system_mac_address_does_not_break_dns_parsing(
         by_mac_state.return_value = {"00:11:22:33:44:55": "foobar"}
         by_mac_init.return_value = {"00:11:22:33:44:55": "foobar"}
         state = network_state.parse_net_config_data(
-            safeyaml.load(
+            yaml.safe_load(
                 """\
 version: 2
 ethernets:

tests/unittests/net/test_net_rendering.py

@@ -29,8 +29,8 @@
 from pathlib import Path
 
 import pytest
+import yaml
 
-from cloudinit import safeyaml
 from cloudinit.net.netplan import Renderer as NetplanRenderer
 from cloudinit.net.network_manager import Renderer as NetworkManagerRenderer
 from cloudinit.net.network_state import NetworkState, parse_net_config_data
@@ -55,7 +55,7 @@ def _check_netplan(
     if network_state.version == 2:
         renderer = NetplanRenderer(config={"netplan_path": netplan_path})
         renderer.render_network_state(network_state)
-        assert safeyaml.load(netplan_path.read_text()) == expected_config, (
+        assert yaml.safe_load(netplan_path.read_text()) == expected_config, (
             f"Netplan config generated at {netplan_path} does not match v2 "
             "config defined for this test."
         )
@@ -89,7 +89,7 @@ def _check_network_manager(network_state: NetworkState, tmp_path: Path):
     [("no_matching_mac_v2", Renderer.Netplan | Renderer.NetworkManager)],
 )
 def test_convert(test_name, renderers, tmp_path):
-    network_config = safeyaml.load(
+    network_config = yaml.safe_load(
         Path(ARTIFACT_DIR, f"{test_name}.yaml").read_text()
     )
     network_state = parse_net_config_data(network_config["network"])