From fc3cacba3d0c27e98086a341450240bfec7dc36f Mon Sep 17 00:00:00 2001 From: minghe Date: Wed, 4 Sep 2024 15:55:04 -0700 Subject: [PATCH 01/11] add azure proxy agent --- ...ProvisionGuestProxyAgent-OVF-setting.patch | 114 +++++ ...-ProvisionGuestProxyAgent-as-bool-51.patch | 54 +++ ...re-add-support-for-azure-proxy-agent.patch | 401 ++++++++++++++++++ SPECS/cloud-init/cloud-init.spec | 10 +- 4 files changed, 577 insertions(+), 2 deletions(-) create mode 100644 SPECS/cloud-init/0001-feat-azure-Add-ProvisionGuestProxyAgent-OVF-setting.patch create mode 100644 SPECS/cloud-init/0002-feat-azure-parse-ProvisionGuestProxyAgent-as-bool-51.patch create mode 100644 SPECS/cloud-init/0003-feat-azure-add-support-for-azure-proxy-agent.patch diff --git a/SPECS/cloud-init/0001-feat-azure-Add-ProvisionGuestProxyAgent-OVF-setting.patch b/SPECS/cloud-init/0001-feat-azure-Add-ProvisionGuestProxyAgent-OVF-setting.patch new file mode 100644 index 00000000000..2031eeeb0c9 --- /dev/null +++ b/SPECS/cloud-init/0001-feat-azure-Add-ProvisionGuestProxyAgent-OVF-setting.patch @@ -0,0 +1,114 @@ +From 402e9331a72d543e779898667488a51ad3e3ec13 Mon Sep 17 00:00:00 2001 +From: Ksenija Stanojevic +Date: Fri, 9 Feb 2024 13:32:19 -0800 +Subject: [PATCH 1/3] feat(azure): Add ProvisionGuestProxyAgent OVF setting + (#4860) + +Add ProvisionGuestProxyAgent Boolean configuration setting into the OvfEnv class. +This PR is only logging the value of ProvisionGuestProxyAgent. +--- + cloudinit/sources/DataSourceAzure.py | 6 ++++++ + cloudinit/sources/helpers/azure.py | 8 ++++++++ + tests/unittests/sources/test_azure.py | 15 +++++++++++++++ + 3 files changed, 29 insertions(+) + +diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py +index 5a82aa34e..dc2b79a3a 100644 +--- a/cloudinit/sources/DataSourceAzure.py ++++ b/cloudinit/sources/DataSourceAzure.py +@@ -1784,6 +1784,12 @@ def read_azure_ovf(contents): + "PreprovisionedVMType: %s" % ovf_env.preprovisioned_vm_type, + logger_func=LOG.info, + ) ++ ++ cfg["ProvisionGuestProxyAgent"] = ovf_env.provision_guest_proxy_agent ++ report_diagnostic_event( ++ "ProvisionGuestProxyAgent: %s" % ovf_env.provision_guest_proxy_agent, ++ logger_func=LOG.info, ++ ) + return (md, ud, cfg) + + +diff --git a/cloudinit/sources/helpers/azure.py b/cloudinit/sources/helpers/azure.py +index 6e5c1f433..2847a9e53 100644 +--- a/cloudinit/sources/helpers/azure.py ++++ b/cloudinit/sources/helpers/azure.py +@@ -1064,6 +1064,7 @@ class OvfEnvXml: + public_keys: Optional[List[dict]] = None, + preprovisioned_vm: bool = False, + preprovisioned_vm_type: Optional[str] = None, ++ provision_guest_proxy_agent: bool = False, + ) -> None: + self.username = username + self.password = password +@@ -1073,6 +1074,7 @@ class OvfEnvXml: + self.public_keys: List[dict] = public_keys or [] + self.preprovisioned_vm = preprovisioned_vm + self.preprovisioned_vm_type = preprovisioned_vm_type ++ self.provision_guest_proxy_agent = provision_guest_proxy_agent + + def __eq__(self, other) -> bool: + return self.__dict__ == other.__dict__ +@@ -1216,6 +1218,12 @@ class OvfEnvXml: + "PreprovisionedVMType", + required=False, + ) ++ self.provision_guest_proxy_agent = self._parse_property( ++ platform_settings, ++ "ProvisionGuestProxyAgent", ++ default=False, ++ required=False, ++ ) + + def _parse_ssh_section(self, config_set): + self.public_keys = [] +diff --git a/tests/unittests/sources/test_azure.py b/tests/unittests/sources/test_azure.py +index 1ddbd3f39..6afde95fd 100644 +--- a/tests/unittests/sources/test_azure.py ++++ b/tests/unittests/sources/test_azure.py +@@ -356,6 +356,7 @@ def construct_ovf_env( + disable_ssh_password_auth=None, + preprovisioned_vm=None, + preprovisioned_vm_type=None, ++ provision_guest_proxy_agent=None, + ): + content = [ + '', +@@ -426,6 +427,11 @@ def construct_ovf_env( + "%s" + % preprovisioned_vm_type + ) ++ if provision_guest_proxy_agent is not None: ++ content.append( ++ "%s" ++ % provision_guest_proxy_agent ++ ) + content += [ + "", + "", +@@ -1316,6 +1322,7 @@ scbus-1 on xpt0 bus 0 + expected_cfg = { + "PreprovisionedVMType": None, + "PreprovisionedVm": False, ++ "ProvisionGuestProxyAgent": False, + "system_info": {"default_user": {"name": "myuser"}}, + } + expected_metadata = { +@@ -2668,6 +2675,14 @@ class TestPreprovisioningReadAzureOvfFlag(CiTestCase): + self.assertTrue(cfg["PreprovisionedVm"]) + self.assertEqual("Savable", cfg["PreprovisionedVMType"]) + ++ def test_read_azure_ovf_with_proxy_guest_agent(self): ++ """The read_azure_ovf method should set ProvisionGuestProxyAgent ++ cfg flag to True.""" ++ content = construct_ovf_env(provision_guest_proxy_agent=True) ++ ret = dsaz.read_azure_ovf(content) ++ cfg = ret[2] ++ self.assertTrue(cfg["ProvisionGuestProxyAgent"]) ++ + + @pytest.mark.parametrize( + "ovf_cfg,imds_md,pps_type", +-- +2.34.1 + diff --git a/SPECS/cloud-init/0002-feat-azure-parse-ProvisionGuestProxyAgent-as-bool-51.patch b/SPECS/cloud-init/0002-feat-azure-parse-ProvisionGuestProxyAgent-as-bool-51.patch new file mode 100644 index 00000000000..f61cbc5b10f --- /dev/null +++ b/SPECS/cloud-init/0002-feat-azure-parse-ProvisionGuestProxyAgent-as-bool-51.patch @@ -0,0 +1,54 @@ +From e3ba5800d26065df9ce03ee2ac58ec6f08506423 Mon Sep 17 00:00:00 2001 +From: Ksenija Stanojevic +Date: Fri, 5 Apr 2024 16:52:26 -0700 +Subject: [PATCH 2/3] feat(azure): parse ProvisionGuestProxyAgent as bool + (#5126) + +--- + cloudinit/sources/helpers/azure.py | 1 + + tests/unittests/sources/test_azure.py | 12 ++++++++++-- + 2 files changed, 11 insertions(+), 2 deletions(-) + +diff --git a/cloudinit/sources/helpers/azure.py b/cloudinit/sources/helpers/azure.py +index 2847a9e53..165f47429 100644 +--- a/cloudinit/sources/helpers/azure.py ++++ b/cloudinit/sources/helpers/azure.py +@@ -1221,6 +1221,7 @@ class OvfEnvXml: + self.provision_guest_proxy_agent = self._parse_property( + platform_settings, + "ProvisionGuestProxyAgent", ++ parse_bool=True, + default=False, + required=False, + ) +diff --git a/tests/unittests/sources/test_azure.py b/tests/unittests/sources/test_azure.py +index 6afde95fd..255991ec3 100644 +--- a/tests/unittests/sources/test_azure.py ++++ b/tests/unittests/sources/test_azure.py +@@ -2675,13 +2675,21 @@ class TestPreprovisioningReadAzureOvfFlag(CiTestCase): + self.assertTrue(cfg["PreprovisionedVm"]) + self.assertEqual("Savable", cfg["PreprovisionedVMType"]) + +- def test_read_azure_ovf_with_proxy_guest_agent(self): ++ def test_read_azure_ovf_with_proxy_guest_agent_true(self): + """The read_azure_ovf method should set ProvisionGuestProxyAgent + cfg flag to True.""" + content = construct_ovf_env(provision_guest_proxy_agent=True) + ret = dsaz.read_azure_ovf(content) + cfg = ret[2] +- self.assertTrue(cfg["ProvisionGuestProxyAgent"]) ++ assert cfg["ProvisionGuestProxyAgent"] is True ++ ++ def test_read_azure_ovf_with_proxy_guest_agent_false(self): ++ """The read_azure_ovf method should set ProvisionGuestProxyAgent ++ cfg flag to False.""" ++ content = construct_ovf_env(provision_guest_proxy_agent=False) ++ ret = dsaz.read_azure_ovf(content) ++ cfg = ret[2] ++ assert cfg["ProvisionGuestProxyAgent"] is False + + + @pytest.mark.parametrize( +-- +2.34.1 + diff --git a/SPECS/cloud-init/0003-feat-azure-add-support-for-azure-proxy-agent.patch b/SPECS/cloud-init/0003-feat-azure-add-support-for-azure-proxy-agent.patch new file mode 100644 index 00000000000..04cb087e2ae --- /dev/null +++ b/SPECS/cloud-init/0003-feat-azure-add-support-for-azure-proxy-agent.patch @@ -0,0 +1,401 @@ +From 8932242a65bae5504ba45134091767f215a441fa Mon Sep 17 00:00:00 2001 +From: Ksenija Stanojevic +Date: Mon, 15 Jul 2024 18:48:19 -0700 +Subject: [PATCH 3/3] feat(azure): add support for azure-proxy-agent + +--- + cloudinit/sources/DataSourceAzure.py | 40 ++++ + cloudinit/sources/azure/errors.py | 19 +- + tests/unittests/sources/test_azure.py | 254 ++++++++++++++++++++++++++ + 3 files changed, 312 insertions(+), 1 deletion(-) + +diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py +index dc2b79a3a..c2f74e173 100644 +--- a/cloudinit/sources/DataSourceAzure.py ++++ b/cloudinit/sources/DataSourceAzure.py +@@ -483,6 +483,41 @@ class DataSourceAzure(sources.DataSource): + or self._ephemeral_dhcp_ctx.lease is None + ) + ++ def _check_azure_proxy_agent_status(self) -> None: ++ """Check if azure-proxy-agent is ready for communication with WS/IMDS. ++ If ProvisionGuestProxyAgent is true, query azure-proxy-agent status, ++ waiting up to 120 seconds for the proxy to negotiate with Wireserver ++ and configure an eBPF proxy. Once azure-proxy-agent is ready, ++ it will exit with code 0 and cloud-init can then expect to be able to ++ communicate with these services. ++ Fail deployment if azure-proxy-agent is not found or otherwise returns ++ an error. ++ For more information, check out: ++ https://github.com/azure/guestproxyagent ++ """ ++ try: ++ cmd = [ ++ "azure-proxy-agent", ++ "--status", ++ "--wait", ++ "120", ++ ] ++ out, err = subp.subp(cmd) ++ report_diagnostic_event( ++ "Running azure-proxy-agent %s resulted" ++ "in stderr output: %s with stdout: %s" % (cmd, err, out), ++ logger_func=LOG.debug, ++ ) ++ except subp.ProcessExecutionError as error: ++ if isinstance(error.reason, FileNotFoundError): ++ report_error = errors.ReportableErrorProxyAgentNotFound() ++ self._report_failure(report_error) ++ else: ++ reportable_error = ( ++ errors.ReportableErrorProxyAgentStatusFailure(error) ++ ) ++ self._report_failure(reportable_error) ++ + @azure_ds_telemetry_reporter + def crawl_metadata(self): + """Walk all instance metadata sources returning a dict on success. +@@ -566,6 +601,11 @@ class DataSourceAzure(sources.DataSource): + + imds_md = {} + if self._is_ephemeral_networking_up(): ++ # check if azure-proxy-agent is enabled in the ovf-env.xml file. ++ # azure-proxy-agent feature is opt-in and disabled by default. ++ if cfg.get("ProvisionGuestProxyAgent"): ++ self._check_azure_proxy_agent_status() ++ + imds_md = self.get_metadata_from_imds(report_failure=True) + + if not imds_md and ovf_source is None: +diff --git a/cloudinit/sources/azure/errors.py b/cloudinit/sources/azure/errors.py +index 966725b00..b331cd686 100644 +--- a/cloudinit/sources/azure/errors.py ++++ b/cloudinit/sources/azure/errors.py +@@ -12,7 +12,7 @@ from typing import Any, Dict, List, Optional + + import requests + +-from cloudinit import version ++from cloudinit import subp, version + from cloudinit.sources.azure import identity + from cloudinit.url_helper import UrlError + +@@ -151,3 +151,20 @@ class ReportableErrorUnhandledException(ReportableError): + + self.supporting_data["exception"] = repr(exception) + self.supporting_data["traceback_base64"] = trace_base64 ++ ++ ++class ReportableErrorProxyAgentNotFound(ReportableError): ++ def __init__(self) -> None: ++ super().__init__( ++ "Unable to activate Azure Guest Proxy Agent." ++ "azure-proxy-agent not found" ++ ) ++ ++ ++class ReportableErrorProxyAgentStatusFailure(ReportableError): ++ def __init__(self, exception: subp.ProcessExecutionError) -> None: ++ super().__init__("azure-proxy-agent status failure") ++ ++ self.supporting_data["exit_code"] = exception.exit_code ++ self.supporting_data["stdout"] = exception.stdout ++ self.supporting_data["stderr"] = exception.stderr +diff --git a/tests/unittests/sources/test_azure.py b/tests/unittests/sources/test_azure.py +index 255991ec3..9b6672e1e 100644 +--- a/tests/unittests/sources/test_azure.py ++++ b/tests/unittests/sources/test_azure.py +@@ -1,6 +1,7 @@ + # This file is part of cloud-init. See LICENSE file for license information. + + import copy ++import datetime + import json + import os + import stat +@@ -48,6 +49,16 @@ def mock_wrapping_setup_ephemeral_networking(azure_ds): + yield m + + ++@pytest.fixture ++def mock_wrapping_report_failure(azure_ds): ++ with mock.patch.object( ++ azure_ds, ++ "_report_failure", ++ wraps=azure_ds._report_failure, ++ ) as m: ++ yield m ++ ++ + @pytest.fixture + def mock_azure_helper_readurl(): + with mock.patch( +@@ -253,6 +264,14 @@ def mock_subp_subp(): + yield m + + ++@pytest.fixture ++def mock_timestamp(): ++ timestamp = datetime.datetime.utcnow() ++ with mock.patch.object(errors, "datetime", autospec=True) as m: ++ m.utcnow.return_value = timestamp ++ yield timestamp ++ ++ + @pytest.fixture + def mock_util_ensure_dir(): + with mock.patch( +@@ -3672,6 +3691,91 @@ class TestProvisioning: + } + + def test_no_pps(self): ++ ovf = construct_ovf_env(provision_guest_proxy_agent=False) ++ md, ud, cfg = dsaz.read_azure_ovf(ovf) ++ self.mock_util_mount_cb.return_value = (md, ud, cfg, {}) ++ self.mock_readurl.side_effect = [ ++ mock.MagicMock(contents=json.dumps(self.imds_md).encode()), ++ ] ++ self.mock_azure_get_metadata_from_fabric.return_value = [] ++ ++ self.azure_ds._check_and_get_data() ++ ++ assert self.mock_subp_subp.mock_calls == [] ++ ++ # Verify DHCP is setup once. ++ assert self.mock_wrapping_setup_ephemeral_networking.mock_calls == [ ++ mock.call(timeout_minutes=20) ++ ] ++ assert self.mock_net_dhcp_maybe_perform_dhcp_discovery.mock_calls == [ ++ mock.call( ++ self.azure_ds.distro, ++ None, ++ dsaz.dhcp_log_cb, ++ ) ++ ] ++ assert self.azure_ds._wireserver_endpoint == "10.11.12.13" ++ assert self.azure_ds._is_ephemeral_networking_up() is False ++ ++ # Verify DMI usage. ++ assert self.mock_dmi_read_dmi_data.mock_calls == [ ++ mock.call("chassis-asset-tag"), ++ mock.call("system-uuid"), ++ ] ++ assert ( ++ self.azure_ds.metadata["instance-id"] ++ == "50109936-ef07-47fe-ac82-890c853f60d5" ++ ) ++ ++ # Verify IMDS metadata. ++ assert self.azure_ds.metadata["imds"] == self.imds_md ++ ++ # Verify reporting ready once. ++ assert self.mock_azure_get_metadata_from_fabric.mock_calls == [ ++ mock.call( ++ endpoint="10.11.12.13", ++ distro=self.azure_ds.distro, ++ iso_dev="/dev/sr0", ++ pubkey_info=None, ++ ) ++ ] ++ ++ # Verify netlink. ++ assert self.mock_netlink.mock_calls == [] ++ ++ # Verify no reported_ready marker written. ++ assert self.wrapped_util_write_file.mock_calls == [] ++ assert self.patched_reported_ready_marker_path.exists() is False ++ ++ # Verify reports via KVP. ++ assert len(self.mock_kvp_report_failure_to_host.mock_calls) == 0 ++ assert len(self.mock_azure_report_failure_to_fabric.mock_calls) == 0 ++ assert len(self.mock_kvp_report_success_to_host.mock_calls) == 1 ++ ++ # Verify dmesg reported via KVP. ++ assert len(self.mock_report_dmesg_to_kvp.mock_calls) == 1 ++ ++ def test_no_pps_gpa(self): ++ """test full provisioning scope when azure-proxy-agent ++ is enabled and running.""" ++ self.mock_subp_subp.side_effect = [ ++ subp.SubpResult("Guest Proxy Agent running", ""), ++ ] ++ ovf = construct_ovf_env(provision_guest_proxy_agent=True) ++ md, ud, cfg = dsaz.read_azure_ovf(ovf) ++ self.mock_util_mount_cb.return_value = (md, ud, cfg, {}) + self.mock_readurl.side_effect = [ + mock.MagicMock(contents=json.dumps(self.imds_md).encode()), + ] +@@ -3679,6 +3783,11 @@ class TestProvisioning: + + self.azure_ds._check_and_get_data() + ++ assert self.mock_subp_subp.mock_calls == [ ++ mock.call( ++ ["azure-proxy-agent", "--status", "--wait", "120"], ++ ), ++ ] + assert self.mock_readurl.mock_calls == [ + mock.call( + "http://169.254.169.254/metadata/instance?" +@@ -3736,6 +3845,93 @@ class TestProvisioning: + + # Verify reports via KVP. + assert len(self.mock_kvp_report_failure_to_host.mock_calls) == 0 ++ assert len(self.mock_azure_report_failure_to_fabric.mock_calls) == 0 ++ assert len(self.mock_kvp_report_success_to_host.mock_calls) == 1 ++ ++ def test_no_pps_gpa_fail(self): ++ """test full provisioning scope when azure-proxy-agent is enabled and ++ throwing an exception during provisioning.""" ++ self.mock_subp_subp.side_effect = [ ++ subp.ProcessExecutionError( ++ cmd=["failed", "azure-proxy-agent"], ++ stdout="test_stdout", ++ stderr="test_stderr", ++ exit_code=4, ++ ), ++ ] ++ ovf = construct_ovf_env(provision_guest_proxy_agent=True) ++ md, ud, cfg = dsaz.read_azure_ovf(ovf) ++ self.mock_util_mount_cb.return_value = (md, ud, cfg, {}) ++ self.mock_readurl.side_effect = [ ++ mock.MagicMock(contents=json.dumps(self.imds_md).encode()), ++ ] ++ self.mock_azure_get_metadata_from_fabric.return_value = [] ++ ++ self.azure_ds._check_and_get_data() ++ ++ assert self.mock_subp_subp.mock_calls == [ ++ mock.call( ++ ["azure-proxy-agent", "--status", "--wait", "120"], ++ ), ++ ] ++ assert self.mock_readurl.mock_calls == [ ++ mock.call( ++ "http://169.254.169.254/metadata/instance?" ++ "api-version=2021-08-01&extended=true", ++ timeout=30, ++ headers={"Metadata": "true"}, ++ exception_cb=mock.ANY, ++ infinite=True, ++ log_req_resp=True, ++ ), ++ ] ++ ++ # Verify DHCP is setup once. ++ assert self.mock_wrapping_setup_ephemeral_networking.mock_calls == [ ++ mock.call(timeout_minutes=20) ++ ] ++ assert self.mock_net_dhcp_maybe_perform_dhcp_discovery.mock_calls == [ ++ mock.call( ++ self.azure_ds.distro, ++ None, ++ dsaz.dhcp_log_cb, ++ ) ++ ] ++ assert self.azure_ds._wireserver_endpoint == "10.11.12.13" ++ assert self.azure_ds._is_ephemeral_networking_up() is False ++ ++ # Verify DMI usage. ++ assert self.mock_dmi_read_dmi_data.mock_calls == [ ++ mock.call("chassis-asset-tag"), ++ mock.call("system-uuid"), ++ mock.call("system-uuid"), ++ ] ++ assert ( ++ self.azure_ds.metadata["instance-id"] ++ == "50109936-ef07-47fe-ac82-890c853f60d5" ++ ) ++ ++ # Verify IMDS metadata. ++ assert self.azure_ds.metadata["imds"] == self.imds_md ++ ++ ### BACKPORT NOTE: 23.3 _will_ report ready later after failure. ++ ### In newer versions there will be no call to report ready after failure. ++ assert self.mock_azure_get_metadata_from_fabric.mock_calls == [ ++ mock.call( ++ endpoint="10.11.12.13", iso_dev="/dev/sr0", pubkey_info=None ++ ) ++ ] ++ ++ # Verify netlink. ++ assert self.mock_netlink.mock_calls == [] ++ ++ # Verify no reported_ready marker written. ++ assert self.wrapped_util_write_file.mock_calls == [] ++ assert self.patched_reported_ready_marker_path.exists() is False ++ ++ # Verify reports via KVP. ++ assert len(self.mock_kvp_report_failure_to_host.mock_calls) == 1 ++ assert len(self.mock_azure_report_failure_to_fabric.mock_calls) == 1 + assert len(self.mock_kvp_report_success_to_host.mock_calls) == 1 + + def test_running_pps(self): +@@ -4315,6 +4511,64 @@ class TestProvisioning: + assert len(self.mock_kvp_report_success_to_host.mock_calls) == 1 + + ++class TestCheckAzureProxyAgent: ++ @pytest.fixture(autouse=True) ++ def proxy_setup( ++ self, ++ azure_ds, ++ mock_subp_subp, ++ caplog, ++ mock_wrapping_report_failure, ++ mock_timestamp, ++ ): ++ self.azure_ds = azure_ds ++ self.mock_subp_subp = mock_subp_subp ++ self.caplog = caplog ++ self.mock_wrapping_report_failure = mock_wrapping_report_failure ++ self.mock_timestamp = mock_timestamp ++ ++ def test_check_azure_proxy_agent_status(self): ++ self.mock_subp_subp.side_effect = [ ++ subp.SubpResult("Guest Proxy Agent running", ""), ++ ] ++ self.azure_ds._check_azure_proxy_agent_status() ++ assert "Running azure-proxy-agent" in self.caplog.text ++ assert self.mock_wrapping_report_failure.mock_calls == [] ++ ++ def test_check_azure_proxy_agent_status_notfound(self): ++ exception = subp.ProcessExecutionError(reason=FileNotFoundError()) ++ self.mock_subp_subp.side_effect = [ ++ exception, ++ ] ++ self.azure_ds._check_azure_proxy_agent_status() ++ assert "azure-proxy-agent not found" in self.caplog.text ++ assert self.mock_wrapping_report_failure.mock_calls == [ ++ mock.call( ++ errors.ReportableErrorProxyAgentNotFound(), ++ ), ++ ] ++ ++ def test_check_azure_proxy_agent_status_failure(self): ++ exception = subp.ProcessExecutionError( ++ cmd=["failed", "azure-proxy-agent"], ++ stdout="test_stdout", ++ stderr="test_stderr", ++ exit_code=4, ++ ) ++ self.mock_subp_subp.side_effect = [ ++ exception, ++ ] ++ self.azure_ds._check_azure_proxy_agent_status() ++ assert "azure-proxy-agent status failure" in self.caplog.text ++ assert self.mock_wrapping_report_failure.mock_calls == [ ++ mock.call( ++ errors.ReportableErrorProxyAgentStatusFailure( ++ exception=exception ++ ), ++ ), ++ ] ++ ++ + class TestGetMetadataFromImds: + @pytest.mark.parametrize("report_failure", [False, True]) + @pytest.mark.parametrize( +-- +2.34.1 + diff --git a/SPECS/cloud-init/cloud-init.spec b/SPECS/cloud-init/cloud-init.spec index ffe61e76ee8..414bc5a49ce 100644 --- a/SPECS/cloud-init/cloud-init.spec +++ b/SPECS/cloud-init/cloud-init.spec @@ -5,7 +5,7 @@ Summary: Cloud instance init scripts Name: cloud-init Epoch: 1 Version: %{package_version} -Release: 3%{?dist} +Release: 4%{?dist} License: GPLv3 Vendor: Microsoft Corporation Distribution: Mariner @@ -16,6 +16,9 @@ Source1: 10-azure-kvp.cfg Patch0: overrideDatasourceDetection.patch Patch1: exec_cmd_error_handling.patch Patch2: Add-Network-Interface-Renaming-Support-for-CAPM3-Met.patch +Patch3: 0001-feat-azure-Add-ProvisionGuestProxyAgent-OVF-setting.patch +Patch4: 0002-feat-azure-parse-ProvisionGuestProxyAgent-as-bool-51.patch +Patch5: 0003-feat-azure-add-support-for-azure-proxy-agent.patch %define cl_services cloud-config.service cloud-config.target cloud-final.service cloud-init.service cloud-init.target cloud-init-local.service BuildRequires: automake BuildRequires: dbus @@ -151,7 +154,10 @@ make check %{?_smp_mflags} %config(noreplace) %{_sysconfdir}/cloud/cloud.cfg.d/10-azure-kvp.cfg %changelog -* Wed May 8 2024 Sharath Srikanth Chellappa - 1:23.3-3 +* Wed Spe 04 2024 Minghe Ren - 1:23.3-4 +- Add patches to support azure-proxy-agent. + +* Wed May 08 2024 Sharath Srikanth Chellappa - 1:23.3-3 - Add patch to add network interface renaming support for CAPM3 Met. * Wed Apr 03 2024 Rachel Menge - 1:23.3-2 From 0ae7df1efd4caba50cb04207345e31c9ceeeb443 Mon Sep 17 00:00:00 2001 From: minghe Date: Wed, 4 Sep 2024 16:50:14 -0700 Subject: [PATCH 02/11] modify patch --- ...re-add-support-for-azure-proxy-agent.patch | 70 +++++++++---------- 1 file changed, 34 insertions(+), 36 deletions(-) diff --git a/SPECS/cloud-init/0003-feat-azure-add-support-for-azure-proxy-agent.patch b/SPECS/cloud-init/0003-feat-azure-add-support-for-azure-proxy-agent.patch index 04cb087e2ae..8dd924daa05 100644 --- a/SPECS/cloud-init/0003-feat-azure-add-support-for-azure-proxy-agent.patch +++ b/SPECS/cloud-init/0003-feat-azure-add-support-for-azure-proxy-agent.patch @@ -101,23 +101,22 @@ index 966725b00..b331cd686 100644 + self.supporting_data["exit_code"] = exception.exit_code + self.supporting_data["stdout"] = exception.stdout + self.supporting_data["stderr"] = exception.stderr -diff --git a/tests/unittests/sources/test_azure.py b/tests/unittests/sources/test_azure.py -index 255991ec3..9b6672e1e 100644 ---- a/tests/unittests/sources/test_azure.py -+++ b/tests/unittests/sources/test_azure.py +diff -ruN a/tests/unittests/sources/test_azure.py b/tests/unittests/sources/test_azure.py +--- a/tests/unittests/sources/test_azure.py 2023-08-28 09:20:24.000000000 -0700 ++++ b/tests/unittests/sources/test_azure.py 2024-09-04 16:47:35.140368594 -0700 @@ -1,6 +1,7 @@ # This file is part of cloud-init. See LICENSE file for license information. - + import copy +import datetime import json import os import stat -@@ -48,6 +49,16 @@ def mock_wrapping_setup_ephemeral_networking(azure_ds): +@@ -48,6 +49,16 @@ yield m - - -+@pytest.fixture + + +++@pytest.fixture +def mock_wrapping_report_failure(azure_ds): + with mock.patch.object( + azure_ds, @@ -130,11 +129,11 @@ index 255991ec3..9b6672e1e 100644 @pytest.fixture def mock_azure_helper_readurl(): with mock.patch( -@@ -253,6 +264,14 @@ def mock_subp_subp(): +@@ -253,6 +264,14 @@ yield m - - -+@pytest.fixture + + +++@pytest.fixture +def mock_timestamp(): + timestamp = datetime.datetime.utcnow() + with mock.patch.object(errors, "datetime", autospec=True) as m: @@ -145,20 +144,20 @@ index 255991ec3..9b6672e1e 100644 @pytest.fixture def mock_util_ensure_dir(): with mock.patch( -@@ -3672,6 +3691,91 @@ class TestProvisioning: +@@ -3649,6 +3668,9 @@ } - + def test_no_pps(self): + ovf = construct_ovf_env(provision_guest_proxy_agent=False) + md, ud, cfg = dsaz.read_azure_ovf(ovf) + self.mock_util_mount_cb.return_value = (md, ud, cfg, {}) -+ self.mock_readurl.side_effect = [ -+ mock.MagicMock(contents=json.dumps(self.imds_md).encode()), -+ ] -+ self.mock_azure_get_metadata_from_fabric.return_value = [] -+ -+ self.azure_ds._check_and_get_data() -+ + self.mock_readurl.side_effect = [ + mock.MagicMock(contents=json.dumps(self.imds_md).encode()), + ] +@@ -3656,6 +3678,81 @@ + + self.azure_ds._check_and_get_data() + + assert self.mock_subp_subp.mock_calls == [] + + # Verify DHCP is setup once. @@ -222,13 +221,13 @@ index 255991ec3..9b6672e1e 100644 + ovf = construct_ovf_env(provision_guest_proxy_agent=True) + md, ud, cfg = dsaz.read_azure_ovf(ovf) + self.mock_util_mount_cb.return_value = (md, ud, cfg, {}) - self.mock_readurl.side_effect = [ - mock.MagicMock(contents=json.dumps(self.imds_md).encode()), - ] -@@ -3679,6 +3783,11 @@ class TestProvisioning: - - self.azure_ds._check_and_get_data() - ++ self.mock_readurl.side_effect = [ ++ mock.MagicMock(contents=json.dumps(self.imds_md).encode()), ++ ] ++ self.mock_azure_get_metadata_from_fabric.return_value = [] ++ ++ self.azure_ds._check_and_get_data() ++ + assert self.mock_subp_subp.mock_calls == [ + mock.call( + ["azure-proxy-agent", "--status", "--wait", "120"], @@ -237,8 +236,8 @@ index 255991ec3..9b6672e1e 100644 assert self.mock_readurl.mock_calls == [ mock.call( "http://169.254.169.254/metadata/instance?" -@@ -3736,6 +3845,93 @@ class TestProvisioning: - +@@ -3713,6 +3810,92 @@ + # Verify reports via KVP. assert len(self.mock_kvp_report_failure_to_host.mock_calls) == 0 + assert len(self.mock_azure_report_failure_to_fabric.mock_calls) == 0 @@ -264,7 +263,6 @@ index 255991ec3..9b6672e1e 100644 + self.mock_azure_get_metadata_from_fabric.return_value = [] + + self.azure_ds._check_and_get_data() -+ + assert self.mock_subp_subp.mock_calls == [ + mock.call( + ["azure-proxy-agent", "--status", "--wait", "120"], @@ -329,12 +327,12 @@ index 255991ec3..9b6672e1e 100644 + assert len(self.mock_kvp_report_failure_to_host.mock_calls) == 1 + assert len(self.mock_azure_report_failure_to_fabric.mock_calls) == 1 assert len(self.mock_kvp_report_success_to_host.mock_calls) == 1 - + def test_running_pps(self): -@@ -4315,6 +4511,64 @@ class TestProvisioning: +@@ -4292,6 +4475,64 @@ assert len(self.mock_kvp_report_success_to_host.mock_calls) == 1 - - + + +class TestCheckAzureProxyAgent: + @pytest.fixture(autouse=True) + def proxy_setup( From 34969c62a051cbfb7b43a3d8e4d409d2da5637ae Mon Sep 17 00:00:00 2001 From: minghe Date: Thu, 5 Sep 2024 13:47:59 -0700 Subject: [PATCH 03/11] update patch --- .../0003-feat-azure-add-support-for-azure-proxy-agent.patch | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/SPECS/cloud-init/0003-feat-azure-add-support-for-azure-proxy-agent.patch b/SPECS/cloud-init/0003-feat-azure-add-support-for-azure-proxy-agent.patch index 8dd924daa05..a02b0b432da 100644 --- a/SPECS/cloud-init/0003-feat-azure-add-support-for-azure-proxy-agent.patch +++ b/SPECS/cloud-init/0003-feat-azure-add-support-for-azure-proxy-agent.patch @@ -116,7 +116,7 @@ diff -ruN a/tests/unittests/sources/test_azure.py b/tests/unittests/sources/test yield m -++@pytest.fixture ++@pytest.fixture +def mock_wrapping_report_failure(azure_ds): + with mock.patch.object( + azure_ds, @@ -133,7 +133,7 @@ diff -ruN a/tests/unittests/sources/test_azure.py b/tests/unittests/sources/test yield m -++@pytest.fixture ++@pytest.fixture +def mock_timestamp(): + timestamp = datetime.datetime.utcnow() + with mock.patch.object(errors, "datetime", autospec=True) as m: From 892286e37af7b83e0de9eca444ecf75b1e340256 Mon Sep 17 00:00:00 2001 From: minghe Date: Thu, 5 Sep 2024 15:36:35 -0700 Subject: [PATCH 04/11] update patch --- ...re-add-support-for-azure-proxy-agent.patch | 139 ++++++++++-------- 1 file changed, 81 insertions(+), 58 deletions(-) diff --git a/SPECS/cloud-init/0003-feat-azure-add-support-for-azure-proxy-agent.patch b/SPECS/cloud-init/0003-feat-azure-add-support-for-azure-proxy-agent.patch index a02b0b432da..0c7125439dd 100644 --- a/SPECS/cloud-init/0003-feat-azure-add-support-for-azure-proxy-agent.patch +++ b/SPECS/cloud-init/0003-feat-azure-add-support-for-azure-proxy-agent.patch @@ -101,21 +101,22 @@ index 966725b00..b331cd686 100644 + self.supporting_data["exit_code"] = exception.exit_code + self.supporting_data["stdout"] = exception.stdout + self.supporting_data["stderr"] = exception.stderr -diff -ruN a/tests/unittests/sources/test_azure.py b/tests/unittests/sources/test_azure.py ---- a/tests/unittests/sources/test_azure.py 2023-08-28 09:20:24.000000000 -0700 -+++ b/tests/unittests/sources/test_azure.py 2024-09-04 16:47:35.140368594 -0700 +diff --git a/tests/unittests/sources/test_azure.py b/tests/unittests/sources/test_azure.py +index 255991ec3..75abf09c6 100644 +--- a/tests/unittests/sources/test_azure.py ++++ b/tests/unittests/sources/test_azure.py @@ -1,6 +1,7 @@ # This file is part of cloud-init. See LICENSE file for license information. - + import copy +import datetime import json import os import stat -@@ -48,6 +49,16 @@ +@@ -48,6 +49,16 @@ def mock_wrapping_setup_ephemeral_networking(azure_ds): yield m - - + + +@pytest.fixture +def mock_wrapping_report_failure(azure_ds): + with mock.patch.object( @@ -129,10 +130,10 @@ diff -ruN a/tests/unittests/sources/test_azure.py b/tests/unittests/sources/test @pytest.fixture def mock_azure_helper_readurl(): with mock.patch( -@@ -253,6 +264,14 @@ +@@ -253,6 +264,14 @@ def mock_subp_subp(): yield m - - + + +@pytest.fixture +def mock_timestamp(): + timestamp = datetime.datetime.utcnow() @@ -144,9 +145,25 @@ diff -ruN a/tests/unittests/sources/test_azure.py b/tests/unittests/sources/test @pytest.fixture def mock_util_ensure_dir(): with mock.patch( -@@ -3649,6 +3668,9 @@ +@@ -288,6 +307,15 @@ def mock_util_mount_cb(): + yield m + + ++@pytest.fixture ++def mock_report_failure(): ++ with mock.patch( ++ MOCKPATH + "_report_failure", ++ autospec=True, ++ ) as m: ++ yield m ++ ++ + @pytest.fixture + def wrapped_util_write_file(): + with mock.patch.object( +@@ -3672,6 +3700,9 @@ class TestProvisioning: } - + def test_no_pps(self): + ovf = construct_ovf_env(provision_guest_proxy_agent=False) + md, ud, cfg = dsaz.read_azure_ovf(ovf) @@ -154,11 +171,54 @@ diff -ruN a/tests/unittests/sources/test_azure.py b/tests/unittests/sources/test self.mock_readurl.side_effect = [ mock.MagicMock(contents=json.dumps(self.imds_md).encode()), ] -@@ -3656,6 +3678,81 @@ - +@@ -3679,6 +3710,8 @@ class TestProvisioning: + self.azure_ds._check_and_get_data() - + + assert self.mock_subp_subp.mock_calls == [] ++ + assert self.mock_readurl.mock_calls == [ + mock.call( + "http://169.254.169.254/metadata/instance?" +@@ -3736,6 +3769,174 @@ class TestProvisioning: + + # Verify reports via KVP. + assert len(self.mock_kvp_report_failure_to_host.mock_calls) == 0 ++ assert len(self.mock_azure_report_failure_to_fabric.mock_calls) == 0 ++ assert len(self.mock_kvp_report_success_to_host.mock_calls) == 1 ++ ++ def test_no_pps_gpa(self): ++ """test full provisioning scope when azure-proxy-agent ++ is enabled and running.""" ++ self.mock_subp_subp.side_effect = [ ++ subp.SubpResult("Guest Proxy Agent running", ""), ++ ] ++ ovf = construct_ovf_env(provision_guest_proxy_agent=True) ++ md, ud, cfg = dsaz.read_azure_ovf(ovf) ++ self.mock_util_mount_cb.return_value = (md, ud, cfg, {}) ++ self.mock_readurl.side_effect = [ ++ mock.MagicMock(contents=json.dumps(self.imds_md).encode()), ++ ] ++ self.mock_azure_get_metadata_from_fabric.return_value = [] ++ ++ self.azure_ds._check_and_get_data() ++ ++ assert self.mock_subp_subp.mock_calls == [ ++ mock.call( ++ ["azure-proxy-agent", "--status", "--wait", "120"], ++ ), ++ ] ++ assert self.mock_readurl.mock_calls == [ ++ mock.call( ++ "http://169.254.169.254/metadata/instance?" ++ "api-version=2021-08-01&extended=true", ++ timeout=30, ++ headers={"Metadata": "true"}, ++ exception_cb=mock.ANY, ++ infinite=True, ++ log_req_resp=True, ++ ), ++ ] + + # Verify DHCP is setup once. + assert self.mock_wrapping_setup_ephemeral_networking.mock_calls == [ @@ -191,7 +251,6 @@ diff -ruN a/tests/unittests/sources/test_azure.py b/tests/unittests/sources/test + assert self.mock_azure_get_metadata_from_fabric.mock_calls == [ + mock.call( + endpoint="10.11.12.13", -+ distro=self.azure_ds.distro, + iso_dev="/dev/sr0", + pubkey_info=None, + ) @@ -209,40 +268,6 @@ diff -ruN a/tests/unittests/sources/test_azure.py b/tests/unittests/sources/test + assert len(self.mock_azure_report_failure_to_fabric.mock_calls) == 0 + assert len(self.mock_kvp_report_success_to_host.mock_calls) == 1 + -+ # Verify dmesg reported via KVP. -+ assert len(self.mock_report_dmesg_to_kvp.mock_calls) == 1 -+ -+ def test_no_pps_gpa(self): -+ """test full provisioning scope when azure-proxy-agent -+ is enabled and running.""" -+ self.mock_subp_subp.side_effect = [ -+ subp.SubpResult("Guest Proxy Agent running", ""), -+ ] -+ ovf = construct_ovf_env(provision_guest_proxy_agent=True) -+ md, ud, cfg = dsaz.read_azure_ovf(ovf) -+ self.mock_util_mount_cb.return_value = (md, ud, cfg, {}) -+ self.mock_readurl.side_effect = [ -+ mock.MagicMock(contents=json.dumps(self.imds_md).encode()), -+ ] -+ self.mock_azure_get_metadata_from_fabric.return_value = [] -+ -+ self.azure_ds._check_and_get_data() -+ -+ assert self.mock_subp_subp.mock_calls == [ -+ mock.call( -+ ["azure-proxy-agent", "--status", "--wait", "120"], -+ ), -+ ] - assert self.mock_readurl.mock_calls == [ - mock.call( - "http://169.254.169.254/metadata/instance?" -@@ -3713,6 +3810,92 @@ - - # Verify reports via KVP. - assert len(self.mock_kvp_report_failure_to_host.mock_calls) == 0 -+ assert len(self.mock_azure_report_failure_to_fabric.mock_calls) == 0 -+ assert len(self.mock_kvp_report_success_to_host.mock_calls) == 1 -+ + def test_no_pps_gpa_fail(self): + """test full provisioning scope when azure-proxy-agent is enabled and + throwing an exception during provisioning.""" @@ -263,6 +288,7 @@ diff -ruN a/tests/unittests/sources/test_azure.py b/tests/unittests/sources/test + self.mock_azure_get_metadata_from_fabric.return_value = [] + + self.azure_ds._check_and_get_data() ++ + assert self.mock_subp_subp.mock_calls == [ + mock.call( + ["azure-proxy-agent", "--status", "--wait", "120"], @@ -327,12 +353,12 @@ diff -ruN a/tests/unittests/sources/test_azure.py b/tests/unittests/sources/test + assert len(self.mock_kvp_report_failure_to_host.mock_calls) == 1 + assert len(self.mock_azure_report_failure_to_fabric.mock_calls) == 1 assert len(self.mock_kvp_report_success_to_host.mock_calls) == 1 - + def test_running_pps(self): -@@ -4292,6 +4475,64 @@ +@@ -4315,6 +4516,61 @@ class TestProvisioning: assert len(self.mock_kvp_report_success_to_host.mock_calls) == 1 - - + + +class TestCheckAzureProxyAgent: + @pytest.fixture(autouse=True) + def proxy_setup( @@ -358,10 +384,7 @@ diff -ruN a/tests/unittests/sources/test_azure.py b/tests/unittests/sources/test + assert self.mock_wrapping_report_failure.mock_calls == [] + + def test_check_azure_proxy_agent_status_notfound(self): -+ exception = subp.ProcessExecutionError(reason=FileNotFoundError()) -+ self.mock_subp_subp.side_effect = [ -+ exception, -+ ] ++ self.mock_subp_subp.side_effect = [FileNotFoundError] + self.azure_ds._check_azure_proxy_agent_status() + assert "azure-proxy-agent not found" in self.caplog.text + assert self.mock_wrapping_report_failure.mock_calls == [ From 81415c595c39381cabb9bf65b9c01b36bc850412 Mon Sep 17 00:00:00 2001 From: minghe Date: Thu, 5 Sep 2024 16:16:56 -0700 Subject: [PATCH 05/11] update patch --- ...re-add-support-for-azure-proxy-agent.patch | 138 ++++++++---------- 1 file changed, 57 insertions(+), 81 deletions(-) diff --git a/SPECS/cloud-init/0003-feat-azure-add-support-for-azure-proxy-agent.patch b/SPECS/cloud-init/0003-feat-azure-add-support-for-azure-proxy-agent.patch index 0c7125439dd..ebfd17e59f5 100644 --- a/SPECS/cloud-init/0003-feat-azure-add-support-for-azure-proxy-agent.patch +++ b/SPECS/cloud-init/0003-feat-azure-add-support-for-azure-proxy-agent.patch @@ -101,22 +101,21 @@ index 966725b00..b331cd686 100644 + self.supporting_data["exit_code"] = exception.exit_code + self.supporting_data["stdout"] = exception.stdout + self.supporting_data["stderr"] = exception.stderr -diff --git a/tests/unittests/sources/test_azure.py b/tests/unittests/sources/test_azure.py -index 255991ec3..75abf09c6 100644 ---- a/tests/unittests/sources/test_azure.py -+++ b/tests/unittests/sources/test_azure.py +diff -ruN a/tests/unittests/sources/test_azure.py b/tests/unittests/sources/test_azure.py +--- a/tests/unittests/sources/test_azure.py 2023-08-28 09:20:24.000000000 -0700 ++++ b/tests/unittests/sources/test_azure.py 2024-09-04 16:47:35.140368594 -0700 @@ -1,6 +1,7 @@ # This file is part of cloud-init. See LICENSE file for license information. - + import copy +import datetime import json import os import stat -@@ -48,6 +49,16 @@ def mock_wrapping_setup_ephemeral_networking(azure_ds): +@@ -48,6 +49,16 @@ yield m - - + + +@pytest.fixture +def mock_wrapping_report_failure(azure_ds): + with mock.patch.object( @@ -130,10 +129,10 @@ index 255991ec3..75abf09c6 100644 @pytest.fixture def mock_azure_helper_readurl(): with mock.patch( -@@ -253,6 +264,14 @@ def mock_subp_subp(): +@@ -253,6 +264,14 @@ yield m - - + + +@pytest.fixture +def mock_timestamp(): + timestamp = datetime.datetime.utcnow() @@ -145,25 +144,9 @@ index 255991ec3..75abf09c6 100644 @pytest.fixture def mock_util_ensure_dir(): with mock.patch( -@@ -288,6 +307,15 @@ def mock_util_mount_cb(): - yield m - - -+@pytest.fixture -+def mock_report_failure(): -+ with mock.patch( -+ MOCKPATH + "_report_failure", -+ autospec=True, -+ ) as m: -+ yield m -+ -+ - @pytest.fixture - def wrapped_util_write_file(): - with mock.patch.object( -@@ -3672,6 +3700,9 @@ class TestProvisioning: +@@ -3649,6 +3668,9 @@ } - + def test_no_pps(self): + ovf = construct_ovf_env(provision_guest_proxy_agent=False) + md, ud, cfg = dsaz.read_azure_ovf(ovf) @@ -171,54 +154,11 @@ index 255991ec3..75abf09c6 100644 self.mock_readurl.side_effect = [ mock.MagicMock(contents=json.dumps(self.imds_md).encode()), ] -@@ -3679,6 +3710,8 @@ class TestProvisioning: - +@@ -3656,6 +3678,81 @@ + self.azure_ds._check_and_get_data() - + + assert self.mock_subp_subp.mock_calls == [] -+ - assert self.mock_readurl.mock_calls == [ - mock.call( - "http://169.254.169.254/metadata/instance?" -@@ -3736,6 +3769,174 @@ class TestProvisioning: - - # Verify reports via KVP. - assert len(self.mock_kvp_report_failure_to_host.mock_calls) == 0 -+ assert len(self.mock_azure_report_failure_to_fabric.mock_calls) == 0 -+ assert len(self.mock_kvp_report_success_to_host.mock_calls) == 1 -+ -+ def test_no_pps_gpa(self): -+ """test full provisioning scope when azure-proxy-agent -+ is enabled and running.""" -+ self.mock_subp_subp.side_effect = [ -+ subp.SubpResult("Guest Proxy Agent running", ""), -+ ] -+ ovf = construct_ovf_env(provision_guest_proxy_agent=True) -+ md, ud, cfg = dsaz.read_azure_ovf(ovf) -+ self.mock_util_mount_cb.return_value = (md, ud, cfg, {}) -+ self.mock_readurl.side_effect = [ -+ mock.MagicMock(contents=json.dumps(self.imds_md).encode()), -+ ] -+ self.mock_azure_get_metadata_from_fabric.return_value = [] -+ -+ self.azure_ds._check_and_get_data() -+ -+ assert self.mock_subp_subp.mock_calls == [ -+ mock.call( -+ ["azure-proxy-agent", "--status", "--wait", "120"], -+ ), -+ ] -+ assert self.mock_readurl.mock_calls == [ -+ mock.call( -+ "http://169.254.169.254/metadata/instance?" -+ "api-version=2021-08-01&extended=true", -+ timeout=30, -+ headers={"Metadata": "true"}, -+ exception_cb=mock.ANY, -+ infinite=True, -+ log_req_resp=True, -+ ), -+ ] + + # Verify DHCP is setup once. + assert self.mock_wrapping_setup_ephemeral_networking.mock_calls == [ @@ -268,6 +208,40 @@ index 255991ec3..75abf09c6 100644 + assert len(self.mock_azure_report_failure_to_fabric.mock_calls) == 0 + assert len(self.mock_kvp_report_success_to_host.mock_calls) == 1 + ++ # Verify dmesg reported via KVP. ++ assert len(self.mock_report_dmesg_to_kvp.mock_calls) == 1 ++ ++ def test_no_pps_gpa(self): ++ """test full provisioning scope when azure-proxy-agent ++ is enabled and running.""" ++ self.mock_subp_subp.side_effect = [ ++ subp.SubpResult("Guest Proxy Agent running", ""), ++ ] ++ ovf = construct_ovf_env(provision_guest_proxy_agent=True) ++ md, ud, cfg = dsaz.read_azure_ovf(ovf) ++ self.mock_util_mount_cb.return_value = (md, ud, cfg, {}) ++ self.mock_readurl.side_effect = [ ++ mock.MagicMock(contents=json.dumps(self.imds_md).encode()), ++ ] ++ self.mock_azure_get_metadata_from_fabric.return_value = [] ++ ++ self.azure_ds._check_and_get_data() ++ ++ assert self.mock_subp_subp.mock_calls == [ ++ mock.call( ++ ["azure-proxy-agent", "--status", "--wait", "120"], ++ ), ++ ] + assert self.mock_readurl.mock_calls == [ + mock.call( + "http://169.254.169.254/metadata/instance?" +@@ -3713,6 +3810,92 @@ + + # Verify reports via KVP. + assert len(self.mock_kvp_report_failure_to_host.mock_calls) == 0 ++ assert len(self.mock_azure_report_failure_to_fabric.mock_calls) == 0 ++ assert len(self.mock_kvp_report_success_to_host.mock_calls) == 1 ++ + def test_no_pps_gpa_fail(self): + """test full provisioning scope when azure-proxy-agent is enabled and + throwing an exception during provisioning.""" @@ -288,7 +262,6 @@ index 255991ec3..75abf09c6 100644 + self.mock_azure_get_metadata_from_fabric.return_value = [] + + self.azure_ds._check_and_get_data() -+ + assert self.mock_subp_subp.mock_calls == [ + mock.call( + ["azure-proxy-agent", "--status", "--wait", "120"], @@ -353,12 +326,12 @@ index 255991ec3..75abf09c6 100644 + assert len(self.mock_kvp_report_failure_to_host.mock_calls) == 1 + assert len(self.mock_azure_report_failure_to_fabric.mock_calls) == 1 assert len(self.mock_kvp_report_success_to_host.mock_calls) == 1 - + def test_running_pps(self): -@@ -4315,6 +4516,61 @@ class TestProvisioning: +@@ -4292,6 +4475,64 @@ assert len(self.mock_kvp_report_success_to_host.mock_calls) == 1 - - + + +class TestCheckAzureProxyAgent: + @pytest.fixture(autouse=True) + def proxy_setup( @@ -384,7 +357,10 @@ index 255991ec3..75abf09c6 100644 + assert self.mock_wrapping_report_failure.mock_calls == [] + + def test_check_azure_proxy_agent_status_notfound(self): -+ self.mock_subp_subp.side_effect = [FileNotFoundError] ++ exception = subp.ProcessExecutionError(reason=FileNotFoundError()) ++ self.mock_subp_subp.side_effect = [ ++ exception, ++ ] + self.azure_ds._check_azure_proxy_agent_status() + assert "azure-proxy-agent not found" in self.caplog.text + assert self.mock_wrapping_report_failure.mock_calls == [ From 1df4acc5fe7819d32c1803b706a097cebc8c406b Mon Sep 17 00:00:00 2001 From: minghe Date: Thu, 5 Sep 2024 16:20:27 -0700 Subject: [PATCH 06/11] update patch --- ...re-add-support-for-azure-proxy-agent.patch | 52 +++++++++---------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/SPECS/cloud-init/0003-feat-azure-add-support-for-azure-proxy-agent.patch b/SPECS/cloud-init/0003-feat-azure-add-support-for-azure-proxy-agent.patch index ebfd17e59f5..f6e38d0e221 100644 --- a/SPECS/cloud-init/0003-feat-azure-add-support-for-azure-proxy-agent.patch +++ b/SPECS/cloud-init/0003-feat-azure-add-support-for-azure-proxy-agent.patch @@ -103,7 +103,7 @@ index 966725b00..b331cd686 100644 + self.supporting_data["stderr"] = exception.stderr diff -ruN a/tests/unittests/sources/test_azure.py b/tests/unittests/sources/test_azure.py --- a/tests/unittests/sources/test_azure.py 2023-08-28 09:20:24.000000000 -0700 -+++ b/tests/unittests/sources/test_azure.py 2024-09-04 16:47:35.140368594 -0700 ++++ b/tests/unittests/sources/test_azure.py 2024-09-05 16:19:14.035134350 -0700 @@ -1,6 +1,7 @@ # This file is part of cloud-init. See LICENSE file for license information. @@ -112,11 +112,10 @@ diff -ruN a/tests/unittests/sources/test_azure.py b/tests/unittests/sources/test import json import os import stat -@@ -48,6 +49,16 @@ - yield m +@@ -49,6 +50,16 @@ -+@pytest.fixture + @pytest.fixture +def mock_wrapping_report_failure(azure_ds): + with mock.patch.object( + azure_ds, @@ -126,14 +125,14 @@ diff -ruN a/tests/unittests/sources/test_azure.py b/tests/unittests/sources/test + yield m + + - @pytest.fixture ++@pytest.fixture def mock_azure_helper_readurl(): with mock.patch( -@@ -253,6 +264,14 @@ - yield m + "cloudinit.sources.helpers.azure.url_helper.readurl", autospec=True +@@ -254,6 +265,14 @@ -+@pytest.fixture + @pytest.fixture +def mock_timestamp(): + timestamp = datetime.datetime.utcnow() + with mock.patch.object(errors, "datetime", autospec=True) as m: @@ -141,23 +140,24 @@ diff -ruN a/tests/unittests/sources/test_azure.py b/tests/unittests/sources/test + yield timestamp + + - @pytest.fixture ++@pytest.fixture def mock_util_ensure_dir(): with mock.patch( -@@ -3649,6 +3668,9 @@ + MOCKPATH + "util.ensure_dir", +@@ -3649,6 +3668,78 @@ } def test_no_pps(self): + ovf = construct_ovf_env(provision_guest_proxy_agent=False) + md, ud, cfg = dsaz.read_azure_ovf(ovf) + self.mock_util_mount_cb.return_value = (md, ud, cfg, {}) - self.mock_readurl.side_effect = [ - mock.MagicMock(contents=json.dumps(self.imds_md).encode()), - ] -@@ -3656,6 +3678,81 @@ - - self.azure_ds._check_and_get_data() - ++ self.mock_readurl.side_effect = [ ++ mock.MagicMock(contents=json.dumps(self.imds_md).encode()), ++ ] ++ self.mock_azure_get_metadata_from_fabric.return_value = [] ++ ++ self.azure_ds._check_and_get_data() ++ + assert self.mock_subp_subp.mock_calls == [] + + # Verify DHCP is setup once. @@ -220,13 +220,13 @@ diff -ruN a/tests/unittests/sources/test_azure.py b/tests/unittests/sources/test + ovf = construct_ovf_env(provision_guest_proxy_agent=True) + md, ud, cfg = dsaz.read_azure_ovf(ovf) + self.mock_util_mount_cb.return_value = (md, ud, cfg, {}) -+ self.mock_readurl.side_effect = [ -+ mock.MagicMock(contents=json.dumps(self.imds_md).encode()), -+ ] -+ self.mock_azure_get_metadata_from_fabric.return_value = [] -+ -+ self.azure_ds._check_and_get_data() -+ + self.mock_readurl.side_effect = [ + mock.MagicMock(contents=json.dumps(self.imds_md).encode()), + ] +@@ -3656,6 +3747,11 @@ + + self.azure_ds._check_and_get_data() + + assert self.mock_subp_subp.mock_calls == [ + mock.call( + ["azure-proxy-agent", "--status", "--wait", "120"], @@ -235,7 +235,7 @@ diff -ruN a/tests/unittests/sources/test_azure.py b/tests/unittests/sources/test assert self.mock_readurl.mock_calls == [ mock.call( "http://169.254.169.254/metadata/instance?" -@@ -3713,6 +3810,92 @@ +@@ -3713,6 +3809,92 @@ # Verify reports via KVP. assert len(self.mock_kvp_report_failure_to_host.mock_calls) == 0 @@ -328,7 +328,7 @@ diff -ruN a/tests/unittests/sources/test_azure.py b/tests/unittests/sources/test assert len(self.mock_kvp_report_success_to_host.mock_calls) == 1 def test_running_pps(self): -@@ -4292,6 +4475,64 @@ +@@ -4292,6 +4474,64 @@ assert len(self.mock_kvp_report_success_to_host.mock_calls) == 1 From d71951b94156793dcdf92462d656438cfccdc7ff Mon Sep 17 00:00:00 2001 From: minghe Date: Fri, 6 Sep 2024 03:09:09 -0700 Subject: [PATCH 07/11] update patch --- .../0003-feat-azure-add-support-for-azure-proxy-agent.patch | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SPECS/cloud-init/0003-feat-azure-add-support-for-azure-proxy-agent.patch b/SPECS/cloud-init/0003-feat-azure-add-support-for-azure-proxy-agent.patch index f6e38d0e221..4c691b40d50 100644 --- a/SPECS/cloud-init/0003-feat-azure-add-support-for-azure-proxy-agent.patch +++ b/SPECS/cloud-init/0003-feat-azure-add-support-for-azure-proxy-agent.patch @@ -103,7 +103,7 @@ index 966725b00..b331cd686 100644 + self.supporting_data["stderr"] = exception.stderr diff -ruN a/tests/unittests/sources/test_azure.py b/tests/unittests/sources/test_azure.py --- a/tests/unittests/sources/test_azure.py 2023-08-28 09:20:24.000000000 -0700 -+++ b/tests/unittests/sources/test_azure.py 2024-09-05 16:19:14.035134350 -0700 ++++ b/tests/unittests/sources/test_azure.py 2024-09-06 03:06:08.563274466 -0700 @@ -1,6 +1,7 @@ # This file is part of cloud-init. See LICENSE file for license information. From d2250c238294cfd1c5a0c08017324380310fa987 Mon Sep 17 00:00:00 2001 From: minghe Date: Fri, 6 Sep 2024 11:31:28 -0700 Subject: [PATCH 08/11] update patch --- ...eat-azure-add-support-for-azure-proxy-agent.patch | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/SPECS/cloud-init/0003-feat-azure-add-support-for-azure-proxy-agent.patch b/SPECS/cloud-init/0003-feat-azure-add-support-for-azure-proxy-agent.patch index 4c691b40d50..2a3895c000d 100644 --- a/SPECS/cloud-init/0003-feat-azure-add-support-for-azure-proxy-agent.patch +++ b/SPECS/cloud-init/0003-feat-azure-add-support-for-azure-proxy-agent.patch @@ -103,7 +103,7 @@ index 966725b00..b331cd686 100644 + self.supporting_data["stderr"] = exception.stderr diff -ruN a/tests/unittests/sources/test_azure.py b/tests/unittests/sources/test_azure.py --- a/tests/unittests/sources/test_azure.py 2023-08-28 09:20:24.000000000 -0700 -+++ b/tests/unittests/sources/test_azure.py 2024-09-06 03:06:08.563274466 -0700 ++++ b/tests/unittests/sources/test_azure.py 2024-09-06 11:30:27.992040291 -0700 @@ -1,6 +1,7 @@ # This file is part of cloud-init. See LICENSE file for license information. @@ -144,7 +144,7 @@ diff -ruN a/tests/unittests/sources/test_azure.py b/tests/unittests/sources/test def mock_util_ensure_dir(): with mock.patch( MOCKPATH + "util.ensure_dir", -@@ -3649,6 +3668,78 @@ +@@ -3649,6 +3668,76 @@ } def test_no_pps(self): @@ -208,8 +208,6 @@ diff -ruN a/tests/unittests/sources/test_azure.py b/tests/unittests/sources/test + assert len(self.mock_azure_report_failure_to_fabric.mock_calls) == 0 + assert len(self.mock_kvp_report_success_to_host.mock_calls) == 1 + -+ # Verify dmesg reported via KVP. -+ assert len(self.mock_report_dmesg_to_kvp.mock_calls) == 1 + + def test_no_pps_gpa(self): + """test full provisioning scope when azure-proxy-agent @@ -223,7 +221,7 @@ diff -ruN a/tests/unittests/sources/test_azure.py b/tests/unittests/sources/test self.mock_readurl.side_effect = [ mock.MagicMock(contents=json.dumps(self.imds_md).encode()), ] -@@ -3656,6 +3747,11 @@ +@@ -3656,6 +3745,11 @@ self.azure_ds._check_and_get_data() @@ -235,7 +233,7 @@ diff -ruN a/tests/unittests/sources/test_azure.py b/tests/unittests/sources/test assert self.mock_readurl.mock_calls == [ mock.call( "http://169.254.169.254/metadata/instance?" -@@ -3713,6 +3809,92 @@ +@@ -3713,6 +3807,92 @@ # Verify reports via KVP. assert len(self.mock_kvp_report_failure_to_host.mock_calls) == 0 @@ -328,7 +326,7 @@ diff -ruN a/tests/unittests/sources/test_azure.py b/tests/unittests/sources/test assert len(self.mock_kvp_report_success_to_host.mock_calls) == 1 def test_running_pps(self): -@@ -4292,6 +4474,64 @@ +@@ -4292,6 +4472,64 @@ assert len(self.mock_kvp_report_success_to_host.mock_calls) == 1 From b43ceab2c1875bc11142a23d8e6a2ed681b8296d Mon Sep 17 00:00:00 2001 From: minghe Date: Mon, 9 Sep 2024 16:29:55 -0700 Subject: [PATCH 09/11] add commentsin patch file --- ...at-azure-add-support-for-azure-proxy-agent.patch | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/SPECS/cloud-init/0003-feat-azure-add-support-for-azure-proxy-agent.patch b/SPECS/cloud-init/0003-feat-azure-add-support-for-azure-proxy-agent.patch index 2a3895c000d..7c7fc18c4f2 100644 --- a/SPECS/cloud-init/0003-feat-azure-add-support-for-azure-proxy-agent.patch +++ b/SPECS/cloud-init/0003-feat-azure-add-support-for-azure-proxy-agent.patch @@ -3,6 +3,19 @@ From: Ksenija Stanojevic Date: Mon, 15 Jul 2024 18:48:19 -0700 Subject: [PATCH 3/3] feat(azure): add support for azure-proxy-agent +NOTE change for azurelinux : + +This patch has some addtional modifications in the test_azure.py. Those changes remove +some tests which would fail to run: +1. removing unit tests for imds.headers_cb in test_no_pps, which is not implement +in our current version of cloud-init +2. remove checking for distro=self.azure_ds.distro in test_no_pps, this is due to +a behavior difference in mock_azure_get_metadata_from_fabric which would not return +self.azure_ds.distro +3. remove mock_report_dmesg_to_kvp test in test_no_pps, as mock_report_dmesg_to_kvp is not +implemented in our current version of cloud-init + +Original Fedora commit message below: --- cloudinit/sources/DataSourceAzure.py | 40 ++++ cloudinit/sources/azure/errors.py | 19 +- From c6d1a91743a51fad6a89ff6a1501ee454f1c5c8a Mon Sep 17 00:00:00 2001 From: minghe Date: Tue, 10 Sep 2024 18:15:21 -0700 Subject: [PATCH 10/11] update patch --- .../0003-feat-azure-add-support-for-azure-proxy-agent.patch | 1 - 1 file changed, 1 deletion(-) diff --git a/SPECS/cloud-init/0003-feat-azure-add-support-for-azure-proxy-agent.patch b/SPECS/cloud-init/0003-feat-azure-add-support-for-azure-proxy-agent.patch index 7c7fc18c4f2..fd1bc8a9417 100644 --- a/SPECS/cloud-init/0003-feat-azure-add-support-for-azure-proxy-agent.patch +++ b/SPECS/cloud-init/0003-feat-azure-add-support-for-azure-proxy-agent.patch @@ -15,7 +15,6 @@ self.azure_ds.distro 3. remove mock_report_dmesg_to_kvp test in test_no_pps, as mock_report_dmesg_to_kvp is not implemented in our current version of cloud-init -Original Fedora commit message below: --- cloudinit/sources/DataSourceAzure.py | 40 ++++ cloudinit/sources/azure/errors.py | 19 +- From 87dbdf1fbea837291268dcd7cb088a5b1630079c Mon Sep 17 00:00:00 2001 From: minghe Date: Wed, 11 Sep 2024 11:06:13 -0700 Subject: [PATCH 11/11] modify date --- SPECS/cloud-init/cloud-init.spec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SPECS/cloud-init/cloud-init.spec b/SPECS/cloud-init/cloud-init.spec index 414bc5a49ce..3b4f4222cef 100644 --- a/SPECS/cloud-init/cloud-init.spec +++ b/SPECS/cloud-init/cloud-init.spec @@ -154,7 +154,7 @@ make check %{?_smp_mflags} %config(noreplace) %{_sysconfdir}/cloud/cloud.cfg.d/10-azure-kvp.cfg %changelog -* Wed Spe 04 2024 Minghe Ren - 1:23.3-4 +* Wed Sep 04 2024 Minghe Ren - 1:23.3-4 - Add patches to support azure-proxy-agent. * Wed May 08 2024 Sharath Srikanth Chellappa - 1:23.3-3