Merge pull request #2635 from Azure/hotfix-2.7.2.0

Merge hotfix 2.7.3.0 to master

azurelinuxagent/common/osutil/factory.py

@@ -34,7 +34,7 @@
 from .nsbsd import NSBSDOSUtil
 from .openbsd import OpenBSDOSUtil
 from .openwrt import OpenWRTOSUtil
-from .redhat import RedhatOSUtil, Redhat6xOSUtil
+from .redhat import RedhatOSUtil, Redhat6xOSUtil, RedhatOSModernUtil
 from .suse import SUSEOSUtil, SUSE11OSUtil
 from .photonos import PhotonOSUtil
 from .ubuntu import UbuntuOSUtil, Ubuntu12OSUtil, Ubuntu14OSUtil, \
@@ -106,6 +106,9 @@ def _get_osutil(distro_name, distro_code_name, distro_version, distro_full_name)
         if Version(distro_version) < Version("7"):
             return Redhat6xOSUtil()
 
+        if Version(distro_version) >= Version("8.6"):
+            return RedhatOSModernUtil()
+
         return RedhatOSUtil()
 
     if distro_name == "euleros":

azurelinuxagent/common/osutil/redhat.py

@@ -142,3 +142,25 @@ def get_dhcp_lease_endpoint(self):
             endpoint = self.get_endpoint_from_leases_path('/var/lib/NetworkManager/dhclient-*.lease')
 
         return endpoint
+
+
+class RedhatOSModernUtil(RedhatOSUtil):
+    def __init__(self):  # pylint: disable=W0235
+        super(RedhatOSModernUtil, self).__init__()
+
+    def restart_if(self, ifname, retries=3, wait=5):
+        """
+        Restart an interface by bouncing the link. systemd-networkd observes
+        this event, and forces a renew of DHCP.
+        """
+        retry_limit = retries + 1
+        for attempt in range(1, retry_limit):
+            return_code = shellutil.run("ip link set {0} down && ip link set {0} up".format(ifname))
+            if return_code == 0:
+                return
+            logger.warn("failed to restart {0}: return code {1}".format(ifname, return_code))
+            if attempt < retry_limit:
+                logger.info("retrying in {0} seconds".format(wait))
+                time.sleep(wait)
+            else:
+                logger.warn("exceeded restart retries")

azurelinuxagent/common/protocol/extensions_goal_state_from_extensions_config.py

@@ -25,6 +25,7 @@
 from azurelinuxagent.common.future import ustr
 from azurelinuxagent.common.protocol.extensions_goal_state import ExtensionsGoalState
 from azurelinuxagent.common.protocol.restapi import ExtensionSettings, Extension, VMAgentManifest, ExtensionState, InVMGoalStateMetaData
+from azurelinuxagent.common.utils import restutil
 from azurelinuxagent.common.utils.textutil import parse_doc, parse_json, findall, find, findtext, getattrib, gettext, format_exception, \
     is_str_none_or_whitespace, is_str_empty
 
@@ -98,7 +99,7 @@ def fetch_direct():
         def fetch_through_host():
             host = wire_client.get_host_plugin()
             uri, headers = host.get_artifact_request(artifacts_profile_blob)
-            content, _ = wire_client.fetch(uri, headers, use_proxy=False)
+            content, _ = wire_client.fetch(uri, headers, use_proxy=False, retry_codes=restutil.HGAP_GET_EXTENSION_ARTIFACT_RETRY_CODES)
             return content
 
         logger.verbose("Retrieving the artifacts profile")

azurelinuxagent/common/protocol/wire.py

@@ -147,7 +147,7 @@ def get_extensions_goal_state(self):
     def _download_ext_handler_pkg_through_host(self, uri, destination):
         host = self.client.get_host_plugin()
         uri, headers = host.get_artifact_request(uri, host.manifest_uri)
-        success = self.client.stream(uri, destination, headers=headers, use_proxy=False, max_retry=1)
+        success = self.client.stream(uri, destination, headers=headers, use_proxy=False, max_retry=1)  # set max_retry to 1 because extension packages already have a retry loop (see ExtHandlerInstance.download())
         return success
 
     def download_ext_handler_pkg(self, uri, destination, headers=None, use_proxy=True):  # pylint: disable=W0613
@@ -652,7 +652,7 @@ def call_storage_service(http_req, *args, **kwargs):
     def fetch_manifest_through_host(self, uri):
         host = self.get_host_plugin()
         uri, headers = host.get_artifact_request(uri)
-        response, _ = self.fetch(uri, headers, use_proxy=False, max_retry=1)
+        response, _ = self.fetch(uri, headers, use_proxy=False, retry_codes=restutil.HGAP_GET_EXTENSION_ARTIFACT_RETRY_CODES)
         return response
 
     def fetch_manifest(self, version_uris, timeout_in_minutes=5, timeout_in_ms=0):
@@ -675,9 +675,11 @@ def fetch_manifest(self, version_uris, timeout_in_minutes=5, timeout_in_ms=0):
                 logger.verbose('The specified manifest URL is empty, ignored.')
                 continue
 
-            direct_func = lambda: self.fetch(version_uri, max_retry=1)[0]  # pylint: disable=W0640
+            # Disable W0640: OK to use version_uri in a lambda within the loop's body
+            direct_func = lambda: self.fetch(version_uri)[0]  # pylint: disable=W0640
             # NOTE: the host_func may be called after refreshing the goal state, be careful about any goal state data
             # in the lambda.
+            # Disable W0640: OK to use version_uri in a lambda within the loop's body
             host_func = lambda: self.fetch_manifest_through_host(version_uri)  # pylint: disable=W0640
 
             try:
@@ -716,7 +718,7 @@ def stream(self, uri, destination, headers=None, use_proxy=None, max_retry=None)
 
         return success
 
-    def fetch(self, uri, headers=None, use_proxy=None, decode=True, max_retry=None, ok_codes=None):
+    def fetch(self, uri, headers=None, use_proxy=None, decode=True, max_retry=None, retry_codes=None, ok_codes=None):
         """
         max_retry indicates the maximum number of retries for the HTTP request; None indicates that the default value should be used
 
@@ -725,14 +727,14 @@ def fetch(self, uri, headers=None, use_proxy=None, decode=True, max_retry=None,
         logger.verbose("Fetch [{0}] with headers [{1}]", uri, headers)
         content = None
         response_headers = None
-        response = self._fetch_response(uri, headers, use_proxy, max_retry=max_retry, ok_codes=ok_codes)
+        response = self._fetch_response(uri, headers, use_proxy, max_retry=max_retry, retry_codes=retry_codes, ok_codes=ok_codes)
         if response is not None and not restutil.request_failed(response, ok_codes=ok_codes):
             response_content = response.read()
             content = self.decode_config(response_content) if decode else response_content
             response_headers = response.getheaders()
         return content, response_headers
 
-    def _fetch_response(self, uri, headers=None, use_proxy=None, max_retry=None, ok_codes=None):
+    def _fetch_response(self, uri, headers=None, use_proxy=None, max_retry=None, retry_codes=None, ok_codes=None):
         """
         max_retry indicates the maximum number of retries for the HTTP request; None indicates that the default value should be used
         """
@@ -743,7 +745,8 @@ def _fetch_response(self, uri, headers=None, use_proxy=None, max_retry=None, ok_
                 uri,
                 headers=headers,
                 use_proxy=use_proxy,
-                max_retry=max_retry)
+                max_retry=max_retry,
+                retry_codes=retry_codes)
 
             host_plugin = self.get_host_plugin()
 

azurelinuxagent/common/utils/restutil.py

@@ -56,6 +56,15 @@
     429,  # Request Rate Limit Exceeded
 ]
 
+#
+# Currently the HostGAPlugin has an issue its cache that may produce a BAD_REQUEST failure for valid URIs when using the extensionArtifact API.
+# Add this status to the retryable codes, but use it only when requesting downloads via the HostGAPlugin. The retry logic in the download code
+# would give enough time to the HGAP to refresh its cache. Once the fix to address that issue is deployed, consider removing the use of
+# HGAP_GET_EXTENSION_ARTIFACT_RETRY_CODES.
+#
+HGAP_GET_EXTENSION_ARTIFACT_RETRY_CODES = RETRY_CODES[:]  # make a copy of RETRY_CODES
+HGAP_GET_EXTENSION_ARTIFACT_RETRY_CODES.append(httpclient.BAD_REQUEST)
+
 RESOURCE_GONE_CODES = [
     httpclient.GONE
 ]

azurelinuxagent/common/version.py

@@ -209,7 +209,7 @@ def has_logrotate():
 #
 #            When doing a release, be sure to use the actual agent version.
 #
-AGENT_VERSION = '2.7.1.0'
+AGENT_VERSION = '2.7.3.0'
 AGENT_LONG_VERSION = "{0}-{1}".format(AGENT_NAME, AGENT_VERSION)
 AGENT_DESCRIPTION = """
 The Azure Linux Agent supports the provisioning and running of Linux

azurelinuxagent/ga/update.py

@@ -1317,7 +1317,7 @@ def _download(self):
 
                 uri, headers = self.host.get_artifact_request(uri, self.host.manifest_uri)
                 try:
-                    if self._fetch(uri, headers=headers, use_proxy=False):
+                    if self._fetch(uri, headers=headers, use_proxy=False, retry_codes=restutil.HGAP_GET_EXTENSION_ARTIFACT_RETRY_CODES):
                         if not HostPluginProtocol.is_default_channel:
                             logger.verbose("Setting host plugin as default channel")
                             HostPluginProtocol.is_default_channel = True
@@ -1344,12 +1344,12 @@ def _download(self):
                 message=msg)
             raise UpdateError(msg)
 
-    def _fetch(self, uri, headers=None, use_proxy=True):
+    def _fetch(self, uri, headers=None, use_proxy=True, retry_codes=None):
         package = None
         try:
             is_healthy = True
             error_response = ''
-            resp = restutil.http_get(uri, use_proxy=use_proxy, headers=headers, max_retry=1)
+            resp = restutil.http_get(uri, use_proxy=use_proxy, headers=headers, max_retry=3, retry_codes=retry_codes)  # Use only 3 retries, since there are usually 5 or 6 URIs and we try all of them
             if restutil.request_succeeded(resp):
                 package = resp.read()
                 fileutil.write_file(self.get_agent_pkg_path(),