diff --git a/azurelinuxagent/agent.py b/azurelinuxagent/agent.py index d05bdbdae9..7ac6fa3d0b 100644 --- a/azurelinuxagent/agent.py +++ b/azurelinuxagent/agent.py @@ -191,11 +191,13 @@ def collect_logs(self, is_full_mode): sys.exit(1) -def main(args=[]): # pylint: disable=R0912,W0102 +def main(args=None): # pylint: disable=R0912 """ Parse command line arguments, exit with usage() on error. Invoke different methods according to different command """ + if args is None: + args = [] if len(args) <= 0: # pylint: disable=len-as-condition args = sys.argv[1:] command, force, verbose, debug, conf_file_path, log_collector_full_mode = parse_args(args) diff --git a/azurelinuxagent/common/osutil/default.py b/azurelinuxagent/common/osutil/default.py index 2159230506..7f994c751a 100644 --- a/azurelinuxagent/common/osutil/default.py +++ b/azurelinuxagent/common/osutil/default.py @@ -31,17 +31,17 @@ import struct import sys import time -from pwd import getpwall # pylint: disable=ungrouped-imports +from pwd import getpwall # pylint: disable=ungrouped-imports import array -#pylint: disable=R0801 +# pylint: disable=R0801 import azurelinuxagent.common.conf as conf import azurelinuxagent.common.logger as logger import azurelinuxagent.common.utils.fileutil as fileutil import azurelinuxagent.common.utils.shellutil as shellutil import azurelinuxagent.common.utils.textutil as textutil -#pylint: enable=R0801 +# pylint: enable=R0801 from azurelinuxagent.common.exception import OSUtilError from azurelinuxagent.common.future import ustr @@ -50,8 +50,8 @@ from azurelinuxagent.common.utils.networkutil import RouteEntry, NetworkInterfaceCard from azurelinuxagent.common.utils.shellutil import CommandError -__RULES_FILES__ = [ "/lib/udev/rules.d/75-persistent-net-generator.rules", - "/etc/udev/rules.d/70-persistent-net.rules" ] +__RULES_FILES__ = ["/lib/udev/rules.d/75-persistent-net-generator.rules", + "/etc/udev/rules.d/70-persistent-net.rules"] """ Define distro specific behavior. OSUtil class defines default behavior @@ -59,7 +59,7 @@ if needed. """ -_IPTABLES_VERSION_PATTERN = re.compile("^[^\d\.]*([\d\.]+).*$") # pylint: disable=W1401 +_IPTABLES_VERSION_PATTERN = re.compile("^[^\d\.]*([\d\.]+).*$") # pylint: disable=W1401 _IPTABLES_LOCKING_VERSION = FlexibleVersion('1.4.21') @@ -77,11 +77,14 @@ def _get_iptables_version_command(): def _get_firewall_accept_command(wait, command, destination, owner_uid): - return _add_wait(wait, ["iptables", "-t", "security", command, "OUTPUT", "-d", destination, "-p", "tcp", "-m", "owner", "--uid-owner", str(owner_uid), "-j" "ACCEPT"]) # pylint: disable=W1404,W1403 + return _add_wait(wait, + ["iptables", "-t", "security", command, "OUTPUT", "-d", destination, "-p", "tcp", "-m", "owner", + "--uid-owner", str(owner_uid), "-j" "ACCEPT"]) # pylint: disable=W1404,W1403 def _get_firewall_drop_command(wait, command, destination): - return _add_wait(wait, ["iptables", "-t", "security", command, "OUTPUT", "-d", destination, "-p", "tcp", "-m", "conntrack", "--ctstate", "INVALID,NEW", "-j", "DROP"]) + return _add_wait(wait, ["iptables", "-t", "security", command, "OUTPUT", "-d", destination, "-p", "tcp", "-m", + "conntrack", "--ctstate", "INVALID,NEW", "-j", "DROP"]) def _get_firewall_list_command(wait): @@ -95,20 +98,26 @@ def _get_firewall_packets_command(wait): # Precisely delete the rules created by the agent. # this rule was used <= 2.2.25. This rule helped to validate our change, and determine impact. def _get_firewall_delete_conntrack_accept_command(wait, destination): - return _add_wait(wait, ["iptables", "-t", "security", "-D", "OUTPUT", "-d", destination, "-p", "tcp", "-m", "conntrack", "--ctstate", "INVALID,NEW", "-j", "ACCEPT"]) + return _add_wait(wait, + ["iptables", "-t", "security", "-D", "OUTPUT", "-d", destination, "-p", "tcp", "-m", "conntrack", + "--ctstate", "INVALID,NEW", "-j", "ACCEPT"]) def _get_firewall_delete_owner_accept_command(wait, destination, owner_uid): - return _add_wait(wait, ["iptables", "-t", "security", "-D", "OUTPUT", "-d", destination, "-p", "tcp", "-m", "owner", "--uid-owner", str(owner_uid), "-j", "ACCEPT"]) + return _add_wait(wait, ["iptables", "-t", "security", "-D", "OUTPUT", "-d", destination, "-p", "tcp", "-m", "owner", + "--uid-owner", str(owner_uid), "-j", "ACCEPT"]) + def _get_firewall_delete_conntrack_drop_command(wait, destination): - return _add_wait(wait, ["iptables", "-t", "security", "-D", "OUTPUT", "-d", destination, "-p", "tcp", "-m", "conntrack", "--ctstate", "INVALID,NEW", "-j", "DROP"]) + return _add_wait(wait, + ["iptables", "-t", "security", "-D", "OUTPUT", "-d", destination, "-p", "tcp", "-m", "conntrack", + "--ctstate", "INVALID,NEW", "-j", "DROP"]) -PACKET_PATTERN = "^\s*(\d+)\s+(\d+)\s+DROP\s+.*{0}[^\d]*$" # pylint: disable=W1401 -ALL_CPUS_REGEX = re.compile('^cpu .*') +PACKET_PATTERN = "^\s*(\d+)\s+(\d+)\s+DROP\s+.*{0}[^\d]*$" # pylint: disable=W1401 +ALL_CPUS_REGEX = re.compile('^cpu .*') -_enable_firewall = True # pylint: disable=C0103 +_enable_firewall = True # pylint: disable=C0103 DMIDECODE_CMD = 'dmidecode --string system-uuid' PRODUCT_ID_FILE = '/sys/class/dmi/id/product_uuid' @@ -121,12 +130,13 @@ def _get_firewall_delete_conntrack_drop_command(wait, destination): IOCTL_SIOCGIFHWADDR = 0x8927 IFNAMSIZ = 16 -IP_COMMAND_OUTPUT = re.compile('^\d+:\s+(\w+):\s+(.*)$') # pylint: disable=W1401 +IP_COMMAND_OUTPUT = re.compile('^\d+:\s+(\w+):\s+(.*)$') # pylint: disable=W1401 STORAGE_DEVICE_PATH = '/sys/bus/vmbus/devices/' GEN2_DEVICE_ID = 'f8b3781a-1e82-4818-a1c3-63d806ec15bb' -class DefaultOSUtil(object): # pylint: disable=R0904 + +class DefaultOSUtil(object): # pylint: disable=R0904 def __init__(self): self.agent_conf_file_path = '/etc/waagent.conf' self.selinux = None @@ -140,7 +150,7 @@ def get_service_name(): def get_firewall_dropped_packets(self, dst_ip=None): # If a previous attempt failed, do not retry - global _enable_firewall # pylint: disable=W0603,C0103 + global _enable_firewall # pylint: disable=W0603,C0103 if not _enable_firewall: return 0 @@ -152,12 +162,12 @@ def get_firewall_dropped_packets(self, dst_ip=None): pattern = re.compile(PACKET_PATTERN.format(dst_ip)) for line in output.split('\n'): - m = pattern.match(line) # pylint: disable=C0103 + m = pattern.match(line) # pylint: disable=C0103 if m is not None: return int(m.group(1)) - except Exception as e: # pylint: disable=C0103 - if isinstance(e, CommandError) and e.returncode == 3: # pylint: disable=E1101 + except Exception as e: # pylint: disable=C0103 + if isinstance(e, CommandError) and e.returncode == 3: # pylint: disable=E1101 # Transient error that we ignore. This code fires every loop # of the daemon (60m), so we will get the value eventually. return 0 @@ -166,7 +176,7 @@ def get_firewall_dropped_packets(self, dst_ip=None): return 0 - except Exception as e: # pylint: disable=C0103 + except Exception as e: # pylint: disable=C0103 _enable_firewall = False logger.warn("Unable to retrieve firewall packets dropped" "{0}".format(ustr(e))) @@ -176,20 +186,20 @@ def get_firewall_will_wait(self): # Determine if iptables will serialize access try: output = shellutil.run_command(_get_iptables_version_command()) - except Exception as e: # pylint: disable=C0103 + except Exception as e: # pylint: disable=C0103 msg = "Unable to determine version of iptables: {0}".format(ustr(e)) logger.warn(msg) raise Exception(msg) - m = _IPTABLES_VERSION_PATTERN.match(output) # pylint: disable=C0103 + m = _IPTABLES_VERSION_PATTERN.match(output) # pylint: disable=C0103 if m is None: msg = "iptables did not return version information: {0}".format(output) logger.warn(msg) raise Exception(msg) wait = "-w" \ - if FlexibleVersion(m.group(1)) >= _IPTABLES_LOCKING_VERSION \ - else "" + if FlexibleVersion(m.group(1)) >= _IPTABLES_LOCKING_VERSION \ + else "" return wait def _delete_rule(self, rule): @@ -197,10 +207,10 @@ def _delete_rule(self, rule): Continually execute the delete operation until the return code is non-zero or the limit has been reached. """ - for i in range(1, 100): # pylint: disable=W0612 + for i in range(1, 100): # pylint: disable=W0612 try: - rc = shellutil.run_command(rule) # pylint: disable=W0612,C0103 - except CommandError as e: # pylint: disable=C0103 + rc = shellutil.run_command(rule) # pylint: disable=W0612,C0103 + except CommandError as e: # pylint: disable=C0103 if e.returncode == 1: return if e.returncode == 2: @@ -208,7 +218,7 @@ def _delete_rule(self, rule): def remove_firewall(self, dst_ip, uid): # If a previous attempt failed, do not retry - global _enable_firewall # pylint: disable=W0603,C0103 + global _enable_firewall # pylint: disable=W0603,C0103 if not _enable_firewall: return False @@ -223,7 +233,7 @@ def remove_firewall(self, dst_ip, uid): return True - except Exception as e: # pylint: disable=C0103 + except Exception as e: # pylint: disable=C0103 _enable_firewall = False logger.info("Unable to remove firewall -- " "no further attempts will be made: " @@ -232,7 +242,7 @@ def remove_firewall(self, dst_ip, uid): def enable_firewall(self, dst_ip, uid): # If a previous attempt failed, do not retry - global _enable_firewall # pylint: disable=W0603,C0103 + global _enable_firewall # pylint: disable=W0603,C0103 if not _enable_firewall: return False @@ -240,13 +250,13 @@ def enable_firewall(self, dst_ip, uid): wait = self.get_firewall_will_wait() # If the DROP rule exists, make no changes - firewall_established = False # pylint: disable=W0612 + firewall_established = False # pylint: disable=W0612 try: drop_rule = _get_firewall_drop_command(wait, "-C", dst_ip) shellutil.run_command(drop_rule) logger.verbose("Firewall appears established") return True - except CommandError as e: # pylint: disable=C0103 + except CommandError as e: # pylint: disable=C0103 if e.returncode == 2: self.remove_firewall(dst_ip, uid) msg = "please upgrade iptables to a version that supports the -C option" @@ -257,7 +267,7 @@ def enable_firewall(self, dst_ip, uid): try: accept_rule = _get_firewall_accept_command(wait, "-A", dst_ip, uid) shellutil.run_command(accept_rule) - except Exception as e: # pylint: disable=C0103 + except Exception as e: # pylint: disable=C0103 msg = "Unable to add ACCEPT firewall rule '{0}' - {1}".format(accept_rule, ustr(e)) logger.warn(msg) raise Exception(msg) @@ -265,7 +275,7 @@ def enable_firewall(self, dst_ip, uid): try: drop_rule = _get_firewall_drop_command(wait, "-A", dst_ip) shellutil.run_command(drop_rule) - except Exception as e: # pylint: disable=C0103 + except Exception as e: # pylint: disable=C0103 msg = "Unable to add DROP firewall rule '{0}' - {1}".format(drop_rule, ustr(e)) logger.warn(msg) raise Exception(msg) @@ -275,12 +285,12 @@ def enable_firewall(self, dst_ip, uid): try: output = shellutil.run_command(_get_firewall_list_command(wait)) logger.info("Firewall rules:\n{0}".format(output)) - except Exception as e: # pylint: disable=C0103 + except Exception as e: # pylint: disable=C0103 logger.warn("Listing firewall rules failed: {0}".format(ustr(e))) return True - except Exception as e: # pylint: disable=C0103 + except Exception as e: # pylint: disable=C0103 _enable_firewall = False logger.info("Unable to establish firewall -- " "no further attempts will be made: " @@ -288,8 +298,8 @@ def enable_firewall(self, dst_ip, uid): return False @staticmethod - def _correct_instance_id(id): # pylint: disable=W0622,C0103 - ''' + def _correct_instance_id(instance_id): + """ Azure stores the instance ID with an incorrect byte ordering for the first parts. For example, the ID returned by the metadata service: @@ -301,49 +311,49 @@ def _correct_instance_id(id): # pylint: disable=W0622,C0103 This code corrects the byte order such that it is consistent with that returned by the metadata service. - ''' + """ - if not UUID_PATTERN.match(id): - return id + if not UUID_PATTERN.match(instance_id): + return instance_id - parts = id.split('-') + parts = instance_id.split('-') return '-'.join([ - textutil.swap_hexstring(parts[0], width=2), - textutil.swap_hexstring(parts[1], width=2), - textutil.swap_hexstring(parts[2], width=2), - parts[3], - parts[4] - ]) + textutil.swap_hexstring(parts[0], width=2), + textutil.swap_hexstring(parts[1], width=2), + textutil.swap_hexstring(parts[2], width=2), + parts[3], + parts[4] + ]) def is_current_instance_id(self, id_that): - ''' + """ Compare two instance IDs for equality, but allow that some IDs may have been persisted using the incorrect byte ordering. - ''' + """ id_this = self.get_instance_id() logger.verbose("current instance id: {0}".format(id_this)) logger.verbose(" former instance id: {0}".format(id_that)) return id_this.lower() == id_that.lower() or \ - id_this.lower() == self._correct_instance_id(id_that).lower() + id_this.lower() == self._correct_instance_id(id_that).lower() def get_agent_conf_file_path(self): return self.agent_conf_file_path def get_instance_id(self): - ''' + """ Azure records a UUID as the instance ID First check /sys/class/dmi/id/product_uuid. If that is missing, then extracts from dmidecode If nothing works (for old VMs), return the empty string - ''' + """ if os.path.isfile(PRODUCT_ID_FILE): - s = fileutil.read_file(PRODUCT_ID_FILE).strip() # pylint: disable=C0103 - + s = fileutil.read_file(PRODUCT_ID_FILE).strip() # pylint: disable=C0103 + else: - rc, s = shellutil.run_get_output(DMIDECODE_CMD) # pylint: disable=C0103 + rc, s = shellutil.run_get_output(DMIDECODE_CMD) # pylint: disable=C0103 if rc != 0 or UUID_PATTERN.match(s) is None: return "" - + return self._correct_instance_id(s.strip()) @staticmethod @@ -369,11 +379,11 @@ def is_sys_user(self, username): "/etc/login.defs") if uidmin_def is not None: uidmin = int(uidmin_def.split()[1]) - except IOError as e: # pylint: disable=W0612,C0103 + except IOError as e: # pylint: disable=W0612,C0103 pass - if uidmin == None: # pylint: disable=C0121 + if uidmin == None: # pylint: disable=C0121 uidmin = 100 - if userentry != None and userentry[2] < uidmin: # pylint: disable=C0121,R1705,R1703 + if userentry != None and userentry[2] < uidmin: # pylint: disable=C0121,R1705,R1703 return True else: return False @@ -391,7 +401,7 @@ def useradd(self, username, expiration=None, comment=None): cmd = ["useradd", "-m", username, "-e", expiration] else: cmd = ["useradd", "-m", username] - + if comment is not None: cmd.extend(["-c", comment]) @@ -405,7 +415,7 @@ def chpasswd(self, username, password, crypt_id=6, salt_len=10): self._run_command_raising_OSUtilError(["usermod", "-p", passwd_hash, username], err_msg="Failed to set password for {0}".format(username)) - + def get_users(self): return getpwall() @@ -439,7 +449,7 @@ def conf_sudoer(self, username, nopasswd=False, remove=False): sudoers = content.split("\n") sudoers = [x for x in sudoers if username not in x] fileutil.write_file(sudoers_wagent, "\n".join(sudoers)) - except IOError as e: # pylint: disable=C0103 + except IOError as e: # pylint: disable=C0103 raise OSUtilError("Failed to remove sudoer: {0}".format(e)) def del_root_password(self): @@ -450,7 +460,7 @@ def del_root_password(self): new_passwd = [x for x in passwd if not x.startswith("root:")] new_passwd.insert(0, "root:*LOCK*:14600::::::") fileutil.write_file(passwd_file_path, "\n".join(new_passwd)) - except IOError as e: # pylint: disable=C0103 + except IOError as e: # pylint: disable=C0103 raise OSUtilError("Failed to delete root password:{0}".format(e)) @staticmethod @@ -528,8 +538,8 @@ def is_selinux_system(self): """ Checks and sets self.selinux = True if SELinux is available on system. """ - if self.selinux == None: # pylint: disable=C0121 - if shellutil.run("which getenforce", chk_err=False) == 0: # pylint: disable=simplifiable-if-statement + if self.selinux == None: # pylint: disable=C0121 + if shellutil.run("which getenforce", chk_err=False) == 0: # pylint: disable=simplifiable-if-statement self.selinux = True else: self.selinux = False @@ -539,13 +549,13 @@ def is_selinux_enforcing(self): """ Calls shell command 'getenforce' and returns True if 'Enforcing'. """ - if self.is_selinux_system(): # pylint: disable=R1705 + if self.is_selinux_system(): # pylint: disable=R1705 output = shellutil.run_get_output("getenforce")[1] return output.startswith("Enforcing") else: return False - def set_selinux_context(self, path, con): # pylint: disable=R1710 + def set_selinux_context(self, path, con): # pylint: disable=R1710 """ Calls shell 'chcon' with 'path' and 'con' context. Returns exit result. @@ -584,7 +594,7 @@ def get_dvd_device(self, dev_dir='/dev'): raise OSUtilError(msg="Failed to get dvd device from {0}".format(dev_dir), inner=inner_detail) - def mount_dvd(self, # pylint: disable=R0913 + def mount_dvd(self, # pylint: disable=R0913 max_retry=6, chk_err=True, dvd_device=None, @@ -611,7 +621,7 @@ def mount_dvd(self, # pylint: disable=R0913 mount_point, option=["-o", "ro", "-t", "udf,iso9660"], chk_err=False) - if return_code == 0: # pylint: disable=R1705 + if return_code == 0: # pylint: disable=R1705 logger.info("Successfully mounted dvd") return else: @@ -630,7 +640,7 @@ def umount_dvd(self, chk_err=True, mount_point=None): mount_point = conf.get_dvd_mount_point() return_code = self.umount(mount_point, chk_err=chk_err) if chk_err and return_code != 0: - raise OSUtilError("Failed to unmount dvd device at {0}", # pylint: disable=W0715 + raise OSUtilError("Failed to unmount dvd device at {0}", # pylint: disable=W0715 mount_point) def eject_dvd(self, chk_err=True): @@ -648,7 +658,7 @@ def eject_dvd(self, chk_err=True): def try_load_atapiix_mod(self): try: self.load_atapiix_mod() - except Exception as e: # pylint: disable=C0103 + except Exception as e: # pylint: disable=C0103 logger.warn("Could not load ATAPI driver: {0}".format(e)) def load_atapiix_mod(self): @@ -663,7 +673,7 @@ def load_atapiix_mod(self): if not os.path.isfile(mod_path): raise Exception("Can't find module file:{0}".format(mod_path)) - ret, output = shellutil.run_get_output("insmod " + mod_path) # pylint: disable=W0612 + ret, output = shellutil.run_get_output("insmod " + mod_path) # pylint: disable=W0612 if ret != 0: raise Exception("Error calling insmod for ATAPI CD-ROM driver") if not self.is_atapiix_mod_loaded(max_retry=3): @@ -710,7 +720,9 @@ def allow_dhcp_broadcast(self): shellutil.run("iptables -I INPUT -p udp --dport 68 -j ACCEPT", chk_err=False) - def remove_rules_files(self, rules_files=__RULES_FILES__): # pylint: disable=W0102 + def remove_rules_files(self, rules_files=None): + if rules_files is None: + rules_files = __RULES_FILES__ lib_dir = conf.get_lib_dir() for src in rules_files: file_name = fileutil.base_name(src) @@ -721,7 +733,9 @@ def remove_rules_files(self, rules_files=__RULES_FILES__): # pylint: disable=W01 logger.warn("Move rules file {0} to {1}", file_name, dest) shutil.move(src, dest) - def restore_rules_files(self, rules_files=__RULES_FILES__): # pylint: disable=W0102 + def restore_rules_files(self, rules_files=None): + if rules_files is None: + rules_files = __RULES_FILES__ lib_dir = conf.get_lib_dir() for dest in rules_files: filename = fileutil.base_name(dest) @@ -748,7 +762,7 @@ def get_if_mac(self, ifname): sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM, socket.IPPROTO_UDP) - param = struct.pack('256s', (ifname[:15]+('\0'*241)).encode('latin-1')) + param = struct.pack('256s', (ifname[:15] + ('\0' * 241)).encode('latin-1')) info = fcntl.ioctl(sock.fileno(), IOCTL_SIOCGIFHWADDR, param) sock.close() return ''.join(['%02X' % textutil.str_to_ord(char) for char in info[18:24]]) @@ -768,7 +782,7 @@ def _get_all_interfaces(self): Return a dictionary mapping from interface name to IPv4 address. Interfaces without a name are ignored. """ - expected=16 # how many devices should I expect... + expected = 16 # how many devices should I expect... struct_size = DefaultOSUtil._get_struct_ifconf_size() array_size = expected * struct_size @@ -788,11 +802,11 @@ def _get_all_interfaces(self): ifaces = {} for i in range(0, array_size, struct_size): - iface = ifconf_buff[i:i+IFNAMSIZ].split(b'\0', 1)[0] - if len(iface) > 0: # pylint: disable=len-as-condition + iface = ifconf_buff[i:i + IFNAMSIZ].split(b'\0', 1)[0] + if len(iface) > 0: # pylint: disable=len-as-condition iface_name = iface.decode('latin-1') if iface_name not in ifaces: - ifaces[iface_name] = socket.inet_ntoa(ifconf_buff[i+20:i+24]) + ifaces[iface_name] = socket.inet_ntoa(ifconf_buff[i + 20:i + 24]) return ifaces def get_first_if(self): @@ -806,7 +820,7 @@ def get_first_if(self): if primary in ifaces: return primary, ifaces[primary] - for iface_name in ifaces.keys(): # pylint: disable=C0201 + for iface_name in ifaces.keys(): # pylint: disable=C0201 if not self.is_loopback(iface_name): logger.info("Choosing non-primary [{0}]".format(iface_name)) return iface_name, ifaces[iface_name] @@ -842,9 +856,9 @@ def _build_route_list(proc_net_route): route_list = [] for entry in proc_net_route[1:]: route = entry.split("\t") - if len(route) > 0: # pylint: disable=len-as-condition + if len(route) > 0: # pylint: disable=len-as-condition route_obj = RouteEntry(route[idx_iface], route[idx_dest], route[idx_gw], route[idx_mask], - route[idx_flags], route[idx_metric]) + route[idx_flags], route[idx_metric]) route_list.append(route_obj) return route_list @@ -860,7 +874,7 @@ def read_route_table(): try: with open('/proc/net/route') as routing_table: return list(map(str.strip, routing_table.readlines())) - except Exception as e: # pylint: disable=C0103 + except Exception as e: # pylint: disable=C0103 logger.error("Cannot read route table [{0}]", ustr(e)) return [] @@ -893,8 +907,8 @@ def get_primary_interface(self): :return: the interface which has the default route """ # from linux/route.h - RTF_GATEWAY = 0x02 # pylint: disable=C0103 - DEFAULT_DEST = "00000000" # pylint: disable=C0103 + RTF_GATEWAY = 0x02 # pylint: disable=C0103 + DEFAULT_DEST = "00000000" # pylint: disable=C0103 primary_interface = None @@ -908,9 +922,10 @@ def is_default(route): candidates = list(filter(is_default, DefaultOSUtil.get_list_of_routes(route_table))) - if len(candidates) > 0: # pylint: disable=len-as-condition + if len(candidates) > 0: # pylint: disable=len-as-condition def get_metric(route): return int(route.metric) + primary_route = min(candidates, key=get_metric) primary_interface = primary_route.interface @@ -941,8 +956,8 @@ def is_loopback(self, ifname): """ Determine if a named interface is loopback. """ - s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM, socket.IPPROTO_UDP) # pylint: disable=C0103 - ifname_buff = ifname + ('\0'*256) + s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM, socket.IPPROTO_UDP) # pylint: disable=C0103 + ifname_buff = ifname + ('\0' * 256) result = fcntl.ioctl(s.fileno(), IOCTL_SIOCGIFFLAGS, ifname_buff) flags, = struct.unpack('H', result[16:18]) isloopback = flags & 8 == 8 @@ -971,15 +986,16 @@ def get_endpoint_from_leases_path(pathglob): # pylint: disable=R0912 """ endpoint = None - HEADER_LEASE = "lease" # pylint: disable=C0103 - HEADER_OPTION_245 = "option unknown-245" # pylint: disable=C0103 - HEADER_EXPIRE = "expire" # pylint: disable=C0103 - FOOTER_LEASE = "}" # pylint: disable=C0103 - FORMAT_DATETIME = "%Y/%m/%d %H:%M:%S" # pylint: disable=C0103 - option_245_re = re.compile(r'\s*option\s+unknown-245\s+([0-9a-fA-F]+):([0-9a-fA-F]+):([0-9a-fA-F]+):([0-9a-fA-F]+);') + HEADER_LEASE = "lease" # pylint: disable=C0103 + HEADER_OPTION_245 = "option unknown-245" # pylint: disable=C0103 + HEADER_EXPIRE = "expire" # pylint: disable=C0103 + FOOTER_LEASE = "}" # pylint: disable=C0103 + FORMAT_DATETIME = "%Y/%m/%d %H:%M:%S" # pylint: disable=C0103 + option_245_re = re.compile( + r'\s*option\s+unknown-245\s+([0-9a-fA-F]+):([0-9a-fA-F]+):([0-9a-fA-F]+):([0-9a-fA-F]+);') logger.info("looking for leases in path [{0}]".format(pathglob)) - for lease_file in glob.glob(pathglob): # pylint: disable=R1702 + for lease_file in glob.glob(pathglob): # pylint: disable=R1702 leases = open(lease_file).read() if HEADER_OPTION_245 in leases: cached_endpoint = None @@ -998,7 +1014,7 @@ def get_endpoint_from_leases_path(pathglob): # pylint: disable=R0912 expire_date = datetime.datetime.strptime(expire_string, FORMAT_DATETIME) if expire_date > datetime.datetime.utcnow(): expired = False - except: # pylint: disable=W0702 + except: # pylint: disable=W0702 logger.error("could not parse expiry token '{0}'".format(line)) elif FOOTER_LEASE in line: logger.info("dhcp entry:{0}, 245:{1}, expired:{2}".format( @@ -1030,7 +1046,7 @@ def is_missing_default_route(self): if route.startswith("0.0.0.0 ") or route.startswith("default "): return False return True - except CommandError as e: # pylint: disable=C0103 + except CommandError as e: # pylint: disable=C0103 logger.warn("Cannot get the routing table. {0} failed: {1}", ustr(route_cmd), ustr(e)) return False @@ -1091,7 +1107,7 @@ def unregister_agent_service(self): def restart_ssh_service(self): pass - def route_add(self, net, mask, gateway): # pylint: disable=W0613 + def route_add(self, net, mask, gateway): # pylint: disable=W0613 """ Add specified route """ @@ -1109,7 +1125,7 @@ def _text_to_pid_list(text): def _get_dhcp_pid(command): try: return DefaultOSUtil._text_to_pid_list(shellutil.run_command(command)) - except CommandError as exception: # pylint: disable=W0612 + except CommandError as exception: # pylint: disable=W0612 return [] def get_dhcp_pid(self): @@ -1126,14 +1142,14 @@ def set_dhcp_hostname(self, hostname): if not os.path.isfile(conf_file): continue if fileutil.findre_in_file(conf_file, autosend): - #Return if auto send host-name is configured + # Return if auto send host-name is configured return fileutil.update_conf_file(conf_file, 'send host-name', 'send host-name "{0}";'.format(hostname)) def restart_if(self, ifname, retries=3, wait=5): - retry_limit=retries+1 + retry_limit = retries + 1 for attempt in range(1, retry_limit): try: shellutil.run_command(["ifdown", ifname]) @@ -1190,9 +1206,9 @@ def get_mount_point(self, mountlist, device): """ if (mountlist and device): for entry in mountlist.split('\n'): - if(re.search(device, entry)): + if (re.search(device, entry)): tokens = entry.split() - #Return the 3rd column of this line + # Return the 3rd column of this line return tokens[2] if len(tokens) > 2 else None return None @@ -1230,10 +1246,10 @@ def search_for_resource_disk(gen1_device_prefix, gen2_device_id): device = None # We have to try device IDs for both Gen1 and Gen2 VMs. logger.info('Searching gen1 prefix {0} or gen2 {1}'.format(gen1_device_prefix, gen2_device_id)) - try: # pylint: disable=R1702 + try: # pylint: disable=R1702 for vmbus, guid in DefaultOSUtil._enumerate_device_id(): if guid.startswith(gen1_device_prefix) or guid == gen2_device_id: - for root, dirs, files in os.walk(STORAGE_DEVICE_PATH + vmbus): # pylint: disable=W0612 + for root, dirs, files in os.walk(STORAGE_DEVICE_PATH + vmbus): # pylint: disable=W0612 root_path_parts = root.split('/') # For Gen1 VMs we only have to check for the block dir in the # current device. But for Gen2 VMs all of the disks (sda, sdb, @@ -1242,14 +1258,14 @@ def search_for_resource_disk(gen1_device_prefix, gen2_device_id): # 0 - OS disk # 1 - Resource disk # 2 - CDROM - if root_path_parts[-1] == 'block' and ( # pylint: disable=R1705 + if root_path_parts[-1] == 'block' and ( # pylint: disable=R1705 guid != gen2_device_id or root_path_parts[-2].split(':')[-1] == '1'): device = dirs[0] return device else: # older distros - for d in dirs: # pylint: disable=C0103 + for d in dirs: # pylint: disable=C0103 if ':' in d and "block" == d.split(':')[0]: device = d.split(':')[1] return device @@ -1263,9 +1279,9 @@ def device_for_ide_port(self, port_id): """ if port_id > 3: return None - g0 = "00000000" # pylint: disable=C0103 + g0 = "00000000" # pylint: disable=C0103 if port_id > 1: - g0 = "00000001" # pylint: disable=C0103 + g0 = "00000001" # pylint: disable=C0103 port_id = port_id - 2 gen1_device_prefix = '{0}-000{1}'.format(g0, port_id) @@ -1305,7 +1321,7 @@ def decode_customdata(self, data): def get_total_mem(self): # Get total memory in bytes and divide by 1024**2 to get the value in MB. - return os.sysconf('SC_PAGE_SIZE') * os.sysconf('SC_PHYS_PAGES') / (1024**2) + return os.sysconf('SC_PAGE_SIZE') * os.sysconf('SC_PHYS_PAGES') / (1024 ** 2) def get_processor_cores(self): return multiprocessing.cpu_count() @@ -1316,7 +1332,7 @@ def check_pid_alive(self, pid): os.kill(pid, 0) except (ValueError, TypeError): return False - except OSError as e: # pylint: disable=C0103 + except OSError as e: # pylint: disable=C0103 if e.errno == errno.EPERM: return True return False @@ -1324,7 +1340,7 @@ def check_pid_alive(self, pid): @property def is_64bit(self): - return sys.maxsize > 2**32 + return sys.maxsize > 2 ** 32 @staticmethod def _get_proc_stat(): @@ -1358,7 +1374,8 @@ def get_total_cpu_ticks_since_boot(): if proc_stat is not None: for line in proc_stat.splitlines(): if ALL_CPUS_REGEX.match(line): - system_cpu = sum(int(i) for i in line.split()[1:8]) # see "man proc" for a description of these fields + system_cpu = sum( + int(i) for i in line.split()[1:8]) # see "man proc" for a description of these fields break return system_cpu @@ -1405,7 +1422,7 @@ def get_nic_state(self): """ # pylint: disable=W1401 1: lo inet6 ::1/128 scope host \ valid_lft forever preferred_lft forever 2: eth0 inet6 fe80::20d:3aff:fe30:c35a/64 scope link \ valid_lft forever preferred_lft forever - """ + """ # pylint: enable=W0105 return state @@ -1452,12 +1469,12 @@ def _run_multiple_commands_without_raising(commands, log_error=True, continue_on break @staticmethod - def _run_command_raising_OSUtilError(cmd, err_msg, cmd_input=None): # pylint: disable=C0103 + def _run_command_raising_OSUtilError(cmd, err_msg, cmd_input=None): # pylint: disable=C0103 # This method runs shell command using the new secure shellutil.run_command and raises OSUtilErrors on failures. try: return shellutil.run_command(cmd, log_error=True, cmd_input=cmd_input) - except shellutil.CommandError as e: # pylint: disable=C0103 + except shellutil.CommandError as e: # pylint: disable=C0103 raise OSUtilError( "{0}, Retcode: {1}, Output: {2}, Error: {3}".format(err_msg, e.returncode, e.stdout, e.stderr)) - except Exception as e: # pylint: disable=C0103 + except Exception as e: # pylint: disable=C0103 raise OSUtilError("{0}, Retcode: {1}, Error: {2}".format(err_msg, -1, ustr(e))) diff --git a/azurelinuxagent/common/rdma.py b/azurelinuxagent/common/rdma.py index 1528961c80..a64b83acb9 100644 --- a/azurelinuxagent/common/rdma.py +++ b/azurelinuxagent/common/rdma.py @@ -29,7 +29,7 @@ import azurelinuxagent.common.utils.shellutil as shellutil from azurelinuxagent.common.utils.textutil import parse_doc, find, getattrib -dapl_config_paths = [ # pylint: disable=invalid-name +dapl_config_paths = [ # pylint: disable=invalid-name '/etc/dat.conf', '/etc/rdma/dat.conf', '/usr/local/etc/dat.conf' @@ -71,16 +71,16 @@ def setup_rdma_device(nd_version, shared_conf): logger.info("RDMA: device is set up") return -class RDMAHandler(object): +class RDMAHandler(object): driver_module_name = 'hv_network_direct' nd_version = None - def get_rdma_version(self): # pylint: disable=R1710 + def get_rdma_version(self): # pylint: disable=R1710 """Retrieve the firmware version information from the system. This depends on information provided by the Linux kernel.""" - if self.nd_version : + if self.nd_version: return self.nd_version kvp_key_size = 512 @@ -94,18 +94,18 @@ def get_rdma_version(self): # pylint: disable=R1710 logger.error(error_msg % driver_info_source) return - f = open(driver_info_source) # pylint: disable=C0103 - while True : + f = open(driver_info_source) # pylint: disable=C0103 + while True: key = f.read(kvp_key_size) value = f.read(kvp_value_size) - if key and value : + if key and value: key_0 = key.split("\x00")[0] value_0 = value.split("\x00")[0] - if key_0 == "NdDriverVersion" : + if key_0 == "NdDriverVersion": f.close() self.nd_version = value_0 return self.nd_version - else : + else: break f.close() @@ -121,18 +121,17 @@ def is_kvp_daemon_running(): # for suse, it uses hv_kvp_daemon kvp_daemon_names = ['hypervkvpd', 'hv_kvp_daemon'] - exitcode, ps_out = shellutil.run_get_output("ps -ef") + exitcode, ps_out = shellutil.run_get_output("ps -ef") if exitcode != 0: raise Exception('RDMA: ps -ef failed: %s' % ps_out) - for n in kvp_daemon_names: # pylint: disable=C0103 - if n in ps_out: # pylint: disable=R1705 + for n in kvp_daemon_names: # pylint: disable=C0103 + if n in ps_out: # pylint: disable=R1705 logger.info('RDMA: kvp daemon (%s) is running' % n) return True else: logger.verbose('RDMA: kvp daemon (%s) is not running' % n) return False - def load_driver_module(self): """Load the kernel driver, this depends on the proper driver to be installed with the install_driver() method""" @@ -165,7 +164,7 @@ def install_driver(self): def is_driver_loaded(self): """Check if the network module is loaded in kernel space""" cmd = 'lsmod | grep ^%s' % self.driver_module_name - status, loaded_modules = shellutil.run_get_output(cmd) # pylint: disable=W0612 + status, loaded_modules = shellutil.run_get_output(cmd) # pylint: disable=W0612 logger.info('RDMA: Checking if the module loaded.') if loaded_modules: logger.info('RDMA: module loaded.') @@ -182,11 +181,11 @@ def reboot_system(self): logger.error('RDMA: Failed to reboot the system') -dapl_config_paths = [ # pylint: disable=invalid-name +dapl_config_paths = [ # pylint: disable=invalid-name '/etc/dat.conf', '/etc/rdma/dat.conf', '/usr/local/etc/dat.conf'] -class RDMADeviceHandler(object): +class RDMADeviceHandler(object): """ Responsible for writing RDMA IP and MAC address to the /dev/hvnd_rdma interface. @@ -215,16 +214,16 @@ def start(self): def process(self): try: - if not self.nd_version : + if not self.nd_version: logger.info("RDMA: provisioning SRIOV RDMA device.") self.provision_sriov_rdma() - else : + else: logger.info("RDMA: provisioning Network Direct RDMA device.") self.provision_network_direct_rdma() - except Exception as e: # pylint: disable=C0103 + except Exception as e: # pylint: disable=C0103 logger.error("RDMA: device processing failed: {0}".format(e)) - def provision_network_direct_rdma(self) : + def provision_network_direct_rdma(self): RDMADeviceHandler.update_dat_conf(dapl_config_paths, self.ipv4_addr) if not conf.enable_check_rdma_driver(): @@ -234,22 +233,22 @@ def provision_network_direct_rdma(self) : skip_rdma_device = False module_name = "hv_network_direct" - retcode,out = shellutil.run_get_output("modprobe -R %s" % module_name, chk_err=False) + retcode, out = shellutil.run_get_output("modprobe -R %s" % module_name, chk_err=False) if retcode == 0: module_name = out.strip() else: logger.info("RDMA: failed to resolve module name. Use original name") - retcode,out = shellutil.run_get_output("modprobe %s" % module_name) + retcode, out = shellutil.run_get_output("modprobe %s" % module_name) if retcode != 0: logger.error("RDMA: failed to load module %s" % module_name) return - retcode,out = shellutil.run_get_output("modinfo %s" % module_name) + retcode, out = shellutil.run_get_output("modinfo %s" % module_name) if retcode == 0: - version = re.search("version:\s+(\d+)\.(\d+)\.(\d+)\D", out, re.IGNORECASE) # pylint: disable=W1401 + version = re.search("version:\s+(\d+)\.(\d+)\.(\d+)\D", out, re.IGNORECASE) # pylint: disable=W1401 if version: - v1 = int(version.groups(0)[0]) # pylint: disable=C0103 - v2 = int(version.groups(0)[1]) # pylint: disable=C0103 - if v1>4 or v1==4 and v2>0: + v1 = int(version.groups(0)[0]) # pylint: disable=C0103 + v2 = int(version.groups(0)[1]) # pylint: disable=C0103 + if v1 > 4 or v1 == 4 and v2 > 0: logger.info("Skip setting /dev/hvnd_rdma on 4.1 or later") skip_rdma_device = True else: @@ -265,17 +264,18 @@ def provision_network_direct_rdma(self) : RDMADeviceHandler.update_network_interface(self.mac_addr, self.ipv4_addr) - def provision_sriov_rdma(self) : # pylint: disable=R1711 + def provision_sriov_rdma(self): # pylint: disable=R1711 RDMADeviceHandler.wait_any_rdma_device( self.sriov_dir, self.device_check_timeout_sec, self.device_check_interval_sec) - RDMADeviceHandler.update_iboip_interface(self.ipv4_addr, self.ipoib_check_timeout_sec, self.ipoib_check_interval_sec) + RDMADeviceHandler.update_iboip_interface(self.ipv4_addr, self.ipoib_check_timeout_sec, + self.ipoib_check_interval_sec) return @staticmethod - def update_iboip_interface(ipv4_addr, timeout_sec, check_interval_sec) : + def update_iboip_interface(ipv4_addr, timeout_sec, check_interval_sec): logger.info("Wait for ib0 become available") - total_retries = timeout_sec/check_interval_sec - n = 0 # pylint: disable=C0103 + total_retries = timeout_sec / check_interval_sec + n = 0 # pylint: disable=C0103 found_ib0 = None while not found_ib0 and n < total_retries: ret, output = shellutil.run_get_output("ifconfig -a") @@ -285,7 +285,7 @@ def update_iboip_interface(ipv4_addr, timeout_sec, check_interval_sec) : if found_ib0: break time.sleep(check_interval_sec) - n += 1 # pylint: disable=C0103 + n += 1 # pylint: disable=C0103 if not found_ib0: raise Exception("ib0 is not available") @@ -304,7 +304,7 @@ def update_dat_conf(paths, ipv4_addr): infiniband interface. """ logger.info("Updating DAPL configuration file") - for f in paths: # pylint: disable=C0103 + for f in paths: # pylint: disable=C0103 logger.info("RDMA: trying {0}".format(f)) if not os.path.isfile(f): logger.info( @@ -322,7 +322,7 @@ def update_dat_conf(paths, ipv4_addr): @staticmethod def replace_dat_conf_contents(cfg, ipv4_addr): - old = "ofa-v2-ib0 u2.0 nonthreadsafe default libdaplofa.so.2 dapl.2.0 \"\S+ 0\"" # pylint: disable=W1401 + old = "ofa-v2-ib0 u2.0 nonthreadsafe default libdaplofa.so.2 dapl.2.0 \"\S+ 0\"" # pylint: disable=W1401 new = "ofa-v2-ib0 u2.0 nonthreadsafe default libdaplofa.so.2 dapl.2.0 \"{0} 0\"".format( ipv4_addr) return re.sub(old, new, cfg) @@ -332,7 +332,7 @@ def write_rdma_config_to_device(path, ipv4_addr, mac_addr): data = RDMADeviceHandler.generate_rdma_config(ipv4_addr, mac_addr) logger.info( "RDMA: Updating device with configuration: {0}".format(data)) - with open(path, "w") as f: # pylint: disable=C0103 + with open(path, "w") as f: # pylint: disable=C0103 logger.info("RDMA: Device opened for writing") f.write(data) logger.info("RDMA: Updated device with IPv4/MAC addr successfully") @@ -344,8 +344,8 @@ def generate_rdma_config(ipv4_addr, mac_addr): @staticmethod def wait_rdma_device(path, timeout_sec, check_interval_sec): logger.info("RDMA: waiting for device={0} timeout={1}s".format(path, timeout_sec)) - total_retries = timeout_sec/check_interval_sec - n = 0 # pylint: disable=C0103 + total_retries = timeout_sec / check_interval_sec + n = 0 # pylint: disable=C0103 while n < total_retries: if os.path.exists(path): logger.info("RDMA: device ready") @@ -353,38 +353,38 @@ def wait_rdma_device(path, timeout_sec, check_interval_sec): logger.verbose( "RDMA: device not ready, sleep {0}s".format(check_interval_sec)) time.sleep(check_interval_sec) - n += 1 # pylint: disable=C0103 + n += 1 # pylint: disable=C0103 logger.error("RDMA device wait timed out") raise Exception("The device did not show up in {0} seconds ({1} retries)".format( timeout_sec, total_retries)) @staticmethod - def wait_any_rdma_device(dir, timeout_sec, check_interval_sec): # pylint: disable=W0622 + def wait_any_rdma_device(directory, timeout_sec, check_interval_sec): logger.info( "RDMA: waiting for any Infiniband device at directory={0} timeout={1}s".format( - dir, timeout_sec)) - total_retries = timeout_sec/check_interval_sec - n = 0 # pylint: disable=C0103 + directory, timeout_sec)) + total_retries = timeout_sec / check_interval_sec + n = 0 # pylint: disable=C0103 while n < total_retries: - r = os.listdir(dir) # pylint: disable=C0103 + r = os.listdir(directory) # pylint: disable=C0103 if r: - logger.info("RDMA: device found in {0}".format(dir)) + logger.info("RDMA: device found in {0}".format(directory)) return logger.verbose( "RDMA: device not ready, sleep {0}s".format(check_interval_sec)) time.sleep(check_interval_sec) - n += 1 # pylint: disable=C0103 + n += 1 # pylint: disable=C0103 logger.error("RDMA device wait timed out") raise Exception("The device did not show up in {0} seconds ({1} retries)".format( timeout_sec, total_retries)) @staticmethod def update_network_interface(mac_addr, ipv4_addr): - netmask=16 - + netmask = 16 + logger.info("RDMA: will update the network interface with IPv4/MAC") - if_name=RDMADeviceHandler.get_interface_by_mac(mac_addr) + if_name = RDMADeviceHandler.get_interface_by_mac(mac_addr) logger.info("RDMA: network interface found: {0}", if_name) logger.info("RDMA: bringing network interface up") if shellutil.run("ifconfig {0} up".format(if_name)) != 0: @@ -402,12 +402,12 @@ def get_interface_by_mac(mac): if ret != 0: raise Exception("Failed to list network interfaces") output = output.replace('\n', '') - match = re.search(r"(eth\d).*(HWaddr|ether) {0}".format(mac), + match = re.search(r"(eth\d).*(HWaddr|ether) {0}".format(mac), output, re.IGNORECASE) if match is None: raise Exception("Failed to get ifname with mac: {0}".format(mac)) output = match.group(0) eths = re.findall(r"eth\d", output) - if eths is None or len(eths) == 0: # pylint: disable=len-as-condition + if eths is None or len(eths) == 0: # pylint: disable=len-as-condition raise Exception("ifname with mac: {0} not found".format(mac)) return eths[-1] diff --git a/azurelinuxagent/common/utils/fileutil.py b/azurelinuxagent/common/utils/fileutil.py index a734c396e2..a4ed595be2 100644 --- a/azurelinuxagent/common/utils/fileutil.py +++ b/azurelinuxagent/common/utils/fileutil.py @@ -224,11 +224,13 @@ def get_all_files(root_path): return result -def clean_ioerror(e, paths=[]): # pylint: disable=W0102,C0103 +def clean_ioerror(e, paths=None): # pylint: disable=C0103 """ Clean-up possibly bad files and directories after an IO error. The code ignores *all* errors since disk state may be unhealthy. """ + if paths is None: + paths = [] if isinstance(e, IOError) and e.errno in KNOWN_IOERRORS: for path in paths: if path is None: diff --git a/azurelinuxagent/common/utils/restutil.py b/azurelinuxagent/common/utils/restutil.py index 6b08c569ef..0e3dbfd558 100644 --- a/azurelinuxagent/common/utils/restutil.py +++ b/azurelinuxagent/common/utils/restutil.py @@ -142,7 +142,9 @@ def _compute_delay(retry_attempt=1, delay=DELAY_IN_SECONDS): return delay*fib[1] -def _is_retry_status(status, retry_codes=RETRY_CODES): # pylint: disable=W0102 +def _is_retry_status(status, retry_codes=None): + if retry_codes is None: + retry_codes = RETRY_CODES return status in retry_codes @@ -333,14 +335,16 @@ def _http_request(method, host, rel_uri, port=None, data=None, secure=False, # p return conn.getresponse() -def http_request(method, # pylint: disable=R0913,R0912,W0102 +def http_request(method, # pylint: disable=R0913,R0912 url, data, headers=None, use_proxy=False, max_retry=DEFAULT_RETRIES, - retry_codes=RETRY_CODES, + retry_codes=None, retry_delay=DELAY_IN_SECONDS, redact_data=False): + if retry_codes is None: + retry_codes = RETRY_CODES global SECURE_WARNING_EMITTED # pylint: disable=W0603 host, port, secure, rel_uri = _parse_url(url) @@ -461,13 +465,15 @@ def http_request(method, # pylint: disable=R0913,R0912,W0102 raise HttpError("{0} -- {1} attempts made".format(msg, attempt)) -def http_get(url, # pylint: disable=R0913,W0102 +def http_get(url, # pylint: disable=R0913 headers=None, use_proxy=False, max_retry=DEFAULT_RETRIES, - retry_codes=RETRY_CODES, + retry_codes=None, retry_delay=DELAY_IN_SECONDS): + if retry_codes is None: + retry_codes = RETRY_CODES return http_request("GET", url, None, headers=headers, use_proxy=use_proxy, @@ -476,13 +482,15 @@ def http_get(url, # pylint: disable=R0913,W0102 retry_delay=retry_delay) -def http_head(url, # pylint: disable=R0913,W0102 +def http_head(url, # pylint: disable=R0913 headers=None, use_proxy=False, max_retry=DEFAULT_RETRIES, - retry_codes=RETRY_CODES, + retry_codes=None, retry_delay=DELAY_IN_SECONDS): + if retry_codes is None: + retry_codes = RETRY_CODES return http_request("HEAD", url, None, headers=headers, use_proxy=use_proxy, @@ -491,14 +499,16 @@ def http_head(url, # pylint: disable=R0913,W0102 retry_delay=retry_delay) -def http_post(url, # pylint: disable=R0913,W0102 +def http_post(url, # pylint: disable=R0913 data, headers=None, use_proxy=False, max_retry=DEFAULT_RETRIES, - retry_codes=RETRY_CODES, + retry_codes=None, retry_delay=DELAY_IN_SECONDS): + if retry_codes is None: + retry_codes = RETRY_CODES return http_request("POST", url, data, headers=headers, use_proxy=use_proxy, @@ -507,15 +517,17 @@ def http_post(url, # pylint: disable=R0913,W0102 retry_delay=retry_delay) -def http_put(url, # pylint: disable=R0913,W0102 +def http_put(url, # pylint: disable=R0913 data, headers=None, use_proxy=False, max_retry=DEFAULT_RETRIES, - retry_codes=RETRY_CODES, + retry_codes=None, retry_delay=DELAY_IN_SECONDS, redact_data=False): + if retry_codes is None: + retry_codes = RETRY_CODES return http_request("PUT", url, data, headers=headers, use_proxy=use_proxy, @@ -525,13 +537,15 @@ def http_put(url, # pylint: disable=R0913,W0102 redact_data=redact_data) -def http_delete(url, # pylint: disable=R0913,W0102 +def http_delete(url, # pylint: disable=R0913 headers=None, use_proxy=False, max_retry=DEFAULT_RETRIES, - retry_codes=RETRY_CODES, + retry_codes=None, retry_delay=DELAY_IN_SECONDS): + if retry_codes is None: + retry_codes = RETRY_CODES return http_request("DELETE", url, None, headers=headers, use_proxy=use_proxy, @@ -540,11 +554,15 @@ def http_delete(url, # pylint: disable=R0913,W0102 retry_delay=retry_delay) -def request_failed(resp, ok_codes=OK_CODES): # pylint: disable=W0102 +def request_failed(resp, ok_codes=None): + if ok_codes is None: + ok_codes = OK_CODES return not request_succeeded(resp, ok_codes=ok_codes) -def request_succeeded(resp, ok_codes=OK_CODES): # pylint: disable=W0102 +def request_succeeded(resp, ok_codes=None): + if ok_codes is None: + ok_codes = OK_CODES return resp is not None and resp.status in ok_codes @@ -552,10 +570,12 @@ def request_not_modified(resp): return resp is not None and resp.status in NOT_MODIFIED_CODES -def request_failed_at_hostplugin(resp, upstream_failure_codes=HOSTPLUGIN_UPSTREAM_FAILURE_CODES): # pylint: disable=W0102 +def request_failed_at_hostplugin(resp, upstream_failure_codes=None): """ Host plugin will return 502 for any upstream issue, so a failure is any 5xx except 502 """ + if upstream_failure_codes is None: + upstream_failure_codes = HOSTPLUGIN_UPSTREAM_FAILURE_CODES return resp is not None and resp.status >= 500 and resp.status not in upstream_failure_codes diff --git a/azurelinuxagent/common/utils/shellutil.py b/azurelinuxagent/common/utils/shellutil.py index c7439b6e2d..593e9c0901 100644 --- a/azurelinuxagent/common/utils/shellutil.py +++ b/azurelinuxagent/common/utils/shellutil.py @@ -69,18 +69,20 @@ def has_command(cmd): return not run(cmd, False) -def run(cmd, chk_err=True, expected_errors=[]): # pylint: disable=W0102 +def run(cmd, chk_err=True, expected_errors=None): """ Note: Deprecating in favour of `azurelinuxagent.common.utils.shellutil.run_command` function. Calls run_get_output on 'cmd', returning only the return code. If chk_err=True then errors will be reported in the log. If chk_err=False then errors will be suppressed from the log. """ + if expected_errors is None: + expected_errors = [] retcode, out = run_get_output(cmd, chk_err=chk_err, expected_errors=expected_errors) # pylint: disable=W0612 return retcode -def run_get_output(cmd, chk_err=True, log_cmd=True, expected_errors=[]): # pylint: disable=W0102 +def run_get_output(cmd, chk_err=True, log_cmd=True, expected_errors=None): """ Wrapper for subprocess.check_output. Execute 'cmd'. Returns return code and STDOUT, trapping expected @@ -90,6 +92,8 @@ def run_get_output(cmd, chk_err=True, log_cmd=True, expected_errors=[]): # pylin For new callers, consider using run_command instead as it separates stdout from stderr, returns only stdout on success, logs both outputs and return code on error and raises an exception. """ + if expected_errors is None: + expected_errors = [] if log_cmd: logger.verbose(u"Command: [{0}]", cmd) try: diff --git a/azurelinuxagent/common/utils/textutil.py b/azurelinuxagent/common/utils/textutil.py index 9265daa6ba..708a0a6042 100644 --- a/azurelinuxagent/common/utils/textutil.py +++ b/azurelinuxagent/common/utils/textutil.py @@ -93,12 +93,12 @@ def getattrib(node, attr_name): return None -def unpack(buf, offset, range): # pylint: disable=W0622 +def unpack(buf, offset, value_range): """ Unpack bytes into python values. """ result = 0 - for i in range: + for i in value_range: result = (result << 8) | str_to_ord(buf[offset + i]) return result @@ -133,9 +133,9 @@ def hex_dump2(buf): def is_in_range(a, low, high): # pylint: disable=C0103 """ - Return True if 'a' in 'low' <= a >= 'high' + Return True if 'a' in 'low' <= a <= 'high' """ - return (a >= low and a <= high) # pylint: disable=R1716 + return low <= a <= high def is_printable(ch): # pylint: disable=C0103 diff --git a/azurelinuxagent/daemon/resourcedisk/default.py b/azurelinuxagent/daemon/resourcedisk/default.py index f3e0c7c6b1..5ab1e107d9 100644 --- a/azurelinuxagent/daemon/resourcedisk/default.py +++ b/azurelinuxagent/daemon/resourcedisk/default.py @@ -222,28 +222,28 @@ def change_partition_type(self, suppress_message, option_str): First try with --part-type; if fails, fall back to -c """ - command_to_use = '--part-type' - input = "sfdisk {0} {1} {2}".format( # pylint: disable=W0622 - command_to_use, '-f' if suppress_message else '', option_str) + option_to_use = '--part-type' + command = "sfdisk {0} {1} {2}".format( + option_to_use, '-f' if suppress_message else '', option_str) err_code, output = shellutil.run_get_output( - input, chk_err=False, log_cmd=True) + command, chk_err=False, log_cmd=True) # fall back to -c if err_code != 0: logger.info( "sfdisk with --part-type failed [{0}], retrying with -c", err_code) - command_to_use = '-c' - input = "sfdisk {0} {1} {2}".format( - command_to_use, '-f' if suppress_message else '', option_str) - err_code, output = shellutil.run_get_output(input, log_cmd=True) + option_to_use = '-c' + command = "sfdisk {0} {1} {2}".format( + option_to_use, '-f' if suppress_message else '', option_str) + err_code, output = shellutil.run_get_output(command, log_cmd=True) if err_code == 0: logger.info('{0} succeeded', - input) + command) else: logger.error('{0} failed [{1}: {2}]', - input, + command, err_code, output) diff --git a/azurelinuxagent/ga/exthandlers.py b/azurelinuxagent/ga/exthandlers.py index 8f24181b4e..eca1702dcd 100644 --- a/azurelinuxagent/ga/exthandlers.py +++ b/azurelinuxagent/ga/exthandlers.py @@ -391,7 +391,7 @@ def handle_ext_handlers(self, etag=None): # If handled successfully, proceed with the current handler. # Otherwise, skip the rest of the extension installation. dep_level = ext_handler.sort_key() - if dep_level >= 0 and dep_level < max_dep_level: # pylint: disable=R1716 + if 0 <= dep_level < max_dep_level: if not self.wait_for_handler_successful_completion(ext_handler, wait_until): logger.warn("An extension failed or timed out, will skip processing the rest of the extensions") break diff --git a/azurelinuxagent/ga/update.py b/azurelinuxagent/ga/update.py index abad70f90d..abae8e77e2 100644 --- a/azurelinuxagent/ga/update.py +++ b/azurelinuxagent/ga/update.py @@ -650,7 +650,9 @@ def _purge_agents(self): # pylint: disable=R1711 logger.warn(u"Purging {0} raised exception: {1}", agent_path, ustr(e)) return - def _set_agents(self, agents=[]): # pylint: disable=W0102,R1711 + def _set_agents(self, agents=None): # pylint: disable=R1711 + if agents is None: + agents = [] self.agents = agents self.agents.sort(key=lambda agent: agent.version, reverse=True) return diff --git a/azurelinuxagent/pa/deprovision/default.py b/azurelinuxagent/pa/deprovision/default.py index d3cf845e13..58a187cbad 100644 --- a/azurelinuxagent/pa/deprovision/default.py +++ b/azurelinuxagent/pa/deprovision/default.py @@ -25,9 +25,7 @@ import azurelinuxagent.common.conf as conf import azurelinuxagent.common.utils.fileutil as fileutil -import azurelinuxagent.common.utils.shellutil as shellutil # pylint: disable=W0611 from azurelinuxagent.common import version - from azurelinuxagent.common.exception import ProtocolError from azurelinuxagent.common.osutil import get_osutil from azurelinuxagent.common.protocol.util import get_protocol_util @@ -43,8 +41,13 @@ def read_input(message): # Suppress it here. return raw_input(message) # pylint: disable=E0602 + class DeprovisionAction(object): # pylint: disable=R0903 - def __init__(self, func, args=[], kwargs={}): # pylint: disable=W0102 + def __init__(self, func, args=None, kwargs=None): + if args is None: + args = [] + if kwargs is None: + kwargs = {} self.func = func self.args = args self.kwargs = kwargs @@ -52,6 +55,7 @@ def __init__(self, func, args=[], kwargs={}): # pylint: disable=W0102 def invoke(self): self.func(*self.args, **self.kwargs) + class DeprovisionHandler(object): def __init__(self): self.osutil = get_osutil() @@ -80,7 +84,6 @@ def del_user(self, warnings, actions): actions.append(DeprovisionAction(self.osutil.del_account, [username])) - def regen_ssh_host_key(self, warnings, actions): warnings.append("WARNING! All SSH host key pairs will be deleted.") actions.append(DeprovisionAction(fileutil.rm_files, diff --git a/azurelinuxagent/pa/rdma/centos.py b/azurelinuxagent/pa/rdma/centos.py index 40fbc941e6..0dc874406f 100644 --- a/azurelinuxagent/pa/rdma/centos.py +++ b/azurelinuxagent/pa/rdma/centos.py @@ -179,8 +179,8 @@ def install_rdma_drivers(self, fw_version): logger.info("RDMA: kernel module is loaded") @staticmethod - def get_file_by_pattern(list, pattern): # pylint: disable=W0622 - for l in list: # pylint: disable=C0103 + def get_file_by_pattern(file_list, pattern): + for l in file_list: # pylint: disable=C0103 if re.match(pattern, l): return l return None diff --git a/ci/2.7.pylintrc b/ci/2.7.pylintrc index ca17cbba37..fc1e4f7adc 100644 --- a/ci/2.7.pylintrc +++ b/ci/2.7.pylintrc @@ -10,7 +10,6 @@ # blacklisted-name: (needs review) Used when the name is listed in the black list (unauthorized names). # cell-var-from-loop: (hi-pri) A variable used in a closure is defined in a loop. This will result in all closures using the same value for the closed-over variable. # cyclic-import: (hi-pri) Used when a cyclic import between two or more modules is detected. -# dangerous-default-value: (hi-pri) Used when a mutable value as list or dictionary is detected in a default value for an argument. # expression-not-assigned: (hi-pri) Used when an expression that is not a function call is assigned to nothing. # duplicate-code: (hi-pri) Indicates that a set of similar lines has been detected among multiple file. (also disabled globally) # function-redefined: (hi-pri) Used when a function / class / method is redefined. @@ -61,7 +60,6 @@ # The source code suppresses the following error codes specifically for python3.4: -# chained-comparison: This message is emitted when pylint encounters boolean operation like"a < b and b < c", suggesting instead to refactor it to "a < b < c" # consider-using-in: To check if a variable is equal to one of many values,combine the values into a tuple and check if the variable is contained "in" it instead of checking for equality against each of the values. # duplicate-string-formatting-argument: (hi-pri) Used when we detect that a string formatting is repeating an argument instead of using named string arguments # implicit-str-concat-in-sequence: (hi-pri) String literals are implicitly concatenated in a literal iterable definition : maybe a comma is missing ? diff --git a/ci/3.6.pylintrc b/ci/3.6.pylintrc index 16907250db..35100e605a 100644 --- a/ci/3.6.pylintrc +++ b/ci/3.6.pylintrc @@ -8,12 +8,10 @@ # bad-super-call: (hi-pri) Used when another argument than the current class is given as first argument of the super builtin. # bare-except: (hi-pri) Used when an except clause doesn't specify exceptions type to catch. # cell-var-from-loop: (hi-pri) A variable used in a closure is defined in a loop. This will result in all closures using the same value for the closed-over variable. -# chained-comparison: This message is emitted when pylint encounters boolean operation like"a < b and b < c", suggesting instead to refactor it to "a < b < c" # consider-merging-isinstance: Used when multiple consecutive isinstance calls can be merged into one. # consider-using-enumerate: Emitted when code that iterates with range and len is encountered. # consider-using-in: To check if a variable is equal to one of many values,combine the values into a tuple and check if the variable is contained "in" it instead of checking for equality against each of the values. # cyclic-import: (hi-pri) Used when a cyclic import between two or more modules is detected. -# dangerous-default-value: (hi-pri) Used when a mutable value as list or dictionary is detected in a default value for an argument. # duplicate-code: (hi-pri) Indicates that a set of similar lines has been detected among multiple file. (also disabled globally) # duplicate-string-formatting-argument: (hi-pri) Used when we detect that a string formatting is repeating an argument instead of using named string arguments # expression-not-assigned: (hi-pri) Used when an expression that is not a function call is assigned to nothing. diff --git a/setup.py b/setup.py index e2bfffdc86..d832fc9f02 100755 --- a/setup.py +++ b/setup.py @@ -38,43 +38,53 @@ def set_files(data_files, dest=None, src=None): data_files.append((dest, src)) -def set_bin_files(data_files, dest="/usr/sbin", # pylint: disable=W0102 - src=["bin/waagent", "bin/waagent2.0"]): +def set_bin_files(data_files, dest="/usr/sbin", src=None): + if src is None: + src = ["bin/waagent", "bin/waagent2.0"] data_files.append((dest, src)) -def set_conf_files(data_files, dest="/etc", src=["config/waagent.conf"]): # pylint: disable=W0102 +def set_conf_files(data_files, dest="/etc", src=None): + if src is None: + src = ["config/waagent.conf"] data_files.append((dest, src)) -def set_logrotate_files(data_files, dest="/etc/logrotate.d", # pylint: disable=W0102 - src=["config/waagent.logrotate", - "config/waagent-extn.logrotate"]): +def set_logrotate_files(data_files, dest="/etc/logrotate.d", src=None): + if src is None: + src = ["config/waagent.logrotate", + "config/waagent-extn.logrotate"] data_files.append((dest, src)) -def set_sysv_files(data_files, dest="/etc/rc.d/init.d", src=["init/waagent"]): # pylint: disable=W0102 +def set_sysv_files(data_files, dest="/etc/rc.d/init.d", src=None): + if src is None: + src = ["init/waagent"] data_files.append((dest, src)) -def set_systemd_files(data_files, dest="/lib/systemd/system", # pylint: disable=W0102 - src=["init/waagent.service"]): +def set_systemd_files(data_files, dest="/lib/systemd/system", src=None): + if src is None: + src = ["init/waagent.service"] data_files.append((dest, src)) -def set_freebsd_rc_files(data_files, dest="/etc/rc.d/", # pylint: disable=W0102 - src=["init/freebsd/waagent"]): +def set_freebsd_rc_files(data_files, dest="/etc/rc.d/", src=None): + if src is None: + src = ["init/freebsd/waagent"] data_files.append((dest, src)) -def set_openbsd_rc_files(data_files, dest="/etc/rc.d/", # pylint: disable=W0102 - src=["init/openbsd/waagent"]): +def set_openbsd_rc_files(data_files, dest="/etc/rc.d/", src=None): + if src is None: + src = ["init/openbsd/waagent"] data_files.append((dest, src)) -def set_udev_files(data_files, dest="/etc/udev/rules.d/", # pylint: disable=W0102 - src=["config/66-azure-storage.rules", - "config/99-azure-product-uuid.rules"]): +def set_udev_files(data_files, dest="/etc/udev/rules.d/", src=None): + if src is None: + src = ["config/66-azure-storage.rules", + "config/99-azure-product-uuid.rules"] data_files.append((dest, src)) diff --git a/tests/ga/test_monitor.py b/tests/ga/test_monitor.py index 83b0d8b4f9..6eb2359c6f 100644 --- a/tests/ga/test_monitor.py +++ b/tests/ga/test_monitor.py @@ -41,8 +41,9 @@ def random_generator(size=6, chars=string.ascii_uppercase + string.digits + string.ascii_lowercase): return ''.join(random.choice(chars) for x in range(size)) + @contextlib.contextmanager -def _create_monitor_handler(enabled_operations=[], iterations=1): # pylint: disable=dangerous-default-value +def _create_monitor_handler(enabled_operations=None, iterations=1): """ Creates an instance of MonitorHandler that * Uses a mock_wire_protocol for network requests, @@ -55,6 +56,9 @@ def _create_monitor_handler(enabled_operations=[], iterations=1): # pylint: disa * run_and_wait() - invokes run() and wait() on the MonitorHandler """ + if enabled_operations is None: + enabled_operations = [] + def run(self): if len(enabled_operations) == 0 or self._name in enabled_operations: # pylint: disable=protected-access,len-as-condition run.original_definition(self) diff --git a/tests/ga/test_update.py b/tests/ga/test_update.py index e0d048fb9f..7eb6372f9a 100644 --- a/tests/ga/test_update.py +++ b/tests/ga/test_update.py @@ -147,13 +147,17 @@ def agent_versions(self): v.sort(reverse=True) return v - def get_error_file(self, error_data=NO_ERROR): # pylint: disable=dangerous-default-value + def get_error_file(self, error_data=None): + if error_data is None: + error_data = NO_ERROR fp = tempfile.NamedTemporaryFile(mode="w") # pylint: disable=invalid-name json.dump(error_data if error_data is not None else NO_ERROR, fp) fp.seek(0) return fp - def create_error(self, error_data=NO_ERROR): # pylint: disable=dangerous-default-value + def create_error(self, error_data=None): + if error_data is None: + error_data = NO_ERROR with self.get_error_file(error_data) as path: err = GuestAgentError(path.name) err.load() @@ -1233,7 +1237,9 @@ def test_run_latest_creates_only_one_signal_handler(self, mock_signal): self._test_run_latest() self.assertEqual(0, mock_signal.call_count) - def _test_run(self, invocations=1, calls=[call.run()], enable_updates=False, sleep_interval=(6,)): # pylint: disable=dangerous-default-value + def _test_run(self, invocations=1, calls=None, enable_updates=False, sleep_interval=(6,)): + if calls is None: + calls = [call.run()] conf.get_autoupdate_enabled = Mock(return_value=enable_updates) # Note: @@ -1494,7 +1500,7 @@ def test_telemetry_heartbeat_creates_event(self, patch_add_event, patch_info, *_ for call_args in patch_info.call_args), "The heartbeat was not written to the agent's log") @contextlib.contextmanager - def _get_update_handler(self, iterations=1, test_data=DATA_FILE): # pylint: disable=dangerous-default-value + def _get_update_handler(self, iterations=1, test_data=None): """ This function returns a mocked version of the UpdateHandler object to be used for testing. It will only run the main loop [iterations] no of times. @@ -1502,6 +1508,8 @@ def _get_update_handler(self, iterations=1, test_data=DATA_FILE): # pylint: disa :param iterations: No of times the UpdateHandler.run() method should run. :return: Mocked object of UpdateHandler() class and object of the MockWireProtocol(). """ + if test_data is None: + test_data = DATA_FILE def _set_iterations(iterations): # This will reset the current iteration and the max iterations to run for this test object. diff --git a/tests/protocol/mockwiredata.py b/tests/protocol/mockwiredata.py index 8c4e6d69b6..c716f95e65 100644 --- a/tests/protocol/mockwiredata.py +++ b/tests/protocol/mockwiredata.py @@ -95,7 +95,9 @@ class WireProtocolData(object): # pylint: disable=too-many-instance-attributes - def __init__(self, data_files=DATA_FILE): # pylint: disable=dangerous-default-value + def __init__(self, data_files=None): + if data_files is None: + data_files = DATA_FILE self.emulate_stale_goal_state = False self.call_counts = { "comp=versions": 0,