fix false positive rule and add ignore list

azdev/operations/command_change/custom.py

@@ -8,7 +8,8 @@
 from knack.log import get_logger
 from azdev.utilities import (CMD_PROPERTY_ADD_BREAK_LIST, CMD_PROPERTY_REMOVE_BREAK_LIST,
                              CMD_PROPERTY_UPDATE_BREAK_LIST, PARA_PROPERTY_REMOVE_BREAK_LIST,
-                             PARA_PROPERTY_ADD_BREAK_LIST, PARA_PROPERTY_UPDATE_BREAK_LIST)
+                             PARA_PROPERTY_ADD_BREAK_LIST, PARA_PROPERTY_UPDATE_BREAK_LIST,
+                             get_azdev_repo_path)
 from .util import extract_cmd_name, extract_cmd_property, extract_subgroup_name, ChangeType
 from .util import get_command_tree
 from .meta_changes import (CmdAdd, CmdRemove, CmdPropAdd, CmdPropRemove, CmdPropUpdate, ParaAdd, ParaRemove,
@@ -42,6 +43,15 @@ def __init__(self, deep_diff=None, base_meta=None, diff_meta=None):
         self.diff_meta = diff_meta
         self.diff_objs = []
         self.cmd_set_with_parameter_change = set()
+        self.__get_meta_change_whitelist__()
+
+    def __get_meta_change_whitelist__(self):
+        self.meta_change_whitelist = set()
+        with open(get_azdev_repo_path() +
+                  "/azdev/operations/command_change/data/meta_change_whitelist.txt", "r") as f_in:
+            for line in f_in:
+                white_key = line.rstrip()
+                self.meta_change_whitelist.add(white_key)
 
     @staticmethod
     def __search_cmd_obj(cmd_name, search_meta):
@@ -88,7 +98,7 @@ def __iter_dict_items(self, dict_items, diff_type):
                     if cmd_property in CMD_PROPERTY_ADD_BREAK_LIST:
                         diff_obj = CmdPropAdd(cmd_name, cmd_property, True)
                     else:
-                        diff_obj = CmdPropAdd(cmd_name, cmd_property, True)
+                        diff_obj = CmdPropAdd(cmd_name, cmd_property, False)
                     self.diff_objs.append(diff_obj)
                 else:
                     if cmd_property in CMD_PROPERTY_REMOVE_BREAK_LIST:
@@ -257,13 +267,25 @@ def check_cmds_parameter_diff(self):
             cmp_parameters = cmd_cmp.get("parameters", [])
             self.check_cmd_parameter_diff(cmd_name, base_parameters, cmp_parameters)
 
+    def filter_diffs_by_whitelist(self):
+        """
+        filter_diffs_by_whitelist
+        """
+        new_diff_objs = []
+        for obj in self.diff_objs:
+            if obj.filter_key and obj.is_break and "\t".join(obj.filter_key) in self.meta_change_whitelist:
+                continue
+            new_diff_objs.append(obj)
+        self.diff_objs = new_diff_objs
+
     def check_deep_diffs(self):
         self.check_dict_item_remove()
         self.check_dict_item_add()
         self.check_list_item_remove()
         self.check_list_item_add()
         self.check_value_change()
         self.check_cmds_parameter_diff()
+        self.filter_diffs_by_whitelist()
 
     @staticmethod
     def fill_subgroup_rules(obj, ret_mod, rule):
@@ -325,6 +347,8 @@ def export_meta_changes(self, only_break, output_type="text"):
         for obj in self.diff_objs:
             if only_break and not obj.is_break:
                 continue
+            if obj.is_ignore:
+                continue
             ret = {}
             for prop in self.EXPORTED_META_PROPERTY:
                 if hasattr(obj, prop):

azdev/operations/command_change/data/meta_change_whitelist.txt

@@ -0,0 +1,9 @@
+1010	ams asset get-sas-urls	expiry_time	default
+1010	mysql flexible-server create	administrator_login	default
+1010	mysql flexible-server restore	restore_point_in_time	default
+1010	mysql flexible-server import create	administrator_login	default
+1010	postgres flexible-server create	administrator_login	default
+1010	postgres flexible-server restore	restore_point_in_time	default
+1010	mysql server create	administrator_login	default
+1010	postgres server create	administrator_login	default
+1010	mariadb server create	administrator_login	default

azdev/operations/command_change/meta_changes.py

@@ -4,18 +4,24 @@
 # license information.
 # -----------------------------------------------------------------------------
 
-from azdev.utilities import get_change_rule_template, get_change_suggest_template, \
-    BREAKING_CHANE_RULE_LINK_URL_PREFIX, BREAKING_CHANE_RULE_LINK_URL_SUFFIX
+# pylint: disable=too-many-instance-attributes
+
+from azdev.utilities import get_change_rule_template, get_change_suggest_template
+from azdev.utilities import CMD_PROPERTY_IGNORED_LIST, PARA_PROPERTY_IGNORED_LIST, PARA_NAME_IGNORED_LIST, \
+    PARA_VALUE_IGNORED_LIST, BREAKING_CHANE_RULE_LINK_URL_PREFIX, BREAKING_CHANE_RULE_LINK_URL_SUFFIX
 
 
 class MetaChange:
 
-    def __init__(self, rule_id="1000", is_break=False, rule_message="", suggest_message=""):
+    def __init__(self, rule_id="1000", is_break=False, rule_message="", suggest_message="",
+                 is_ignore=False, filter_key=None):
         self.rule_id = rule_id
         self.rule_link_url = BREAKING_CHANE_RULE_LINK_URL_PREFIX + self.rule_id + BREAKING_CHANE_RULE_LINK_URL_SUFFIX
         self.is_break = is_break
         self.rule_message = rule_message
         self.suggest_message = suggest_message
+        self.is_ignore = is_ignore
+        self.filter_key = filter_key
 
     def __str__(self):
         res = [self.rule_message]
@@ -80,36 +86,46 @@ def __init__(self, cmd_name, cmd_property, is_break=False):
         if not cmd_name or not cmd_property:
             raise Exception("cmd name needed")
         self.rule_id = "1003"
+        self.is_ignore = False
         self.cmd_name = cmd_name
         self.cmd_property = cmd_property
         self.is_break = is_break
         self.rule_message = get_change_rule_template(self.rule_id).format(self.cmd_name, self.cmd_property)
         self.suggest_message = get_change_suggest_template(self.rule_id).format(self.cmd_property, self.cmd_name) \
             if self.is_break else ""
-        super().__init__(self.rule_id, is_break, self.rule_message, self.suggest_message)
+        if cmd_property in CMD_PROPERTY_IGNORED_LIST:
+            self.is_ignore = True
+        self.filter_key = [self.rule_id, self.cmd_name, self.cmd_property]
+        super().__init__(self.rule_id, is_break, self.rule_message, self.suggest_message, self.is_ignore,
+                         self.filter_key)
 
 
 class CmdPropRemove(MetaChange):
     def __init__(self, cmd_name, cmd_property, is_break=False):
         if not cmd_name or not cmd_property:
             raise Exception("cmd name needed")
         self.rule_id = "1004"
+        self.is_ignore = False
         self.cmd_name = cmd_name
         self.cmd_property = cmd_property
         self.is_break = is_break
         self.rule_message = get_change_rule_template(self.rule_id).format(self.cmd_name, self.cmd_property)
         self.suggest_message = get_change_suggest_template(self.rule_id).format(self.cmd_property, self.cmd_name) \
             if self.is_break else ""
-        super().__init__(self.rule_id, is_break, self.rule_message, self.suggest_message)
+        if cmd_property in CMD_PROPERTY_IGNORED_LIST:
+            self.is_ignore = True
+        self.filter_key = [self.rule_id, self.cmd_name, self.cmd_property]
+        super().__init__(self.rule_id, is_break, self.rule_message, self.suggest_message,
+                         self.is_ignore, self.filter_key)
 
 
 class CmdPropUpdate(MetaChange):
 
-    # pylint: disable=too-many-instance-attributes
     def __init__(self, cmd_name, cmd_property, is_break=False, old_value=None, new_value=None):
         if not cmd_name or not cmd_property:
             raise Exception("cmd name and cmd prop needed")
         self.rule_id = "1005"
+        self.is_ignore = False
         self.cmd_name = cmd_name
         self.is_break = is_break
         self.cmd_prop_updated = cmd_property
@@ -125,7 +141,11 @@ def __init__(self, cmd_name, cmd_property, is_break=False, old_value=None, new_v
         self.suggest_message = get_change_suggest_template(self.rule_id).format(self.cmd_prop_updated,
                                                                                 self.new_value, self.old_value,
                                                                                 self.cmd_name) if self.is_break else ""
-        super().__init__(self.rule_id, is_break, self.rule_message, self.suggest_message)
+        if cmd_property in CMD_PROPERTY_IGNORED_LIST:
+            self.is_ignore = True
+        self.filter_key = [self.rule_id, self.cmd_name, self.cmd_prop_updated]
+        super().__init__(self.rule_id, is_break, self.rule_message, self.suggest_message,
+                         self.is_ignore, self.filter_key)
 
 
 class ParaAdd(MetaChange):
@@ -164,6 +184,7 @@ def __init__(self, cmd_name, para_name, para_property, is_break=False):
         if not cmd_name or not para_name or not para_property:
             raise Exception("cmd name, parameter name and parameter property needed")
         self.rule_id = "1008"
+        self.is_ignore = False
         self.cmd_name = cmd_name
         self.para_name = para_name
         self.para_prop = para_property
@@ -174,7 +195,9 @@ def __init__(self, cmd_name, para_name, para_property, is_break=False):
         self.suggest_message = get_change_suggest_template(self.rule_id).format(self.para_prop,
                                                                                 self.para_name,
                                                                                 self.cmd_name) if self.is_break else ""
-        super().__init__(self.rule_id, is_break, self.rule_message, self.suggest_message)
+        if para_property in PARA_PROPERTY_IGNORED_LIST or para_name in PARA_NAME_IGNORED_LIST:
+            self.is_ignore = True
+        super().__init__(self.rule_id, is_break, self.rule_message, self.suggest_message, self.is_ignore)
 
 
 class ParaPropRemove(MetaChange):
@@ -183,6 +206,7 @@ def __init__(self, cmd_name, para_name, para_property, is_break=False):
         if not cmd_name or not para_name or not para_property:
             raise Exception("cmd name, parameter name and parameter property needed")
         self.rule_id = "1009"
+        self.is_ignore = False
         self.cmd_name = cmd_name
         self.para_name = para_name
         self.para_prop = para_property
@@ -193,16 +217,18 @@ def __init__(self, cmd_name, para_name, para_property, is_break=False):
         self.suggest_message = get_change_suggest_template(self.rule_id).format(self.para_prop,
                                                                                 self.para_name,
                                                                                 self.cmd_name) if self.is_break else ""
-        super().__init__(self.rule_id, is_break, self.rule_message, self.suggest_message)
+        if para_property in PARA_PROPERTY_IGNORED_LIST or para_name in PARA_NAME_IGNORED_LIST:
+            self.is_ignore = True
+        super().__init__(self.rule_id, is_break, self.rule_message, self.suggest_message, self.is_ignore)
 
 
 class ParaPropUpdate(MetaChange):
 
-    # pylint: disable=too-many-instance-attributes
     def __init__(self, cmd_name, para_name, para_property, is_break=False, old_value=None, new_value=None):
         if not cmd_name or not para_name or not para_property:
             raise Exception("cmd name, parameter name and parameter property needed")
         self.rule_id = "1010"
+        self.is_ignore = False
         self.cmd_name = cmd_name
         self.para_name = para_name
         self.para_prop_updated = para_property
@@ -221,4 +247,10 @@ def __init__(self, cmd_name, para_name, para_property, is_break=False, old_value
                                                                                 self.new_value, self.old_value,
                                                                                 self.para_name,
                                                                                 self.cmd_name) if self.is_break else ""
-        super().__init__(self.rule_id, is_break, self.rule_message, self.suggest_message)
+        if para_property in PARA_PROPERTY_IGNORED_LIST or para_name in PARA_NAME_IGNORED_LIST:
+            self.is_ignore = True
+        if self.new_value in PARA_VALUE_IGNORED_LIST or self.old_value in PARA_VALUE_IGNORED_LIST:
+            self.is_ignore = True
+        self.filter_key = [self.rule_id, self.cmd_name, self.para_name, self.para_prop_updated]
+        super().__init__(self.rule_id, is_break, self.rule_message, self.suggest_message,
+                         self.is_ignore, self.filter_key)

azdev/operations/command_change/util.py

@@ -68,7 +68,8 @@ def process_aaz_argument(az_arguments_schema, argument_settings, para):
     aaz_type = _fields.get(argument_settings["dest"], None)
     if aaz_type:
         para["aaz_type"] = aaz_type.__class__.__name__
-        para["type"] = aaz_type._type_in_help  # pylint: disable=protected-access
+        if aaz_type._type_in_help and aaz_type._type_in_help.lower() != "undefined":  # pylint: disable=protected-access
+            para["type"] = aaz_type._type_in_help  # pylint: disable=protected-access
         if has_value(aaz_type._default):  # pylint: disable=protected-access
             para["aaz_default"] = aaz_type._default  # pylint: disable=protected-access
         if para["aaz_type"] in ["AAZArgEnum"] and aaz_type.get("enum", None) and aaz_type.enum.get("items", None):
@@ -95,6 +96,21 @@ def process_arg_options(argument_settings, para):
     para["options"] = sorted(option_list)
 
 
+def process_arg_type(argument_settings, para):
+    if not argument_settings.get("type", None):
+        return
+    configured_type = argument_settings["type"]
+    raw_type = None
+    if hasattr(configured_type, "__name__"):
+        raw_type = configured_type.__name__
+    elif hasattr(configured_type, "__class__"):
+        raw_type = configured_type.__class__.__name__
+    else:
+        print("unsupported type", configured_type)
+        return
+    para["type"] = raw_type if raw_type in ["str", "int", "float", "bool", "file_type"] else "custom_type"
+
+
 def gen_command_meta(command_info, with_help=False, with_example=False):
     stored_property_when_exist = ["confirmation", "supports_no_wait", "is_preview"]
     command_meta = {
@@ -128,6 +144,7 @@ def gen_command_meta(command_info, with_help=False, with_example=False):
             "name": settings["dest"],
         }
         process_arg_options(settings, para)
+        process_arg_type(settings, para)
         if settings.get("required", False):
             para["required"] = True
         if settings.get("choices", None):

azdev/operations/tests/jsons/az_ams_meta_after.json

@@ -0,0 +1,49 @@
+{
+    "module_name": "ams",
+    "name": "az",
+    "commands": {},
+    "sub_groups": {
+        "ams": {
+            "name": "ams",
+            "commands": {},
+            "sub_groups": {
+                "ams asset": {
+                    "name": "ams asset",
+                    "commands": {
+                        "ams asset get-sas-urls": {
+                            "name": "ams asset get-sas-urls",
+                            "is_aaz": false,
+                            "parameters": [{
+                                "name": "resource_group_name",
+                                "options": ["--resource-group", "-g"],
+                                "required": true,
+                                "id_part": "resource_group"
+                            }, {
+                                "name": "account_name",
+                                "options": ["--account-name", "-a"],
+                                "required": true,
+                                "id_part": "name"
+                            }, {
+                                "name": "asset_name",
+                                "options": ["--name", "-n"],
+                                "required": true,
+                                "id_part": "child_name_1"
+                            }, {
+                                "name": "permissions",
+                                "options": ["--permissions"],
+                                "choices": ["Read", "ReadWrite", "ReadWriteDelete"],
+                                "default": "Read"
+                            }, {
+                                "name": "expiry_time",
+                                "options": ["--expiry"],
+                                "type": "custom_type",
+                                "default": "2023-07-10 18:47:05.586776"
+                            }]
+                        }
+                    },
+                    "sub_groups": {}
+                }
+            }
+        }
+    }
+}

azdev/operations/tests/jsons/az_ams_meta_before.json

@@ -0,0 +1,49 @@
+{
+    "module_name": "ams",
+    "name": "az",
+    "commands": {},
+    "sub_groups": {
+        "ams": {
+            "name": "ams",
+            "commands": {},
+            "sub_groups": {
+                "ams asset": {
+                    "name": "ams asset",
+                    "commands": {
+                        "ams asset get-sas-urls": {
+                            "name": "ams asset get-sas-urls",
+                            "is_aaz": false,
+                            "parameters": [{
+                                "name": "resource_group_name",
+                                "options": ["--resource-group", "-g"],
+                                "required": true,
+                                "id_part": "resource_group"
+                            }, {
+                                "name": "account_name",
+                                "options": ["--account-name", "-a"],
+                                "required": true,
+                                "id_part": "name"
+                            }, {
+                                "name": "asset_name",
+                                "options": ["--name", "-n"],
+                                "required": true,
+                                "id_part": "child_name_1"
+                            }, {
+                                "name": "permissions",
+                                "options": ["--permissions"],
+                                "choices": ["Read", "ReadWrite", "ReadWriteDelete"],
+                                "default": "Read"
+                            }, {
+                                "name": "expiry_time",
+                                "options": ["--expiry"],
+                                "type": "custom_type",
+                                "default": "2023-07-10 18:43:29.693229"
+                            }]
+                        }
+                    },
+                    "sub_groups": {}
+                }
+            }
+        }
+    }
+}