[Mellanox] Optimize thermal policies (#9665)

- Why I did it
Optimize thermal control policies to simplify the logic and add more protection code in policies to make sure it works even if kernel algorithm does not work.

- How I did it
Reduce unused thermal policies
Add timely ASIC temperature check in thermal policy to make sure ASIC temperature and fan speed is coordinated
Minimum allowed fan speed now is calculated by max of the expected fan speed among all policies
Move some logic from fan.py to thermal.py to make it more readable

- How to verify it
1. Manual test
2. Regression

platform/mellanox/mlnx-platform-api/sonic_platform/fan.py

@@ -13,6 +13,7 @@
     from sonic_platform_base.fan_base import FanBase
     from .led import FanLed, ComponentFaultyIndicator
     from .utils import read_int_from_file, read_str_from_file, write_file
+    from .thermal import Thermal
 except ImportError as e:
     raise ImportError (str(e) + "- required module not found")
 
@@ -24,22 +25,19 @@
 FAN_DIR = "/var/run/hw-management/thermal/fan{}_dir"
 FAN_DIR_VALUE_EXHAUST = 0
 FAN_DIR_VALUE_INTAKE = 1
-COOLING_STATE_PATH = "/var/run/hw-management/thermal/cooling_cur_state"
 
 class Fan(FanBase):
     """Platform-specific Fan class"""
 
     STATUS_LED_COLOR_ORANGE = "orange"
-    min_cooling_level = 2
-    MIN_VALID_COOLING_LEVEL = 1
-    MAX_VALID_COOLING_LEVEL = 10
+
     # PSU fan speed vector
     PSU_FAN_SPEED = ['0x3c', '0x3c', '0x3c', '0x3c', '0x3c',
                      '0x3c', '0x3c', '0x46', '0x50', '0x5a', '0x64']
 
     def __init__(self, fan_index, fan_drawer, position, psu_fan = False, psu=None):
         super(Fan, self).__init__()
-    
+
         # API index is starting from 0, Mellanox platform index is starting from 1
         self.index = fan_index + 1
         self.fan_drawer = fan_drawer
@@ -84,7 +82,7 @@ def get_direction(self):
             depending on fan direction
 
         Notes:
-            What Mellanox calls forward: 
+            What Mellanox calls forward:
             Air flows from fans side to QSFP side, for example: MSN2700-CS2F
             which means intake in community
             What Mellanox calls reverse:
@@ -161,7 +159,7 @@ def get_target_speed(self):
         if self.is_psu_fan:
             try:
                 # Get PSU fan target speed according to current system cooling level
-                cooling_level = self.get_cooling_level()
+                cooling_level = Thermal.get_cooling_level()
                 return int(self.PSU_FAN_SPEED[cooling_level], 16)
             except Exception:
                 return self.get_speed()
@@ -179,7 +177,7 @@ def set_speed(self, speed):
                    in the range 0 (off) to 100 (full speed)
 
         Returns:
-            bool: True if set success, False if fail. 
+            bool: True if set success, False if fail.
         """
         status = True
 
@@ -203,11 +201,6 @@ def set_speed(self, speed):
                 return False
 
         try:
-            cooling_level = int(speed // 10)
-            if cooling_level < self.min_cooling_level:
-                cooling_level = self.min_cooling_level
-                speed = self.min_cooling_level * 10
-            self.set_cooling_level(cooling_level, cooling_level)
             pwm = int(round(PWM_MAX*speed/100.0))
             write_file(os.path.join(FAN_PATH, self.fan_speed_set_path), pwm, raise_exception=True)
         except (ValueError, IOError):
@@ -225,7 +218,7 @@ def set_status_led(self, color):
                    fan module status LED
 
         Returns:
-            bool: True if set success, False if fail. 
+            bool: True if set success, False if fail.
         """
         return self.led.set_status(color)
 
@@ -267,36 +260,4 @@ def is_replaceable(self):
         """
         return False
 
-    @classmethod
-    def set_cooling_level(cls, level, cur_state):
-        """
-        Change cooling level. The input level should be an integer value [1, 10].
-        1 means 10%, 2 means 20%, 10 means 100%.
-        """
-        if not isinstance(level, int):
-            raise RuntimeError("Failed to set cooling level, input parameter must be integer")
 
-        if level < cls.MIN_VALID_COOLING_LEVEL or level > cls.MAX_VALID_COOLING_LEVEL:
-            raise RuntimeError("Failed to set cooling level, level value must be in range [{}, {}], got {}".format(
-                cls.MIN_VALID_COOLING_LEVEL,
-                cls.MAX_VALID_COOLING_LEVEL,
-                level
-                ))
-
-        try:
-            # Reset FAN cooling level vector. According to low level team,
-            # if we need set cooling level to X, we need first write a (10+X) 
-            # to cooling_cur_state file to reset the cooling level vector.
-            write_file(COOLING_STATE_PATH, level + 10, raise_exception=True)
-
-            # We need set cooling level after resetting the cooling level vector 
-            write_file(COOLING_STATE_PATH, cur_state, raise_exception=True)
-        except (ValueError, IOError) as e:
-            raise RuntimeError("Failed to set cooling level - {}".format(e))
-
-    @classmethod
-    def get_cooling_level(cls):
-        try:
-            return read_int_from_file(COOLING_STATE_PATH, raise_exception=True)
-        except (ValueError, IOError) as e:
-            raise RuntimeError("Failed to get cooling level - {}".format(e))

platform/mellanox/mlnx-platform-api/sonic_platform/thermal.py

@@ -13,6 +13,9 @@
     from os.path import isfile, join
     import io
     import os.path
+    import glob
+
+    from . import utils
 except ImportError as e:
     raise ImportError (str(e) + "- required module not found")
 
@@ -46,7 +49,17 @@
 THERMAL_ZONE_MODE = "thermal_zone_mode"
 THERMAL_ZONE_POLICY = "thermal_zone_policy"
 THERMAL_ZONE_TEMPERATURE = "thermal_zone_temp"
-THERMAL_ZONE_NORMAL_TEMPERATURE = "temp_trip_high"
+THERMAL_ZONE_HOT_THRESHOLD = "temp_trip_hot"
+THERMAL_ZONE_HIGH_THRESHOLD = "temp_trip_high"
+THERMAL_ZONE_NORMAL_THRESHOLD = "temp_trip_norm"
+THERMAL_ZONE_FOLDER_WILDCARD = '/run/hw-management/thermal/mlxsw*'
+THERMAL_ZONE_HYSTERESIS = 5000
+COOLING_STATE_PATH = "/var/run/hw-management/thermal/cooling_cur_state"
+# Min allowed cooling level when all thermal zones are in normal state
+MIN_COOLING_LEVEL_FOR_NORMAL = 2
+# Min allowed cooling level when any thermal zone is in high state but no thermal zone is in emergency state
+MIN_COOLING_LEVEL_FOR_HIGH = 4
+MAX_COOLING_LEVEL = 10
 
 MODULE_TEMPERATURE_FAULT_PATH = "/var/run/hw-management/thermal/module{}_temp_fault"
 
@@ -96,14 +109,14 @@
     THERMAL_DEV_BOARD_AMBIENT : "Ambient Board Temp"
 }
 thermal_api_handlers = {
-    THERMAL_DEV_CATEGORY_CPU_CORE : thermal_api_handler_cpu_core, 
+    THERMAL_DEV_CATEGORY_CPU_CORE : thermal_api_handler_cpu_core,
     THERMAL_DEV_CATEGORY_CPU_PACK : thermal_api_handler_cpu_pack,
     THERMAL_DEV_CATEGORY_MODULE : thermal_api_handler_module,
     THERMAL_DEV_CATEGORY_PSU : thermal_api_handler_psu,
     THERMAL_DEV_CATEGORY_GEARBOX : thermal_api_handler_gearbox
 }
 thermal_name = {
-    THERMAL_DEV_CATEGORY_CPU_CORE : "CPU Core {} Temp", 
+    THERMAL_DEV_CATEGORY_CPU_CORE : "CPU Core {} Temp",
     THERMAL_DEV_CATEGORY_CPU_PACK : "CPU Pack Temp",
     THERMAL_DEV_CATEGORY_MODULE : "xSFP module {} Temp",
     THERMAL_DEV_CATEGORY_PSU : "PSU-{} Temp",
@@ -351,7 +364,7 @@ def initialize_sfp_thermals(platform, thermal_list, sfp_index):
     thermal = Thermal(THERMAL_DEV_CATEGORY_MODULE, sfp_index, True, 1)
     thermal_list.append(thermal)
 
-    
+
 def initialize_chassis_thermals(platform, thermal_list):
     # create thermal objects for all categories of sensors
     tp_index = platform_dict_thermal[platform]
@@ -386,6 +399,14 @@ def initialize_chassis_thermals(platform, thermal_list):
 class Thermal(ThermalBase):
     thermal_profile = None
     thermal_algorithm_status = False
+    # Expect cooling level, used for caching the cooling level value before commiting to hardware
+    expect_cooling_level = None
+    # Expect cooling state
+    expect_cooling_state = None
+    # Last committed cooling level
+    last_set_cooling_level = None
+    last_set_cooling_state = None
+    last_set_psu_cooling_level = None
 
     def __init__(self, category, index, has_index, position, dependency = None):
         """
@@ -464,7 +485,7 @@ def get_temperature(self):
 
         Returns:
             A float number of current temperature in Celsius up to nearest thousandth
-            of one degree Celsius, e.g. 30.125 
+            of one degree Celsius, e.g. 30.125
         """
         if self.dependency:
             status, hint = self.dependency()
@@ -546,7 +567,7 @@ def is_replaceable(self):
     @classmethod
     def _write_generic_file(cls, filename, content):
         """
-        Generic functions to write content to a specified file path if 
+        Generic functions to write content to a specified file path if
         the content has changed.
         """
         try:
@@ -566,8 +587,8 @@ def set_thermal_algorithm_status(cls, status, force=True):
         only adjust fan speed when temperature across some "edge", e.g temperature
         changes to exceed high threshold.
         When disable kernel thermal algorithm, kernel no longer adjust fan speed.
-        We usually disable the algorithm when we want to set a fix speed. E.g, when 
-        a fan unit is removed from system, we will set fan speed to 100% and disable 
+        We usually disable the algorithm when we want to set a fix speed. E.g, when
+        a fan unit is removed from system, we will set fan speed to 100% and disable
         the algorithm to avoid it adjust the speed.
 
         Returns:
@@ -601,51 +622,38 @@ def set_thermal_algorithm_status(cls, status, force=True):
         return True
 
     @classmethod
-    def check_thermal_zone_temperature(cls):
-        """
-        Check thermal zone current temperature with normal temperature
+    def get_min_allowed_cooling_level_by_thermal_zone(cls):
+        """Get min allowed cooling level according to thermal zone status:
+            1. If temperature of all thermal zones is less than normal threshold, min allowed cooling level is
+               $MIN_COOLING_LEVEL_FOR_NORMAL = 2
+            2. If temperature of any thermal zone is greater than normal threshold, but no thermal zone temperature
+               is greater than high threshold, min allowed cooling level is $MIN_COOLING_LEVEL_FOR_HIGH = 4
+            3. Otherwise, there is no minimum allowed value and policy should not adjust cooling level
 
         Returns:
-            True if all thermal zones current temperature less or equal than normal temperature
+            int: minimum allowed cooling level
         """
-        if not cls.thermal_profile:
-            raise Exception("Fail to get thermal profile for this switch")
-
-        if not cls._check_thermal_zone_temperature(THERMAL_ZONE_ASIC_PATH):
-            return False
-
-        if THERMAL_DEV_CATEGORY_MODULE in cls.thermal_profile:
-            start, count = cls.thermal_profile[THERMAL_DEV_CATEGORY_MODULE]
-            if count != 0:
-                for index in range(count):
-                    if not cls._check_thermal_zone_temperature(THERMAL_ZONE_MODULE_PATH.format(start + index)):
-                        return False
-
-        if THERMAL_DEV_CATEGORY_GEARBOX in cls.thermal_profile:
-            start, count = cls.thermal_profile[THERMAL_DEV_CATEGORY_GEARBOX]
-            if count != 0:
-                for index in range(count):
-                    if not cls._check_thermal_zone_temperature(THERMAL_ZONE_GEARBOX_PATH.format(start + index)):
-                        return False
-
-        return True
+        min_allowed = MIN_COOLING_LEVEL_FOR_NORMAL
+        thermal_zone_present = False
+        try:
+            for thermal_zone_folder in glob.iglob(THERMAL_ZONE_FOLDER_WILDCARD):
+                thermal_zone_present = True
+                normal_thresh = utils.read_int_from_file(os.path.join(thermal_zone_folder, THERMAL_ZONE_NORMAL_THRESHOLD))
+                current = utils.read_int_from_file(os.path.join(thermal_zone_folder, THERMAL_ZONE_TEMPERATURE))
+                if current < normal_thresh - THERMAL_ZONE_HYSTERESIS:
+                    continue
 
-    @classmethod
-    def _check_thermal_zone_temperature(cls, thermal_zone_path):
-        normal_temp_path = join(thermal_zone_path, THERMAL_ZONE_NORMAL_TEMPERATURE)
-        current_temp_path = join(thermal_zone_path, THERMAL_ZONE_TEMPERATURE)
-        normal = None
-        current = None
-        try:    
-            with open(normal_temp_path, 'r') as file_obj:
-                normal = float(file_obj.read())
-
-            with open(current_temp_path, 'r') as file_obj:
-                current = float(file_obj.read())
-
-            return current <= normal
+                hot_thresh = utils.read_int_from_file(os.path.join(thermal_zone_folder, THERMAL_ZONE_HIGH_THRESHOLD))
+                if current < hot_thresh - THERMAL_ZONE_HYSTERESIS:
+                    min_allowed = MIN_COOLING_LEVEL_FOR_HIGH
+                else:
+                    min_allowed = None
+                    break
         except Exception as e:
-            logger.log_info("Fail to check thermal zone temperature for file {} due to {}".format(thermal_zone_path, repr(e)))
+            logger.log_error('Failed to get thermal zone status for {} - {}'.format(thermal_zone_folder, repr(e)))
+            return None
+
+        return min_allowed if thermal_zone_present else None
 
     @classmethod
     def check_module_temperature_trustable(cls):
@@ -669,3 +677,85 @@ def get_min_amb_temperature(cls):
         fan_ambient_temp = int(cls._read_generic_file(fan_ambient_path, 0))
         port_ambient_temp = int(cls._read_generic_file(port_ambient_path, 0))
         return fan_ambient_temp if fan_ambient_temp < port_ambient_temp else port_ambient_temp
+
+    @classmethod
+    def set_cooling_level(cls, level):
+        """
+        Change cooling level. The input level should be an integer value [1, 10].
+        1 means 10%, 2 means 20%, 10 means 100%.
+        """
+        if cls.last_set_cooling_level != level:
+            utils.write_file(COOLING_STATE_PATH, level + 10, raise_exception=True)
+            cls.last_set_cooling_level = level
+
+    @classmethod
+    def set_cooling_state(cls, state):
+        """Change cooling state.
+
+        Args:
+            state (int): cooling state
+        """
+        if cls.last_set_cooling_state != state:
+            utils.write_file(COOLING_STATE_PATH, state, raise_exception=True)
+            cls.last_set_cooling_state = state
+
+    @classmethod
+    def get_cooling_level(cls):
+        try:
+            return utils.read_int_from_file(COOLING_STATE_PATH, raise_exception=True)
+        except (ValueError, IOError) as e:
+            raise RuntimeError("Failed to get cooling level - {}".format(e))
+
+    @classmethod
+    def set_expect_cooling_level(cls, expect_value):
+        """During thermal policy running, cache the expect cooling level generated by policies. The max expect
+           cooling level will be committed to hardware.
+
+        Args:
+            expect_value (int): Expected cooling level value
+        """
+        if cls.expect_cooling_level is None or cls.expect_cooling_level < expect_value:
+            cls.expect_cooling_level = int(expect_value)
+
+    @classmethod
+    def commit_cooling_level(cls, thermal_info_dict):
+        """Commit cooling level to hardware. This will affect system fan and PSU fan speed.
+
+        Args:
+            thermal_info_dict (dict): Thermal information dictionary
+        """
+        if cls.expect_cooling_level is not None:
+            cls.set_cooling_level(cls.expect_cooling_level)
+
+        if cls.expect_cooling_state is not None:
+            cls.set_cooling_state(cls.expect_cooling_state)
+        elif cls.expect_cooling_level is not None:
+            cls.set_cooling_state(cls.expect_cooling_level)
+
+        cls.expect_cooling_level = None
+        # We need to set system fan speed here because kernel will automaticlly adjust fan speed according to cooling level and cooling state
+
+        # Commit PSU fan speed with current state
+        from .thermal_infos import ChassisInfo
+        if ChassisInfo.INFO_NAME in thermal_info_dict and isinstance(thermal_info_dict[ChassisInfo.INFO_NAME], ChassisInfo):
+            cooling_level = cls.get_cooling_level()
+            if cls.last_set_psu_cooling_level == cooling_level:
+                return
+            speed = cooling_level * 10
+            chassis = thermal_info_dict[ChassisInfo.INFO_NAME].get_chassis()
+            for psu in chassis.get_all_psus():
+                for psu_fan in psu.get_all_fans():
+                    psu_fan.set_speed(speed)
+            cls.last_set_psu_cooling_level = cooling_level
+
+    @classmethod
+    def monitor_asic_themal_zone(cls):
+        """This is a protection for asic thermal zone, if asic temperature is greater than hot threshold + THERMAL_ZONE_HYSTERESIS,
+           and if cooling state is not MAX, we need enforce the cooling state to MAX
+        """
+        asic_temp = utils.read_int_from_file(os.path.join(THERMAL_ZONE_ASIC_PATH, THERMAL_ZONE_TEMPERATURE), raise_exception=True)
+        hot_thresh = utils.read_int_from_file(os.path.join(THERMAL_ZONE_ASIC_PATH, THERMAL_ZONE_HOT_THRESHOLD), raise_exception=True)
+        if asic_temp >= hot_thresh + THERMAL_ZONE_HYSTERESIS:
+            cls.expect_cooling_state = MAX_COOLING_LEVEL
+        else:
+            cls.expect_cooling_state = None