[Mellanox] Adjust thermal control policies

device/mellanox/x86_64-mlnx_msn2700-r0/thermal_policy.json

@@ -1,6 +1,6 @@
 {   
     "thermal_control_algorithm": {
-        "run_at_boot_up": "false",
+        "run_at_boot_up": "true",
         "fan_speed_when_suspend": "60"
     },
     "info_types": [
@@ -51,6 +51,24 @@
                 }
             ]
         },
+        {
+            "name": "any fan broken",
+            "conditions": [
+                {
+                    "type": "fan.any.fault"
+                }
+            ],
+            "actions": [
+                {
+                    "type": "thermal_control.control",
+                    "status": "false"
+                },
+                {
+                    "type": "fan.all.set_speed",
+                    "speed": "100"
+                }
+            ]
+        },
         {
             "name": "all fan and psu presence",
             "conditions": [
@@ -59,12 +77,15 @@
                 },
                 {
                     "type": "psu.all.presence"
+                },
+                {
+                    "type": "fan.all.good"
                 }
             ],
             "actions": [
                 {
-                    "type": "fan.all.set_speed",
-                    "speed": "60"
+                    "type": "thermal_control.control",
+                    "status": "true"
                 }
             ]
         }

dockers/docker-platform-monitor/Dockerfile.j2

@@ -18,7 +18,8 @@ RUN apt-get update &&   \
         rrdtool         \
         python-smbus    \
         ethtool         \
-        dmidecode
+        dmidecode       \
+        i2c-tools
 
 {% if docker_platform_monitor_debs.strip() -%}
 # Copy locally-built Debian package dependencies

platform/mellanox/mlnx-platform-api/sonic_platform/device_data.py

@@ -0,0 +1,134 @@
+DEVICE_DATA = {
+    'ACS-MSN2700': {
+        'thermal': {
+            'minimum_table': {
+                "p2c_trust":   {"-127:40":13, "41:120":15},
+                "p2c_untrust": {"-127:25":13, "26:30":14 , "31:35":15, "36:120":16},
+                "c2p_trust":   {"-127:20":13, "21:25":14 , "26:30":15, "31:120":16},
+                "c2p_untrust": {"-127:20":13, "21:25":14 , "26:30":15, "31:120":16},
+                "unk_trust":   {"-127:20":13, "21:25":14 , "26:30":15, "31:120":16},
+                "unk_untrust": {"-127:20":13, "21:25":14 , "26:30":15, "31:120":16}
+            }
+        }
+    },
+    'LS-SN2700': {
+        'thermal': {
+            'minimum_table': {
+                "p2c_trust":   {"-127:40":13, "41:120":15},
+                "p2c_untrust": {"-127:25":13, "26:30":14 , "31:35":15, "36:120":16},
+                "c2p_trust":   {"-127:20":13, "21:25":14 , "26:30":15, "31:120":16},
+                "c2p_untrust": {"-127:20":13, "21:25":14 , "26:30":15, "31:120":16},
+                "unk_trust":   {"-127:20":13, "21:25":14 , "26:30":15, "31:120":16},
+                "unk_untrust": {"-127:20":13, "21:25":14 , "26:30":15, "31:120":16}
+            }
+        }
+    },
+    'ACS-MSN2740': {
+        'thermal': {
+            'minimum_table': {
+                "p2c_trust":   {"-127:120":13},
+                "p2c_untrust": {"-127:35":13, "36:40":14 , "41:120":15},
+                "c2p_trust":   {"-127:120":13},
+                "c2p_untrust": {"-127:15":13, "16:30":14 , "31:35":15, "36:120":17},
+                "unk_trust":   {"-127:120":13},
+                "unk_untrust": {"-127:15":13, "16:30":14 , "31:35":15, "36:120":17},
+            }
+        }
+    },
+    'ACS-MSN2410': {
+        'thermal': {
+            'minimum_table': {
+                "p2c_trust":   {"-127:40":13, "41:120":15},
+                "p2c_untrust": {"-127:25":13, "26:30":14 , "31:35":15, "36:120":16},
+                "c2p_trust":   {"-127:20":13, "21:25":14 , "26:30":15, "31:120":16},
+                "c2p_untrust": {"-127:20":13, "21:25":14 , "26:30":15, "31:120":16},
+                "unk_trust":   {"-127:20":13, "21:25":14 , "26:30":15, "31:120":16},
+                "unk_untrust": {"-127:20":13, "21:25":14 , "26:30":15, "31:120":16}
+            }
+        }
+    },
+    'Mellanox-SN2700': {
+        'thermal': {
+            'minimum_table': {
+                "p2c_trust":   {"-127:40":13, "41:120":15},
+                "p2c_untrust": {"-127:25":13, "26:30":14 , "31:35":15, "36:120":16},
+                "c2p_trust":   {"-127:20":13, "21:25":14 , "26:30":15, "31:120":16},
+                "c2p_untrust": {"-127:20":13, "21:25":14 , "26:30":15, "31:120":16},
+                "unk_trust":   {"-127:20":13, "21:25":14 , "26:30":15, "31:120":16},
+                "unk_untrust": {"-127:20":13, "21:25":14 , "26:30":15, "31:120":16}
+            }
+        }
+    },
+    'Mellanox-SN2700-D48C8': {
+        'thermal': {
+            'minimum_table': {
+                "p2c_trust":   {"-127:40":13, "41:120":15},
+                "p2c_untrust": {"-127:25":13, "26:30":14 , "31:35":15, "36:120":16},
+                "c2p_trust":   {"-127:20":13, "21:25":14 , "26:30":15, "31:120":16},
+                "c2p_untrust": {"-127:20":13, "21:25":14 , "26:30":15, "31:120":16},
+                "unk_trust":   {"-127:20":13, "21:25":14 , "26:30":15, "31:120":16},
+                "unk_untrust": {"-127:20":13, "21:25":14 , "26:30":15, "31:120":16}
+            }
+        }
+    },
+    'ACS-MSN2100': {
+        'thermal': {
+            'minimum_table': {
+                "p2c_trust":  {"-127:120":12},
+                "p2c_untrust": {"-127:15":12, "16:25":13, "26:30":14, "31:35":15, "36:120":16},
+                "c2p_trust":   {"-127:40":12, "41:120":13},
+                "c2p_untrust": {"-127:40":12, "41:120":13},
+                "unk_trust":   {"-127:40":12, "41:120":13},
+                "unk_untrust": {"-127:15":12, "16:25":13, "26:30":14, "31:35":15, "36:120":16}
+            }
+        }
+    },
+    'ACS-MSN2010': {
+        'thermal': {
+            'minimum_table': {
+                "p2c_trust":   {"-127:120":12},
+                "p2c_untrust": {"-127:15":12, "16:20":13, "21:30":14, "31:35":15, "36:120":16},
+                "c2p_trust":   {"-127:120":12},
+                "c2p_untrust": {"-127:20":12, "21:25":13 , "26:30":14, "31:35":15, "36:120":16},
+                "unk_trust":   {"-127:120":12},
+                "unk_untrust": {"-127:15":12, "16:20":13 , "21:30":14, "31:35":15, "36:120":16}
+            }
+        }
+    },
+    'ACS-MSN3700': {
+        'thermal': {
+            'minimum_table': {
+                "p2c_trust":   {"-127:25":12, "26:40":13 , "41:120":14},
+                "p2c_untrust": {"-127:15":12, "16:30":13 , "31:35":14, "36:40":15, "41:120":16},
+                "c2p_trust":   {"-127:25":12, "26:40":13 , "41:120":14},
+                "c2p_untrust":   {"-127:25":12, "26:40":13 , "41:120":14},
+                "unk_trust":   {"-127:25":12, "26:40":13 , "41:120":14},
+                "unk_untrust": {"-127:15":12, "16:30":13 , "31:35":14, "36:40":15, "41:120":16},
+            }
+        }
+    },
+    'ACS-MSN3800': {
+        'thermal': {
+            'minimum_table': {
+                "p2c_trust":   {"-127:35":12, "36:120":13},
+                "p2c_untrust": {"-127:0":12, "1:10":13 , "11:15":14, "16:20":15, "21:35":16, "36:120":17},
+                "c2p_trust":   {"-127:30":12, "31:40":13 , "41:120":14},
+                "c2p_untrust":   {"-127:20":12, "21:30":13 , "31:35":14, "36:40":15, "41:120":16},
+                "unk_trust":   {"-127:30":12, "31:40":13 , "41:120":14},
+                "unk_untrust": {"-127:0":12, "1:10":13 , "11:15":14, "16:20":15, "21:35":16, "36:120":17},
+            }
+        }
+    },
+    'Mellanox-SN3800-D112C8': {
+        'thermal': {
+            'minimum_table': {
+                "p2c_trust":   {"-127:35":12, "36:120":13},
+                "p2c_untrust": {"-127:0":12, "1:10":13 , "11:15":14, "16:20":15, "21:35":16, "36:120":17},
+                "c2p_trust":   {"-127:30":12, "31:40":13 , "41:120":14},
+                "c2p_untrust":   {"-127:20":12, "21:30":13 , "31:35":14, "36:40":15, "41:120":16},
+                "unk_trust":   {"-127:30":12, "31:40":13 , "41:120":14},
+                "unk_untrust": {"-127:0":12, "1:10":13 , "11:15":14, "16:20":15, "21:35":16, "36:120":17},
+            }
+        }
+    },
+}

platform/mellanox/mlnx-platform-api/sonic_platform/fan.py

@@ -9,6 +9,7 @@
 #############################################################################
 
 import os.path
+import subprocess
 
 try:
     from sonic_platform_base.fan_base import FanBase
@@ -22,8 +23,14 @@
 
 FAN_PATH = "/var/run/hw-management/thermal/"
 LED_PATH = "/var/run/hw-management/led/"
+CONFIG_PATH = "/var/run/hw-management/config"
 # fan_dir isn't supported on Spectrum 1. It is supported on Spectrum 2 and later switches
 FAN_DIR = "/var/run/hw-management/system/fan_dir"
+COOLING_STATE_PATH = "/var/run/hw-management/thermal/cooling_cur_state"
+
+# SKUs with unplugable FANs:
+# 1. don't have fanX_status and should be treated as always present
+hwsku_dict_with_unplugable_fan = ['ACS-MSN2010', 'ACS-MSN2100']
 
 # SKUs with unplugable FANs:
 # 1. don't have fanX_status and should be treated as always present
@@ -33,6 +40,10 @@ class Fan(FanBase):
     """Platform-specific Fan class"""
 
     STATUS_LED_COLOR_ORANGE = "orange"
+    min_cooling_level = 2
+    # PSU fan speed vector
+    PSU_FAN_SPEED = ['0x3c', '0x3c', '0x3c', '0x3c', '0x3c',
+                     '0x3c', '0x3c', '0x46', '0x50', '0x5a', '0x64']
 
     def __init__(self, has_fan_dir, fan_index, drawer_index = 1, psu_fan = False, sku = None):
         # API index is starting from 0, Mellanox platform index is starting from 1
@@ -54,6 +65,10 @@ def __init__(self, has_fan_dir, fan_index, drawer_index = 1, psu_fan = False, sk
             self.fan_presence_path = "psu{}_fan1_speed_get".format(self.index)
             self._name = 'psu_{}_fan_{}'.format(self.index, 1)
             self.fan_max_speed_path = None
+            self.psu_i2c_bus_path = os.path.join(CONFIG_PATH, 'psu{0}_i2c_bus'.format(self.index))
+            self.psu_i2c_addr_path = os.path.join(CONFIG_PATH, 'psu{0}_i2c_addr'.format(self.index))
+            self.psu_i2c_command_path = os.path.join(CONFIG_PATH, 'fan_command')
+
         self.fan_status_path = "fan{}_fault".format(self.index)
         self.fan_green_led_path = "led_fan{}_green".format(self.drawer_index)
         self.fan_red_led_path = "led_fan{}_red".format(self.drawer_index)
@@ -231,13 +246,28 @@ def set_speed(self, speed):
             bool: True if set success, False if fail. 
         """
         status = True
-        pwm = int(round(PWM_MAX*speed/100.0))
 
         if self.is_psu_fan:
-            #PSU fan speed is not setable.
-            return False
-
+            try:
+                with open(self.psu_i2c_bus_path, 'r') as f:
+                    bus = f.read().strip()
+                with open(self.psu_i2c_addr_path, 'r') as f:
+                    addr = f.read().strip()
+                with open(self.psu_i2c_command_path, 'r') as f:
+                    command = f.read().strip()
+                speed = Fan.PSU_FAN_SPEED[int(speed / 10)]
+                subprocess.call("i2cset -f -y {0} {1} {2} {3} wp".format(bus, addr, command, speed), shell = True)
+                return True
+            except Exception as e:
+                return False
+
         try:
+            cooling_level = int(speed / 10)
+            if cooling_level < self.min_cooling_level:
+                cooling_level = self.min_cooling_level
+                speed = self.min_cooling_level * 10
+            self.set_cooling_level(cooling_level)
+            pwm = int(round(PWM_MAX*speed/100.0))
             with open(os.path.join(FAN_PATH, self.fan_speed_set_path), 'w') as fan_pwm:
                 fan_pwm.write(str(pwm))
         except (ValueError, IOError):
@@ -352,3 +382,36 @@ def get_speed_tolerance(self):
         """
         # The tolerance value is fixed as 20% for all the Mellanox platform
         return 20
+
+    @classmethod
+    def set_cooling_level(cls, level):
+        """
+        Change cooling level. The input level should be an integer value [1, 10].
+        1 means 10%, 2 means 20%, 10 means 100%.
+        """
+        if not isinstance(level, int):
+            raise RuntimeError("Failed to set cooling level, input parameter must be integer")
+
+        if level < 1 or level > 10:
+            raise RuntimeError("Failed to set cooling level, level value must be in range [1, 10], got {}".format(level))
+
+        try:
+            # reset FAN driver and change cooling state
+            with open(COOLING_STATE_PATH, 'w') as cooling_state:
+                cooling_state.write(str(level + 10))
+
+            # make cooling state diplay correct value
+            with open(COOLING_STATE_PATH, 'w') as cooling_state:
+                cooling_state.write(str(level))
+        except (ValueError, IOError) as e:
+            raise RuntimeError("Failed to set cooling level - {}".format(e))
+
+    @classmethod
+    def get_cooling_level(cls):
+        try:
+            with open(COOLING_STATE_PATH, 'r') as cooling_state:
+                cooling_level = int(cooling_state.read())
+                return cooling_level
+        except (ValueError, IOError) as e:
+            raise RuntimeError("Failed to get cooling level - {}".format(e))
+