U5.1 hotfix

src/freenas/usr/local/lib/middlewared_truenas/plugins/enclosure.py

@@ -1,9 +1,3 @@
-# Copyright (c) 2019 iXsystems, Inc.
-# All rights reserved.
-# This file is a part of TrueNAS
-# and may not be copied and/or distributed
-# without the express permission of iXsystems.
-
 from collections import OrderedDict
 import logging
 import os
@@ -148,8 +142,23 @@ async def do_update(self, id, data):
 
         return await self._get_instance(id)
 
-    def _get_slot(self, slot_filter, enclosure_query=None):
-        for enclosure in self.middleware.call_sync("enclosure.query", enclosure_query or []):
+    async def _build_slot_for_disks_dict(self):
+        enclosure_info = await self.middleware.call('enclosure.query')
+
+        results = {}
+        for enc in enclosure_info:
+            slots = next(filter(lambda x: x["name"] == "Array Device Slot", enc["elements"]))["elements"]
+            results.update({
+                j["slot"] * 1000 + enc["number"]: j["data"]["Device"] or None for j in slots
+            })
+
+        return results
+
+    def _get_slot(self, slot_filter, enclosure_query=None, enclosure_info=None):
+        if enclosure_info is None:
+            enclosure_info = self.middleware.call_sync("enclosure.query", enclosure_query or [])
+
+        for enclosure in enclosure_info:
             try:
                 elements = next(filter(lambda element: element["name"] == "Array Device Slot",
                                        enclosure["elements"]))["elements"]
@@ -160,8 +169,8 @@ def _get_slot(self, slot_filter, enclosure_query=None):
 
         raise MatchNotFound()
 
-    def _get_slot_for_disk(self, disk):
-        return self._get_slot(lambda element: element["data"]["Device"] == disk)
+    def _get_slot_for_disk(self, disk, enclosure_info=None):
+        return self._get_slot(lambda element: element["data"]["Device"] == disk, enclosure_info=enclosure_info)
 
     def _get_ses_slot(self, enclosure, element):
         if "original" in element:
@@ -206,15 +215,27 @@ def set_slot_status(self, enclosure_id, slot, status):
             raise CallError("Error setting slot status")
 
     @private
-    def sync_disk(self, id):
+    async def sync_disks(self):
+        curr_slot_info = await self._build_slot_for_disks_dict()
+        for db_entry in await self.middleware.call('datastore.query', 'storage.disk'):
+            for curr_slot, curr_disk in curr_slot_info.items():
+                if db_entry['disk_name'] == curr_disk and db_entry['disk_enclosure_slot'] != curr_slot:
+                    await self.middleware.call(
+                        'datastore.update', 'storage.disk',
+                        db_entry['disk_identifier'],
+                        {'disk_enclosure_slot': curr_slot},
+                    )
+
+    @private
+    def sync_disk(self, id, enclosure_info=None):
         disk = self.middleware.call_sync(
             'disk.query',
             [['identifier', '=', id]],
             {'get': True, "extra": {'include_expired': True}}
         )
 
         try:
-            enclosure, element = self._get_slot_for_disk(disk["name"])
+            enclosure, element = self._get_slot_for_disk(disk["name"], enclosure_info)
         except MatchNotFound:
             disk_enclosure = None
         else:
@@ -245,14 +266,16 @@ def sync_zpool(self, pool):
 
         seen_devs = []
         label2disk = {}
+        cache = self.middleware.call_sync("disk.label_to_dev_disk_cache")
+        hardware = self.middleware.call_sync("truenas.get_chassis_hardware")
         for pool in pools:
             try:
                 pool = self.middleware.call_sync("zfs.pool.query", [["name", "=", pool]], {"get": True})
             except IndexError:
                 continue
 
             label2disk.update({
-                label: self.middleware.call_sync("disk.label_to_disk", label)
+                label: self.middleware.call_sync("disk.label_to_disk", label, False, cache)
                 for label in self.middleware.call_sync("zfs.pool.get_devices", pool["id"])
             })
 
@@ -267,9 +290,12 @@ def sync_zpool(self, pool):
                 if disk is None:
                     continue
 
-            # We want spares to only identify slot for Z-series
-            # See #32706
-            if self.middleware.call_sync("truenas.get_chassis_hardware").startswith("TRUENAS-Z"):
+            if hardware.startswith("TRUENAS-Z"):
+                # We want spares to have identify set on the enclosure slot for
+                # Z-series systems only see #32706 for details. Gist is that
+                # the other hardware platforms "identify" light is red which
+                # causes customer confusion because they see red and think
+                # something is wrong.
                 spare_value = "identify"
             else:
                 spare_value = "clear"
@@ -306,11 +332,14 @@ def sync_zpool(self, pool):
                 seen_devs.append(label)
 
                 try:
-                    element = self._get_ses_slot_for_disk(disk)
-                except MatchNotFound:
-                    pass
-                else:
-                    element.device_slot_set("clear")
+                    element = encs.find_device_slot(disk)
+                    if element:
+                        element.device_slot_set("clear")
+                except AssertionError:
+                    # happens for pmem devices since those
+                    # are NVDIMM sticks internal to each
+                    # controller
+                    continue
 
         disks = []
         for label in seen_devs:
@@ -339,27 +368,25 @@ def sync_zpool(self, pool):
                     element.device_slot_set("clear")
 
     def __get_enclosures(self):
-        return Enclosures(self.middleware.call_sync("enclosure.get_ses_enclosures"),
-                          self.middleware.call_sync("system.info"))
+        return Enclosures(
+            self.middleware.call_sync("enclosure.get_ses_enclosures"),
+            self.middleware.call_sync("system.dmidecode_info")["system-product-name"]
+        )
 
 
 class Enclosures(object):
 
-    def __init__(self, stat, system_info):
+    def __init__(self, stat, product):
         blacklist = [
             "VirtualSES",
         ]
-        if (
-            system_info["system_product"] and
-            system_info["system_product"].startswith("TRUENAS-") and
-            "-MINI-" not in system_info["system_product"] and
-            system_info["system_product"] not in ["TRUENAS-R20", "TRUENAS-R20A"]
-        ):
-            blacklist.append("AHCI SGPIO Enclosure 2.00")
+        if product is not None and product.startswith("TRUENAS-"):
+            if "-MINI-" not in product and product not in ["TRUENAS-R20", "TRUENAS-R20A"]:
+                blacklist.append("AHCI SGPIO Enclosure 2.00")
 
         self.__enclosures = []
         for num, data in stat.items():
-            enclosure = Enclosure(num, data, stat, system_info)
+            enclosure = Enclosure(num, data, stat, product)
             if any(s in enclosure.encname for s in blacklist):
                 continue
 
@@ -394,14 +421,11 @@ def get_by_encid(self, _id):
 
 class Enclosure(object):
 
-    def __init__(self, num, data, stat, system_info):
+    def __init__(self, num, data, stat, product):
         self.num = num
         self.stat = stat
-        self.system_info = system_info
-        if IS_FREEBSD:
-            self.devname = f"ses{num}"
-        else:
-            self.devname, data = data
+        self.product = product
+        self.devname = f"ses{num}"
         self.encname = ""
         self.encid = ""
         self.model = ""
@@ -528,15 +552,15 @@ def _set_model(self, data):
             self.model = "M Series"
             self.controller = True
         elif R_SERIES_REGEX.match(self.encname) or R20_REGEX.match(self.encname) or R50_REGEX.match(self.encname):
-            self.model = self.system_info["system_product"].replace("TRUENAS-", "")
+            self.model = self.product.replace("TRUENAS-", "")
             self.controller = True
         elif self.encname == "AHCI SGPIO Enclosure 2.00":
-            if self.system_info["system_product"] in ["TRUENAS-R20", "TRUENAS-R20A"]:
-                self.model = self.system_info["system_product"].replace("TRUENAS-", "")
+            if self.product in ["TRUENAS-R20", "TRUENAS-R20A"]:
+                self.model = self.product.replace("TRUENAS-", "")
                 self.controller = True
-            elif MINI_REGEX.match(self.system_info["system_product"]):
+            elif MINI_REGEX.match(self.product):
                 # TrueNAS Mini's do not have their product name stripped
-                self.model = self.system_info["system_product"]
+                self.model = self.product
                 self.controller = True
         elif X_SERIES_REGEX.match(self.encname):
             self.model = "X Series"

src/freenas/usr/local/lib/middlewared_truenas/plugins/enclosure_/ses_enclosure.py

@@ -0,0 +1,36 @@
+import subprocess
+import pathlib
+
+from middlewared.service import Service, private
+
+
+class EnclosureService(Service):
+
+    @private
+    def get_ses_enclosures(self):
+        """
+        Return `getencstat` output for all detected enclosures devices.
+        """
+        output = {}
+        cmd = ['getencstat', '-V']
+        for ses in pathlib.Path('/dev').iterdir():
+            if not ses.name.startswith('ses'):
+                continue
+
+            cmd.append(str(ses))
+
+        idx = None
+        with subprocess.Popen(cmd, stdout=subprocess.PIPE) as proc:
+            while True:
+                line = proc.stdout.readline().decode('utf8', 'ignore')
+                if not line:
+                    break
+                elif line.find('Enclosure Name:') != -1:
+                    idx = int(line.split(':')[0].split('ses')[-1])
+                    # line looks like /dev/ses0: Enclosure Name:...
+                    output[idx] = line
+                else:
+                    # now we just append the line as is
+                    output[idx] += line
+
+        return output

src/middlewared/middlewared/common/smart/smartctl.py

@@ -85,9 +85,10 @@ async def get_smartctl_args(middleware, devices, disk):
         return [f"/dev/{driver}{controller_id}", "-d", f"3ware,{port}"]
 
     args = [f"/dev/{disk}"]
-    p = await smartctl(args + ["-i"], stderr=subprocess.STDOUT, check=False, encoding="utf8", errors="ignore")
-    if "Unknown USB bridge" in p.stdout:
-        args = args + ["-d", "sat"]
+    if disk.startswith("da") and not await middleware.call("system.is_enterprise_ix_hardware"):
+        p = await smartctl(args + ["-i"], stderr=subprocess.STDOUT, check=False, encoding="utf8", errors="ignore")
+        if "Unknown USB bridge" in p.stdout:
+            args = args + ["-d", "sat"]
 
     return args
 

src/middlewared/middlewared/etc_files/smartd.py

@@ -12,7 +12,12 @@
 async def annotate_disk_for_smart(middleware, devices, disk):
     args = await get_smartctl_args(middleware, devices, disk)
     if args:
-        if await ensure_smart_enabled(args):
+        if (
+            # On Enterprise hardware we only use S.M.A.R.T.-enabled disks, there is no need to check for this
+            # every time.
+            await middleware.call('system.is_enterprise_ix_hardware') or
+            await ensure_smart_enabled(args)
+        ):
             args.extend(["-a"])
             args.extend(["-d", "removable"])
             return disk, dict(smartctl_args=args)

src/middlewared/middlewared/plugins/disk_/disk_info_base.py

@@ -33,11 +33,11 @@ async def get_swap_part_type(self):
     async def get_swap_devices(self):
         raise NotImplementedError()
 
-    async def label_to_dev(self, label, *args):
+    async def label_to_dev(self, label, geom_scan=True, cache=None):
         raise NotImplementedError()
 
     @private
-    async def label_to_disk(self, label, *args):
+    async def label_to_disk(self, label, geom_scan=True, cache=None):
         raise NotImplementedError()
 
     @private