Use SSS to perform SID conversion when enabled

IPA domains provide users and groups SIDs for SMB protocol
support. When the LDAP service is enabled we should wrap
around the python sss nss idmap module to resolve SIDs for
cases where we need them.

This commit also extends the output of get_user_obj and
get_group_obj to include explicit information about the source
providing the ID, which helps to streamline authentication.

src/middlewared/middlewared/plugins/account.py

@@ -10,10 +10,11 @@
 import subprocess
 import time
 import warnings
+import wbclient
 from pathlib import Path
 from contextlib import suppress
 
-from middlewared.schema import accepts, Bool, Dict, Int, List, Password, Patch, returns, Str, LocalUsername
+from middlewared.schema import accepts, Bool, Dict, Int, List, Password, Patch, returns, SID, Str, LocalUsername
 from middlewared.service import (
     CallError, CRUDService, ValidationErrors, no_auth_required, no_authz_required, pass_app, private, filterable, job
 )
@@ -22,15 +23,18 @@
 from middlewared.utils import run, filter_list
 from middlewared.utils.crypto import sha512_crypt
 from middlewared.utils.nss import pwd, grp
+from middlewared.utils.nss.nss_common import NssModule
 from middlewared.utils.privilege import credential_has_full_admin, privileges_group_mapping
 from middlewared.validators import Email, Range
 from middlewared.async_validators import check_path_resides_within_volume
 from middlewared.plugins.smb_.constants import SMBBuiltin
-from middlewared.plugins.idmap_.utils import (
+from middlewared.plugins.idmap_.idmap_constants import (
     TRUENAS_IDMAP_DEFAULT_LOW,
     SID_LOCAL_USER_PREFIX,
     SID_LOCAL_GROUP_PREFIX
 )
+from middlewared.plugins.idmap_ import idmap_winbind
+from middlewared.plugins.idmap_ import idmap_sss
 
 ADMIN_UID = 950  # When googled, does not conflict with anything
 ADMIN_GID = 950
@@ -1053,7 +1057,8 @@ def shell_choices(self, group_ids):
         Int('pw_uid'),
         Int('pw_gid'),
         List('grouplist'),
-        Dict('sid_info'),
+        SID('sid', null=True),
+        Str('source', enum=['LOCAL', 'ACTIVEDIRECTORY', 'LDAP']),
         Bool('local'),
         register=True,
     ))
@@ -1089,14 +1094,9 @@ def get_user_obj(self, data):
         `grouplist` - optional list of group ids for groups of which this account is a member. If `get_groups`
         is not specified, this value will be null.
 
-        `sid_info - optional dictionary object containing details of SID and domain information. If `sid_info`
-        is not specified, this value will be null.
+        `sid` - optional SID value for the accoun that is present if `sid_info` is specified in payload.
 
-        NOTE: in some pathological scenarios this may make the operation hang until
-        the winbindd request timeout has been reached if the winbindd connection manager
-        has not yet marked the domain as offline. The TrueNAS middleware is more aggressive
-        about marking AD domains as FAULTED and so it may be advisable to first check the
-        Active Directory service state prior to batch operations using this option.
+        `source` - the source for the user account.
         """
         verrors = ValidationErrors()
         if not data['username'] and data['uid'] is None:
@@ -1123,40 +1123,70 @@ def get_user_obj(self, data):
             except KeyError:
                 raise KeyError(f'{data["uid"]}: user with this id does not exist') from None
 
-        source = user_obj.pop('source')
-        user_obj['local'] = source == 'FILES'
+        match user_obj['source']:
+            case NssModule.FILES.name:
+                user_obj['source'] = 'LOCAL'
+            case NssModule.WINBIND.name:
+                user_obj['source'] = 'ACTIVEDIRECTORY'
+            case NssModule.SSS.name:
+                user_obj['source'] = 'LDAP'
+            case _:
+                raise ValueError(f'{user_obj["source"]}: unknown ID source. Please file a bug report.')
+
+        user_obj['local'] = user_obj['source'] == 'LOCAL'
 
         if data['get_groups']:
             user_obj['grouplist'] = os.getgrouplist(user_obj['pw_name'], user_obj['pw_gid'])
 
         if data['sid_info']:
-            try:
-                if (idmap := self.middleware.call_sync('idmap.convert_unixids', [{
-                    'id_type': 'USER',
-                    'id': user_obj['pw_uid'],
-                }])['mapped']):
-                    sid = idmap[f'UID:{user_obj["pw_uid"]}']['sid']
-                else:
-                    sid = SID_LOCAL_USER_PREFIX + str(user_obj['pw_uid'])
-            except CallError as e:
-                # ENOENT means no winbindd entry for user
-                # ENOTCONN means winbindd is stopped / can't be started
-                # EAGAIN means the system dataset is hosed and needs to be fixed,
-                # but we need to let it through so that it's very clear in logs
-                if e.errno not in (errno.ENOENT, errno.ENOTCONN):
-                    self.logger.error('Failed to retrieve SID for uid: %d', user_obj['pw_uid'], exc_info=True)
-                sid = None
-            except Exception:
-                self.logger.error('Failed to retrieve SID for uid: %d', user_obj['pw_uid'], exc_info=True)
-                sid = None
+            match user_obj['source']:
+                case 'LOCAL' | 'ACTIVEDIRECTORY':
+                    # winbind provides idmapping for local and AD users
+                    try:
+                        idmap_ctx = idmap_winbind.WBClient()
+                    except wbclient.WBCError as e:
+                        if e.error_code != wbclient.WBC_ERR_WINBIND_NOT_AVAILABLE:
+                            self.logger.error('Failed to retrieve SID for uid: %d',
+                                              user_obj['pw_uid'], exc_info=True)
+
+                        idmap_ctx = None
+                case 'LDAP':
+                    # SSSD provides ID mapping for IPA domains
+                    idmap_ctx = idmap_sss.SSSClient()
+                case _:
+                    # We're not raising an exception here since it
+                    # can be a critical areai
+                    self.logger.error(
+                        '%s: unknown ID source. Please file a bug report.',
+                        user_obj['source']
+                    )
+                    idmap_ctx = None
 
-            if sid:
-                user_obj['sid_info'] = {
-                    'sid': sid,
-                    'domain_information': self.middleware.call_sync('idmap.parse_domain_info', sid)
-                }
+            if idmap_ctx is not None:
+                try:
+                    sid = idmap_ctx.uidgid_to_idmap_entry({
+                        'id_type': 'USER',
+                        'id': user_obj['pw_uid']
+                    })['sid']
+                except MatchNotFound:
+                    if user_obj['source'] == 'LOCAL':
+                        # Local user that doesn't have passdb entry
+                        # we can simply apply default prefix
+                        sid = SID_LOCAL_USER_PREFIX + str(user_obj['pw_uid'])
+                    else:
+                        # This is a more odd situation. The user accout exists
+                        # in IPA but doesn't have a SID assigned to it.
+                        sid = None
             else:
-                user_obj['sid_info'] = None
+                # We were unable to establish an idmap client context even
+                # though we were able to retrieve the user account info. This
+                # most likely means that we're dealing with a local account and
+                # winbindd is not running.
+                sid = None
+
+            user_obj['sid'] = sid
+        else:
+            user_obj['sid'] = None
 
         return user_obj
 
@@ -2052,7 +2082,8 @@ async def get_next_gid(self):
         Str('gr_name'),
         Int('gr_gid'),
         List('gr_mem'),
-        Dict('sid_info'),
+        SID('sid', null=True),
+        Str('source', enum=['LOCAL', 'ACTIVEDIRECTORY', 'LDAP']),
         Bool('local'),
     ))
     def get_group_obj(self, data):
@@ -2071,11 +2102,14 @@ def get_group_obj(self, data):
 
         `gr_mem` - list of gids that are members of the group
 
-        `sid_info` - optional SMB information if `sid_info` is specified specified, otherwise
-        this field will be null.
-
         `local` - boolean indicating whether this group is local to the NAS or provided by a
         directory service.
+
+        `sid` - optional SID value for the accoun that is present if `sid_info` is specified in payload.
+
+        `source` - the name server switch module that provided the user. Options are:
+        FILES - local user in passwd file of server, WINBIND - user provided by winbindd, SSS - user
+        provided by SSSD.
         """
         verrors = ValidationErrors()
         if not data['groupname'] and data['gid'] is None:
@@ -2101,33 +2135,66 @@ def get_group_obj(self, data):
             except KeyError:
                 raise KeyError(f'{data["gid"]}: group with this id does not exist') from None
 
-        source = grp_obj.pop('source')
-        grp_obj['local'] = source == 'FILES'
+        grp_obj['local'] = grp_obj['source'] == NssModule.FILES.name
+        match grp_obj['source']:
+            case NssModule.FILES.name:
+                grp_obj['source'] = 'LOCAL'
+            case NssModule.WINBIND.name:
+                grp_obj['source'] = 'ACTIVEDIRECTORY'
+            case NssModule.SSS.name:
+                grp_obj['source'] = 'LDAP'
+            case _:
+                raise ValueError(f'{grp_obj["source"]}: unknown ID source. Please file a bug report.')
+
+        grp_obj['local'] = grp_obj['source'] == 'LOCAL'
 
         if data['sid_info']:
-            try:
-                if (idmap := self.middleware.call_sync('idmap.convert_unixids', [{
-                    'id_type': 'GROUP',
-                    'id': grp_obj['gr_gid'],
-                }])['mapped']):
-                    sid = idmap[f'GID:{grp_obj["gr_gid"]}']['sid']
-                else:
-                    sid = SID_LOCAL_GROUP_PREFIX + str(grp_obj['gr_gid'])
-            except CallError as e:
-                if e.errno not in (errno.ENOENT, errno.ENOTCONN):
-                    self.logger.error('Failed to retrieve SID for gid: %d', grp_obj['gr_gid'], exc_info=True)
-                sid = None
-            except Exception:
-                self.logger.error('Failed to retrieve SID for gid: %d', grp['gr_gid'], exc_info=True)
-                sid = None
+            match grp_obj['source']:
+                case 'LOCAL' | 'ACTIVEDIRECTORY':
+                    # winbind provides idmapping for local and AD users
+                    try:
+                        idmap_ctx = idmap_winbind.WBClient()
+                    except wbclient.WBCError as e:
+                        # Library error from libwbclient.
+                        # Don't bother logging if winbind isn't running since
+                        # we have plenty of other places that are logging that
+                        # error condition
+                        if e.error_code != wbclient.WBC_ERR_WINBIND_NOT_AVAILABLE:
+                            self.logger.error('Failed to retrieve SID for gid: %d',
+                                              grp_obj['gr_gid'], exc_info=True)
+
+                        idmap_ctx = None
+                case 'LDAP':
+                    # SSSD provides ID mapping for IPA domains
+                    idmap_ctx = idmap_sss.SSSClient()
+                case _:
+                    # We're not raising an exception here since it
+                    # can be a critical areai
+                    self.logger.error(
+                        '%s: unknown ID source. Please file a bug report.',
+                        grp_obj['source']
+                    )
+                    idmap_ctx = None
 
-            if sid:
-                grp_obj['sid_info'] = {
-                    'sid': sid,
-                    'domain_information': self.middleware.call_sync('idmap.parse_domain_info', sid)
-                }
+            if idmap_ctx is not None:
+                try:
+                    sid = idmap_ctx.uidgid_to_idmap_entry({
+                        'id_type': 'GROUP',
+                        'id': grp_obj['gr_gid']
+                    })['sid']
+                except MatchNotFound:
+                    if grp_obj['source'] == 'LOCAL':
+                        # Local user that doesn't have groupmap entry
+                        # we can simply apply default prefix
+                        sid = SID_LOCAL_GROUP_PREFIX + str(grp_obj['gr_gid'])
+                    else:
+                        sid = None
             else:
-                grp_obj['sid_info'] = None
+                sid = None
+
+            grp_obj['sid'] = sid
+        else:
+            grp_obj['sid'] = None
 
         return grp_obj
 

src/middlewared/middlewared/plugins/activedirectory_/cache.py

@@ -1,4 +1,4 @@
-from middlewared.plugins.idmap_.utils import (
+from middlewared.plugins.idmap_.idmap_constants import (
     IDType,
     SID_LOCAL_USER_PREFIX,
     SID_LOCAL_GROUP_PREFIX,

src/middlewared/middlewared/plugins/activedirectory_/health.py

@@ -9,7 +9,7 @@
 from middlewared.service import private, Service, ValidationErrors
 from middlewared.service_exception import CallError, MatchNotFound
 from middlewared.plugins.directoryservices import DSStatus
-from middlewared.plugins.idmap_.utils import WBClient, WBCErr
+from middlewared.plugins.idmap_.idmap_winbind import WBClient, WBCErr
 from middlewared.utils import filter_list
 
 

src/middlewared/middlewared/plugins/auth_/authenticate.py

@@ -86,18 +86,39 @@ async def authenticate_user(self, query, user_info):
             )
             return None
 
-        if user_info['local']:
-            twofactor_id = user_info['id']
-            groups_key = 'local_groups'
-            account_flags = ['LOCAL']
-        else:
-            twofactor_id = user['sid_info']['sid']
-            groups_key = 'ds_groups'
-            account_flags = ['DIRECTORY_SERVICE']
-            if user['sid_info']['domain_information']['activedirectory']:
-                account_flags.append('ACTIVE_DIRECTORY')
-            else:
-                account_flags.append('LDAP')
+        match user['source']:
+            case 'LOCAL':
+                # Local user
+                twofactor_id = user_info['id']
+                groups_key = 'local_groups'
+                account_flags = ['LOCAL']
+            case 'ACTIVEDIRECTORY':
+                # Active directory user
+                twofactor_id = user['sid']
+                groups_key = 'ds_groups'
+                account_flags = ['DIRECTORY_SERVICE', 'ACTIVE_DIRECTORY']
+            case 'LDAP':
+                # This includes both OpenLDAP and IPA domains
+                # Since IPA domains may have cross-realm trusts with separate
+                # idmap configuration we will preferentially use the SID if it is
+                # available (since it should be static and universally unique)
+                twofactor_id = user['sid'] or user_info['id']
+                groups_key = 'ds_groups'
+                account_flags = ['DIRECTORY_SERVICE', 'LDAP']
+            case _:
+                self.logger.error('[%s]: unknown user source. Rejecting access.', user['source'])
+                return None
+
+        if user['local'] != user_info['local']:
+            # There is a disagreement between our expectation of user account source
+            # based on our database and what NSS _actually_ returned.
+            self.logger.error(
+                'Rejecting access due to potenital collision between local and directory '
+                'service user account. TrueNAS configuration expected a %s user account '
+                'but received an account provided by %s.',
+                'local' if user_info['local'] else 'non-local', user['source']
+            )
+            return None
 
         # Two-factor authentication token is keyed by SID for activedirectory
         # users.
@@ -112,6 +133,16 @@ async def authenticate_user(self, query, user_info):
             account_flags.append('2FA')
 
         if user['pw_uid'] in (0, 950):
+            if not user['local']:
+                # Although this should be covered in above check for mismatch in
+                # value of `local`, perform an extra explicit check for the case
+                # of root / root-equivalent accounts.
+                self.logger.error(
+                    'Rejecting admin account access due to collision with acccount provided '
+                    'by a directory service.'
+                )
+                return None
+
             account_flags.append('SYS_ADMIN')
 
         privileges = await self.middleware.call('privilege.privileges_for_groups', groups_key, groups)