NAS-118851 / 23.10 / API authentication using Directory Services

src/freenas/debian/rules

@@ -18,6 +18,7 @@ override_dh_auto_install:
 		cp etc/iso_3166_2_countries.csv debian/truenas-files/etc/; \
 		cp -a etc/logrotate.d debian/truenas-files/etc/; \
 		cp etc/nsswitch.conf debian/truenas-files/etc/; \
+		cp -a etc/pam.d debian/truenas-files/etc/; \
 		cp -a etc/syslog-ng debian/truenas-files/etc/; \
 		cp -a etc/systemd debian/truenas-files/etc/; \
 		mkdir debian/truenas-files/home; \

src/freenas/etc/pam.d/common-auth-unix

@@ -0,0 +1,2 @@
+auth	requisite			pam_deny.so
+auth	required			pam_permit.so

src/middlewared/debian/control

@@ -53,6 +53,7 @@ Build-Depends: alembic,
                python3-onedrivesdk,
                python3-packaging,
                python3-parted,
+               python3-pampy,
                python3-passlib,
                python3-prctl,
                python3-psutil,
@@ -159,6 +160,7 @@ Depends: alembic,
          python3-onedrivesdk,
          python3-packaging,
          python3-parted,
+         python3-pampy,
          python3-passlib,
          python3-prctl,
          python3-psutil,

src/middlewared/middlewared/etc_files/pam.d/common-auth.mako

@@ -13,6 +13,5 @@
 %>\
 
 ${'\n'.join(dsp['primary'])}
-auth	requisite			pam_deny.so
-auth	required			pam_permit.so
+@include common-auth-unix
 ${'\n'.join(dsp['additional'])}

src/middlewared/middlewared/etc_files/pam.d/middleware.mako

@@ -0,0 +1,18 @@
+<%
+    ds_auth = middleware.call_sync('datastore.config', 'system.settings')['stg_ds_auth']
+%>\
+# PAM configuration for the middleware (Web UI login)
+
+%if ds_auth:
+@include common-auth
+%else:
+<%namespace name="pam" file="pam.inc.mako" />\
+<%
+        dsp = pam.getNoDirectoryServicePam().pam_auth()
+%>\
+${'\n'.join(dsp['primary'])}
+@include common-auth-unix
+%endif
+account	required	pam_deny.so
+password	required	pam_deny.so
+session	required	pam_deny.so

src/middlewared/middlewared/etc_files/pam.d/pam.inc.mako

@@ -260,3 +260,6 @@
 <%def name="getDirectoryServicePam(**kwargs)">
   <% return DirectoryServicePam(**kwargs) %>
 </%def>
+<%def name="getNoDirectoryServicePam()">
+  <% return DirectoryServicePamBase() %>
+</%def>

src/middlewared/middlewared/main.py

@@ -191,6 +191,9 @@ def send_error(self, message, errno, reason=None, exc_info=None, etype=None, ext
     async def call_method(self, message, serviceobj, methodobj):
         params = message.get('params') or []
 
+        if mock := self.middleware._mock_method(message['method'], params):
+            methodobj = mock
+
         try:
             async with self._softhardsemaphore:
                 result = await self.middleware._call(message['method'], serviceobj, methodobj, params, app=self)
@@ -887,7 +890,7 @@ def __init__(
         self.__console_io = False if os.path.exists(self.CONSOLE_ONCE_PATH) else None
         self.__terminate_task = None
         self.jobs = JobsQueue(self)
-        self.mocks = {}
+        self.mocks = defaultdict(list)
         self.socket_messages_queue = deque(maxlen=200)
         self.tasks = set()
 
@@ -1330,7 +1333,7 @@ async def _call(
             self.logger.trace('Calling %r in current IO loop', name)
             return await methodobj(*prepared_call.args)
 
-        if name not in self.mocks and serviceobj._config.process_pool:
+        if not self.mocks.get(name) and serviceobj._config.process_pool:
             self.logger.trace('Calling %r in process pool', name)
             if isinstance(serviceobj, middlewared.service.CRUDService):
                 service_name, method_name = name.rsplit('.', 1)
@@ -1352,6 +1355,9 @@ def dump_args(self, args, method=None, method_name=None):
                 except Exception:
                     return args
 
+                if mock := self._mock_method(method_name, args):
+                    method = mock
+
         if (not hasattr(method, 'accepts') and
                 method.__name__ in ['create', 'update', 'delete'] and
                 hasattr(method, '__self__')):
@@ -1376,6 +1382,9 @@ def dump_result(self, result, method):
     async def call(self, name, *params, pipes=None, job_on_progress_cb=None, app=None, profile=False):
         serviceobj, methodobj = self._method_lookup(name)
 
+        if mock := self._mock_method(name, params):
+            methodobj = mock
+
         if profile:
             methodobj = profile_wrap(methodobj)
 
@@ -1390,6 +1399,9 @@ def call_sync(self, name, *params, job_on_progress_cb=None, background=False):
 
         serviceobj, methodobj = self._method_lookup(name)
 
+        if mock := self._mock_method(name, params):
+            methodobj = mock
+
         prepared_call = self._call_prepare(name, serviceobj, methodobj, params, job_on_progress_cb=job_on_progress_cb,
                                            in_event_loop=False)
 
@@ -1570,9 +1582,10 @@ def _tracemalloc_start(self, limit, interval):
 
             time.sleep(interval)
 
-    def set_mock(self, name, mock):
-        if name in self.mocks:
-            raise ValueError(f'{name!r} is already mocked')
+    def set_mock(self, name, args, mock):
+        for _args, _mock in self.mocks[name]:
+            if args == _args:
+                raise ValueError(f'{name!r} is already mocked with {args!r}')
 
         serviceobj, methodobj = self._method_lookup(name)
 
@@ -1587,18 +1600,22 @@ def f(*args, **kwargs):
             f._job = methodobj._job
         copy_function_metadata(mock, f)
 
-        self.mocks[name] = f
+        self.mocks[name].append((args, f))
 
-    def remove_mock(self, name):
-        self.mocks.pop(name)
-
-    def _method_lookup(self, name):
-        serviceobj, methodobj = super()._method_lookup(name)
-
-        if mock := self.mocks.get(name):
-            return serviceobj, mock
+    def remove_mock(self, name, args):
+        for i, (_args, _mock) in enumerate(self.mocks[name]):
+            if args == _args:
+                del self.mocks[name][i]
+                break
 
-        return serviceobj, methodobj
+    def _mock_method(self, name, params):
+        if mocks := self.mocks.get(name):
+            for args, mock in mocks:
+                if args == list(params):
+                    return mock
+            for args, mock in mocks:
+                if args is None:
+                    return mock
 
     async def ws_handler(self, request):
         ws = web.WebSocketResponse()

src/middlewared/middlewared/plugins/account.py

@@ -715,6 +715,7 @@ async def do_delete(self, pk, options):
 
         await self.middleware.call('datastore.delete', 'account.bsdusers', pk)
         await self.middleware.call('service.reload', 'user')
+        await self.middleware.call('idmap.flush_gencache')
 
         return pk
 
@@ -1566,6 +1567,7 @@ async def do_delete(self, pk, options):
             await gm_job.wait()
 
         await self.middleware.call('service.reload', 'user')
+        await self.middleware.call('idmap.flush_gencache')
 
         return pk
 

src/middlewared/middlewared/plugins/auth.py

@@ -651,16 +651,20 @@ def check_permission(middleware, app):
             user = middleware.call_sync('auth.authenticate_root')
         else:
             try:
-                local_user = middleware.call_sync(
-                    'datastore.query',
-                    'account.bsdusers',
-                    [['bsdusr_uid', '=', origin.uid]],
-                    {'get': True, 'prefix': 'bsdusr_'},
-                )
+                query = {
+                    'username': middleware.call_sync(
+                        'datastore.query',
+                        'account.bsdusers',
+                        [['uid', '=', origin.uid]],
+                        {'get': True, 'prefix': 'bsdusr_'},
+                    )['username'],
+                }
+                local = True
             except MatchNotFound:
-                return
+                query = {'uid': origin.uid}
+                local = False
 
-            user = middleware.call_sync('auth.authenticate_local_user', local_user['id'], local_user['username'])
+            user = middleware.call_sync('auth.authenticate_user', query, local)
             if user is None:
                 return
 

src/middlewared/middlewared/plugins/auth_/authenticate.py

@@ -1,6 +1,8 @@
 import crypt
 import hmac
 
+import pam
+
 from middlewared.service import Service, private
 
 
@@ -12,62 +14,44 @@ class Config:
     @private
     async def authenticate(self, username, password):
         if '@' in username:
-            if (await self.middleware.call('datastore.config', 'system.settings'))['stg_ds_auth']:
-                return await self.ds_authenticate(username, password)
-            else:
-                return None
+            username = username.split('@')[0]
+            local = False
         else:
-            return await self.local_authenticate(username, password)
+            local = True
 
-    @private
-    async def local_authenticate(self, username, password):
-        try:
-            user = await self.middleware.call(
+        if username == 'root' and await self.middleware.call('privilege.always_has_root_password_enabled'):
+            root = await self.middleware.call(
                 'datastore.query',
                 'account.bsdusers',
-                [
-                    ('username', '=', username),
-                    ('locked', '=', False),
-                ],
+                [('username', '=', 'root')],
                 {'get': True, 'prefix': 'bsdusr_'},
             )
-        except IndexError:
-            return None
-
-        if user['unixhash'] in ('x', '*'):
-            return None
 
-        if user['password_disabled']:
-            if user['username'] == 'root':
-                if not await self.middleware.call('privilege.always_has_root_password_enabled'):
-                    return None
-            else:
+            if root['unixhash'] in ('x', '*'):
                 return None
 
-        if not hmac.compare_digest(crypt.crypt(password, user['unixhash']), user['unixhash']):
+            if not hmac.compare_digest(crypt.crypt(password, root['unixhash']), root['unixhash']):
+                return None
+        elif not await self.middleware.call('auth.libpam_authenticate', username, password):
             return None
 
-        return await self.authenticate_local_user(user['id'], username)
+        return await self.authenticate_user({'username': username}, local)
 
     @private
-    async def authenticate_local_user(self, user_id, username):
-        gids = {
-            member['bsdgrpmember_group']['bsdgrp_gid']
-            for member in await self.middleware.call(
-                'datastore.query',
-                'account.bsdgroupmembership',
-                [['bsdgrpmember_user', '=', user_id]],
-            )
-        }
-
-        return await self.common_authenticate(username, 'local_groups', gids)
+    def libpam_authenticate(self, username, password):
+        p = pam.pam()
+        return p.authenticate(username, password, service='middleware')
 
     @private
-    async def ds_authenticate(self, username, password):
-        return None  # FIXME
+    async def authenticate_user(self, query, local):
+        try:
+            user = await self.middleware.call('user.get_user_obj', {**query, 'get_groups': True})
+        except KeyError:
+            return None
+
+        groups = set(user['grouplist'])
+        groups_key = 'local_groups' if local else 'ds_groups'
 
-    @private
-    async def common_authenticate(self, username, groups_key, groups):
         privileges = [
             privilege for privilege in await self.middleware.call('datastore.query', 'account.privilege')
             if set(privilege[groups_key]) & groups
@@ -76,7 +60,7 @@ async def common_authenticate(self, username, groups_key, groups):
             return None
 
         return {
-            'username': username,
+            'username': user['pw_name'],
             'privilege': await self.middleware.call('privilege.compose_privilege', privileges),
         }
 

src/middlewared/middlewared/plugins/etc.py

@@ -139,6 +139,9 @@ class EtcService(Service):
                 {'type': 'mako', 'path': 'pam.d/common-session'},
             ]
         },
+        'pam_middleware': [
+            {'type': 'mako', 'path': 'pam.d/middleware'},
+        ],
         'ftp': [
             {'type': 'mako', 'path': 'proftpd/proftpd.conf',
              'local_path': 'local/proftpd.conf'},

src/middlewared/middlewared/plugins/idmap.py

@@ -414,6 +414,12 @@ async def get_sssd_low_range(self, domain, sssd_config=None, seed=0xdeadbeef):
 
         return (hash % max_slices) * range_size + range_size
 
+    @private
+    async def flush_gencache(self):
+        gencache_flush = await run(['net', 'cache', 'flush'], check=False)
+        if gencache_flush.returncode != 0:
+            raise CallError(f'Attempt to flush gencache failed with error: {gencache_flush.stderr.decode().strip()}')
+
     @accepts()
     @job(lock='clear_idmap_cache', lock_queue_size=1)
     async def clear_idmap_cache(self, job):
@@ -441,9 +447,7 @@ async def clear_idmap_cache(self, job):
         except Exception:
             self.logger.debug("Failed to remove winbindd_cache.tdb.", exc_info=True)
 
-        gencache_flush = await run(['net', 'cache', 'flush'], check=False)
-        if gencache_flush.returncode != 0:
-            raise CallError(f'Attempt to flush gencache failed with error: {gencache_flush.stderr.decode().strip()}')
+        await self.flush_gencache()
 
         await self.middleware.call('service.start', 'idmap')
         if smb_started:

src/middlewared/middlewared/plugins/system_general/update.py

@@ -277,6 +277,9 @@ async def do_update(self, data):
         if config['crash_reporting'] != new_config['crash_reporting']:
             await self.middleware.call('system.general.set_crash_reporting')
 
+        if config['ds_auth'] != new_config['ds_auth']:
+            await self.middleware.call('etc.generate', 'pam_middleware')
+
         await self.middleware.call('service.start', 'ssl')
 
         if rollback_timeout is not None:

src/middlewared/middlewared/plugins/test/mock.py

@@ -5,7 +5,7 @@ class TestService(Service):
     class Config:
         private = True
 
-    async def set_mock(self, name, description):
+    async def set_mock(self, name, args, description):
         if isinstance(description, str):
             exec(description)
             try:
@@ -25,10 +25,10 @@ def method(*args, **kwargs):
         else:
             raise CallError("Invalid mock declaration")
 
-        self.middleware.set_mock(name, method)
+        self.middleware.set_mock(name, args, method)
 
-    async def remove_mock(self, name):
-        self.middleware.remove_mock(name)
+    async def remove_mock(self, name, args):
+        self.middleware.remove_mock(name, args)
 
     # Dummy methods to mock for internal infrastructure testing (i.e. jobs manager)
 

src/middlewared/middlewared/test/integration/utils/call.py

@@ -5,5 +5,5 @@
 
 
 def call(*args, **kwargs):
-    with client() as c:
+    with client(**kwargs.pop("client_kwargs", {})) as c:
         return c.call(*args, **kwargs)