Issue 4666 - BUG - cb_ping_farm can fail with anonymous binds disabled (

#4669)

Bug Description: cb_ping_farm had a combination of issues that made it
possible to fail in high load or odd situations. First it used anonymous
binds instead of the same credentials as the chaining process. Second
it used a NULL search DN, meaning it would use the default BASE configured
in /etc/openldap/ldap.conf. Depending on per-site configuration this
could cause the cb_ping_farm check to fail infinitly until restart
of the instance.

Fix Description: Change chaining cb_ping_farm to bind with the same
credentials as the chaining configuration, and change the target base
dn to the DN of the suffix that we are chaining to.

fixes: #4666

Author: William Brown <william@blackhats.net.au>

Review by: @progier389

dirsrvtests/tests/suites/chaining_plugin/anonymous_access_denied_basic.py

@@ -0,0 +1,149 @@
+# --- BEGIN COPYRIGHT BLOCK ---
+# Copyright (C) 2021 William Brown <william@blackhats.net.au>
+# All rights reserved.
+#
+# License: GPL (version 3 or any later version).
+# See LICENSE for details.
+# --- END COPYRIGHT BLOCK ---
+
+import ldap
+import pytest
+import time
+import shutil
+from lib389.idm.account import Accounts, Account
+from lib389.topologies import topology_i2 as topology
+from lib389.backend import Backends
+from lib389._constants import DEFAULT_SUFFIX
+from lib389.plugins import ChainingBackendPlugin
+from lib389.chaining import ChainingLinks
+from lib389.mappingTree import MappingTrees
+from lib389.idm.services import ServiceAccounts, ServiceAccount
+from lib389.idm.domain import Domain
+
+PW = 'thnaoehtnuaoenhtuaoehtnu'
+
+pytestmark = pytest.mark.tier1
+
+def test_chaining_paged_search(topology):
+    """ Check that when the chaining target has anonymous access
+    disabled that the ping still functions and allows the search
+    to continue with an appropriate bind user.
+
+    :id: 00bf31db-d93b-4224-8e70-86abb2d4cd17
+    :setup: Two standalones in chaining.
+    :steps:
+        1. Configure chaining between the nodes
+        2. Do a chaining search (w anon allow) to assert it works
+        3. Configure anon dis allowed on st2
+        4. Restart both
+        5. Check search still works
+
+    :expectedresults:
+        1. Success
+        2. Success
+        3. Success
+        4. Success
+        5. Success
+    """
+    st1 = topology.ins["standalone1"]
+    st2 = topology.ins["standalone2"]
+
+    ### We setup so that st1 -> st2
+
+    # Setup a chaining user on st2 to authenticate to.
+    sa = ServiceAccounts(st2, DEFAULT_SUFFIX).create(properties = {
+        'cn': 'sa',
+        'userPassword': PW
+    })
+
+    # Add a proxy user.
+    sproxy = ServiceAccounts(st2, DEFAULT_SUFFIX).create(properties = {
+        'cn': 'proxy',
+        'userPassword': PW
+    })
+
+    # Add the read and proxy ACI
+    dc = Domain(st2, DEFAULT_SUFFIX)
+    dc.add('aci',
+        f"""(targetattr="objectClass || cn || uid")(version 3.0; acl "Enable sa read"; allow (read, search, compare)(userdn="ldap:///{sa.dn}");)"""
+    )
+    # Add the proxy ACI
+    dc.add('aci',
+        f"""(targetattr="*")(version 3.0; acl "Enable proxy access"; allow (proxy)(userdn="ldap:///{sproxy.dn}");)"""
+    )
+
+    # Clear all the BE in st1
+    bes1 = Backends(st1)
+    for be in bes1.list():
+        be.delete()
+
+    # Setup st1 to chain to st2
+    chain_plugin_1 = ChainingBackendPlugin(st1)
+    chain_plugin_1.enable()
+
+    # Chain with the proxy user.
+    chains = ChainingLinks(st1)
+    chain = chains.create(properties={
+        'cn': 'demochain',
+        'nsfarmserverurl': st2.toLDAPURL(),
+        'nsslapd-suffix': DEFAULT_SUFFIX,
+        'nsmultiplexorbinddn': sproxy.dn,
+        'nsmultiplexorcredentials': PW,
+        'nsCheckLocalACI': 'on',
+        'nsConnectionLife': '30',
+    })
+
+    mts = MappingTrees(st1)
+    # Due to a bug in lib389, we need to delete and recreate the mt.
+    for mt in mts.list():
+        mt.delete()
+    mts.ensure_state(properties={
+        'cn': DEFAULT_SUFFIX,
+        'nsslapd-state': 'backend',
+        'nsslapd-backend': 'demochain',
+        'nsslapd-distribution-plugin': 'libreplication-plugin',
+        'nsslapd-distribution-funct': 'repl_chain_on_update',
+    })
+
+    # Enable pwpolicy (Not sure if part of the issue).
+    st1.config.set('passwordIsGlobalPolicy', 'on')
+    st2.config.set('passwordIsGlobalPolicy', 'on')
+
+    # Restart to enable everything.
+    st1.restart()
+
+    # Get a proxy auth connection.
+    sa1 = ServiceAccount(st1, sa.dn)
+    sa1_conn = sa1.bind(password=PW)
+
+    # Now do a search from st1 -> st2
+    sa1_dc = Domain(sa1_conn, DEFAULT_SUFFIX)
+    assert sa1_dc.exists()
+
+    # Now on st2 disable anonymous access.
+    st2.config.set('nsslapd-allow-anonymous-access', 'rootdse')
+
+    # Stop st2 to force the connection to be dead.
+    st2.stop()
+    # Restart st1 - this means it must re-do the ping/keepalive.
+    st1.restart()
+
+    # do a bind - this should fail, and forces the conn offline.
+    with pytest.raises(ldap.OPERATIONS_ERROR):
+        sa1.bind(password=PW)
+
+    # Allow time to attach lldb if needed.
+    # print("🔥🔥🔥")
+    # time.sleep(45)
+
+    # Bring st2 online.
+    st2.start()
+
+    # Wait a bit
+    time.sleep(5)
+
+    # Get a proxy auth connection (again)
+    sa1_conn = sa1.bind(password=PW)
+    # Now do a search from st1 -> st2
+    sa1_dc = Domain(sa1_conn, DEFAULT_SUFFIX)
+    assert sa1_dc.exists()

ldap/servers/plugins/chainingdb/cb_conn_stateless.c

@@ -846,40 +846,84 @@ cb_stale_all_connections(cb_backend_instance *cb)
 int
 cb_ping_farm(cb_backend_instance *cb, cb_outgoing_conn *cnx, time_t end_time)
 {
-
+    int version = LDAP_VERSION3;
     char *attrs[] = {"1.1", NULL};
     int rc;
+    int ret;
     struct timeval timeout;
     LDAP *ld;
     LDAPMessage *result;
     time_t now;
     int secure;
-    if (cb->max_idle_time <= 0) /* Heart-beat disabled */
+    char *plain = NULL;
+    const char *target = NULL;
+
+    if (cb->max_idle_time <= 0) {
+        /* Heart-beat disabled */
         return LDAP_SUCCESS;
+    }
+
+    const Slapi_DN *target_sdn = slapi_be_getsuffix(cb->inst_be, 0);
+
+    if (target_sdn == NULL) {
+        return LDAP_NO_SUCH_ATTRIBUTE;
+    }
+
+    target = slapi_sdn_get_dn(target_sdn);
 
-    if (cnx && (cnx->status != CB_CONNSTATUS_OK)) /* Known problem */
+    if (cnx && (cnx->status != CB_CONNSTATUS_OK)) {
+        /* Known problem */
         return LDAP_SERVER_DOWN;
+    }
 
     now = slapi_current_rel_time_t();
-    if (end_time && ((now <= end_time) || (end_time < 0)))
+    if (end_time && ((now <= end_time) || (end_time < 0))) {
         return LDAP_SUCCESS;
+    }
+
+    ret = pw_rever_decode(cb->pool->password, &plain, CB_CONFIG_USERPASSWORD);
+    if (ret == -1) {
+        return LDAP_INVALID_CREDENTIALS;
+    }
 
     secure = cb->pool->secure;
     if (cb->pool->starttls) {
         secure = 2;
     }
     ld = slapi_ldap_init(cb->pool->hostname, cb->pool->port, secure, 0);
     if (NULL == ld) {
+        if (ret == 0) {
+            slapi_ch_free_string(&plain);
+        }
+        cb_update_failed_conn_cpt(cb);
+        return LDAP_SERVER_DOWN;
+    }
+    ldap_set_option(ld, LDAP_OPT_PROTOCOL_VERSION, &version);
+    /* Don't chase referrals */
+    ldap_set_option(ld, LDAP_OPT_REFERRALS, LDAP_OPT_OFF);
+
+    /* Authenticate as the multiplexor user */
+    LDAPControl **serverctrls = NULL;
+    rc = slapi_ldap_bind(ld, cb->pool->binddn, plain,
+                         cb->pool->mech, NULL, &serverctrls,
+                         &(cb->pool->conn.bind_timeout), NULL);
+    if (ret == 0) {
+        slapi_ch_free_string(&plain);
+    }
+
+    if (LDAP_SUCCESS != rc) {
+        slapi_ldap_unbind(ld);
         cb_update_failed_conn_cpt(cb);
         return LDAP_SERVER_DOWN;
     }
 
+    ldap_controls_free(serverctrls);
+
+    /* Setup to search the base of the suffix */
     timeout.tv_sec = cb->max_test_time;
     timeout.tv_usec = 0;
 
-    /* NOTE: This will fail if we implement the ability to disable
-       anonymous bind */
-    rc = ldap_search_ext_s(ld, NULL, LDAP_SCOPE_BASE, "objectclass=*", attrs, 1, NULL,
+    rc = ldap_search_ext_s(ld, target, LDAP_SCOPE_BASE, "objectclass=*", attrs, 1, NULL,
                            NULL, &timeout, 1, &result);
     if (LDAP_SUCCESS != rc) {
         slapi_ldap_unbind(ld);