From 3d1db97770a23ab8a9d9dd5700544518f865647a Mon Sep 17 00:00:00 2001 From: Rakesh Radhakrishnan Date: Fri, 17 Mar 2017 20:55:56 +0530 Subject: [PATCH] ZOOKEEPER-2712: MiniKdc test case intermittently failing due to principal not found in Kerberos database --- .../zookeeper/server/quorum/QuorumPeer.java | 11 ++++ .../zookeeper/server/quorum/auth/MiniKdc.java | 2 +- .../quorum/auth/QuorumAuthTestBase.java | 60 +++++++++++++++---- .../quorum/auth/QuorumAuthUpgradeTest.java | 10 ++-- .../quorum/auth/QuorumDigestAuthTest.java | 10 ++-- .../quorum/auth/QuorumKerberosAuthTest.java | 11 +++- .../auth/QuorumKerberosHostBasedAuthTest.java | 20 +++++-- 7 files changed, 92 insertions(+), 32 deletions(-) diff --git a/src/java/main/org/apache/zookeeper/server/quorum/QuorumPeer.java b/src/java/main/org/apache/zookeeper/server/quorum/QuorumPeer.java index 889ee62a851..9eeeb5d3ee1 100644 --- a/src/java/main/org/apache/zookeeper/server/quorum/QuorumPeer.java +++ b/src/java/main/org/apache/zookeeper/server/quorum/QuorumPeer.java @@ -98,6 +98,10 @@ public class QuorumPeer extends ZooKeeperThread implements QuorumStats.Provider QuorumCnxManager qcm; QuorumAuthServer authServer; QuorumAuthLearner authLearner; + // VisibleForTesting. This flag is used to know whether qLearner's and + // qServer's login context has been initialized as ApacheDS has concurrency + // issues. Refer https://issues.apache.org/jira/browse/ZOOKEEPER-2712 + private boolean authInitialized = false; /* ZKDatabase is a top level member of quorumpeer * which will be used in all the zookeeperservers @@ -571,6 +575,7 @@ public void initialize() throws SaslException { quorumServerLoginContext, authzHosts); authLearner = new SaslQuorumAuthLearner(isQuorumLearnerSaslAuthRequired(), quorumServicePrincipal, quorumLearnerLoginContext); + authInitialized = true; } else { authServer = new NullQuorumAuthServer(); authLearner = new NullQuorumAuthLearner(); @@ -1455,6 +1460,12 @@ private boolean isQuorumLearnerSaslAuthRequired() { return quorumLearnerSaslAuthRequired; } + // VisibleForTesting. Returns true if both the quorumlearner and + // quorumserver login has been finished. Otherwse, false. + public boolean hasAuthInitialized(){ + return authInitialized; + } + public QuorumCnxManager createCnxnManager() { return new QuorumCnxManager(this.getId(), this.getView(), diff --git a/src/java/test/org/apache/zookeeper/server/quorum/auth/MiniKdc.java b/src/java/test/org/apache/zookeeper/server/quorum/auth/MiniKdc.java index 8e3cb1bed04..ebe541db554 100644 --- a/src/java/test/org/apache/zookeeper/server/quorum/auth/MiniKdc.java +++ b/src/java/test/org/apache/zookeeper/server/quorum/auth/MiniKdc.java @@ -224,7 +224,7 @@ public void run() { DEFAULT_CONFIG.setProperty(TRANSPORT, "TCP"); DEFAULT_CONFIG.setProperty(MAX_TICKET_LIFETIME, "86400000"); DEFAULT_CONFIG.setProperty(MAX_RENEWABLE_LIFETIME, "604800000"); - DEFAULT_CONFIG.setProperty(DEBUG, "false"); + DEFAULT_CONFIG.setProperty(DEBUG, "true"); } /** diff --git a/src/java/test/org/apache/zookeeper/server/quorum/auth/QuorumAuthTestBase.java b/src/java/test/org/apache/zookeeper/server/quorum/auth/QuorumAuthTestBase.java index 4d4b071b304..219d5bc6f85 100644 --- a/src/java/test/org/apache/zookeeper/server/quorum/auth/QuorumAuthTestBase.java +++ b/src/java/test/org/apache/zookeeper/server/quorum/auth/QuorumAuthTestBase.java @@ -66,10 +66,11 @@ public static void cleanupJaasConfig() { } protected String startQuorum(final int serverCount, - Map authConfigs, int authServerCount) throws IOException { + Map authConfigs, int authServerCount, + boolean delayedServerStartup) throws IOException { StringBuilder connectStr = new StringBuilder(); final int[] clientPorts = startQuorum(serverCount, 0, connectStr, - authConfigs, authServerCount); + authConfigs, authServerCount, delayedServerStartup); for (int i = 0; i < serverCount; i++) { Assert.assertTrue("waiting for server " + i + " being up", ClientBase.waitForServerUp("127.0.0.1:" + clientPorts[i], @@ -98,7 +99,7 @@ protected String startQuorum(final int serverCount, int observerCount, throws IOException { StringBuilder connectStr = new StringBuilder(); final int[] clientPorts = startQuorum(serverCount, observerCount, - connectStr, authConfigs, authServerCount); + connectStr, authConfigs, authServerCount, false); for (int i = 0; i < serverCount; i++) { Assert.assertTrue("waiting for server " + i + " being up", ClientBase.waitForServerUp("127.0.0.1:" + clientPorts[i], @@ -122,12 +123,15 @@ protected String startQuorum(final int serverCount, int observerCount, * configuration parameters for authentication * @param authServerCount * number of auth enabled servers + * @param delayedServerStartup + * true flag value to add delay between server's startup, false otherwise. * @return client port for the respective servers * @throws IOException */ protected int[] startQuorum(final int serverCount, int observerCount, StringBuilder connectStr, Map authConfigs, - int authServerCount) throws IOException { + int authServerCount, boolean delayedServerStartup) + throws IOException { final int clientPorts[] = new int[serverCount]; StringBuilder sb = new StringBuilder(); @@ -159,7 +163,7 @@ protected int[] startQuorum(final int serverCount, int observerCount, String obsCfgSection = quorumCfg + "\npeerType=observer"; quorumCfg = obsCfgSection; } - startServer(authConfigs, clientPorts, quorumCfg, i); + startServer(authConfigs, clientPorts[i], quorumCfg, i, delayedServerStartup); } // servers without any authentication configured for (int j = 0; j < serverCount - authServerCount; j++, i++) { @@ -167,20 +171,52 @@ protected int[] startQuorum(final int serverCount, int observerCount, String obsCfgSection = quorumCfg + "\npeerType=observer"; quorumCfg = obsCfgSection; } - MainThread mthread = new MainThread(i, clientPorts[i], quorumCfg); - mt.add(mthread); - mthread.start(); + startServer(null, clientPorts[i], quorumCfg, i, delayedServerStartup); } return clientPorts; } private void startServer(Map authConfigs, - final int[] clientPorts, String quorumCfg, int i) - throws IOException { - MainThread mthread = new MainThread(i, clientPorts[i], quorumCfg, - authConfigs); + final int clientPort, String quorumCfg, int i, + boolean delayedServerStartup) throws IOException { + MainThread mthread; + if (authConfigs != null) { + mthread = new MainThread(i, clientPort, quorumCfg, authConfigs); + } else { + mthread = new MainThread(i, clientPort, quorumCfg); + } mt.add(mthread); mthread.start(); + + if (delayedServerStartup) { + addDelayBeforeStartingNextServer(mthread); + } + } + + private void addDelayBeforeStartingNextServer(MainThread mThread) { + // Refer https://issues.apache.org/jira/browse/ZOOKEEPER-2712 + LOG.info("Waiting to finish login context init(Krb login), " + + "as there are potential concurrency issues in ApacheDS " + + "if multiple servers starts together!"); + int retries = 60; // 15secs delay + while (retries > 0) { + if (mThread.getQuorumPeer() != null + && mThread.getQuorumPeer().hasAuthInitialized()) { + try { + Thread.sleep(1000); // adding 1sec grace period. + } catch (InterruptedException e) { + LOG.info("Ignore InterruptedException"); + } + break; + } + // moving to next retry cycle + retries--; + try { + Thread.sleep(250); + } catch (InterruptedException e) { + LOG.info("Ignore InterruptedException"); + } + } } protected void startServer(MainThread restartPeer, diff --git a/src/java/test/org/apache/zookeeper/server/quorum/auth/QuorumAuthUpgradeTest.java b/src/java/test/org/apache/zookeeper/server/quorum/auth/QuorumAuthUpgradeTest.java index 359324549e2..4eeccf34a90 100644 --- a/src/java/test/org/apache/zookeeper/server/quorum/auth/QuorumAuthUpgradeTest.java +++ b/src/java/test/org/apache/zookeeper/server/quorum/auth/QuorumAuthUpgradeTest.java @@ -83,7 +83,7 @@ public void testNullAuthLearnerServer() throws Exception { Map authConfigs = new HashMap(); authConfigs.put(QuorumAuth.QUORUM_SASL_AUTH_ENABLED, "false"); - String connectStr = startQuorum(2, authConfigs, 0); + String connectStr = startQuorum(2, authConfigs, 0, false); CountdownWatcher watcher = new CountdownWatcher(); ZooKeeper zk = new ZooKeeper(connectStr, ClientBase.CONNECTION_TIMEOUT, watcher); @@ -103,7 +103,7 @@ public void testAuthLearnerAgainstNullAuthServer() throws Exception { Map authConfigs = new HashMap(); authConfigs.put(QuorumAuth.QUORUM_SASL_AUTH_ENABLED, "true"); - String connectStr = startQuorum(2, authConfigs, 1); + String connectStr = startQuorum(2, authConfigs, 1, false); CountdownWatcher watcher = new CountdownWatcher(); ZooKeeper zk = new ZooKeeper(connectStr, ClientBase.CONNECTION_TIMEOUT, watcher); @@ -123,7 +123,7 @@ public void testAuthLearnerAgainstNoAuthRequiredServer() throws Exception { Map authConfigs = new HashMap(); authConfigs.put(QuorumAuth.QUORUM_SASL_AUTH_ENABLED, "true"); - String connectStr = startQuorum(2, authConfigs, 2); + String connectStr = startQuorum(2, authConfigs, 2, false); CountdownWatcher watcher = new CountdownWatcher(); ZooKeeper zk = new ZooKeeper(connectStr, ClientBase.CONNECTION_TIMEOUT, watcher); @@ -145,7 +145,7 @@ public void testAuthLearnerServer() throws Exception { authConfigs.put(QuorumAuth.QUORUM_SERVER_SASL_AUTH_REQUIRED, "true"); authConfigs.put(QuorumAuth.QUORUM_LEARNER_SASL_AUTH_REQUIRED, "true"); - String connectStr = startQuorum(2, authConfigs, 2); + String connectStr = startQuorum(2, authConfigs, 2, false); CountdownWatcher watcher = new CountdownWatcher(); ZooKeeper zk = new ZooKeeper(connectStr, ClientBase.CONNECTION_TIMEOUT, watcher); @@ -178,7 +178,7 @@ public void testRollingUpgrade() throws Exception { Map authConfigs = new HashMap(); authConfigs.put(QuorumAuth.QUORUM_SASL_AUTH_ENABLED, "false"); - String connectStr = startQuorum(3, authConfigs, 0); + String connectStr = startQuorum(3, authConfigs, 0, false); CountdownWatcher watcher = new CountdownWatcher(); ZooKeeper zk = new ZooKeeper(connectStr, ClientBase.CONNECTION_TIMEOUT, watcher); diff --git a/src/java/test/org/apache/zookeeper/server/quorum/auth/QuorumDigestAuthTest.java b/src/java/test/org/apache/zookeeper/server/quorum/auth/QuorumDigestAuthTest.java index 18d1b92f5cf..c2f4cc319b9 100644 --- a/src/java/test/org/apache/zookeeper/server/quorum/auth/QuorumDigestAuthTest.java +++ b/src/java/test/org/apache/zookeeper/server/quorum/auth/QuorumDigestAuthTest.java @@ -87,7 +87,7 @@ public void testValidCredentials() throws Exception { authConfigs.put(QuorumAuth.QUORUM_SERVER_SASL_AUTH_REQUIRED, "true"); authConfigs.put(QuorumAuth.QUORUM_LEARNER_SASL_AUTH_REQUIRED, "true"); - String connectStr = startQuorum(3, authConfigs, 3); + String connectStr = startQuorum(3, authConfigs, 3, false); CountdownWatcher watcher = new CountdownWatcher(); zk = new ZooKeeper(connectStr, ClientBase.CONNECTION_TIMEOUT, watcher); watcher.waitForConnected(ClientBase.CONNECTION_TIMEOUT); @@ -108,7 +108,7 @@ public void testSaslNotRequiredWithInvalidCredentials() throws Exception { authConfigs.put(QuorumAuth.QUORUM_LEARNER_SASL_LOGIN_CONTEXT, "QuorumLearnerInvalid"); authConfigs.put(QuorumAuth.QUORUM_SASL_AUTH_ENABLED, "false"); authConfigs.put(QuorumAuth.QUORUM_SERVER_SASL_AUTH_REQUIRED, "false"); - String connectStr = startQuorum(3, authConfigs, 3); + String connectStr = startQuorum(3, authConfigs, 3, false); CountdownWatcher watcher = new CountdownWatcher(); zk = new ZooKeeper(connectStr, ClientBase.CONNECTION_TIMEOUT, watcher); watcher.waitForConnected(ClientBase.CONNECTION_TIMEOUT); @@ -132,7 +132,7 @@ public void testSaslRequiredInvalidCredentials() throws Exception { authConfigs.put(QuorumAuth.QUORUM_LEARNER_SASL_AUTH_REQUIRED, "true"); int serverCount = 2; final int[] clientPorts = startQuorum(serverCount, 0, - new StringBuilder(), authConfigs, serverCount); + new StringBuilder(), authConfigs, serverCount, false); for (int i = 0; i < serverCount; i++) { boolean waitForServerUp = ClientBase.waitForServerUp( "127.0.0.1:" + clientPorts[i], QuorumPeerTestBase.TIMEOUT); @@ -262,7 +262,7 @@ public void testNonAuthEnabledObserverJoiningAuthEnabledQuorum() // Starting auth enabled 3-node cluster. int totalServerCount = 3; String connectStr = startQuorum(totalServerCount, authConfigs, - totalServerCount); + totalServerCount, false); CountdownWatcher watcher = new CountdownWatcher(); zk = new ZooKeeper(connectStr.toString(), ClientBase.CONNECTION_TIMEOUT, @@ -310,7 +310,7 @@ public void testRelectionWithValidCredentials() throws Exception { authConfigs.put(QuorumAuth.QUORUM_SERVER_SASL_AUTH_REQUIRED, "true"); authConfigs.put(QuorumAuth.QUORUM_LEARNER_SASL_AUTH_REQUIRED, "true"); - String connectStr = startQuorum(3, authConfigs, 3); + String connectStr = startQuorum(3, authConfigs, 3, false); CountdownWatcher watcher = new CountdownWatcher(); zk = new ZooKeeper(connectStr, ClientBase.CONNECTION_TIMEOUT, watcher); watcher.waitForConnected(ClientBase.CONNECTION_TIMEOUT); diff --git a/src/java/test/org/apache/zookeeper/server/quorum/auth/QuorumKerberosAuthTest.java b/src/java/test/org/apache/zookeeper/server/quorum/auth/QuorumKerberosAuthTest.java index 2cc56a76794..e3eddf7df21 100644 --- a/src/java/test/org/apache/zookeeper/server/quorum/auth/QuorumKerberosAuthTest.java +++ b/src/java/test/org/apache/zookeeper/server/quorum/auth/QuorumKerberosAuthTest.java @@ -46,7 +46,9 @@ public class QuorumKerberosAuthTest extends KerberosSecurityTestcase { + " keyTab=\"" + keytabFilePath + "\"\n" + " storeKey=true\n" + " useTicketCache=false\n" - + " debug=false\n" + + " debug=true\n" + + " doNotPrompt=true\n" + + " refreshKrb5Config=true\n" + " principal=\"" + KerberosTestUtils.getServerPrincipal() + "\";\n" + "};\n" + "QuorumLearner {\n" + " com.sun.security.auth.module.Krb5LoginModule required\n" @@ -54,7 +56,10 @@ public class QuorumKerberosAuthTest extends KerberosSecurityTestcase { + " keyTab=\"" + keytabFilePath + "\"\n" + " storeKey=true\n" + " useTicketCache=false\n" - + " debug=false\n" + + " debug=true\n" + + " doNotPrompt=true\n" + + " refreshKrb5Config=true\n" + + " isInitiator=true\n" + " principal=\"" + KerberosTestUtils.getLearnerPrincipal() + "\";\n" + "};\n"); setupJaasConfig(jaasEntries); } @@ -98,7 +103,7 @@ public void testValidCredentials() throws Exception { authConfigs.put(QuorumAuth.QUORUM_SERVER_SASL_AUTH_REQUIRED, "true"); authConfigs.put(QuorumAuth.QUORUM_LEARNER_SASL_AUTH_REQUIRED, "true"); authConfigs.put(QuorumAuth.QUORUM_KERBEROS_SERVICE_PRINCIPAL, serverPrincipal); - String connectStr = startQuorum(3, authConfigs, 3); + String connectStr = startQuorum(3, authConfigs, 3, true); CountdownWatcher watcher = new CountdownWatcher(); ZooKeeper zk = new ZooKeeper(connectStr, ClientBase.CONNECTION_TIMEOUT, watcher); watcher.waitForConnected(ClientBase.CONNECTION_TIMEOUT); diff --git a/src/java/test/org/apache/zookeeper/server/quorum/auth/QuorumKerberosHostBasedAuthTest.java b/src/java/test/org/apache/zookeeper/server/quorum/auth/QuorumKerberosHostBasedAuthTest.java index fcb76919f1b..55deefb8326 100644 --- a/src/java/test/org/apache/zookeeper/server/quorum/auth/QuorumKerberosHostBasedAuthTest.java +++ b/src/java/test/org/apache/zookeeper/server/quorum/auth/QuorumKerberosHostBasedAuthTest.java @@ -43,7 +43,7 @@ public class QuorumKerberosHostBasedAuthTest extends KerberosSecurityTestcase { private static File keytabFile; private static String hostServerPrincipal = KerberosTestUtils.getHostServerPrincipal(); private static String hostLearnerPrincipal = KerberosTestUtils.getHostLearnerPrincipal(); - private static String hostNamedLearnerPrincipal = KerberosTestUtils.getHostNamedLearnerPrincipal("myHost"); + private static String hostNamedLearnerPrincipal = KerberosTestUtils.getHostNamedLearnerPrincipal("myhost"); static { setupJaasConfigEntries(hostServerPrincipal, hostLearnerPrincipal, hostNamedLearnerPrincipal); } @@ -58,7 +58,9 @@ private static void setupJaasConfigEntries(String hostServerPrincipal, + " keyTab=\"" + keytabFilePath + "\"\n" + " storeKey=true\n" + " useTicketCache=false\n" - + " debug=false\n" + + " debug=true\n" + + " doNotPrompt=true\n" + + " refreshKrb5Config=true\n" + " principal=\"" + KerberosTestUtils.replaceHostPattern(hostServerPrincipal) + "\";\n" + "};\n" + "QuorumLearner {\n" + " com.sun.security.auth.module.Krb5LoginModule required\n" @@ -66,7 +68,10 @@ private static void setupJaasConfigEntries(String hostServerPrincipal, + " keyTab=\"" + keytabFilePath + "\"\n" + " storeKey=true\n" + " useTicketCache=false\n" - + " debug=false\n" + + " debug=true\n" + + " doNotPrompt=true\n" + + " refreshKrb5Config=true\n" + + " isInitiator=true\n" + " principal=\"" + KerberosTestUtils.replaceHostPattern(hostLearnerPrincipal) + "\";\n" + "};\n" + "QuorumLearnerMyHost {\n" + " com.sun.security.auth.module.Krb5LoginModule required\n" @@ -74,7 +79,10 @@ private static void setupJaasConfigEntries(String hostServerPrincipal, + " keyTab=\"" + keytabFilePath + "\"\n" + " storeKey=true\n" + " useTicketCache=false\n" - + " debug=false\n" + + " debug=true\n" + + " doNotPrompt=true\n" + + " refreshKrb5Config=true\n" + + " isInitiator=true\n" + " principal=\"" + hostNamedLearnerPrincipal + "\";\n" + "};\n"); setupJaasConfig(jaasEntries); } @@ -122,7 +130,7 @@ public void testValidCredentials() throws Exception { authConfigs.put(QuorumAuth.QUORUM_SERVER_SASL_AUTH_REQUIRED, "true"); authConfigs.put(QuorumAuth.QUORUM_LEARNER_SASL_AUTH_REQUIRED, "true"); authConfigs.put(QuorumAuth.QUORUM_KERBEROS_SERVICE_PRINCIPAL, serverPrincipal); - String connectStr = startQuorum(3, authConfigs, 3); + String connectStr = startQuorum(3, authConfigs, 3, true); CountdownWatcher watcher = new CountdownWatcher(); ZooKeeper zk = new ZooKeeper(connectStr, ClientBase.CONNECTION_TIMEOUT, watcher); watcher.waitForConnected(ClientBase.CONNECTION_TIMEOUT); @@ -143,7 +151,7 @@ public void testConnectBadServer() throws Exception { authConfigs.put(QuorumAuth.QUORUM_SERVER_SASL_AUTH_REQUIRED, "true"); authConfigs.put(QuorumAuth.QUORUM_LEARNER_SASL_AUTH_REQUIRED, "true"); authConfigs.put(QuorumAuth.QUORUM_KERBEROS_SERVICE_PRINCIPAL, serverPrincipal); - String connectStr = startQuorum(3, authConfigs, 3); + String connectStr = startQuorum(3, authConfigs, 3, true); CountdownWatcher watcher = new CountdownWatcher(); ZooKeeper zk = new ZooKeeper(connectStr, ClientBase.CONNECTION_TIMEOUT, watcher); watcher.waitForConnected(ClientBase.CONNECTION_TIMEOUT);