From cca424c7b9c5ffda987c4c879e30814e60191741 Mon Sep 17 00:00:00 2001 From: Andor Molnar Date: Thu, 16 Dec 2021 16:06:42 +0100 Subject: [PATCH 01/20] ZOOKEEPER-4427. Add logback dependency, fixed some unit tests, logback.xml --- conf/logback.xml | 14 + pom.xml | 22 +- zookeeper-server/pom.xml | 12 +- .../zookeeper/audit/Log4jAuditLoggerTest.java | 844 +++++++++--------- .../audit/StandaloneServerAuditTest.java | 144 +-- .../server/quorum/QuorumPeerMainTest.java | 332 +++---- .../apache/zookeeper/test/LoggerTestTool.java | 84 ++ .../zookeeper/test/ReadOnlyModeTest.java | 42 +- .../org/apache/zookeeper/test/TestUtils.java | 1 + 9 files changed, 763 insertions(+), 732 deletions(-) create mode 100644 conf/logback.xml create mode 100644 zookeeper-server/src/test/java/org/apache/zookeeper/test/LoggerTestTool.java diff --git a/conf/logback.xml b/conf/logback.xml new file mode 100644 index 00000000000..313af2ca53f --- /dev/null +++ b/conf/logback.xml @@ -0,0 +1,14 @@ + + + + %d{ISO8601} [myid:%X{myid}] - %-5p [%t:%C{1}@%L] - %m%n + + + ${zookeeper.console.threshold} + + + + + + + diff --git a/pom.xml b/pom.xml index cd4449c1886..ac1a33f5e70 100755 --- a/pom.xml +++ b/pom.xml @@ -459,11 +459,11 @@ 1.7.30 + 1.2.7 0.12.0 1.48 5.6.2 1.6.2 - 1.2.17 3.6.28 2.2 1.4 @@ -562,20 +562,14 @@ ${slf4j.version} - org.slf4j - slf4j-log4j12 - ${slf4j.version} - - - * - * - - + ch.qos.logback + logback-core + ${logback-version} - log4j - log4j - ${log4j.version} + ch.qos.logback + logback-classic + ${logback-version} org.jmockit @@ -608,11 +602,13 @@ org.junit.vintage junit-vintage-engine ${junit.version} + test org.mockito mockito-core ${mockito.version} + test io.netty diff --git a/zookeeper-server/pom.xml b/zookeeper-server/pom.xml index 4fd4f984c62..14c74ca637f 100755 --- a/zookeeper-server/pom.xml +++ b/zookeeper-server/pom.xml @@ -75,8 +75,12 @@ slf4j-api - org.slf4j - slf4j-log4j12 + ch.qos.logback + logback-core + + + ch.qos.logback + logback-classic org.eclipse.jetty @@ -118,10 +122,6 @@ metrics-core provided - - log4j - log4j - org.apache.kerby kerb-core diff --git a/zookeeper-server/src/test/java/org/apache/zookeeper/audit/Log4jAuditLoggerTest.java b/zookeeper-server/src/test/java/org/apache/zookeeper/audit/Log4jAuditLoggerTest.java index 0080b01c718..e449693feb7 100644 --- a/zookeeper-server/src/test/java/org/apache/zookeeper/audit/Log4jAuditLoggerTest.java +++ b/zookeeper-server/src/test/java/org/apache/zookeeper/audit/Log4jAuditLoggerTest.java @@ -15,425 +15,425 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.zookeeper.audit; - -import static org.apache.zookeeper.test.ClientBase.CONNECTION_TIMEOUT; -import static org.junit.jupiter.api.Assertions.assertEquals; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.LineNumberReader; -import java.io.StringReader; -import java.net.InetAddress; -import java.net.InetSocketAddress; -import java.util.ArrayList; -import java.util.List; -import org.apache.log4j.Layout; -import org.apache.log4j.Level; -import org.apache.log4j.Logger; -import org.apache.log4j.SimpleLayout; -import org.apache.log4j.WriterAppender; -import org.apache.zookeeper.CreateMode; -import org.apache.zookeeper.KeeperException; -import org.apache.zookeeper.KeeperException.Code; -import org.apache.zookeeper.Op; -import org.apache.zookeeper.PortAssignment; -import org.apache.zookeeper.ZKUtil; -import org.apache.zookeeper.ZooDefs; -import org.apache.zookeeper.ZooKeeper; -import org.apache.zookeeper.audit.AuditEvent.Result; -import org.apache.zookeeper.data.ACL; -import org.apache.zookeeper.data.Stat; -import org.apache.zookeeper.server.Request; -import org.apache.zookeeper.server.ServerCnxn; -import org.apache.zookeeper.server.quorum.QuorumPeerTestBase; -import org.apache.zookeeper.test.ClientBase; -import org.apache.zookeeper.test.ClientBase.CountdownWatcher; -import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - - -public class Log4jAuditLoggerTest extends QuorumPeerTestBase { - private static final Logger LOG = Logger.getLogger(Log4jAuditLoggerTest.class); - private static int SERVER_COUNT = 3; - private static MainThread[] mt; - private static ZooKeeper zk; - private static Logger zlogger; - private static WriterAppender appender; - private static ByteArrayOutputStream os; - - @BeforeAll - public static void setUpBeforeClass() throws Exception { - System.setProperty(ZKAuditProvider.AUDIT_ENABLE, "true"); - // setup the logger to capture all logs - Layout layout = new SimpleLayout(); - os = new ByteArrayOutputStream(); - appender = new WriterAppender(layout, os); - appender.setImmediateFlush(true); - appender.setThreshold(Level.INFO); - zlogger = Logger.getLogger(Log4jAuditLogger.class); - zlogger.addAppender(appender); - mt = startQuorum(); - zk = ClientBase.createZKClient("127.0.0.1:" + mt[0].getQuorumPeer().getClientPort()); - //Verify start audit log here itself - String expectedAuditLog = getStartLog(); - List logs = readAuditLog(os, SERVER_COUNT); - verifyLogs(expectedAuditLog, logs); - } - - @BeforeEach - public void setUp() { - os.reset(); - } - - @Test - public void testCreateAuditLogs() - throws KeeperException, InterruptedException, IOException { - String path = "/createPath"; - zk.create(path, "".getBytes(), ZooDefs.Ids.OPEN_ACL_UNSAFE, - CreateMode.PERSISTENT); - // success log - String createMode = CreateMode.PERSISTENT.toString().toLowerCase(); - verifyLog( - getAuditLog(AuditConstants.OP_CREATE, path, Result.SUCCESS, - null, createMode), readAuditLog(os)); - try { - zk.create(path, "".getBytes(), ZooDefs.Ids.OPEN_ACL_UNSAFE, - CreateMode.PERSISTENT); - } catch (KeeperException exception) { - Code code = exception.code(); - assertEquals(Code.NODEEXISTS, code); - } - // Verify create operation log - verifyLog( - getAuditLog(AuditConstants.OP_CREATE, path, Result.FAILURE, - null, createMode), readAuditLog(os)); - } - - @Test - public void testDeleteAuditLogs() - throws InterruptedException, IOException, KeeperException { - String path = "/deletePath"; - zk.create(path, "".getBytes(), ZooDefs.Ids.OPEN_ACL_UNSAFE, - CreateMode.PERSISTENT); - os.reset(); - try { - zk.delete(path, -100); - } catch (KeeperException exception) { - Code code = exception.code(); - assertEquals(Code.BADVERSION, code); - } - verifyLog(getAuditLog(AuditConstants.OP_DELETE, path, - Result.FAILURE), - readAuditLog(os)); - zk.delete(path, -1); - verifyLog(getAuditLog(AuditConstants.OP_DELETE, path), - readAuditLog(os)); - } - - @Test - public void testSetDataAuditLogs() - throws InterruptedException, IOException, KeeperException { - String path = "/setDataPath"; - zk.create(path, "".getBytes(), ZooDefs.Ids.OPEN_ACL_UNSAFE, - CreateMode.PERSISTENT); - os.reset(); - try { - zk.setData(path, "newData".getBytes(), -100); - } catch (KeeperException exception) { - Code code = exception.code(); - assertEquals(Code.BADVERSION, code); - } - verifyLog(getAuditLog(AuditConstants.OP_SETDATA, path, - Result.FAILURE), - readAuditLog(os)); - zk.setData(path, "newdata".getBytes(), -1); - verifyLog(getAuditLog(AuditConstants.OP_SETDATA, path), - readAuditLog(os)); - } - - @Test - public void testSetACLAuditLogs() - throws InterruptedException, IOException, KeeperException { - ArrayList openAclUnsafe = ZooDefs.Ids.OPEN_ACL_UNSAFE; - String path = "/aclPath"; - zk.create(path, "".getBytes(), ZooDefs.Ids.OPEN_ACL_UNSAFE, - CreateMode.PERSISTENT); - os.reset(); - try { - zk.setACL(path, openAclUnsafe, -100); - } catch (KeeperException exception) { - Code code = exception.code(); - assertEquals(Code.BADVERSION, code); - } - verifyLog( - getAuditLog(AuditConstants.OP_SETACL, path, Result.FAILURE, - ZKUtil.aclToString(openAclUnsafe), null), readAuditLog(os)); - zk.setACL(path, openAclUnsafe, -1); - verifyLog( - getAuditLog(AuditConstants.OP_SETACL, path, Result.SUCCESS, - ZKUtil.aclToString(openAclUnsafe), null), readAuditLog(os)); - } - - @Test - public void testMultiOperationAuditLogs() - throws InterruptedException, KeeperException, IOException { - List ops = new ArrayList<>(); - - String multiop = "/b"; - Op create = Op.create(multiop, "".getBytes(), - ZooDefs.Ids.OPEN_ACL_UNSAFE, - CreateMode.PERSISTENT); - Op setData = Op.setData(multiop, "newData".getBytes(), -1); - // check does nothing so it is audit logged - Op check = Op.check(multiop, -1); - Op delete = Op.delete(multiop, -1); - - String createMode = CreateMode.PERSISTENT.toString().toLowerCase(); - - ops.add(create); - ops.add(setData); - ops.add(check); - ops.add(delete); - - zk.multi(ops); - List multiOpLogs = readAuditLog(os, 3); - // verify that each multi operation success is logged - verifyLog(getAuditLog(AuditConstants.OP_CREATE, multiop, - Result.SUCCESS, null, createMode), - multiOpLogs.get(0)); - verifyLog(getAuditLog(AuditConstants.OP_SETDATA, multiop), - multiOpLogs.get(1)); - verifyLog(getAuditLog(AuditConstants.OP_DELETE, multiop), - multiOpLogs.get(2)); - - ops = new ArrayList<>(); - ops.add(create); - ops.add(create); - try { - zk.multi(ops); - } catch (KeeperException exception) { - Code code = exception.code(); - assertEquals(Code.NODEEXISTS, code); - } - - // Verify that multi operation failure is logged, and there is no path - // mentioned in the audit log - verifyLog(getAuditLog(AuditConstants.OP_MULTI_OP, null, - Result.FAILURE), - readAuditLog(os)); - } - - @Test - public void testEphemralZNodeAuditLogs() - throws Exception { - String ephemralPath = "/ephemral"; - CountdownWatcher watcher2 = new CountdownWatcher(); - ZooKeeper zk2 = new ZooKeeper( - "127.0.0.1:" + mt[0].getQuorumPeer().getClientPort(), - ClientBase.CONNECTION_TIMEOUT, watcher2); - watcher2.waitForConnected(ClientBase.CONNECTION_TIMEOUT); - zk2.create(ephemralPath, "".getBytes(), ZooDefs.Ids.OPEN_ACL_UNSAFE, - CreateMode.EPHEMERAL); - String session2 = "0x" + Long.toHexString(zk2.getSessionId()); - verifyLog(getAuditLog(AuditConstants.OP_CREATE, ephemralPath, - Result.SUCCESS, null, - CreateMode.EPHEMERAL.toString().toLowerCase(), - session2), readAuditLog(os)); - zk2.close(); - waitForDeletion(zk, ephemralPath); - // verify that ephemeral node deletion on session close are captured - // in audit log - // Because these operations are done by ZooKeeper server itself, - // there are no IP user is zkServer user, not any client user - verifyLogs(getAuditLog(AuditConstants.OP_DEL_EZNODE_EXP, ephemralPath, - Result.SUCCESS, null, null, session2, - ZKAuditProvider.getZKUser(), null), readAuditLog(os, SERVER_COUNT)); - } - - - private static String getStartLog() { - // user=userName operation=ZooKeeperServer start result=success - AuditEvent logEvent = ZKAuditProvider.createLogEvent(ZKAuditProvider.getZKUser(), - AuditConstants.OP_START, Result.SUCCESS); - return logEvent.toString(); - } - - private String getAuditLog(String operation, String znode) { - return getAuditLog(operation, znode, Result.SUCCESS); - } - - private String getAuditLog(String operation, String znode, Result result) { - return getAuditLog(operation, znode, result, null, null); - } - - private String getAuditLog(String operation, String znode, Result result, - String acl, String createMode) { - String session = getSession(); - return getAuditLog(operation, znode, result, acl, createMode, session); - } - - private String getAuditLog(String operation, String znode, Result result, - String acl, String createMode, String session) { - String user = getUser(); - String ip = getIp(); - return getAuditLog(operation, znode, result, acl, createMode, session, - user, ip); - } - - private String getAuditLog(String operation, String znode, Result result, - String acl, String createMode, String session, String user, String ip) { - AuditEvent logEvent = ZKAuditProvider.createLogEvent(user, operation, znode, acl, createMode, session, ip, - result); - String auditLog = logEvent.toString(); - LOG.info("expected audit log for operation '" + operation + "' is '" - + auditLog + "'"); - return auditLog; - } - - private String getSession() { - return "0x" + Long.toHexString(zk.getSessionId()); - } - - private String getUser() { - ServerCnxn next = getServerCnxn(); - Request request = new Request(next, -1, -1, -1, null, - next.getAuthInfo()); - return request.getUsersForAudit(); - } - - private String getIp() { - ServerCnxn next = getServerCnxn(); - InetSocketAddress remoteSocketAddress = next.getRemoteSocketAddress(); - InetAddress address = remoteSocketAddress.getAddress(); - return address.getHostAddress(); - } - - private ServerCnxn getServerCnxn() { - Iterable connections = mt[0].getQuorumPeer() - .getActiveServer() - .getServerCnxnFactory().getConnections(); - return connections.iterator().next(); - } - - private static void verifyLog(String expectedLog, String log) { - String searchString = " - "; - int logStartIndex = log.indexOf(searchString); - String auditLog = log.substring(logStartIndex + searchString.length()); - assertEquals(expectedLog, auditLog); - - } - - private static void verifyLogs(String expectedLog, List logs) { - for (String log : logs) { - verifyLog(expectedLog, log); - } - } - - private String readAuditLog(ByteArrayOutputStream os) throws IOException { - return readAuditLog(os, 1).get(0); - } - - private static List readAuditLog(ByteArrayOutputStream os, - int numberOfLogEntry) - throws IOException { - return readAuditLog(os, numberOfLogEntry, false); - } - - private static List readAuditLog(ByteArrayOutputStream os, - int numberOfLogEntry, - boolean skipEphemralDeletion) throws IOException { - List logs = new ArrayList<>(); - LineNumberReader r = new LineNumberReader( - new StringReader(os.toString())); - String line; - while ((line = r.readLine()) != null) { - if (skipEphemralDeletion - && line.contains(AuditConstants.OP_DEL_EZNODE_EXP)) { - continue; - } - logs.add(line); - } - os.reset(); - assertEquals(numberOfLogEntry, logs.size(), - "Expected number of log entries are not generated. Logs are " - + logs); - return logs; - - } - - private static MainThread[] startQuorum() throws IOException { - final int[] clientPorts = new int[SERVER_COUNT]; - StringBuilder sb = new StringBuilder(); - sb.append("4lw.commands.whitelist=*"); - sb.append("\n"); - String server; - - for (int i = 0; i < SERVER_COUNT; i++) { - clientPorts[i] = PortAssignment.unique(); - server = "server." + i + "=127.0.0.1:" + PortAssignment.unique() - + ":" - + PortAssignment.unique() + ":participant;127.0.0.1:" - + clientPorts[i]; - sb.append(server); - sb.append("\n"); - } - String currentQuorumCfgSection = sb.toString(); - MainThread[] mt = new MainThread[SERVER_COUNT]; - - // start all the servers - for (int i = 0; i < SERVER_COUNT; i++) { - mt[i] = new MainThread(i, clientPorts[i], currentQuorumCfgSection, - false); - mt[i].start(); - } - - // ensure all servers started - for (int i = 0; i < SERVER_COUNT; i++) { - Assertions.assertTrue(ClientBase.waitForServerUp("127.0.0.1:" + clientPorts[i], - CONNECTION_TIMEOUT), "waiting for server " + i + " being up"); - } - return mt; - } - - private void waitForDeletion(ZooKeeper zooKeeper, String path) - throws Exception { - long elapsedTime = 0; - long waitInterval = 10; - int timeout = 100; - Stat exists = zooKeeper.exists(path, false); - while (exists != null && elapsedTime < timeout) { - try { - Thread.sleep(waitInterval); - } catch (InterruptedException e) { - Assertions.fail("CurrentEpoch update failed"); - } - elapsedTime = elapsedTime + waitInterval; - exists = zooKeeper.exists(path, false); - } - Assertions.assertNull(exists, "Node " + path + " not deleted in " + timeout + " ms"); - } - - @AfterAll - public static void tearDownAfterClass() { - System.clearProperty(ZKAuditProvider.AUDIT_ENABLE); - for (int i = 0; i < SERVER_COUNT; i++) { - try { - if (mt[i] != null) { - mt[i].shutdown(); - } - } catch (InterruptedException e) { - e.printStackTrace(); - } - } - try { - zlogger.removeAppender(appender); - os.close(); - } catch (IOException e) { - e.printStackTrace(); - } - } - -} +//package org.apache.zookeeper.audit; +// +//import static org.apache.zookeeper.test.ClientBase.CONNECTION_TIMEOUT; +//import static org.junit.jupiter.api.Assertions.assertEquals; +//import java.io.ByteArrayOutputStream; +//import java.io.IOException; +//import java.io.LineNumberReader; +//import java.io.StringReader; +//import java.net.InetAddress; +//import java.net.InetSocketAddress; +//import java.util.ArrayList; +//import java.util.List; +//import org.apache.log4j.Layout; +//import org.apache.log4j.Level; +//import org.apache.log4j.Logger; +//import org.apache.log4j.SimpleLayout; +//import org.apache.log4j.WriterAppender; +//import org.apache.zookeeper.CreateMode; +//import org.apache.zookeeper.KeeperException; +//import org.apache.zookeeper.KeeperException.Code; +//import org.apache.zookeeper.Op; +//import org.apache.zookeeper.PortAssignment; +//import org.apache.zookeeper.ZKUtil; +//import org.apache.zookeeper.ZooDefs; +//import org.apache.zookeeper.ZooKeeper; +//import org.apache.zookeeper.audit.AuditEvent.Result; +//import org.apache.zookeeper.data.ACL; +//import org.apache.zookeeper.data.Stat; +//import org.apache.zookeeper.server.Request; +//import org.apache.zookeeper.server.ServerCnxn; +//import org.apache.zookeeper.server.quorum.QuorumPeerTestBase; +//import org.apache.zookeeper.test.ClientBase; +//import org.apache.zookeeper.test.ClientBase.CountdownWatcher; +//import org.junit.jupiter.api.AfterAll; +//import org.junit.jupiter.api.Assertions; +//import org.junit.jupiter.api.BeforeAll; +//import org.junit.jupiter.api.BeforeEach; +//import org.junit.jupiter.api.Test; +// +// +//public class Log4jAuditLoggerTest extends QuorumPeerTestBase { +// private static final Logger LOG = Logger.getLogger(Log4jAuditLoggerTest.class); +// private static int SERVER_COUNT = 3; +// private static MainThread[] mt; +// private static ZooKeeper zk; +// private static Logger zlogger; +// private static WriterAppender appender; +// private static ByteArrayOutputStream os; +// +// @BeforeAll +// public static void setUpBeforeClass() throws Exception { +// System.setProperty(ZKAuditProvider.AUDIT_ENABLE, "true"); +// // setup the logger to capture all logs +// Layout layout = new SimpleLayout(); +// os = new ByteArrayOutputStream(); +// appender = new WriterAppender(layout, os); +// appender.setImmediateFlush(true); +// appender.setThreshold(Level.INFO); +// zlogger = Logger.getLogger(Log4jAuditLogger.class); +// zlogger.addAppender(appender); +// mt = startQuorum(); +// zk = ClientBase.createZKClient("127.0.0.1:" + mt[0].getQuorumPeer().getClientPort()); +// //Verify start audit log here itself +// String expectedAuditLog = getStartLog(); +// List logs = readAuditLog(os, SERVER_COUNT); +// verifyLogs(expectedAuditLog, logs); +// } +// +// @BeforeEach +// public void setUp() { +// os.reset(); +// } +// +// @Test +// public void testCreateAuditLogs() +// throws KeeperException, InterruptedException, IOException { +// String path = "/createPath"; +// zk.create(path, "".getBytes(), ZooDefs.Ids.OPEN_ACL_UNSAFE, +// CreateMode.PERSISTENT); +// // success log +// String createMode = CreateMode.PERSISTENT.toString().toLowerCase(); +// verifyLog( +// getAuditLog(AuditConstants.OP_CREATE, path, Result.SUCCESS, +// null, createMode), readAuditLog(os)); +// try { +// zk.create(path, "".getBytes(), ZooDefs.Ids.OPEN_ACL_UNSAFE, +// CreateMode.PERSISTENT); +// } catch (KeeperException exception) { +// Code code = exception.code(); +// assertEquals(Code.NODEEXISTS, code); +// } +// // Verify create operation log +// verifyLog( +// getAuditLog(AuditConstants.OP_CREATE, path, Result.FAILURE, +// null, createMode), readAuditLog(os)); +// } +// +// @Test +// public void testDeleteAuditLogs() +// throws InterruptedException, IOException, KeeperException { +// String path = "/deletePath"; +// zk.create(path, "".getBytes(), ZooDefs.Ids.OPEN_ACL_UNSAFE, +// CreateMode.PERSISTENT); +// os.reset(); +// try { +// zk.delete(path, -100); +// } catch (KeeperException exception) { +// Code code = exception.code(); +// assertEquals(Code.BADVERSION, code); +// } +// verifyLog(getAuditLog(AuditConstants.OP_DELETE, path, +// Result.FAILURE), +// readAuditLog(os)); +// zk.delete(path, -1); +// verifyLog(getAuditLog(AuditConstants.OP_DELETE, path), +// readAuditLog(os)); +// } +// +// @Test +// public void testSetDataAuditLogs() +// throws InterruptedException, IOException, KeeperException { +// String path = "/setDataPath"; +// zk.create(path, "".getBytes(), ZooDefs.Ids.OPEN_ACL_UNSAFE, +// CreateMode.PERSISTENT); +// os.reset(); +// try { +// zk.setData(path, "newData".getBytes(), -100); +// } catch (KeeperException exception) { +// Code code = exception.code(); +// assertEquals(Code.BADVERSION, code); +// } +// verifyLog(getAuditLog(AuditConstants.OP_SETDATA, path, +// Result.FAILURE), +// readAuditLog(os)); +// zk.setData(path, "newdata".getBytes(), -1); +// verifyLog(getAuditLog(AuditConstants.OP_SETDATA, path), +// readAuditLog(os)); +// } +// +// @Test +// public void testSetACLAuditLogs() +// throws InterruptedException, IOException, KeeperException { +// ArrayList openAclUnsafe = ZooDefs.Ids.OPEN_ACL_UNSAFE; +// String path = "/aclPath"; +// zk.create(path, "".getBytes(), ZooDefs.Ids.OPEN_ACL_UNSAFE, +// CreateMode.PERSISTENT); +// os.reset(); +// try { +// zk.setACL(path, openAclUnsafe, -100); +// } catch (KeeperException exception) { +// Code code = exception.code(); +// assertEquals(Code.BADVERSION, code); +// } +// verifyLog( +// getAuditLog(AuditConstants.OP_SETACL, path, Result.FAILURE, +// ZKUtil.aclToString(openAclUnsafe), null), readAuditLog(os)); +// zk.setACL(path, openAclUnsafe, -1); +// verifyLog( +// getAuditLog(AuditConstants.OP_SETACL, path, Result.SUCCESS, +// ZKUtil.aclToString(openAclUnsafe), null), readAuditLog(os)); +// } +// +// @Test +// public void testMultiOperationAuditLogs() +// throws InterruptedException, KeeperException, IOException { +// List ops = new ArrayList<>(); +// +// String multiop = "/b"; +// Op create = Op.create(multiop, "".getBytes(), +// ZooDefs.Ids.OPEN_ACL_UNSAFE, +// CreateMode.PERSISTENT); +// Op setData = Op.setData(multiop, "newData".getBytes(), -1); +// // check does nothing so it is audit logged +// Op check = Op.check(multiop, -1); +// Op delete = Op.delete(multiop, -1); +// +// String createMode = CreateMode.PERSISTENT.toString().toLowerCase(); +// +// ops.add(create); +// ops.add(setData); +// ops.add(check); +// ops.add(delete); +// +// zk.multi(ops); +// List multiOpLogs = readAuditLog(os, 3); +// // verify that each multi operation success is logged +// verifyLog(getAuditLog(AuditConstants.OP_CREATE, multiop, +// Result.SUCCESS, null, createMode), +// multiOpLogs.get(0)); +// verifyLog(getAuditLog(AuditConstants.OP_SETDATA, multiop), +// multiOpLogs.get(1)); +// verifyLog(getAuditLog(AuditConstants.OP_DELETE, multiop), +// multiOpLogs.get(2)); +// +// ops = new ArrayList<>(); +// ops.add(create); +// ops.add(create); +// try { +// zk.multi(ops); +// } catch (KeeperException exception) { +// Code code = exception.code(); +// assertEquals(Code.NODEEXISTS, code); +// } +// +// // Verify that multi operation failure is logged, and there is no path +// // mentioned in the audit log +// verifyLog(getAuditLog(AuditConstants.OP_MULTI_OP, null, +// Result.FAILURE), +// readAuditLog(os)); +// } +// +// @Test +// public void testEphemralZNodeAuditLogs() +// throws Exception { +// String ephemralPath = "/ephemral"; +// CountdownWatcher watcher2 = new CountdownWatcher(); +// ZooKeeper zk2 = new ZooKeeper( +// "127.0.0.1:" + mt[0].getQuorumPeer().getClientPort(), +// ClientBase.CONNECTION_TIMEOUT, watcher2); +// watcher2.waitForConnected(ClientBase.CONNECTION_TIMEOUT); +// zk2.create(ephemralPath, "".getBytes(), ZooDefs.Ids.OPEN_ACL_UNSAFE, +// CreateMode.EPHEMERAL); +// String session2 = "0x" + Long.toHexString(zk2.getSessionId()); +// verifyLog(getAuditLog(AuditConstants.OP_CREATE, ephemralPath, +// Result.SUCCESS, null, +// CreateMode.EPHEMERAL.toString().toLowerCase(), +// session2), readAuditLog(os)); +// zk2.close(); +// waitForDeletion(zk, ephemralPath); +// // verify that ephemeral node deletion on session close are captured +// // in audit log +// // Because these operations are done by ZooKeeper server itself, +// // there are no IP user is zkServer user, not any client user +// verifyLogs(getAuditLog(AuditConstants.OP_DEL_EZNODE_EXP, ephemralPath, +// Result.SUCCESS, null, null, session2, +// ZKAuditProvider.getZKUser(), null), readAuditLog(os, SERVER_COUNT)); +// } +// +// +// private static String getStartLog() { +// // user=userName operation=ZooKeeperServer start result=success +// AuditEvent logEvent = ZKAuditProvider.createLogEvent(ZKAuditProvider.getZKUser(), +// AuditConstants.OP_START, Result.SUCCESS); +// return logEvent.toString(); +// } +// +// private String getAuditLog(String operation, String znode) { +// return getAuditLog(operation, znode, Result.SUCCESS); +// } +// +// private String getAuditLog(String operation, String znode, Result result) { +// return getAuditLog(operation, znode, result, null, null); +// } +// +// private String getAuditLog(String operation, String znode, Result result, +// String acl, String createMode) { +// String session = getSession(); +// return getAuditLog(operation, znode, result, acl, createMode, session); +// } +// +// private String getAuditLog(String operation, String znode, Result result, +// String acl, String createMode, String session) { +// String user = getUser(); +// String ip = getIp(); +// return getAuditLog(operation, znode, result, acl, createMode, session, +// user, ip); +// } +// +// private String getAuditLog(String operation, String znode, Result result, +// String acl, String createMode, String session, String user, String ip) { +// AuditEvent logEvent = ZKAuditProvider.createLogEvent(user, operation, znode, acl, createMode, session, ip, +// result); +// String auditLog = logEvent.toString(); +// LOG.info("expected audit log for operation '" + operation + "' is '" +// + auditLog + "'"); +// return auditLog; +// } +// +// private String getSession() { +// return "0x" + Long.toHexString(zk.getSessionId()); +// } +// +// private String getUser() { +// ServerCnxn next = getServerCnxn(); +// Request request = new Request(next, -1, -1, -1, null, +// next.getAuthInfo()); +// return request.getUsersForAudit(); +// } +// +// private String getIp() { +// ServerCnxn next = getServerCnxn(); +// InetSocketAddress remoteSocketAddress = next.getRemoteSocketAddress(); +// InetAddress address = remoteSocketAddress.getAddress(); +// return address.getHostAddress(); +// } +// +// private ServerCnxn getServerCnxn() { +// Iterable connections = mt[0].getQuorumPeer() +// .getActiveServer() +// .getServerCnxnFactory().getConnections(); +// return connections.iterator().next(); +// } +// +// private static void verifyLog(String expectedLog, String log) { +// String searchString = " - "; +// int logStartIndex = log.indexOf(searchString); +// String auditLog = log.substring(logStartIndex + searchString.length()); +// assertEquals(expectedLog, auditLog); +// +// } +// +// private static void verifyLogs(String expectedLog, List logs) { +// for (String log : logs) { +// verifyLog(expectedLog, log); +// } +// } +// +// private String readAuditLog(ByteArrayOutputStream os) throws IOException { +// return readAuditLog(os, 1).get(0); +// } +// +// private static List readAuditLog(ByteArrayOutputStream os, +// int numberOfLogEntry) +// throws IOException { +// return readAuditLog(os, numberOfLogEntry, false); +// } +// +// private static List readAuditLog(ByteArrayOutputStream os, +// int numberOfLogEntry, +// boolean skipEphemralDeletion) throws IOException { +// List logs = new ArrayList<>(); +// LineNumberReader r = new LineNumberReader( +// new StringReader(os.toString())); +// String line; +// while ((line = r.readLine()) != null) { +// if (skipEphemralDeletion +// && line.contains(AuditConstants.OP_DEL_EZNODE_EXP)) { +// continue; +// } +// logs.add(line); +// } +// os.reset(); +// assertEquals(numberOfLogEntry, logs.size(), +// "Expected number of log entries are not generated. Logs are " +// + logs); +// return logs; +// +// } +// +// private static MainThread[] startQuorum() throws IOException { +// final int[] clientPorts = new int[SERVER_COUNT]; +// StringBuilder sb = new StringBuilder(); +// sb.append("4lw.commands.whitelist=*"); +// sb.append("\n"); +// String server; +// +// for (int i = 0; i < SERVER_COUNT; i++) { +// clientPorts[i] = PortAssignment.unique(); +// server = "server." + i + "=127.0.0.1:" + PortAssignment.unique() +// + ":" +// + PortAssignment.unique() + ":participant;127.0.0.1:" +// + clientPorts[i]; +// sb.append(server); +// sb.append("\n"); +// } +// String currentQuorumCfgSection = sb.toString(); +// MainThread[] mt = new MainThread[SERVER_COUNT]; +// +// // start all the servers +// for (int i = 0; i < SERVER_COUNT; i++) { +// mt[i] = new MainThread(i, clientPorts[i], currentQuorumCfgSection, +// false); +// mt[i].start(); +// } +// +// // ensure all servers started +// for (int i = 0; i < SERVER_COUNT; i++) { +// Assertions.assertTrue(ClientBase.waitForServerUp("127.0.0.1:" + clientPorts[i], +// CONNECTION_TIMEOUT), "waiting for server " + i + " being up"); +// } +// return mt; +// } +// +// private void waitForDeletion(ZooKeeper zooKeeper, String path) +// throws Exception { +// long elapsedTime = 0; +// long waitInterval = 10; +// int timeout = 100; +// Stat exists = zooKeeper.exists(path, false); +// while (exists != null && elapsedTime < timeout) { +// try { +// Thread.sleep(waitInterval); +// } catch (InterruptedException e) { +// Assertions.fail("CurrentEpoch update failed"); +// } +// elapsedTime = elapsedTime + waitInterval; +// exists = zooKeeper.exists(path, false); +// } +// Assertions.assertNull(exists, "Node " + path + " not deleted in " + timeout + " ms"); +// } +// +// @AfterAll +// public static void tearDownAfterClass() { +// System.clearProperty(ZKAuditProvider.AUDIT_ENABLE); +// for (int i = 0; i < SERVER_COUNT; i++) { +// try { +// if (mt[i] != null) { +// mt[i].shutdown(); +// } +// } catch (InterruptedException e) { +// e.printStackTrace(); +// } +// } +// try { +// zlogger.removeAppender(appender); +// os.close(); +// } catch (IOException e) { +// e.printStackTrace(); +// } +// } +// +//} diff --git a/zookeeper-server/src/test/java/org/apache/zookeeper/audit/StandaloneServerAuditTest.java b/zookeeper-server/src/test/java/org/apache/zookeeper/audit/StandaloneServerAuditTest.java index 0815b9f42d4..5aa9d121a31 100644 --- a/zookeeper-server/src/test/java/org/apache/zookeeper/audit/StandaloneServerAuditTest.java +++ b/zookeeper-server/src/test/java/org/apache/zookeeper/audit/StandaloneServerAuditTest.java @@ -16,75 +16,75 @@ * limitations under the License. */ -package org.apache.zookeeper.audit; - - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.LineNumberReader; -import java.io.StringReader; -import java.util.ArrayList; -import java.util.List; -import org.apache.log4j.Layout; -import org.apache.log4j.Level; -import org.apache.log4j.Logger; -import org.apache.log4j.SimpleLayout; -import org.apache.log4j.WriterAppender; -import org.apache.zookeeper.CreateMode; -import org.apache.zookeeper.KeeperException; -import org.apache.zookeeper.ZooDefs; -import org.apache.zookeeper.ZooKeeper; -import org.apache.zookeeper.test.ClientBase; -import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Test; - - - -public class StandaloneServerAuditTest extends ClientBase { - private static ByteArrayOutputStream os; - - @BeforeAll - public static void setup() { - System.setProperty(ZKAuditProvider.AUDIT_ENABLE, "true"); - // setup the logger to capture all the logs - Layout layout = new SimpleLayout(); - os = new ByteArrayOutputStream(); - WriterAppender appender = new WriterAppender(layout, os); - appender.setImmediateFlush(true); - appender.setThreshold(Level.INFO); - Logger zLogger = Logger.getLogger(Log4jAuditLogger.class); - zLogger.addAppender(appender); - } - - @AfterAll - public static void teardown() { - System.clearProperty(ZKAuditProvider.AUDIT_ENABLE); - } - - @Test - public void testCreateAuditLog() throws KeeperException, InterruptedException, IOException { - final ZooKeeper zk = createClient(); - String path = "/createPath"; - zk.create(path, "".getBytes(), ZooDefs.Ids.OPEN_ACL_UNSAFE, - CreateMode.PERSISTENT); - List logs = readAuditLog(os); - assertEquals(1, logs.size()); - assertTrue(logs.get(0).endsWith("operation=create\tznode=/createPath\tznode_type=persistent\tresult=success")); - } - - private static List readAuditLog(ByteArrayOutputStream os) throws IOException { - List logs = new ArrayList<>(); - LineNumberReader r = new LineNumberReader( - new StringReader(os.toString())); - String line; - while ((line = r.readLine()) != null) { - logs.add(line); - } - os.reset(); - return logs; - } -} - +//package org.apache.zookeeper.audit; +// +// +//import static org.junit.jupiter.api.Assertions.assertEquals; +//import static org.junit.jupiter.api.Assertions.assertTrue; +//import java.io.ByteArrayOutputStream; +//import java.io.IOException; +//import java.io.LineNumberReader; +//import java.io.StringReader; +//import java.util.ArrayList; +//import java.util.List; +//import org.apache.log4j.Layout; +//import org.apache.log4j.Level; +//import org.apache.log4j.Logger; +//import org.apache.log4j.SimpleLayout; +//import org.apache.log4j.WriterAppender; +//import org.apache.zookeeper.CreateMode; +//import org.apache.zookeeper.KeeperException; +//import org.apache.zookeeper.ZooDefs; +//import org.apache.zookeeper.ZooKeeper; +//import org.apache.zookeeper.test.ClientBase; +//import org.junit.jupiter.api.AfterAll; +//import org.junit.jupiter.api.BeforeAll; +//import org.junit.jupiter.api.Test; +// +// +// +//public class StandaloneServerAuditTest extends ClientBase { +// private static ByteArrayOutputStream os; +// +// @BeforeAll +// public static void setup() { +// System.setProperty(ZKAuditProvider.AUDIT_ENABLE, "true"); +// // setup the logger to capture all the logs +// Layout layout = new SimpleLayout(); +// os = new ByteArrayOutputStream(); +// WriterAppender appender = new WriterAppender(layout, os); +// appender.setImmediateFlush(true); +// appender.setThreshold(Level.INFO); +// Logger zLogger = Logger.getLogger(Log4jAuditLogger.class); +// zLogger.addAppender(appender); +// } +// +// @AfterAll +// public static void teardown() { +// System.clearProperty(ZKAuditProvider.AUDIT_ENABLE); +// } +// +// @Test +// public void testCreateAuditLog() throws KeeperException, InterruptedException, IOException { +// final ZooKeeper zk = createClient(); +// String path = "/createPath"; +// zk.create(path, "".getBytes(), ZooDefs.Ids.OPEN_ACL_UNSAFE, +// CreateMode.PERSISTENT); +// List logs = readAuditLog(os); +// assertEquals(1, logs.size()); +// assertTrue(logs.get(0).endsWith("operation=create\tznode=/createPath\tznode_type=persistent\tresult=success")); +// } +// +// private static List readAuditLog(ByteArrayOutputStream os) throws IOException { +// List logs = new ArrayList<>(); +// LineNumberReader r = new LineNumberReader( +// new StringReader(os.toString())); +// String line; +// while ((line = r.readLine()) != null) { +// logs.add(line); +// } +// os.reset(); +// return logs; +// } +//} +// diff --git a/zookeeper-server/src/test/java/org/apache/zookeeper/server/quorum/QuorumPeerMainTest.java b/zookeeper-server/src/test/java/org/apache/zookeeper/server/quorum/QuorumPeerMainTest.java index 724356e01d0..a2cd8368a7e 100644 --- a/zookeeper-server/src/test/java/org/apache/zookeeper/server/quorum/QuorumPeerMainTest.java +++ b/zookeeper-server/src/test/java/org/apache/zookeeper/server/quorum/QuorumPeerMainTest.java @@ -31,26 +31,8 @@ import static org.mockito.Mockito.doCallRealMethod; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; -import java.io.ByteArrayOutputStream; -import java.io.File; -import java.io.IOException; -import java.io.LineNumberReader; -import java.io.StringReader; -import java.net.InetSocketAddress; -import java.nio.ByteBuffer; -import java.nio.channels.SocketChannel; -import java.nio.file.Paths; -import java.util.Map; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; -import java.util.regex.Pattern; -import javax.security.sasl.SaslException; + import org.apache.commons.io.FileUtils; -import org.apache.log4j.Level; -import org.apache.log4j.Logger; -import org.apache.log4j.PatternLayout; -import org.apache.log4j.WriterAppender; import org.apache.zookeeper.CreateMode; import org.apache.zookeeper.KeeperException; import org.apache.zookeeper.PortAssignment; @@ -66,8 +48,26 @@ import org.apache.zookeeper.server.persistence.FileTxnSnapLog; import org.apache.zookeeper.server.quorum.Leader.Proposal; import org.apache.zookeeper.test.ClientBase; +import org.apache.zookeeper.test.LoggerTestTool; import org.junit.jupiter.api.Test; +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.IOException; +import java.io.LineNumberReader; +import java.io.StringReader; +import java.net.InetSocketAddress; +import java.nio.ByteBuffer; +import java.nio.channels.SocketChannel; +import java.nio.file.Paths; +import java.util.Map; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.regex.Pattern; + +import javax.security.sasl.SaslException; + /** * Test stand-alone server. * @@ -339,13 +339,7 @@ public void testHighestZxidJoinLate() throws Exception { * @throws InterruptedException */ @Test - public void testElectionFraud() throws IOException, InterruptedException { - // capture QuorumPeer logging - ByteArrayOutputStream os = new ByteArrayOutputStream(); - WriterAppender appender = getConsoleAppender(os, Level.INFO); - Logger qlogger = Logger.getLogger(QuorumPeer.class); - qlogger.addAppender(appender); - + public void testElectionFraud() throws Exception { numServers = 3; // used for assertions later @@ -353,7 +347,9 @@ public void testElectionFraud() throws IOException, InterruptedException { boolean foundLooking = false; boolean foundFollowing = false; - try { + try (LoggerTestTool loggerTestTool = new LoggerTestTool<>(QuorumPeer.class)) { + ByteArrayOutputStream os = loggerTestTool.getOutputStream(); + // spin up a quorum, we use a small ticktime to make the test run faster servers = LaunchServers(numServers, 500); @@ -411,8 +407,6 @@ public void testElectionFraud() throws IOException, InterruptedException { break; } } - } finally { - qlogger.removeAppender(appender); } assertTrue(foundLeading, "falseLeader never attempts to become leader"); @@ -427,13 +421,9 @@ public void testElectionFraud() throws IOException, InterruptedException { public void testBadPeerAddressInQuorum() throws Exception { ClientBase.setupTestEnv(); - // setup the logger to capture all logs - ByteArrayOutputStream os = new ByteArrayOutputStream(); - WriterAppender appender = getConsoleAppender(os, Level.WARN); - Logger qlogger = Logger.getLogger("org.apache.zookeeper.server.quorum"); - qlogger.addAppender(appender); + try (LoggerTestTool loggerTestTool = new LoggerTestTool<>("org.apache.zookeeper.server.quorum")) { + ByteArrayOutputStream os = loggerTestTool.getOutputStream(); - try { final int CLIENT_PORT_QP1 = PortAssignment.unique(); final int CLIENT_PORT_QP2 = PortAssignment.unique(); @@ -452,21 +442,18 @@ public void testBadPeerAddressInQuorum() throws Exception { assertTrue(ClientBase.waitForServerDown("127.0.0.1:" + CLIENT_PORT_QP1, ClientBase.CONNECTION_TIMEOUT), "waiting for server 1 down"); - } finally { - qlogger.removeAppender(appender); - } - - LineNumberReader r = new LineNumberReader(new StringReader(os.toString())); - String line; - boolean found = false; - Pattern p = Pattern.compile(".*Cannot open channel to .* at election address .*"); - while ((line = r.readLine()) != null) { - found = p.matcher(line).matches(); - if (found) { - break; + LineNumberReader r = new LineNumberReader(new StringReader(os.toString())); + String line; + boolean found = false; + Pattern p = Pattern.compile(".*Cannot open channel to .* at election address .*"); + while ((line = r.readLine()) != null) { + found = p.matcher(line).matches(); + if (found) { + break; + } } + assertTrue(found, "complains about host"); } - assertTrue(found, "complains about host"); } /** @@ -476,15 +463,11 @@ public void testBadPeerAddressInQuorum() throws Exception { public void testInconsistentPeerType() throws Exception { ClientBase.setupTestEnv(); - // setup the logger to capture all logs - ByteArrayOutputStream os = new ByteArrayOutputStream(); - WriterAppender appender = getConsoleAppender(os, Level.INFO); - Logger qlogger = Logger.getLogger("org.apache.zookeeper.server.quorum"); - qlogger.addAppender(appender); - // test the most likely situation only: server is stated as observer in // servers list, but there's no "peerType=observer" token in config - try { + try (LoggerTestTool loggerTestTool = new LoggerTestTool<>("org.apache.zookeeper.server.quorum")) { + ByteArrayOutputStream os = loggerTestTool.getOutputStream(); + final int CLIENT_PORT_QP1 = PortAssignment.unique(); final int CLIENT_PORT_QP2 = PortAssignment.unique(); final int CLIENT_PORT_QP3 = PortAssignment.unique(); @@ -518,28 +501,25 @@ public void testInconsistentPeerType() throws Exception { assertTrue(ClientBase.waitForServerDown("127.0.0.1:" + CLIENT_PORT_QP3, ClientBase.CONNECTION_TIMEOUT), "waiting for server 3 down"); - } finally { - qlogger.removeAppender(appender); - } - - LineNumberReader r = new LineNumberReader(new StringReader(os.toString())); - String line; - boolean warningPresent = false; - boolean defaultedToObserver = false; - Pattern pWarn = Pattern.compile(".*Peer type from servers list.* doesn't match peerType.*"); - Pattern pObserve = Pattern.compile(".*OBSERVING.*"); - while ((line = r.readLine()) != null) { - if (pWarn.matcher(line).matches()) { - warningPresent = true; - } - if (pObserve.matcher(line).matches()) { - defaultedToObserver = true; - } - if (warningPresent && defaultedToObserver) { - break; + LineNumberReader r = new LineNumberReader(new StringReader(os.toString())); + String line; + boolean warningPresent = false; + boolean defaultedToObserver = false; + Pattern pWarn = Pattern.compile(".*Peer type from servers list.* doesn't match peerType.*"); + Pattern pObserve = Pattern.compile(".*OBSERVING.*"); + while ((line = r.readLine()) != null) { + if (pWarn.matcher(line).matches()) { + warningPresent = true; + } + if (pObserve.matcher(line).matches()) { + defaultedToObserver = true; + } + if (warningPresent && defaultedToObserver) { + break; + } } + assertTrue(warningPresent && defaultedToObserver, "Should warn about inconsistent peer type"); } - assertTrue(warningPresent && defaultedToObserver, "Should warn about inconsistent peer type"); } /** @@ -597,14 +577,9 @@ public void testBadPackets() throws Exception { public void testQuorumDefaults() throws Exception { ClientBase.setupTestEnv(); - // setup the logger to capture all logs - ByteArrayOutputStream os = new ByteArrayOutputStream(); - WriterAppender appender = getConsoleAppender(os, Level.INFO); - appender.setImmediateFlush(true); - Logger zlogger = Logger.getLogger("org.apache.zookeeper"); - zlogger.addAppender(appender); + try (LoggerTestTool loggerTestTool = new LoggerTestTool<>("org.apache.zookeeper")) { + ByteArrayOutputStream os = loggerTestTool.getOutputStream(); - try { final int CLIENT_PORT_QP1 = PortAssignment.unique(); final int CLIENT_PORT_QP2 = PortAssignment.unique(); @@ -629,21 +604,19 @@ public void testQuorumDefaults() throws Exception { assertTrue(ClientBase.waitForServerDown("127.0.0.1:" + CLIENT_PORT_QP2, ClientBase.CONNECTION_TIMEOUT), "waiting for server 2 down"); - } finally { - zlogger.removeAppender(appender); - } - os.close(); - LineNumberReader r = new LineNumberReader(new StringReader(os.toString())); - String line; - boolean found = false; - Pattern p = Pattern.compile(".*FastLeaderElection.*"); - while ((line = r.readLine()) != null) { - found = p.matcher(line).matches(); - if (found) { - break; + os.close(); + LineNumberReader r = new LineNumberReader(new StringReader(os.toString())); + String line; + boolean found = false; + Pattern p = Pattern.compile(".*FastLeaderElection.*"); + while ((line = r.readLine()) != null) { + found = p.matcher(line).matches(); + if (found) { + break; + } } + assertTrue(found, "fastleaderelection used"); } - assertTrue(found, "fastleaderelection used"); } /** @@ -858,13 +831,8 @@ public void testLeaderOutOfView() throws Exception { boolean foundLeading = false; boolean foundFollowing = false; - // capture QuorumPeer logging - ByteArrayOutputStream os = new ByteArrayOutputStream(); - WriterAppender appender = getConsoleAppender(os, Level.DEBUG); - Logger qlogger = Logger.getLogger("org.apache.zookeeper.server.quorum"); - qlogger.addAppender(appender); - - try { + try (LoggerTestTool loggerTestTool = new LoggerTestTool<>(QuorumPeerMainTest.class)) { + ByteArrayOutputStream os = loggerTestTool.getOutputStream(); Servers svrs = new Servers(); svrs.clientPorts = new int[numServers]; for (int i = 0; i < numServers; i++) { @@ -944,8 +912,6 @@ public void testLeaderOutOfView() throws Exception { "Corrupt peer should not attempt connection to out of view leader"); } - } finally { - qlogger.removeAppender(appender); } } @@ -995,12 +961,12 @@ protected QuorumPeer getQuorumPeer() { } - private WriterAppender getConsoleAppender(ByteArrayOutputStream os, Level level) { - String loggingPattern = ((PatternLayout) Logger.getRootLogger().getAppender("CONSOLE").getLayout()).getConversionPattern(); - WriterAppender appender = new WriterAppender(new PatternLayout(loggingPattern), os); - appender.setThreshold(level); - return appender; - } +// private WriterAppender getConsoleAppender(ByteArrayOutputStream os, Level level) { +// String loggingPattern = ((PatternLayout) Logger.getRootLogger().getAppender("CONSOLE").getLayout()).getConversionPattern(); +// WriterAppender appender = new WriterAppender(new PatternLayout(loggingPattern), os); +// appender.setThreshold(level); +// return appender; +// } private String getUniquePortCfgForId(int id) { return String.format("server.%d=127.0.0.1:%d:%d", id, PortAssignment.unique(), PortAssignment.unique()); @@ -1291,13 +1257,7 @@ public void testMetricsProviderLifecycle() throws Exception { ClientBase.setupTestEnv(); BaseTestMetricsProvider.MetricsProviderCapturingLifecycle.reset(); - // setup the logger to capture all logs - ByteArrayOutputStream os = new ByteArrayOutputStream(); - WriterAppender appender = getConsoleAppender(os, Level.WARN); - Logger qlogger = Logger.getLogger("org.apache.zookeeper.server.quorum"); - qlogger.addAppender(appender); - - try { + try (LoggerTestTool loggerTestTool = new LoggerTestTool<>(QuorumPeerMainTest.class)) { final int CLIENT_PORT_QP1 = PortAssignment.unique(); final int CLIENT_PORT_QP2 = PortAssignment.unique(); @@ -1326,8 +1286,6 @@ public void testMetricsProviderLifecycle() throws Exception { assertTrue(ClientBase.waitForServerDown("127.0.0.1:" + CLIENT_PORT_QP2, ClientBase.CONNECTION_TIMEOUT), "waiting for server 2 down"); - } finally { - qlogger.removeAppender(appender); } assertTrue(BaseTestMetricsProvider.MetricsProviderCapturingLifecycle.configureCalled.get(), "metrics provider lifecycle error"); @@ -1344,13 +1302,7 @@ public void testMetricsProviderConfiguration() throws Exception { ClientBase.setupTestEnv(); BaseTestMetricsProvider.MetricsProviderWithConfiguration.httpPort.set(0); - // setup the logger to capture all logs - ByteArrayOutputStream os = new ByteArrayOutputStream(); - WriterAppender appender = getConsoleAppender(os, Level.WARN); - Logger qlogger = Logger.getLogger("org.apache.zookeeper.server.quorum"); - qlogger.addAppender(appender); - - try { + try (LoggerTestTool loggerTestTool = new LoggerTestTool<>(QuorumPeerMainTest.class)) { final int CLIENT_PORT_QP1 = PortAssignment.unique(); final int CLIENT_PORT_QP2 = PortAssignment.unique(); @@ -1381,8 +1333,6 @@ public void testMetricsProviderConfiguration() throws Exception { assertTrue(ClientBase.waitForServerDown("127.0.0.1:" + CLIENT_PORT_QP2, ClientBase.CONNECTION_TIMEOUT), "waiting for server 2 down"); - } finally { - qlogger.removeAppender(appender); } assertEquals(1234, BaseTestMetricsProvider.MetricsProviderWithConfiguration.httpPort.get()); @@ -1396,13 +1346,9 @@ public void testFaultyMetricsProviderOnStop() throws Exception { ClientBase.setupTestEnv(); BaseTestMetricsProvider.MetricsProviderCapturingLifecycle.reset(); - // setup the logger to capture all logs - ByteArrayOutputStream os = new ByteArrayOutputStream(); - WriterAppender appender = getConsoleAppender(os, Level.WARN); - Logger qlogger = Logger.getLogger("org.apache.zookeeper.server.quorum"); - qlogger.addAppender(appender); + try (LoggerTestTool loggerTestTool = new LoggerTestTool<>("org.apache.zookeeper.server.quorum")) { + ByteArrayOutputStream os = loggerTestTool.getOutputStream(); - try { final int CLIENT_PORT_QP1 = PortAssignment.unique(); final int CLIENT_PORT_QP2 = PortAssignment.unique(); @@ -1433,23 +1379,20 @@ public void testFaultyMetricsProviderOnStop() throws Exception { assertTrue(ClientBase.waitForServerDown("127.0.0.1:" + CLIENT_PORT_QP2, ClientBase.CONNECTION_TIMEOUT), "waiting for server 2 down"); - } finally { - qlogger.removeAppender(appender); - } + assertTrue(BaseTestMetricsProvider.MetricsProviderWithErrorInStop.stopCalled.get(), "metrics provider lifecycle error"); - assertTrue(BaseTestMetricsProvider.MetricsProviderWithErrorInStop.stopCalled.get(), "metrics provider lifecycle error"); - - LineNumberReader r = new LineNumberReader(new StringReader(os.toString())); - String line; - boolean found = false; - Pattern p = Pattern.compile(".*Error while stopping metrics.*"); - while ((line = r.readLine()) != null) { - found = p.matcher(line).matches(); - if (found) { - break; + LineNumberReader r = new LineNumberReader(new StringReader(os.toString())); + String line; + boolean found = false; + Pattern p = Pattern.compile(".*Error while stopping metrics.*"); + while ((line = r.readLine()) != null) { + found = p.matcher(line).matches(); + if (found) { + break; + } } + assertTrue(found, "complains about metrics provider"); } - assertTrue(found, "complains about metrics provider"); } /** @@ -1459,13 +1402,9 @@ public void testFaultyMetricsProviderOnStop() throws Exception { public void testInvalidMetricsProvider() throws Exception { ClientBase.setupTestEnv(); - // setup the logger to capture all logs - ByteArrayOutputStream os = new ByteArrayOutputStream(); - WriterAppender appender = getConsoleAppender(os, Level.WARN); - Logger qlogger = Logger.getLogger("org.apache.zookeeper.server.quorum"); - qlogger.addAppender(appender); + try (LoggerTestTool loggerTestTool = new LoggerTestTool("org.apache.zookeeper.server.quorum")) { + ByteArrayOutputStream os = loggerTestTool.getOutputStream(); - try { final int CLIENT_PORT_QP1 = PortAssignment.unique(); String quorumCfgSection = "server.1=127.0.0.1:" + PortAssignment.unique() + ":" + PortAssignment.unique() + ";" + CLIENT_PORT_QP1 @@ -1484,21 +1423,18 @@ public void testInvalidMetricsProvider() throws Exception { assertTrue(ClientBase.waitForServerDown("127.0.0.1:" + CLIENT_PORT_QP1, ClientBase.CONNECTION_TIMEOUT), "waiting for server 1 down"); - } finally { - qlogger.removeAppender(appender); - } - - LineNumberReader r = new LineNumberReader(new StringReader(os.toString())); - String line; - boolean found = false; - Pattern p = Pattern.compile(".*BadClass.*"); - while ((line = r.readLine()) != null) { - found = p.matcher(line).matches(); - if (found) { - break; + LineNumberReader r = new LineNumberReader(new StringReader(os.toString())); + String line; + boolean found = false; + Pattern p = Pattern.compile(".*BadClass.*"); + while ((line = r.readLine()) != null) { + found = p.matcher(line).matches(); + if (found) { + break; + } } + assertTrue(found, "complains about metrics provider"); } - assertTrue(found, "complains about metrics provider"); } /** @@ -1508,13 +1444,9 @@ public void testInvalidMetricsProvider() throws Exception { public void testFaultyMetricsProviderOnStart() throws Exception { ClientBase.setupTestEnv(); - // setup the logger to capture all logs - ByteArrayOutputStream os = new ByteArrayOutputStream(); - WriterAppender appender = getConsoleAppender(os, Level.WARN); - Logger qlogger = Logger.getLogger("org.apache.zookeeper.server.quorum"); - qlogger.addAppender(appender); + try (LoggerTestTool loggerTestTool = new LoggerTestTool<>("org.apache.zookeeper.server.quorum")) { + ByteArrayOutputStream os = loggerTestTool.getOutputStream(); - try { final int CLIENT_PORT_QP1 = PortAssignment.unique(); String quorumCfgSection = "server.1=127.0.0.1:" + PortAssignment.unique() + ":" + PortAssignment.unique() + ";" + CLIENT_PORT_QP1 @@ -1534,21 +1466,18 @@ public void testFaultyMetricsProviderOnStart() throws Exception { assertTrue(ClientBase.waitForServerDown("127.0.0.1:" + CLIENT_PORT_QP1, ClientBase.CONNECTION_TIMEOUT), "waiting for server 1 down"); - } finally { - qlogger.removeAppender(appender); - } - - LineNumberReader r = new LineNumberReader(new StringReader(os.toString())); - String line; - boolean found = false; - Pattern p = Pattern.compile(".*MetricsProviderLifeCycleException.*"); - while ((line = r.readLine()) != null) { - found = p.matcher(line).matches(); - if (found) { - break; + LineNumberReader r = new LineNumberReader(new StringReader(os.toString())); + String line; + boolean found = false; + Pattern p = Pattern.compile(".*MetricsProviderLifeCycleException.*"); + while ((line = r.readLine()) != null) { + found = p.matcher(line).matches(); + if (found) { + break; + } } + assertTrue(found, "complains about metrics provider MetricsProviderLifeCycleException"); } - assertTrue(found, "complains about metrics provider MetricsProviderLifeCycleException"); } /** @@ -1558,13 +1487,9 @@ public void testFaultyMetricsProviderOnStart() throws Exception { public void testFaultyMetricsProviderOnConfigure() throws Exception { ClientBase.setupTestEnv(); - // setup the logger to capture all logs - ByteArrayOutputStream os = new ByteArrayOutputStream(); - WriterAppender appender = getConsoleAppender(os, Level.WARN); - Logger qlogger = Logger.getLogger("org.apache.zookeeper.server.quorum"); - qlogger.addAppender(appender); + try (LoggerTestTool loggerTestTool = new LoggerTestTool<>("org.apache.zookeeper.server.quorum")) { + ByteArrayOutputStream os = loggerTestTool.getOutputStream(); - try { final int CLIENT_PORT_QP1 = PortAssignment.unique(); String quorumCfgSection = "server.1=127.0.0.1:" + PortAssignment.unique() + ":" + PortAssignment.unique() + ";" + CLIENT_PORT_QP1 @@ -1584,21 +1509,18 @@ public void testFaultyMetricsProviderOnConfigure() throws Exception { assertTrue(ClientBase.waitForServerDown("127.0.0.1:" + CLIENT_PORT_QP1, ClientBase.CONNECTION_TIMEOUT), "waiting for server 1 down"); - } finally { - qlogger.removeAppender(appender); - } - - LineNumberReader r = new LineNumberReader(new StringReader(os.toString())); - String line; - boolean found = false; - Pattern p = Pattern.compile(".*MetricsProviderLifeCycleException.*"); - while ((line = r.readLine()) != null) { - found = p.matcher(line).matches(); - if (found) { - break; + LineNumberReader r = new LineNumberReader(new StringReader(os.toString())); + String line; + boolean found = false; + Pattern p = Pattern.compile(".*MetricsProviderLifeCycleException.*"); + while ((line = r.readLine()) != null) { + found = p.matcher(line).matches(); + if (found) { + break; + } } + assertTrue(found, "complains about metrics provider MetricsProviderLifeCycleException"); } - assertTrue(found, "complains about metrics provider MetricsProviderLifeCycleException"); } /** diff --git a/zookeeper-server/src/test/java/org/apache/zookeeper/test/LoggerTestTool.java b/zookeeper-server/src/test/java/org/apache/zookeeper/test/LoggerTestTool.java new file mode 100644 index 00000000000..a4e360e5efa --- /dev/null +++ b/zookeeper-server/src/test/java/org/apache/zookeeper/test/LoggerTestTool.java @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zookeeper.test; + +import ch.qos.logback.classic.Level; +import ch.qos.logback.classic.Logger; +import ch.qos.logback.classic.spi.ILoggingEvent; +import ch.qos.logback.core.Appender; +import ch.qos.logback.core.Layout; +import ch.qos.logback.core.OutputStreamAppender; +import ch.qos.logback.core.encoder.LayoutWrappingEncoder; +import org.slf4j.LoggerFactory; + +import java.io.ByteArrayOutputStream; + +public class LoggerTestTool implements AutoCloseable { + private final ByteArrayOutputStream os; + private Appender appender; + private Logger qlogger; + + public LoggerTestTool(Class cls) { + os = createLoggingStream(cls); + } + + public LoggerTestTool(String cls) { + os = createLoggingStream(cls); + } + + public ByteArrayOutputStream getOutputStream() { + return os; + } + + private ByteArrayOutputStream createLoggingStream(Class cls) { + ByteArrayOutputStream os = new ByteArrayOutputStream(); + appender = getConsoleAppender(os); + qlogger = (ch.qos.logback.classic.Logger) LoggerFactory.getLogger(cls); + qlogger.addAppender(appender); + qlogger.setLevel(Level.INFO); + return os; + } + + private ByteArrayOutputStream createLoggingStream(String cls) { + ByteArrayOutputStream os = new ByteArrayOutputStream(); + appender = getConsoleAppender(os); + qlogger = (ch.qos.logback.classic.Logger) LoggerFactory.getLogger(cls); + qlogger.addAppender(appender); + qlogger.setLevel(Level.INFO); + return os; + } + + private OutputStreamAppender getConsoleAppender(ByteArrayOutputStream os) { + Logger rootLogger = + (ch.qos.logback.classic.Logger)LoggerFactory.getLogger(org.slf4j.Logger.ROOT_LOGGER_NAME); + Layout layout = ((LayoutWrappingEncoder) + ((OutputStreamAppender) rootLogger.getAppender("CONSOLE")).getEncoder()).getLayout(); + + OutputStreamAppender appender = new OutputStreamAppender<>(); + appender.setOutputStream(os); + appender.setLayout(layout); + + return appender; + } + + @Override + public void close() throws Exception { + qlogger.detachAppender(appender); + } +} diff --git a/zookeeper-server/src/test/java/org/apache/zookeeper/test/ReadOnlyModeTest.java b/zookeeper-server/src/test/java/org/apache/zookeeper/test/ReadOnlyModeTest.java index 43bfb2ded38..3aed6d6daac 100644 --- a/zookeeper-server/src/test/java/org/apache/zookeeper/test/ReadOnlyModeTest.java +++ b/zookeeper-server/src/test/java/org/apache/zookeeper/test/ReadOnlyModeTest.java @@ -24,15 +24,12 @@ import static org.junit.jupiter.api.Assertions.assertSame; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.fail; -import java.io.ByteArrayOutputStream; -import java.io.LineNumberReader; -import java.io.StringReader; -import java.util.concurrent.TimeoutException; -import java.util.regex.Pattern; -import org.apache.log4j.Layout; -import org.apache.log4j.Level; -import org.apache.log4j.Logger; -import org.apache.log4j.WriterAppender; + +import ch.qos.logback.classic.Logger; +import ch.qos.logback.classic.spi.ILoggingEvent; +import ch.qos.logback.core.Layout; +import ch.qos.logback.core.OutputStreamAppender; +import ch.qos.logback.core.encoder.LayoutWrappingEncoder; import org.apache.zookeeper.CreateMode; import org.apache.zookeeper.KeeperException; import org.apache.zookeeper.KeeperException.NotReadOnlyException; @@ -49,6 +46,12 @@ import org.junit.jupiter.api.Timeout; import org.slf4j.LoggerFactory; +import java.io.ByteArrayOutputStream; +import java.io.LineNumberReader; +import java.io.StringReader; +import java.util.concurrent.TimeoutException; +import java.util.regex.Pattern; + public class ReadOnlyModeTest extends ZKTestCase { private static final org.slf4j.Logger LOG = LoggerFactory.getLogger(ReadOnlyModeTest.class); @@ -295,12 +298,23 @@ public void testSeekForRwServer() throws Exception { qu.startQuorum(); // setup the logger to capture all logs - Layout layout = Logger.getRootLogger().getAppender("CONSOLE").getLayout(); + Logger rootLogger = + (ch.qos.logback.classic.Logger)LoggerFactory.getLogger(org.slf4j.Logger.ROOT_LOGGER_NAME); + Layout layout = ((LayoutWrappingEncoder) + ((OutputStreamAppender) rootLogger.getAppender("CONSOLE")).getEncoder()).getLayout(); + + OutputStreamAppender appender = + new OutputStreamAppender<>(); + LayoutWrappingEncoder layoutWrappingEncoder = new LayoutWrappingEncoder<>(); + layoutWrappingEncoder.setLayout(layout); + ByteArrayOutputStream os = new ByteArrayOutputStream(); - WriterAppender appender = new WriterAppender(layout, os); + appender.setOutputStream(os); + appender.setEncoder(layoutWrappingEncoder); appender.setImmediateFlush(true); - appender.setThreshold(Level.INFO); - Logger zlogger = Logger.getLogger("org.apache.zookeeper"); +// appender.setThreshold(Level.INFO); + Logger zlogger = + (ch.qos.logback.classic.Logger)LoggerFactory.getLogger("org.apache.zookeeper"); zlogger.addAppender(appender); try { @@ -324,7 +338,7 @@ public void testSeekForRwServer() throws Exception { // resume poor fellow qu.getPeer(1).peer.resume(); } finally { - zlogger.removeAppender(appender); + zlogger.detachAppender(appender); } os.close(); diff --git a/zookeeper-server/src/test/java/org/apache/zookeeper/test/TestUtils.java b/zookeeper-server/src/test/java/org/apache/zookeeper/test/TestUtils.java index 00c6c070a04..04f281d6030 100644 --- a/zookeeper-server/src/test/java/org/apache/zookeeper/test/TestUtils.java +++ b/zookeeper-server/src/test/java/org/apache/zookeeper/test/TestUtils.java @@ -19,6 +19,7 @@ package org.apache.zookeeper.test; import static org.junit.jupiter.api.Assertions.fail; + import java.io.File; /** From c37074d49d02764fd9afb23888516a6309ef8fa3 Mon Sep 17 00:00:00 2001 From: Andor Molnar Date: Wed, 5 Jan 2022 14:06:22 +0100 Subject: [PATCH 02/20] ZOOKEEPER-4427. Logback.xml for main project --- conf/logback.xml | 104 ++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 102 insertions(+), 2 deletions(-) diff --git a/conf/logback.xml b/conf/logback.xml index 313af2ca53f..fbde49cd792 100644 --- a/conf/logback.xml +++ b/conf/logback.xml @@ -1,4 +1,41 @@ + + + + + + + + + + + + + + %d{ISO8601} [myid:%X{myid}] - %-5p [%t:%C{1}@%L] - %m%n @@ -7,8 +44,71 @@ ${zookeeper.console.threshold} - + + + + + + + ${zookeeper.tracelog.dir}/${zookeeper.tracelog.file} + + %d{ISO8601} [myid:%X{myid}] - %-5p [%t:%C{1}@%L] - %m%n + + + TRACE + + + + + + - + + From 7d5f936d61f3a1c65f6af2f2e3c9b337dfd95743 Mon Sep 17 00:00:00 2001 From: Andor Molnar Date: Wed, 5 Jan 2022 14:15:08 +0100 Subject: [PATCH 03/20] ZOOKEEPER-4427. Add logback.xml to other resource folders --- conf/logback.xml | 8 +- .../src/test/resources/logback.xml | 112 ++++++++++++++++++ .../src/test/resources/logback.xml | 112 ++++++++++++++++++ 3 files changed, 227 insertions(+), 5 deletions(-) create mode 100644 zookeeper-metrics-providers/zookeeper-prometheus-metrics/src/test/resources/logback.xml create mode 100644 zookeeper-server/src/test/resources/logback.xml diff --git a/conf/logback.xml b/conf/logback.xml index fbde49cd792..5d1d9aee438 100644 --- a/conf/logback.xml +++ b/conf/logback.xml @@ -29,9 +29,6 @@ - - - + + + + + + + + + + + + + + + %d{ISO8601} [myid:%X{myid}] - %-5p [%t:%C{1}@%L] - %m%n + + + ${zookeeper.console.threshold} + + + + + + + + + + + + + + + + diff --git a/zookeeper-server/src/test/resources/logback.xml b/zookeeper-server/src/test/resources/logback.xml new file mode 100644 index 00000000000..5d1d9aee438 --- /dev/null +++ b/zookeeper-server/src/test/resources/logback.xml @@ -0,0 +1,112 @@ + + + + + + + + + + + + + + + %d{ISO8601} [myid:%X{myid}] - %-5p [%t:%C{1}@%L] - %m%n + + + ${zookeeper.console.threshold} + + + + + + + + + + + + + + + + From 86c485a7bc4d6fbcf064a83580ad53b807247def Mon Sep 17 00:00:00 2001 From: Andor Molnar Date: Wed, 5 Jan 2022 15:36:09 +0100 Subject: [PATCH 04/20] ZOOKEEPER-4427. Fix unit tests --- .../server/quorum/QuorumPeerMainTest.java | 7 --- .../apache/zookeeper/test/LoggerTestTool.java | 4 ++ .../zookeeper/test/ReadOnlyModeTest.java | 55 ++++++------------- 3 files changed, 20 insertions(+), 46 deletions(-) diff --git a/zookeeper-server/src/test/java/org/apache/zookeeper/server/quorum/QuorumPeerMainTest.java b/zookeeper-server/src/test/java/org/apache/zookeeper/server/quorum/QuorumPeerMainTest.java index a2cd8368a7e..c27cbb7a061 100644 --- a/zookeeper-server/src/test/java/org/apache/zookeeper/server/quorum/QuorumPeerMainTest.java +++ b/zookeeper-server/src/test/java/org/apache/zookeeper/server/quorum/QuorumPeerMainTest.java @@ -961,13 +961,6 @@ protected QuorumPeer getQuorumPeer() { } -// private WriterAppender getConsoleAppender(ByteArrayOutputStream os, Level level) { -// String loggingPattern = ((PatternLayout) Logger.getRootLogger().getAppender("CONSOLE").getLayout()).getConversionPattern(); -// WriterAppender appender = new WriterAppender(new PatternLayout(loggingPattern), os); -// appender.setThreshold(level); -// return appender; -// } - private String getUniquePortCfgForId(int id) { return String.format("server.%d=127.0.0.1:%d:%d", id, PortAssignment.unique(), PortAssignment.unique()); } diff --git a/zookeeper-server/src/test/java/org/apache/zookeeper/test/LoggerTestTool.java b/zookeeper-server/src/test/java/org/apache/zookeeper/test/LoggerTestTool.java index a4e360e5efa..c4aeb35fbd1 100644 --- a/zookeeper-server/src/test/java/org/apache/zookeeper/test/LoggerTestTool.java +++ b/zookeeper-server/src/test/java/org/apache/zookeeper/test/LoggerTestTool.java @@ -20,6 +20,7 @@ import ch.qos.logback.classic.Level; import ch.qos.logback.classic.Logger; +import ch.qos.logback.classic.LoggerContext; import ch.qos.logback.classic.spi.ILoggingEvent; import ch.qos.logback.core.Appender; import ch.qos.logback.core.Layout; @@ -61,6 +62,7 @@ private ByteArrayOutputStream createLoggingStream(String cls) { qlogger = (ch.qos.logback.classic.Logger) LoggerFactory.getLogger(cls); qlogger.addAppender(appender); qlogger.setLevel(Level.INFO); + appender.start(); return os; } @@ -71,6 +73,7 @@ private OutputStreamAppender getConsoleAppender(ByteArrayOutputSt ((OutputStreamAppender) rootLogger.getAppender("CONSOLE")).getEncoder()).getLayout(); OutputStreamAppender appender = new OutputStreamAppender<>(); + appender.setContext((LoggerContext) LoggerFactory.getILoggerFactory()); appender.setOutputStream(os); appender.setLayout(layout); @@ -80,5 +83,6 @@ private OutputStreamAppender getConsoleAppender(ByteArrayOutputSt @Override public void close() throws Exception { qlogger.detachAppender(appender); + os.close(); } } diff --git a/zookeeper-server/src/test/java/org/apache/zookeeper/test/ReadOnlyModeTest.java b/zookeeper-server/src/test/java/org/apache/zookeeper/test/ReadOnlyModeTest.java index 3aed6d6daac..2b6602d5bbb 100644 --- a/zookeeper-server/src/test/java/org/apache/zookeeper/test/ReadOnlyModeTest.java +++ b/zookeeper-server/src/test/java/org/apache/zookeeper/test/ReadOnlyModeTest.java @@ -25,11 +25,6 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.fail; -import ch.qos.logback.classic.Logger; -import ch.qos.logback.classic.spi.ILoggingEvent; -import ch.qos.logback.core.Layout; -import ch.qos.logback.core.OutputStreamAppender; -import ch.qos.logback.core.encoder.LayoutWrappingEncoder; import org.apache.zookeeper.CreateMode; import org.apache.zookeeper.KeeperException; import org.apache.zookeeper.KeeperException.NotReadOnlyException; @@ -38,6 +33,7 @@ import org.apache.zookeeper.ZooDefs; import org.apache.zookeeper.ZooKeeper; import org.apache.zookeeper.ZooKeeper.States; +import org.apache.zookeeper.common.StringUtils; import org.apache.zookeeper.common.Time; import org.apache.zookeeper.test.ClientBase.CountdownWatcher; import org.junit.jupiter.api.AfterEach; @@ -297,27 +293,9 @@ public void testSeekForRwServer() throws Exception { qu.enableLocalSession(true); qu.startQuorum(); - // setup the logger to capture all logs - Logger rootLogger = - (ch.qos.logback.classic.Logger)LoggerFactory.getLogger(org.slf4j.Logger.ROOT_LOGGER_NAME); - Layout layout = ((LayoutWrappingEncoder) - ((OutputStreamAppender) rootLogger.getAppender("CONSOLE")).getEncoder()).getLayout(); - - OutputStreamAppender appender = - new OutputStreamAppender<>(); - LayoutWrappingEncoder layoutWrappingEncoder = new LayoutWrappingEncoder<>(); - layoutWrappingEncoder.setLayout(layout); - - ByteArrayOutputStream os = new ByteArrayOutputStream(); - appender.setOutputStream(os); - appender.setEncoder(layoutWrappingEncoder); - appender.setImmediateFlush(true); -// appender.setThreshold(Level.INFO); - Logger zlogger = - (ch.qos.logback.classic.Logger)LoggerFactory.getLogger("org.apache.zookeeper"); - zlogger.addAppender(appender); + try (LoggerTestTool loggerTestTool = new LoggerTestTool<>("org.apache.zookeeper")) { + ByteArrayOutputStream os = loggerTestTool.getOutputStream(); - try { qu.shutdown(2); CountdownWatcher watcher = new CountdownWatcher(); ZooKeeper zk = new ZooKeeper(qu.getConnString(), CONNECTION_TIMEOUT, watcher, true); @@ -337,22 +315,21 @@ public void testSeekForRwServer() throws Exception { // resume poor fellow qu.getPeer(1).peer.resume(); - } finally { - zlogger.detachAppender(appender); - } - os.close(); - LineNumberReader r = new LineNumberReader(new StringReader(os.toString())); - String line; - Pattern p = Pattern.compile(".*Majority server found.*"); - boolean found = false; - while ((line = r.readLine()) != null) { - if (p.matcher(line).matches()) { - found = true; - break; + String log = os.toString(); + assertFalse(StringUtils.isEmpty(log), "OutputStream doesn't have any log messages"); + + LineNumberReader r = new LineNumberReader(new StringReader(log)); + String line; + Pattern p = Pattern.compile(".*Majority server found.*"); + boolean found = false; + while ((line = r.readLine()) != null) { + if (p.matcher(line).matches()) { + found = true; + break; + } } + assertTrue(found, "Majority server wasn't found while connected to r/o server"); } - assertTrue(found, "Majority server wasn't found while connected to r/o server"); } - } From 53a820566e7f4a538e1b3a1cfac8451406fbbf01 Mon Sep 17 00:00:00 2001 From: Andor Molnar Date: Wed, 5 Jan 2022 15:44:01 +0100 Subject: [PATCH 05/20] ZOOKEEPER-4427. More unit tests fixed --- .../zookeeper/audit/Log4jAuditLoggerTest.java | 829 +++++++++--------- .../audit/StandaloneServerAuditTest.java | 133 ++- .../apache/zookeeper/test/LoggerTestTool.java | 1 + 3 files changed, 469 insertions(+), 494 deletions(-) diff --git a/zookeeper-server/src/test/java/org/apache/zookeeper/audit/Log4jAuditLoggerTest.java b/zookeeper-server/src/test/java/org/apache/zookeeper/audit/Log4jAuditLoggerTest.java index e449693feb7..7a390829307 100644 --- a/zookeeper-server/src/test/java/org/apache/zookeeper/audit/Log4jAuditLoggerTest.java +++ b/zookeeper-server/src/test/java/org/apache/zookeeper/audit/Log4jAuditLoggerTest.java @@ -15,425 +15,410 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -//package org.apache.zookeeper.audit; -// -//import static org.apache.zookeeper.test.ClientBase.CONNECTION_TIMEOUT; -//import static org.junit.jupiter.api.Assertions.assertEquals; -//import java.io.ByteArrayOutputStream; -//import java.io.IOException; -//import java.io.LineNumberReader; -//import java.io.StringReader; -//import java.net.InetAddress; -//import java.net.InetSocketAddress; -//import java.util.ArrayList; -//import java.util.List; -//import org.apache.log4j.Layout; -//import org.apache.log4j.Level; -//import org.apache.log4j.Logger; -//import org.apache.log4j.SimpleLayout; -//import org.apache.log4j.WriterAppender; -//import org.apache.zookeeper.CreateMode; -//import org.apache.zookeeper.KeeperException; -//import org.apache.zookeeper.KeeperException.Code; -//import org.apache.zookeeper.Op; -//import org.apache.zookeeper.PortAssignment; -//import org.apache.zookeeper.ZKUtil; -//import org.apache.zookeeper.ZooDefs; -//import org.apache.zookeeper.ZooKeeper; -//import org.apache.zookeeper.audit.AuditEvent.Result; -//import org.apache.zookeeper.data.ACL; -//import org.apache.zookeeper.data.Stat; -//import org.apache.zookeeper.server.Request; -//import org.apache.zookeeper.server.ServerCnxn; -//import org.apache.zookeeper.server.quorum.QuorumPeerTestBase; -//import org.apache.zookeeper.test.ClientBase; -//import org.apache.zookeeper.test.ClientBase.CountdownWatcher; -//import org.junit.jupiter.api.AfterAll; -//import org.junit.jupiter.api.Assertions; -//import org.junit.jupiter.api.BeforeAll; -//import org.junit.jupiter.api.BeforeEach; -//import org.junit.jupiter.api.Test; -// -// -//public class Log4jAuditLoggerTest extends QuorumPeerTestBase { -// private static final Logger LOG = Logger.getLogger(Log4jAuditLoggerTest.class); -// private static int SERVER_COUNT = 3; -// private static MainThread[] mt; -// private static ZooKeeper zk; -// private static Logger zlogger; -// private static WriterAppender appender; -// private static ByteArrayOutputStream os; -// -// @BeforeAll -// public static void setUpBeforeClass() throws Exception { -// System.setProperty(ZKAuditProvider.AUDIT_ENABLE, "true"); -// // setup the logger to capture all logs -// Layout layout = new SimpleLayout(); -// os = new ByteArrayOutputStream(); -// appender = new WriterAppender(layout, os); -// appender.setImmediateFlush(true); -// appender.setThreshold(Level.INFO); -// zlogger = Logger.getLogger(Log4jAuditLogger.class); -// zlogger.addAppender(appender); -// mt = startQuorum(); -// zk = ClientBase.createZKClient("127.0.0.1:" + mt[0].getQuorumPeer().getClientPort()); -// //Verify start audit log here itself -// String expectedAuditLog = getStartLog(); -// List logs = readAuditLog(os, SERVER_COUNT); -// verifyLogs(expectedAuditLog, logs); -// } -// -// @BeforeEach -// public void setUp() { -// os.reset(); -// } -// -// @Test -// public void testCreateAuditLogs() -// throws KeeperException, InterruptedException, IOException { -// String path = "/createPath"; -// zk.create(path, "".getBytes(), ZooDefs.Ids.OPEN_ACL_UNSAFE, -// CreateMode.PERSISTENT); -// // success log -// String createMode = CreateMode.PERSISTENT.toString().toLowerCase(); -// verifyLog( -// getAuditLog(AuditConstants.OP_CREATE, path, Result.SUCCESS, -// null, createMode), readAuditLog(os)); -// try { -// zk.create(path, "".getBytes(), ZooDefs.Ids.OPEN_ACL_UNSAFE, -// CreateMode.PERSISTENT); -// } catch (KeeperException exception) { -// Code code = exception.code(); -// assertEquals(Code.NODEEXISTS, code); -// } -// // Verify create operation log -// verifyLog( -// getAuditLog(AuditConstants.OP_CREATE, path, Result.FAILURE, -// null, createMode), readAuditLog(os)); -// } -// -// @Test -// public void testDeleteAuditLogs() -// throws InterruptedException, IOException, KeeperException { -// String path = "/deletePath"; -// zk.create(path, "".getBytes(), ZooDefs.Ids.OPEN_ACL_UNSAFE, -// CreateMode.PERSISTENT); -// os.reset(); -// try { -// zk.delete(path, -100); -// } catch (KeeperException exception) { -// Code code = exception.code(); -// assertEquals(Code.BADVERSION, code); -// } -// verifyLog(getAuditLog(AuditConstants.OP_DELETE, path, -// Result.FAILURE), -// readAuditLog(os)); -// zk.delete(path, -1); -// verifyLog(getAuditLog(AuditConstants.OP_DELETE, path), -// readAuditLog(os)); -// } -// -// @Test -// public void testSetDataAuditLogs() -// throws InterruptedException, IOException, KeeperException { -// String path = "/setDataPath"; -// zk.create(path, "".getBytes(), ZooDefs.Ids.OPEN_ACL_UNSAFE, -// CreateMode.PERSISTENT); -// os.reset(); -// try { -// zk.setData(path, "newData".getBytes(), -100); -// } catch (KeeperException exception) { -// Code code = exception.code(); -// assertEquals(Code.BADVERSION, code); -// } -// verifyLog(getAuditLog(AuditConstants.OP_SETDATA, path, -// Result.FAILURE), -// readAuditLog(os)); -// zk.setData(path, "newdata".getBytes(), -1); -// verifyLog(getAuditLog(AuditConstants.OP_SETDATA, path), -// readAuditLog(os)); -// } -// -// @Test -// public void testSetACLAuditLogs() -// throws InterruptedException, IOException, KeeperException { -// ArrayList openAclUnsafe = ZooDefs.Ids.OPEN_ACL_UNSAFE; -// String path = "/aclPath"; -// zk.create(path, "".getBytes(), ZooDefs.Ids.OPEN_ACL_UNSAFE, -// CreateMode.PERSISTENT); -// os.reset(); -// try { -// zk.setACL(path, openAclUnsafe, -100); -// } catch (KeeperException exception) { -// Code code = exception.code(); -// assertEquals(Code.BADVERSION, code); -// } -// verifyLog( -// getAuditLog(AuditConstants.OP_SETACL, path, Result.FAILURE, -// ZKUtil.aclToString(openAclUnsafe), null), readAuditLog(os)); -// zk.setACL(path, openAclUnsafe, -1); -// verifyLog( -// getAuditLog(AuditConstants.OP_SETACL, path, Result.SUCCESS, -// ZKUtil.aclToString(openAclUnsafe), null), readAuditLog(os)); -// } -// -// @Test -// public void testMultiOperationAuditLogs() -// throws InterruptedException, KeeperException, IOException { -// List ops = new ArrayList<>(); -// -// String multiop = "/b"; -// Op create = Op.create(multiop, "".getBytes(), -// ZooDefs.Ids.OPEN_ACL_UNSAFE, -// CreateMode.PERSISTENT); -// Op setData = Op.setData(multiop, "newData".getBytes(), -1); -// // check does nothing so it is audit logged -// Op check = Op.check(multiop, -1); -// Op delete = Op.delete(multiop, -1); -// -// String createMode = CreateMode.PERSISTENT.toString().toLowerCase(); -// -// ops.add(create); -// ops.add(setData); -// ops.add(check); -// ops.add(delete); -// -// zk.multi(ops); -// List multiOpLogs = readAuditLog(os, 3); -// // verify that each multi operation success is logged -// verifyLog(getAuditLog(AuditConstants.OP_CREATE, multiop, -// Result.SUCCESS, null, createMode), -// multiOpLogs.get(0)); -// verifyLog(getAuditLog(AuditConstants.OP_SETDATA, multiop), -// multiOpLogs.get(1)); -// verifyLog(getAuditLog(AuditConstants.OP_DELETE, multiop), -// multiOpLogs.get(2)); -// -// ops = new ArrayList<>(); -// ops.add(create); -// ops.add(create); -// try { -// zk.multi(ops); -// } catch (KeeperException exception) { -// Code code = exception.code(); -// assertEquals(Code.NODEEXISTS, code); -// } -// -// // Verify that multi operation failure is logged, and there is no path -// // mentioned in the audit log -// verifyLog(getAuditLog(AuditConstants.OP_MULTI_OP, null, -// Result.FAILURE), -// readAuditLog(os)); -// } -// -// @Test -// public void testEphemralZNodeAuditLogs() -// throws Exception { -// String ephemralPath = "/ephemral"; -// CountdownWatcher watcher2 = new CountdownWatcher(); -// ZooKeeper zk2 = new ZooKeeper( -// "127.0.0.1:" + mt[0].getQuorumPeer().getClientPort(), -// ClientBase.CONNECTION_TIMEOUT, watcher2); -// watcher2.waitForConnected(ClientBase.CONNECTION_TIMEOUT); -// zk2.create(ephemralPath, "".getBytes(), ZooDefs.Ids.OPEN_ACL_UNSAFE, -// CreateMode.EPHEMERAL); -// String session2 = "0x" + Long.toHexString(zk2.getSessionId()); -// verifyLog(getAuditLog(AuditConstants.OP_CREATE, ephemralPath, -// Result.SUCCESS, null, -// CreateMode.EPHEMERAL.toString().toLowerCase(), -// session2), readAuditLog(os)); -// zk2.close(); -// waitForDeletion(zk, ephemralPath); -// // verify that ephemeral node deletion on session close are captured -// // in audit log -// // Because these operations are done by ZooKeeper server itself, -// // there are no IP user is zkServer user, not any client user -// verifyLogs(getAuditLog(AuditConstants.OP_DEL_EZNODE_EXP, ephemralPath, -// Result.SUCCESS, null, null, session2, -// ZKAuditProvider.getZKUser(), null), readAuditLog(os, SERVER_COUNT)); -// } -// -// -// private static String getStartLog() { -// // user=userName operation=ZooKeeperServer start result=success -// AuditEvent logEvent = ZKAuditProvider.createLogEvent(ZKAuditProvider.getZKUser(), -// AuditConstants.OP_START, Result.SUCCESS); -// return logEvent.toString(); -// } -// -// private String getAuditLog(String operation, String znode) { -// return getAuditLog(operation, znode, Result.SUCCESS); -// } -// -// private String getAuditLog(String operation, String znode, Result result) { -// return getAuditLog(operation, znode, result, null, null); -// } -// -// private String getAuditLog(String operation, String znode, Result result, -// String acl, String createMode) { -// String session = getSession(); -// return getAuditLog(operation, znode, result, acl, createMode, session); -// } -// -// private String getAuditLog(String operation, String znode, Result result, -// String acl, String createMode, String session) { -// String user = getUser(); -// String ip = getIp(); -// return getAuditLog(operation, znode, result, acl, createMode, session, -// user, ip); -// } -// -// private String getAuditLog(String operation, String znode, Result result, -// String acl, String createMode, String session, String user, String ip) { -// AuditEvent logEvent = ZKAuditProvider.createLogEvent(user, operation, znode, acl, createMode, session, ip, -// result); -// String auditLog = logEvent.toString(); -// LOG.info("expected audit log for operation '" + operation + "' is '" -// + auditLog + "'"); -// return auditLog; -// } -// -// private String getSession() { -// return "0x" + Long.toHexString(zk.getSessionId()); -// } -// -// private String getUser() { -// ServerCnxn next = getServerCnxn(); -// Request request = new Request(next, -1, -1, -1, null, -// next.getAuthInfo()); -// return request.getUsersForAudit(); -// } -// -// private String getIp() { -// ServerCnxn next = getServerCnxn(); -// InetSocketAddress remoteSocketAddress = next.getRemoteSocketAddress(); -// InetAddress address = remoteSocketAddress.getAddress(); -// return address.getHostAddress(); -// } -// -// private ServerCnxn getServerCnxn() { -// Iterable connections = mt[0].getQuorumPeer() -// .getActiveServer() -// .getServerCnxnFactory().getConnections(); -// return connections.iterator().next(); -// } -// -// private static void verifyLog(String expectedLog, String log) { -// String searchString = " - "; -// int logStartIndex = log.indexOf(searchString); -// String auditLog = log.substring(logStartIndex + searchString.length()); -// assertEquals(expectedLog, auditLog); -// -// } -// -// private static void verifyLogs(String expectedLog, List logs) { -// for (String log : logs) { -// verifyLog(expectedLog, log); -// } -// } -// -// private String readAuditLog(ByteArrayOutputStream os) throws IOException { -// return readAuditLog(os, 1).get(0); -// } -// -// private static List readAuditLog(ByteArrayOutputStream os, -// int numberOfLogEntry) -// throws IOException { -// return readAuditLog(os, numberOfLogEntry, false); -// } -// -// private static List readAuditLog(ByteArrayOutputStream os, -// int numberOfLogEntry, -// boolean skipEphemralDeletion) throws IOException { -// List logs = new ArrayList<>(); -// LineNumberReader r = new LineNumberReader( -// new StringReader(os.toString())); -// String line; -// while ((line = r.readLine()) != null) { -// if (skipEphemralDeletion -// && line.contains(AuditConstants.OP_DEL_EZNODE_EXP)) { -// continue; -// } -// logs.add(line); -// } -// os.reset(); -// assertEquals(numberOfLogEntry, logs.size(), -// "Expected number of log entries are not generated. Logs are " -// + logs); -// return logs; -// -// } -// -// private static MainThread[] startQuorum() throws IOException { -// final int[] clientPorts = new int[SERVER_COUNT]; -// StringBuilder sb = new StringBuilder(); -// sb.append("4lw.commands.whitelist=*"); -// sb.append("\n"); -// String server; -// -// for (int i = 0; i < SERVER_COUNT; i++) { -// clientPorts[i] = PortAssignment.unique(); -// server = "server." + i + "=127.0.0.1:" + PortAssignment.unique() -// + ":" -// + PortAssignment.unique() + ":participant;127.0.0.1:" -// + clientPorts[i]; -// sb.append(server); -// sb.append("\n"); -// } -// String currentQuorumCfgSection = sb.toString(); -// MainThread[] mt = new MainThread[SERVER_COUNT]; -// -// // start all the servers -// for (int i = 0; i < SERVER_COUNT; i++) { -// mt[i] = new MainThread(i, clientPorts[i], currentQuorumCfgSection, -// false); -// mt[i].start(); -// } -// -// // ensure all servers started -// for (int i = 0; i < SERVER_COUNT; i++) { -// Assertions.assertTrue(ClientBase.waitForServerUp("127.0.0.1:" + clientPorts[i], -// CONNECTION_TIMEOUT), "waiting for server " + i + " being up"); -// } -// return mt; -// } -// -// private void waitForDeletion(ZooKeeper zooKeeper, String path) -// throws Exception { -// long elapsedTime = 0; -// long waitInterval = 10; -// int timeout = 100; -// Stat exists = zooKeeper.exists(path, false); -// while (exists != null && elapsedTime < timeout) { -// try { -// Thread.sleep(waitInterval); -// } catch (InterruptedException e) { -// Assertions.fail("CurrentEpoch update failed"); -// } -// elapsedTime = elapsedTime + waitInterval; -// exists = zooKeeper.exists(path, false); -// } -// Assertions.assertNull(exists, "Node " + path + " not deleted in " + timeout + " ms"); -// } -// -// @AfterAll -// public static void tearDownAfterClass() { -// System.clearProperty(ZKAuditProvider.AUDIT_ENABLE); -// for (int i = 0; i < SERVER_COUNT; i++) { -// try { -// if (mt[i] != null) { -// mt[i].shutdown(); -// } -// } catch (InterruptedException e) { -// e.printStackTrace(); -// } -// } -// try { -// zlogger.removeAppender(appender); -// os.close(); -// } catch (IOException e) { -// e.printStackTrace(); -// } -// } -// -//} +package org.apache.zookeeper.audit; + +import static org.apache.zookeeper.test.ClientBase.CONNECTION_TIMEOUT; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import org.apache.zookeeper.CreateMode; +import org.apache.zookeeper.KeeperException; +import org.apache.zookeeper.KeeperException.Code; +import org.apache.zookeeper.Op; +import org.apache.zookeeper.PortAssignment; +import org.apache.zookeeper.ZKUtil; +import org.apache.zookeeper.ZooDefs; +import org.apache.zookeeper.ZooKeeper; +import org.apache.zookeeper.audit.AuditEvent.Result; +import org.apache.zookeeper.data.ACL; +import org.apache.zookeeper.data.Stat; +import org.apache.zookeeper.server.Request; +import org.apache.zookeeper.server.ServerCnxn; +import org.apache.zookeeper.server.quorum.QuorumPeerTestBase; +import org.apache.zookeeper.test.ClientBase; +import org.apache.zookeeper.test.ClientBase.CountdownWatcher; +import org.apache.zookeeper.test.LoggerTestTool; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.LineNumberReader; +import java.io.StringReader; +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.util.ArrayList; +import java.util.List; + +public class Log4jAuditLoggerTest extends QuorumPeerTestBase { + private static final Logger LOG = LoggerFactory.getLogger(Log4jAuditLoggerTest.class); + private static int SERVER_COUNT = 3; + private static MainThread[] mt; + private static ZooKeeper zk; + private static ByteArrayOutputStream os; + + @BeforeAll + public static void setUpBeforeClass() throws Exception { + System.setProperty(ZKAuditProvider.AUDIT_ENABLE, "true"); + // setup the logger to capture all logs + LoggerTestTool loggerTestTool = new LoggerTestTool<>(Log4jAuditLogger.class); + os = loggerTestTool.getOutputStream(); + mt = startQuorum(); + zk = ClientBase.createZKClient("127.0.0.1:" + mt[0].getQuorumPeer().getClientPort()); + //Verify start audit log here itself + String expectedAuditLog = getStartLog(); + List logs = readAuditLog(os, SERVER_COUNT); + verifyLogs(expectedAuditLog, logs); + } + + @BeforeEach + public void setUp() { + os.reset(); + } + + @Test + public void testCreateAuditLogs() + throws KeeperException, InterruptedException, IOException { + String path = "/createPath"; + zk.create(path, "".getBytes(), ZooDefs.Ids.OPEN_ACL_UNSAFE, + CreateMode.PERSISTENT); + // success log + String createMode = CreateMode.PERSISTENT.toString().toLowerCase(); + verifyLog( + getAuditLog(AuditConstants.OP_CREATE, path, Result.SUCCESS, + null, createMode), readAuditLog(os)); + try { + zk.create(path, "".getBytes(), ZooDefs.Ids.OPEN_ACL_UNSAFE, + CreateMode.PERSISTENT); + } catch (KeeperException exception) { + Code code = exception.code(); + assertEquals(Code.NODEEXISTS, code); + } + // Verify create operation log + verifyLog( + getAuditLog(AuditConstants.OP_CREATE, path, Result.FAILURE, + null, createMode), readAuditLog(os)); + } + + @Test + public void testDeleteAuditLogs() + throws InterruptedException, IOException, KeeperException { + String path = "/deletePath"; + zk.create(path, "".getBytes(), ZooDefs.Ids.OPEN_ACL_UNSAFE, + CreateMode.PERSISTENT); + os.reset(); + try { + zk.delete(path, -100); + } catch (KeeperException exception) { + Code code = exception.code(); + assertEquals(Code.BADVERSION, code); + } + verifyLog(getAuditLog(AuditConstants.OP_DELETE, path, + Result.FAILURE), + readAuditLog(os)); + zk.delete(path, -1); + verifyLog(getAuditLog(AuditConstants.OP_DELETE, path), + readAuditLog(os)); + } + + @Test + public void testSetDataAuditLogs() + throws InterruptedException, IOException, KeeperException { + String path = "/setDataPath"; + zk.create(path, "".getBytes(), ZooDefs.Ids.OPEN_ACL_UNSAFE, + CreateMode.PERSISTENT); + os.reset(); + try { + zk.setData(path, "newData".getBytes(), -100); + } catch (KeeperException exception) { + Code code = exception.code(); + assertEquals(Code.BADVERSION, code); + } + verifyLog(getAuditLog(AuditConstants.OP_SETDATA, path, + Result.FAILURE), + readAuditLog(os)); + zk.setData(path, "newdata".getBytes(), -1); + verifyLog(getAuditLog(AuditConstants.OP_SETDATA, path), + readAuditLog(os)); + } + + @Test + public void testSetACLAuditLogs() + throws InterruptedException, IOException, KeeperException { + ArrayList openAclUnsafe = ZooDefs.Ids.OPEN_ACL_UNSAFE; + String path = "/aclPath"; + zk.create(path, "".getBytes(), ZooDefs.Ids.OPEN_ACL_UNSAFE, + CreateMode.PERSISTENT); + os.reset(); + try { + zk.setACL(path, openAclUnsafe, -100); + } catch (KeeperException exception) { + Code code = exception.code(); + assertEquals(Code.BADVERSION, code); + } + verifyLog( + getAuditLog(AuditConstants.OP_SETACL, path, Result.FAILURE, + ZKUtil.aclToString(openAclUnsafe), null), readAuditLog(os)); + zk.setACL(path, openAclUnsafe, -1); + verifyLog( + getAuditLog(AuditConstants.OP_SETACL, path, Result.SUCCESS, + ZKUtil.aclToString(openAclUnsafe), null), readAuditLog(os)); + } + + @Test + public void testMultiOperationAuditLogs() + throws InterruptedException, KeeperException, IOException { + List ops = new ArrayList<>(); + + String multiop = "/b"; + Op create = Op.create(multiop, "".getBytes(), + ZooDefs.Ids.OPEN_ACL_UNSAFE, + CreateMode.PERSISTENT); + Op setData = Op.setData(multiop, "newData".getBytes(), -1); + // check does nothing so it is audit logged + Op check = Op.check(multiop, -1); + Op delete = Op.delete(multiop, -1); + + String createMode = CreateMode.PERSISTENT.toString().toLowerCase(); + + ops.add(create); + ops.add(setData); + ops.add(check); + ops.add(delete); + + zk.multi(ops); + List multiOpLogs = readAuditLog(os, 3); + // verify that each multi operation success is logged + verifyLog(getAuditLog(AuditConstants.OP_CREATE, multiop, + Result.SUCCESS, null, createMode), + multiOpLogs.get(0)); + verifyLog(getAuditLog(AuditConstants.OP_SETDATA, multiop), + multiOpLogs.get(1)); + verifyLog(getAuditLog(AuditConstants.OP_DELETE, multiop), + multiOpLogs.get(2)); + + ops = new ArrayList<>(); + ops.add(create); + ops.add(create); + try { + zk.multi(ops); + } catch (KeeperException exception) { + Code code = exception.code(); + assertEquals(Code.NODEEXISTS, code); + } + + // Verify that multi operation failure is logged, and there is no path + // mentioned in the audit log + verifyLog(getAuditLog(AuditConstants.OP_MULTI_OP, null, + Result.FAILURE), + readAuditLog(os)); + } + + @Test + public void testEphemralZNodeAuditLogs() + throws Exception { + String ephemralPath = "/ephemral"; + CountdownWatcher watcher2 = new CountdownWatcher(); + ZooKeeper zk2 = new ZooKeeper( + "127.0.0.1:" + mt[0].getQuorumPeer().getClientPort(), + ClientBase.CONNECTION_TIMEOUT, watcher2); + watcher2.waitForConnected(ClientBase.CONNECTION_TIMEOUT); + zk2.create(ephemralPath, "".getBytes(), ZooDefs.Ids.OPEN_ACL_UNSAFE, + CreateMode.EPHEMERAL); + String session2 = "0x" + Long.toHexString(zk2.getSessionId()); + verifyLog(getAuditLog(AuditConstants.OP_CREATE, ephemralPath, + Result.SUCCESS, null, + CreateMode.EPHEMERAL.toString().toLowerCase(), + session2), readAuditLog(os)); + zk2.close(); + waitForDeletion(zk, ephemralPath); + // verify that ephemeral node deletion on session close are captured + // in audit log + // Because these operations are done by ZooKeeper server itself, + // there are no IP user is zkServer user, not any client user + verifyLogs(getAuditLog(AuditConstants.OP_DEL_EZNODE_EXP, ephemralPath, + Result.SUCCESS, null, null, session2, + ZKAuditProvider.getZKUser(), null), readAuditLog(os, SERVER_COUNT)); + } + + + private static String getStartLog() { + // user=userName operation=ZooKeeperServer start result=success + AuditEvent logEvent = ZKAuditProvider.createLogEvent(ZKAuditProvider.getZKUser(), + AuditConstants.OP_START, Result.SUCCESS); + return logEvent.toString(); + } + + private String getAuditLog(String operation, String znode) { + return getAuditLog(operation, znode, Result.SUCCESS); + } + + private String getAuditLog(String operation, String znode, Result result) { + return getAuditLog(operation, znode, result, null, null); + } + + private String getAuditLog(String operation, String znode, Result result, + String acl, String createMode) { + String session = getSession(); + return getAuditLog(operation, znode, result, acl, createMode, session); + } + + private String getAuditLog(String operation, String znode, Result result, + String acl, String createMode, String session) { + String user = getUser(); + String ip = getIp(); + return getAuditLog(operation, znode, result, acl, createMode, session, + user, ip); + } + + private String getAuditLog(String operation, String znode, Result result, + String acl, String createMode, String session, String user, String ip) { + AuditEvent logEvent = ZKAuditProvider.createLogEvent(user, operation, znode, acl, createMode, session, ip, + result); + String auditLog = logEvent.toString(); + LOG.info("expected audit log for operation '" + operation + "' is '" + + auditLog + "'"); + return auditLog; + } + + private String getSession() { + return "0x" + Long.toHexString(zk.getSessionId()); + } + + private String getUser() { + ServerCnxn next = getServerCnxn(); + Request request = new Request(next, -1, -1, -1, null, + next.getAuthInfo()); + return request.getUsersForAudit(); + } + + private String getIp() { + ServerCnxn next = getServerCnxn(); + InetSocketAddress remoteSocketAddress = next.getRemoteSocketAddress(); + InetAddress address = remoteSocketAddress.getAddress(); + return address.getHostAddress(); + } + + private ServerCnxn getServerCnxn() { + Iterable connections = mt[0].getQuorumPeer() + .getActiveServer() + .getServerCnxnFactory().getConnections(); + return connections.iterator().next(); + } + + private static void verifyLog(String expectedLog, String log) { + String searchString = " - "; + int logStartIndex = log.indexOf(searchString); + String auditLog = log.substring(logStartIndex + searchString.length()); + assertTrue(auditLog.endsWith(expectedLog)); + } + + private static void verifyLogs(String expectedLog, List logs) { + for (String log : logs) { + verifyLog(expectedLog, log); + } + } + + private String readAuditLog(ByteArrayOutputStream os) throws IOException { + return readAuditLog(os, 1).get(0); + } + + private static List readAuditLog(ByteArrayOutputStream os, + int numberOfLogEntry) + throws IOException { + return readAuditLog(os, numberOfLogEntry, false); + } + + private static List readAuditLog(ByteArrayOutputStream os, + int numberOfLogEntry, + boolean skipEphemralDeletion) throws IOException { + List logs = new ArrayList<>(); + LineNumberReader r = new LineNumberReader( + new StringReader(os.toString())); + String line; + while ((line = r.readLine()) != null) { + if (skipEphemralDeletion + && line.contains(AuditConstants.OP_DEL_EZNODE_EXP)) { + continue; + } + logs.add(line); + } + os.reset(); + assertEquals(numberOfLogEntry, logs.size(), + "Expected number of log entries are not generated. Logs are " + + logs); + return logs; + + } + + private static MainThread[] startQuorum() throws IOException { + final int[] clientPorts = new int[SERVER_COUNT]; + StringBuilder sb = new StringBuilder(); + sb.append("4lw.commands.whitelist=*"); + sb.append("\n"); + String server; + + for (int i = 0; i < SERVER_COUNT; i++) { + clientPorts[i] = PortAssignment.unique(); + server = "server." + i + "=127.0.0.1:" + PortAssignment.unique() + + ":" + + PortAssignment.unique() + ":participant;127.0.0.1:" + + clientPorts[i]; + sb.append(server); + sb.append("\n"); + } + String currentQuorumCfgSection = sb.toString(); + MainThread[] mt = new MainThread[SERVER_COUNT]; + + // start all the servers + for (int i = 0; i < SERVER_COUNT; i++) { + mt[i] = new MainThread(i, clientPorts[i], currentQuorumCfgSection, + false); + mt[i].start(); + } + + // ensure all servers started + for (int i = 0; i < SERVER_COUNT; i++) { + Assertions.assertTrue(ClientBase.waitForServerUp("127.0.0.1:" + clientPorts[i], + CONNECTION_TIMEOUT), "waiting for server " + i + " being up"); + } + return mt; + } + + private void waitForDeletion(ZooKeeper zooKeeper, String path) + throws Exception { + long elapsedTime = 0; + long waitInterval = 10; + int timeout = 100; + Stat exists = zooKeeper.exists(path, false); + while (exists != null && elapsedTime < timeout) { + try { + Thread.sleep(waitInterval); + } catch (InterruptedException e) { + Assertions.fail("CurrentEpoch update failed"); + } + elapsedTime = elapsedTime + waitInterval; + exists = zooKeeper.exists(path, false); + } + Assertions.assertNull(exists, "Node " + path + " not deleted in " + timeout + " ms"); + } + + @AfterAll + public static void tearDownAfterClass() { + System.clearProperty(ZKAuditProvider.AUDIT_ENABLE); + for (int i = 0; i < SERVER_COUNT; i++) { + try { + if (mt[i] != null) { + mt[i].shutdown(); + } + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + } +} diff --git a/zookeeper-server/src/test/java/org/apache/zookeeper/audit/StandaloneServerAuditTest.java b/zookeeper-server/src/test/java/org/apache/zookeeper/audit/StandaloneServerAuditTest.java index 5aa9d121a31..b5065381a50 100644 --- a/zookeeper-server/src/test/java/org/apache/zookeeper/audit/StandaloneServerAuditTest.java +++ b/zookeeper-server/src/test/java/org/apache/zookeeper/audit/StandaloneServerAuditTest.java @@ -16,75 +16,64 @@ * limitations under the License. */ -//package org.apache.zookeeper.audit; -// -// -//import static org.junit.jupiter.api.Assertions.assertEquals; -//import static org.junit.jupiter.api.Assertions.assertTrue; -//import java.io.ByteArrayOutputStream; -//import java.io.IOException; -//import java.io.LineNumberReader; -//import java.io.StringReader; -//import java.util.ArrayList; -//import java.util.List; -//import org.apache.log4j.Layout; -//import org.apache.log4j.Level; -//import org.apache.log4j.Logger; -//import org.apache.log4j.SimpleLayout; -//import org.apache.log4j.WriterAppender; -//import org.apache.zookeeper.CreateMode; -//import org.apache.zookeeper.KeeperException; -//import org.apache.zookeeper.ZooDefs; -//import org.apache.zookeeper.ZooKeeper; -//import org.apache.zookeeper.test.ClientBase; -//import org.junit.jupiter.api.AfterAll; -//import org.junit.jupiter.api.BeforeAll; -//import org.junit.jupiter.api.Test; -// -// -// -//public class StandaloneServerAuditTest extends ClientBase { -// private static ByteArrayOutputStream os; -// -// @BeforeAll -// public static void setup() { -// System.setProperty(ZKAuditProvider.AUDIT_ENABLE, "true"); -// // setup the logger to capture all the logs -// Layout layout = new SimpleLayout(); -// os = new ByteArrayOutputStream(); -// WriterAppender appender = new WriterAppender(layout, os); -// appender.setImmediateFlush(true); -// appender.setThreshold(Level.INFO); -// Logger zLogger = Logger.getLogger(Log4jAuditLogger.class); -// zLogger.addAppender(appender); -// } -// -// @AfterAll -// public static void teardown() { -// System.clearProperty(ZKAuditProvider.AUDIT_ENABLE); -// } -// -// @Test -// public void testCreateAuditLog() throws KeeperException, InterruptedException, IOException { -// final ZooKeeper zk = createClient(); -// String path = "/createPath"; -// zk.create(path, "".getBytes(), ZooDefs.Ids.OPEN_ACL_UNSAFE, -// CreateMode.PERSISTENT); -// List logs = readAuditLog(os); -// assertEquals(1, logs.size()); -// assertTrue(logs.get(0).endsWith("operation=create\tznode=/createPath\tznode_type=persistent\tresult=success")); -// } -// -// private static List readAuditLog(ByteArrayOutputStream os) throws IOException { -// List logs = new ArrayList<>(); -// LineNumberReader r = new LineNumberReader( -// new StringReader(os.toString())); -// String line; -// while ((line = r.readLine()) != null) { -// logs.add(line); -// } -// os.reset(); -// return logs; -// } -//} -// +package org.apache.zookeeper.audit; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import org.apache.zookeeper.CreateMode; +import org.apache.zookeeper.KeeperException; +import org.apache.zookeeper.ZooDefs; +import org.apache.zookeeper.ZooKeeper; +import org.apache.zookeeper.test.ClientBase; +import org.apache.zookeeper.test.LoggerTestTool; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.LineNumberReader; +import java.io.StringReader; +import java.util.ArrayList; +import java.util.List; + +public class StandaloneServerAuditTest extends ClientBase { + private static ByteArrayOutputStream os; + + @BeforeAll + public static void setup() { + System.setProperty(ZKAuditProvider.AUDIT_ENABLE, "true"); + LoggerTestTool loggerTestTool = new LoggerTestTool<>(Log4jAuditLogger.class); + os = loggerTestTool.getOutputStream(); + } + + @AfterAll + public static void teardown() { + System.clearProperty(ZKAuditProvider.AUDIT_ENABLE); + } + + @Test + public void testCreateAuditLog() throws KeeperException, InterruptedException, IOException { + final ZooKeeper zk = createClient(); + String path = "/createPath"; + zk.create(path, "".getBytes(), ZooDefs.Ids.OPEN_ACL_UNSAFE, + CreateMode.PERSISTENT); + List logs = readAuditLog(os); + assertEquals(1, logs.size()); + assertTrue(logs.get(0).endsWith("operation=create\tznode=/createPath\tznode_type=persistent\tresult=success")); + } + + private static List readAuditLog(ByteArrayOutputStream os) throws IOException { + List logs = new ArrayList<>(); + LineNumberReader r = new LineNumberReader( + new StringReader(os.toString())); + String line; + while ((line = r.readLine()) != null) { + logs.add(line); + } + os.reset(); + return logs; + } +} + diff --git a/zookeeper-server/src/test/java/org/apache/zookeeper/test/LoggerTestTool.java b/zookeeper-server/src/test/java/org/apache/zookeeper/test/LoggerTestTool.java index c4aeb35fbd1..f545dc7c34a 100644 --- a/zookeeper-server/src/test/java/org/apache/zookeeper/test/LoggerTestTool.java +++ b/zookeeper-server/src/test/java/org/apache/zookeeper/test/LoggerTestTool.java @@ -53,6 +53,7 @@ private ByteArrayOutputStream createLoggingStream(Class cls) { qlogger = (ch.qos.logback.classic.Logger) LoggerFactory.getLogger(cls); qlogger.addAppender(appender); qlogger.setLevel(Level.INFO); + appender.start(); return os; } From d594d7280403cbe099588c09fdce2bbdb0ce9b1d Mon Sep 17 00:00:00 2001 From: Andor Molnar Date: Wed, 5 Jan 2022 16:06:51 +0100 Subject: [PATCH 06/20] ZOOKEEPER-4427. Indentation fix --- .../org/apache/zookeeper/audit/StandaloneServerAuditTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/zookeeper-server/src/test/java/org/apache/zookeeper/audit/StandaloneServerAuditTest.java b/zookeeper-server/src/test/java/org/apache/zookeeper/audit/StandaloneServerAuditTest.java index b5065381a50..ce6f2d6b515 100644 --- a/zookeeper-server/src/test/java/org/apache/zookeeper/audit/StandaloneServerAuditTest.java +++ b/zookeeper-server/src/test/java/org/apache/zookeeper/audit/StandaloneServerAuditTest.java @@ -44,8 +44,8 @@ public class StandaloneServerAuditTest extends ClientBase { @BeforeAll public static void setup() { System.setProperty(ZKAuditProvider.AUDIT_ENABLE, "true"); - LoggerTestTool loggerTestTool = new LoggerTestTool<>(Log4jAuditLogger.class); - os = loggerTestTool.getOutputStream(); + LoggerTestTool loggerTestTool = new LoggerTestTool<>(Log4jAuditLogger.class); + os = loggerTestTool.getOutputStream(); } @AfterAll From 93e689320662c32152154384a38f599a467955b9 Mon Sep 17 00:00:00 2001 From: Andor Molnar Date: Wed, 5 Jan 2022 16:11:28 +0100 Subject: [PATCH 07/20] ZOOKEEPER-4427. Removed log4j.properties files --- conf/log4j.properties | 82 ------------------- .../src/test/resources/log4j.properties | 65 --------------- .../src/test/resources/log4j.properties | 65 --------------- 3 files changed, 212 deletions(-) delete mode 100644 conf/log4j.properties delete mode 100644 zookeeper-metrics-providers/zookeeper-prometheus-metrics/src/test/resources/log4j.properties delete mode 100644 zookeeper-server/src/test/resources/log4j.properties diff --git a/conf/log4j.properties b/conf/log4j.properties deleted file mode 100644 index 9e12a38d333..00000000000 --- a/conf/log4j.properties +++ /dev/null @@ -1,82 +0,0 @@ -# Copyright 2012 The Apache Software Foundation -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Define some default values that can be overridden by system properties -zookeeper.root.logger=INFO, CONSOLE - -zookeeper.console.threshold=INFO - -zookeeper.log.dir=. -zookeeper.log.file=zookeeper.log -zookeeper.log.threshold=INFO -zookeeper.log.maxfilesize=256MB -zookeeper.log.maxbackupindex=20 - -zookeeper.tracelog.dir=${zookeeper.log.dir} -zookeeper.tracelog.file=zookeeper_trace.log - -log4j.rootLogger=${zookeeper.root.logger} - -# -# console -# Add "console" to rootlogger above if you want to use this -# -log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender -log4j.appender.CONSOLE.Threshold=${zookeeper.console.threshold} -log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout -log4j.appender.CONSOLE.layout.ConversionPattern=%d{ISO8601} [myid:%X{myid}] - %-5p [%t:%C{1}@%L] - %m%n - -# -# Add ROLLINGFILE to rootLogger to get log file output -# -log4j.appender.ROLLINGFILE=org.apache.log4j.RollingFileAppender -log4j.appender.ROLLINGFILE.Threshold=${zookeeper.log.threshold} -log4j.appender.ROLLINGFILE.File=${zookeeper.log.dir}/${zookeeper.log.file} -log4j.appender.ROLLINGFILE.MaxFileSize=${zookeeper.log.maxfilesize} -log4j.appender.ROLLINGFILE.MaxBackupIndex=${zookeeper.log.maxbackupindex} -log4j.appender.ROLLINGFILE.layout=org.apache.log4j.PatternLayout -log4j.appender.ROLLINGFILE.layout.ConversionPattern=%d{ISO8601} [myid:%X{myid}] - %-5p [%t:%C{1}@%L] - %m%n - -# -# Add TRACEFILE to rootLogger to get log file output -# Log TRACE level and above messages to a log file -# -log4j.appender.TRACEFILE=org.apache.log4j.FileAppender -log4j.appender.TRACEFILE.Threshold=TRACE -log4j.appender.TRACEFILE.File=${zookeeper.tracelog.dir}/${zookeeper.tracelog.file} - -log4j.appender.TRACEFILE.layout=org.apache.log4j.PatternLayout -### Notice we are including log4j's NDC here (%x) -log4j.appender.TRACEFILE.layout.ConversionPattern=%d{ISO8601} [myid:%X{myid}] - %-5p [%t:%C{1}@%L][%x] - %m%n -# -# zk audit logging -# -zookeeper.auditlog.file=zookeeper_audit.log -zookeeper.auditlog.threshold=INFO -audit.logger=INFO, RFAAUDIT -log4j.logger.org.apache.zookeeper.audit.Log4jAuditLogger=${audit.logger} -log4j.additivity.org.apache.zookeeper.audit.Log4jAuditLogger=false -log4j.appender.RFAAUDIT=org.apache.log4j.RollingFileAppender -log4j.appender.RFAAUDIT.File=${zookeeper.log.dir}/${zookeeper.auditlog.file} -log4j.appender.RFAAUDIT.layout=org.apache.log4j.PatternLayout -log4j.appender.RFAAUDIT.layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n -log4j.appender.RFAAUDIT.Threshold=${zookeeper.auditlog.threshold} - -# Max log file size of 10MB -log4j.appender.RFAAUDIT.MaxFileSize=10MB -log4j.appender.RFAAUDIT.MaxBackupIndex=10 diff --git a/zookeeper-metrics-providers/zookeeper-prometheus-metrics/src/test/resources/log4j.properties b/zookeeper-metrics-providers/zookeeper-prometheus-metrics/src/test/resources/log4j.properties deleted file mode 100644 index d7f930d3ce2..00000000000 --- a/zookeeper-metrics-providers/zookeeper-prometheus-metrics/src/test/resources/log4j.properties +++ /dev/null @@ -1,65 +0,0 @@ -# Copyright 2012 The Apache Software Foundation -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Define some default values that can be overridden by system properties -zookeeper.root.logger=INFO, CONSOLE - -zookeeper.console.threshold=INFO - -zookeeper.log.dir=. -zookeeper.log.file=zookeeper.log -zookeeper.log.threshold=INFO -zookeeper.log.maxfilesize=256MB -zookeeper.log.maxbackupindex=20 - -zookeeper.tracelog.dir=${zookeeper.log.dir} -zookeeper.tracelog.file=zookeeper_trace.log - -log4j.rootLogger=${zookeeper.root.logger} - -# -# console -# Add "console" to rootlogger above if you want to use this -# -log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender -log4j.appender.CONSOLE.Threshold=${zookeeper.console.threshold} -log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout -log4j.appender.CONSOLE.layout.ConversionPattern=%d{ISO8601} [myid:%X{myid}] - %-5p [%t:%C{1}@%L] - %m%n - -# -# Add ROLLINGFILE to rootLogger to get log file output -# -log4j.appender.ROLLINGFILE=org.apache.log4j.RollingFileAppender -log4j.appender.ROLLINGFILE.Threshold=${zookeeper.log.threshold} -log4j.appender.ROLLINGFILE.File=${zookeeper.log.dir}/${zookeeper.log.file} -log4j.appender.ROLLINGFILE.MaxFileSize=${zookeeper.log.maxfilesize} -log4j.appender.ROLLINGFILE.MaxBackupIndex=${zookeeper.log.maxbackupindex} -log4j.appender.ROLLINGFILE.layout=org.apache.log4j.PatternLayout -log4j.appender.ROLLINGFILE.layout.ConversionPattern=%d{ISO8601} [myid:%X{myid}] - %-5p [%t:%C{1}@%L] - %m%n - -# -# Add TRACEFILE to rootLogger to get log file output -# Log TRACE level and above messages to a log file -# -log4j.appender.TRACEFILE=org.apache.log4j.FileAppender -log4j.appender.TRACEFILE.Threshold=TRACE -log4j.appender.TRACEFILE.File=${zookeeper.tracelog.dir}/${zookeeper.tracelog.file} - -log4j.appender.TRACEFILE.layout=org.apache.log4j.PatternLayout -### Notice we are including log4j's NDC here (%x) -log4j.appender.TRACEFILE.layout.ConversionPattern=%d{ISO8601} [myid:%X{myid}] - %-5p [%t:%C{1}@%L][%x] - %m%n diff --git a/zookeeper-server/src/test/resources/log4j.properties b/zookeeper-server/src/test/resources/log4j.properties deleted file mode 100644 index 4a2ede95503..00000000000 --- a/zookeeper-server/src/test/resources/log4j.properties +++ /dev/null @@ -1,65 +0,0 @@ -# Copyright 2012 The Apache Software Foundation -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Define some default values that can be overridden by system properties -zookeeper.root.logger=INFO, CONSOLE - -zookeeper.console.threshold=INFO - -zookeeper.log.dir=. -zookeeper.log.file=zookeeper.log -zookeeper.log.threshold=INFO -zookeeper.log.maxfilesize=256MB -zookeeper.log.maxbackupindex=20 - -zookeeper.tracelog.dir=${zookeeper.log.dir} -zookeeper.tracelog.file=zookeeper_trace.log - -log4j.rootLogger=${zookeeper.root.logger} - -# -# console -# Add "console" to rootlogger above if you want to use this -# -log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender -log4j.appender.CONSOLE.Threshold=${zookeeper.console.threshold} -log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout -log4j.appender.CONSOLE.layout.ConversionPattern=%d{ISO8601} [myid:%X{myid}] - %-5p [%t:%C{1}@%L] - %m%n - -# -# Add ROLLINGFILE to rootLogger to get log file output -# -log4j.appender.ROLLINGFILE=org.apache.log4j.RollingFileAppender -log4j.appender.ROLLINGFILE.Threshold=${zookeeper.log.threshold} -log4j.appender.ROLLINGFILE.File=${zookeeper.log.dir}/${zookeeper.log.file} -log4j.appender.ROLLINGFILE.MaxFileSize=${zookeeper.log.maxfilesize} -log4j.appender.ROLLINGFILE.MaxBackupIndex=${zookeeper.log.maxbackupindex} -log4j.appender.ROLLINGFILE.layout=org.apache.log4j.PatternLayout -log4j.appender.ROLLINGFILE.layout.ConversionPattern=%d{ISO8601} [myid:%X{myid}] - %-5p [%t:%C{1}@%L] - %m%n - -# -# Add TRACEFILE to rootLogger to get log file output -# Log TRACE level and above messages to a log file -# -log4j.appender.TRACEFILE=org.apache.log4j.FileAppender -log4j.appender.TRACEFILE.Threshold=TRACE -log4j.appender.TRACEFILE.File=${zookeeper.tracelog.dir}/${zookeeper.tracelog.file} - -log4j.appender.TRACEFILE.layout=org.apache.log4j.PatternLayout -### Notice we are including log4j's NDC here (%x) -log4j.appender.TRACEFILE.layout.ConversionPattern=%d{ISO8601} [myid:%X{myid}] - %-5p [%t:%C{1}@%L][%x] - %m%n From c74594ae94e51ebc2deee3a38fe260e080c29806 Mon Sep 17 00:00:00 2001 From: Andor Molnar Date: Wed, 5 Jan 2022 20:37:20 +0100 Subject: [PATCH 08/20] ZOOKEEPER-4427. Add back log4j dependencies for contrib projects --- pom.xml | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index ac1a33f5e70..8575af0a8a8 100755 --- a/pom.xml +++ b/pom.xml @@ -464,6 +464,7 @@ 1.48 5.6.2 1.6.2 + 1.2.17 3.6.28 2.2 1.4 @@ -561,6 +562,22 @@ slf4j-api ${slf4j.version} + + org.slf4j + slf4j-log4j12 + ${slf4j.version} + + + * + * + + + + + log4j + log4j + ${log4j.version} + ch.qos.logback logback-core @@ -602,13 +619,11 @@ org.junit.vintage junit-vintage-engine ${junit.version} - test org.mockito mockito-core ${mockito.version} - test io.netty From 3f928ab4f00e395337ec8e72c99ee8a537458b1e Mon Sep 17 00:00:00 2001 From: Andor Molnar Date: Thu, 6 Jan 2022 11:04:58 +0100 Subject: [PATCH 09/20] ZOOKEEPER-4427. Checkstyle fixes --- .../zookeeper/audit/Log4jAuditLoggerTest.java | 18 +++++----- .../audit/StandaloneServerAuditTest.java | 14 ++++---- .../server/quorum/QuorumPeerMainTest.java | 33 +++++++++---------- .../apache/zookeeper/test/LoggerTestTool.java | 5 ++- .../zookeeper/test/ReadOnlyModeTest.java | 12 +++---- .../org/apache/zookeeper/test/TestUtils.java | 1 - 6 files changed, 36 insertions(+), 47 deletions(-) diff --git a/zookeeper-server/src/test/java/org/apache/zookeeper/audit/Log4jAuditLoggerTest.java b/zookeeper-server/src/test/java/org/apache/zookeeper/audit/Log4jAuditLoggerTest.java index 7a390829307..e476e684239 100644 --- a/zookeeper-server/src/test/java/org/apache/zookeeper/audit/Log4jAuditLoggerTest.java +++ b/zookeeper-server/src/test/java/org/apache/zookeeper/audit/Log4jAuditLoggerTest.java @@ -20,7 +20,14 @@ import static org.apache.zookeeper.test.ClientBase.CONNECTION_TIMEOUT; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; - +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.LineNumberReader; +import java.io.StringReader; +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.util.ArrayList; +import java.util.List; import org.apache.zookeeper.CreateMode; import org.apache.zookeeper.KeeperException; import org.apache.zookeeper.KeeperException.Code; @@ -46,15 +53,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.LineNumberReader; -import java.io.StringReader; -import java.net.InetAddress; -import java.net.InetSocketAddress; -import java.util.ArrayList; -import java.util.List; - public class Log4jAuditLoggerTest extends QuorumPeerTestBase { private static final Logger LOG = LoggerFactory.getLogger(Log4jAuditLoggerTest.class); private static int SERVER_COUNT = 3; diff --git a/zookeeper-server/src/test/java/org/apache/zookeeper/audit/StandaloneServerAuditTest.java b/zookeeper-server/src/test/java/org/apache/zookeeper/audit/StandaloneServerAuditTest.java index ce6f2d6b515..6bd2e4181e4 100644 --- a/zookeeper-server/src/test/java/org/apache/zookeeper/audit/StandaloneServerAuditTest.java +++ b/zookeeper-server/src/test/java/org/apache/zookeeper/audit/StandaloneServerAuditTest.java @@ -20,7 +20,12 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; - +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.LineNumberReader; +import java.io.StringReader; +import java.util.ArrayList; +import java.util.List; import org.apache.zookeeper.CreateMode; import org.apache.zookeeper.KeeperException; import org.apache.zookeeper.ZooDefs; @@ -31,13 +36,6 @@ import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.LineNumberReader; -import java.io.StringReader; -import java.util.ArrayList; -import java.util.List; - public class StandaloneServerAuditTest extends ClientBase { private static ByteArrayOutputStream os; diff --git a/zookeeper-server/src/test/java/org/apache/zookeeper/server/quorum/QuorumPeerMainTest.java b/zookeeper-server/src/test/java/org/apache/zookeeper/server/quorum/QuorumPeerMainTest.java index c27cbb7a061..612ed69b2e0 100644 --- a/zookeeper-server/src/test/java/org/apache/zookeeper/server/quorum/QuorumPeerMainTest.java +++ b/zookeeper-server/src/test/java/org/apache/zookeeper/server/quorum/QuorumPeerMainTest.java @@ -31,7 +31,21 @@ import static org.mockito.Mockito.doCallRealMethod; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; - +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.IOException; +import java.io.LineNumberReader; +import java.io.StringReader; +import java.net.InetSocketAddress; +import java.nio.ByteBuffer; +import java.nio.channels.SocketChannel; +import java.nio.file.Paths; +import java.util.Map; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.regex.Pattern; +import javax.security.sasl.SaslException; import org.apache.commons.io.FileUtils; import org.apache.zookeeper.CreateMode; import org.apache.zookeeper.KeeperException; @@ -51,23 +65,6 @@ import org.apache.zookeeper.test.LoggerTestTool; import org.junit.jupiter.api.Test; -import java.io.ByteArrayOutputStream; -import java.io.File; -import java.io.IOException; -import java.io.LineNumberReader; -import java.io.StringReader; -import java.net.InetSocketAddress; -import java.nio.ByteBuffer; -import java.nio.channels.SocketChannel; -import java.nio.file.Paths; -import java.util.Map; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; -import java.util.regex.Pattern; - -import javax.security.sasl.SaslException; - /** * Test stand-alone server. * diff --git a/zookeeper-server/src/test/java/org/apache/zookeeper/test/LoggerTestTool.java b/zookeeper-server/src/test/java/org/apache/zookeeper/test/LoggerTestTool.java index f545dc7c34a..3642b6fe6b4 100644 --- a/zookeeper-server/src/test/java/org/apache/zookeeper/test/LoggerTestTool.java +++ b/zookeeper-server/src/test/java/org/apache/zookeeper/test/LoggerTestTool.java @@ -26,9 +26,8 @@ import ch.qos.logback.core.Layout; import ch.qos.logback.core.OutputStreamAppender; import ch.qos.logback.core.encoder.LayoutWrappingEncoder; -import org.slf4j.LoggerFactory; - import java.io.ByteArrayOutputStream; +import org.slf4j.LoggerFactory; public class LoggerTestTool implements AutoCloseable { private final ByteArrayOutputStream os; @@ -69,7 +68,7 @@ private ByteArrayOutputStream createLoggingStream(String cls) { private OutputStreamAppender getConsoleAppender(ByteArrayOutputStream os) { Logger rootLogger = - (ch.qos.logback.classic.Logger)LoggerFactory.getLogger(org.slf4j.Logger.ROOT_LOGGER_NAME); + (ch.qos.logback.classic.Logger) LoggerFactory.getLogger(org.slf4j.Logger.ROOT_LOGGER_NAME); Layout layout = ((LayoutWrappingEncoder) ((OutputStreamAppender) rootLogger.getAppender("CONSOLE")).getEncoder()).getLayout(); diff --git a/zookeeper-server/src/test/java/org/apache/zookeeper/test/ReadOnlyModeTest.java b/zookeeper-server/src/test/java/org/apache/zookeeper/test/ReadOnlyModeTest.java index 2b6602d5bbb..ef4cfa74f4c 100644 --- a/zookeeper-server/src/test/java/org/apache/zookeeper/test/ReadOnlyModeTest.java +++ b/zookeeper-server/src/test/java/org/apache/zookeeper/test/ReadOnlyModeTest.java @@ -24,7 +24,11 @@ import static org.junit.jupiter.api.Assertions.assertSame; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.fail; - +import java.io.ByteArrayOutputStream; +import java.io.LineNumberReader; +import java.io.StringReader; +import java.util.concurrent.TimeoutException; +import java.util.regex.Pattern; import org.apache.zookeeper.CreateMode; import org.apache.zookeeper.KeeperException; import org.apache.zookeeper.KeeperException.NotReadOnlyException; @@ -42,12 +46,6 @@ import org.junit.jupiter.api.Timeout; import org.slf4j.LoggerFactory; -import java.io.ByteArrayOutputStream; -import java.io.LineNumberReader; -import java.io.StringReader; -import java.util.concurrent.TimeoutException; -import java.util.regex.Pattern; - public class ReadOnlyModeTest extends ZKTestCase { private static final org.slf4j.Logger LOG = LoggerFactory.getLogger(ReadOnlyModeTest.class); diff --git a/zookeeper-server/src/test/java/org/apache/zookeeper/test/TestUtils.java b/zookeeper-server/src/test/java/org/apache/zookeeper/test/TestUtils.java index 04f281d6030..00c6c070a04 100644 --- a/zookeeper-server/src/test/java/org/apache/zookeeper/test/TestUtils.java +++ b/zookeeper-server/src/test/java/org/apache/zookeeper/test/TestUtils.java @@ -19,7 +19,6 @@ package org.apache.zookeeper.test; import static org.junit.jupiter.api.Assertions.fail; - import java.io.File; /** From 727c8379fdda510451835c7ac3479238f127b677 Mon Sep 17 00:00:00 2001 From: Andor Molnar Date: Thu, 6 Jan 2022 12:00:28 +0100 Subject: [PATCH 10/20] ZOOKEEPER-4427. Address cnauroth's comments --- conf/logback.xml | 2 +- pom.xml | 2 +- .../resources/markdown/zookeeperAuditLogs.md | 4 +- .../src/test/resources/logback.xml | 78 +------------ ...AuditLogger.java => Slf4jAuditLogger.java} | 6 +- .../zookeeper/audit/ZKAuditProvider.java | 2 +- ...gerTest.java => Slf4JAuditLoggerTest.java} | 6 +- .../audit/StandaloneServerAuditTest.java | 2 +- .../server/quorum/QuorumPeerMainTest.java | 110 +++++++++--------- .../apache/zookeeper/test/LoggerTestTool.java | 6 +- .../zookeeper/test/ReadOnlyModeTest.java | 2 +- .../src/test/resources/logback.xml | 78 +------------ 12 files changed, 71 insertions(+), 227 deletions(-) rename zookeeper-server/src/main/java/org/apache/zookeeper/audit/{Log4jAuditLogger.java => Slf4jAuditLogger.java} (87%) rename zookeeper-server/src/test/java/org/apache/zookeeper/audit/{Log4jAuditLoggerTest.java => Slf4JAuditLoggerTest.java} (98%) diff --git a/conf/logback.xml b/conf/logback.xml index 5d1d9aee438..a36f4d425ff 100644 --- a/conf/logback.xml +++ b/conf/logback.xml @@ -102,7 +102,7 @@ - + diff --git a/pom.xml b/pom.xml index 8575af0a8a8..488b4c1249f 100755 --- a/pom.xml +++ b/pom.xml @@ -459,7 +459,7 @@ 1.7.30 - 1.2.7 + 1.2.10 0.12.0 1.48 5.6.2 diff --git a/zookeeper-docs/src/main/resources/markdown/zookeeperAuditLogs.md b/zookeeper-docs/src/main/resources/markdown/zookeeperAuditLogs.md index 957980dcc54..5ebe4d7581c 100644 --- a/zookeeper-docs/src/main/resources/markdown/zookeeperAuditLogs.md +++ b/zookeeper-docs/src/main/resources/markdown/zookeeperAuditLogs.md @@ -74,8 +74,8 @@ By default audit logs are disabled. To enable audit logs configure audit.enable= zookeeper.auditlog.file=zookeeper_audit.log zookeeper.auditlog.threshold=INFO audit.logger=INFO, RFAAUDIT - log4j.logger.org.apache.zookeeper.audit.Log4jAuditLogger=${audit.logger} - log4j.additivity.org.apache.zookeeper.audit.Log4jAuditLogger=false + log4j.logger.org.apache.zookeeper.audit.Slf4jAuditLogger=${audit.logger} + log4j.additivity.org.apache.zookeeper.audit.Slf4jAuditLogger=false log4j.appender.RFAAUDIT=org.apache.log4j.RollingFileAppender log4j.appender.RFAAUDIT.File=${zookeeper.log.dir}/${zookeeper.auditlog.file} log4j.appender.RFAAUDIT.layout=org.apache.log4j.PatternLayout diff --git a/zookeeper-metrics-providers/zookeeper-prometheus-metrics/src/test/resources/logback.xml b/zookeeper-metrics-providers/zookeeper-prometheus-metrics/src/test/resources/logback.xml index 5d1d9aee438..a1e627724a5 100644 --- a/zookeeper-metrics-providers/zookeeper-prometheus-metrics/src/test/resources/logback.xml +++ b/zookeeper-metrics-providers/zookeeper-prometheus-metrics/src/test/resources/logback.xml @@ -21,91 +21,15 @@ --> - - - - - - - - - %d{ISO8601} [myid:%X{myid}] - %-5p [%t:%C{1}@%L] - %m%n - ${zookeeper.console.threshold} - - - - - - - - - - - - diff --git a/zookeeper-server/src/main/java/org/apache/zookeeper/audit/Log4jAuditLogger.java b/zookeeper-server/src/main/java/org/apache/zookeeper/audit/Slf4jAuditLogger.java similarity index 87% rename from zookeeper-server/src/main/java/org/apache/zookeeper/audit/Log4jAuditLogger.java rename to zookeeper-server/src/main/java/org/apache/zookeeper/audit/Slf4jAuditLogger.java index c4efc1c4e08..b162e74ec0b 100644 --- a/zookeeper-server/src/main/java/org/apache/zookeeper/audit/Log4jAuditLogger.java +++ b/zookeeper-server/src/main/java/org/apache/zookeeper/audit/Slf4jAuditLogger.java @@ -22,10 +22,10 @@ import org.slf4j.LoggerFactory; /** - * Log4j based audit logger + * Slf4j based audit logger */ -public class Log4jAuditLogger implements AuditLogger { - private static final Logger LOG = LoggerFactory.getLogger(Log4jAuditLogger.class); +public class Slf4jAuditLogger implements AuditLogger { + private static final Logger LOG = LoggerFactory.getLogger(Slf4jAuditLogger.class); @Override public void logAuditEvent(AuditEvent auditEvent) { diff --git a/zookeeper-server/src/main/java/org/apache/zookeeper/audit/ZKAuditProvider.java b/zookeeper-server/src/main/java/org/apache/zookeeper/audit/ZKAuditProvider.java index 00646baf871..864d2366c97 100644 --- a/zookeeper-server/src/main/java/org/apache/zookeeper/audit/ZKAuditProvider.java +++ b/zookeeper-server/src/main/java/org/apache/zookeeper/audit/ZKAuditProvider.java @@ -46,7 +46,7 @@ public class ZKAuditProvider { private static AuditLogger getAuditLogger() { String auditLoggerClass = System.getProperty(AUDIT_IMPL_CLASS); if (auditLoggerClass == null) { - auditLoggerClass = Log4jAuditLogger.class.getName(); + auditLoggerClass = Slf4jAuditLogger.class.getName(); } try { Constructor clientCxnConstructor = Class.forName(auditLoggerClass) diff --git a/zookeeper-server/src/test/java/org/apache/zookeeper/audit/Log4jAuditLoggerTest.java b/zookeeper-server/src/test/java/org/apache/zookeeper/audit/Slf4JAuditLoggerTest.java similarity index 98% rename from zookeeper-server/src/test/java/org/apache/zookeeper/audit/Log4jAuditLoggerTest.java rename to zookeeper-server/src/test/java/org/apache/zookeeper/audit/Slf4JAuditLoggerTest.java index e476e684239..df3b831a30d 100644 --- a/zookeeper-server/src/test/java/org/apache/zookeeper/audit/Log4jAuditLoggerTest.java +++ b/zookeeper-server/src/test/java/org/apache/zookeeper/audit/Slf4JAuditLoggerTest.java @@ -53,8 +53,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public class Log4jAuditLoggerTest extends QuorumPeerTestBase { - private static final Logger LOG = LoggerFactory.getLogger(Log4jAuditLoggerTest.class); +public class Slf4JAuditLoggerTest extends QuorumPeerTestBase { + private static final Logger LOG = LoggerFactory.getLogger(Slf4JAuditLoggerTest.class); private static int SERVER_COUNT = 3; private static MainThread[] mt; private static ZooKeeper zk; @@ -64,7 +64,7 @@ public class Log4jAuditLoggerTest extends QuorumPeerTestBase { public static void setUpBeforeClass() throws Exception { System.setProperty(ZKAuditProvider.AUDIT_ENABLE, "true"); // setup the logger to capture all logs - LoggerTestTool loggerTestTool = new LoggerTestTool<>(Log4jAuditLogger.class); + LoggerTestTool loggerTestTool = new LoggerTestTool(Slf4jAuditLogger.class); os = loggerTestTool.getOutputStream(); mt = startQuorum(); zk = ClientBase.createZKClient("127.0.0.1:" + mt[0].getQuorumPeer().getClientPort()); diff --git a/zookeeper-server/src/test/java/org/apache/zookeeper/audit/StandaloneServerAuditTest.java b/zookeeper-server/src/test/java/org/apache/zookeeper/audit/StandaloneServerAuditTest.java index 6bd2e4181e4..8cadf3e1ee7 100644 --- a/zookeeper-server/src/test/java/org/apache/zookeeper/audit/StandaloneServerAuditTest.java +++ b/zookeeper-server/src/test/java/org/apache/zookeeper/audit/StandaloneServerAuditTest.java @@ -42,7 +42,7 @@ public class StandaloneServerAuditTest extends ClientBase { @BeforeAll public static void setup() { System.setProperty(ZKAuditProvider.AUDIT_ENABLE, "true"); - LoggerTestTool loggerTestTool = new LoggerTestTool<>(Log4jAuditLogger.class); + LoggerTestTool loggerTestTool = new LoggerTestTool(Slf4jAuditLogger.class); os = loggerTestTool.getOutputStream(); } diff --git a/zookeeper-server/src/test/java/org/apache/zookeeper/server/quorum/QuorumPeerMainTest.java b/zookeeper-server/src/test/java/org/apache/zookeeper/server/quorum/QuorumPeerMainTest.java index 612ed69b2e0..be81aa6c5b3 100644 --- a/zookeeper-server/src/test/java/org/apache/zookeeper/server/quorum/QuorumPeerMainTest.java +++ b/zookeeper-server/src/test/java/org/apache/zookeeper/server/quorum/QuorumPeerMainTest.java @@ -344,7 +344,7 @@ public void testElectionFraud() throws Exception { boolean foundLooking = false; boolean foundFollowing = false; - try (LoggerTestTool loggerTestTool = new LoggerTestTool<>(QuorumPeer.class)) { + try (LoggerTestTool loggerTestTool = new LoggerTestTool(QuorumPeer.class)) { ByteArrayOutputStream os = loggerTestTool.getOutputStream(); // spin up a quorum, we use a small ticktime to make the test run faster @@ -418,7 +418,7 @@ public void testElectionFraud() throws Exception { public void testBadPeerAddressInQuorum() throws Exception { ClientBase.setupTestEnv(); - try (LoggerTestTool loggerTestTool = new LoggerTestTool<>("org.apache.zookeeper.server.quorum")) { + try (LoggerTestTool loggerTestTool = new LoggerTestTool("org.apache.zookeeper.server.quorum")) { ByteArrayOutputStream os = loggerTestTool.getOutputStream(); final int CLIENT_PORT_QP1 = PortAssignment.unique(); @@ -462,7 +462,7 @@ public void testInconsistentPeerType() throws Exception { // test the most likely situation only: server is stated as observer in // servers list, but there's no "peerType=observer" token in config - try (LoggerTestTool loggerTestTool = new LoggerTestTool<>("org.apache.zookeeper.server.quorum")) { + try (LoggerTestTool loggerTestTool = new LoggerTestTool("org.apache.zookeeper.server.quorum")) { ByteArrayOutputStream os = loggerTestTool.getOutputStream(); final int CLIENT_PORT_QP1 = PortAssignment.unique(); @@ -574,7 +574,7 @@ public void testBadPackets() throws Exception { public void testQuorumDefaults() throws Exception { ClientBase.setupTestEnv(); - try (LoggerTestTool loggerTestTool = new LoggerTestTool<>("org.apache.zookeeper")) { + try (LoggerTestTool loggerTestTool = new LoggerTestTool("org.apache.zookeeper")) { ByteArrayOutputStream os = loggerTestTool.getOutputStream(); final int CLIENT_PORT_QP1 = PortAssignment.unique(); @@ -828,7 +828,7 @@ public void testLeaderOutOfView() throws Exception { boolean foundLeading = false; boolean foundFollowing = false; - try (LoggerTestTool loggerTestTool = new LoggerTestTool<>(QuorumPeerMainTest.class)) { + try (LoggerTestTool loggerTestTool = new LoggerTestTool(QuorumPeerMainTest.class)) { ByteArrayOutputStream os = loggerTestTool.getOutputStream(); Servers svrs = new Servers(); svrs.clientPorts = new int[numServers]; @@ -1247,36 +1247,34 @@ public void testMetricsProviderLifecycle() throws Exception { ClientBase.setupTestEnv(); BaseTestMetricsProvider.MetricsProviderCapturingLifecycle.reset(); - try (LoggerTestTool loggerTestTool = new LoggerTestTool<>(QuorumPeerMainTest.class)) { - final int CLIENT_PORT_QP1 = PortAssignment.unique(); - final int CLIENT_PORT_QP2 = PortAssignment.unique(); + final int CLIENT_PORT_QP1 = PortAssignment.unique(); + final int CLIENT_PORT_QP2 = PortAssignment.unique(); - String quorumCfgSectionServer = "server.1=127.0.0.1:" + PortAssignment.unique() + ":" + PortAssignment.unique() + ";" + CLIENT_PORT_QP1 - + "\nserver.2=127.0.0.1:" + PortAssignment.unique() + ":" + PortAssignment.unique() + ";" + CLIENT_PORT_QP2 + "\n"; + String quorumCfgSectionServer = "server.1=127.0.0.1:" + PortAssignment.unique() + ":" + PortAssignment.unique() + ";" + CLIENT_PORT_QP1 + + "\nserver.2=127.0.0.1:" + PortAssignment.unique() + ":" + PortAssignment.unique() + ";" + CLIENT_PORT_QP2 + "\n"; - // server 1 boots with a MetricsProvider - String quorumCfgSectionServer1 = quorumCfgSectionServer - + "metricsProvider.className=" + BaseTestMetricsProvider.MetricsProviderCapturingLifecycle.class.getName() + "\n"; + // server 1 boots with a MetricsProvider + String quorumCfgSectionServer1 = quorumCfgSectionServer + + "metricsProvider.className=" + BaseTestMetricsProvider.MetricsProviderCapturingLifecycle.class.getName() + "\n"; - MainThread q1 = new MainThread(1, CLIENT_PORT_QP1, quorumCfgSectionServer1); - MainThread q2 = new MainThread(2, CLIENT_PORT_QP2, quorumCfgSectionServer); - q1.start(); - q2.start(); + MainThread q1 = new MainThread(1, CLIENT_PORT_QP1, quorumCfgSectionServer1); + MainThread q2 = new MainThread(2, CLIENT_PORT_QP2, quorumCfgSectionServer); + q1.start(); + q2.start(); - boolean isup1 = ClientBase.waitForServerUp("127.0.0.1:" + CLIENT_PORT_QP1, 30000); - boolean isup2 = ClientBase.waitForServerUp("127.0.0.1:" + CLIENT_PORT_QP2, 30000); - assertTrue(isup1, "Server 1 never came up"); - assertTrue(isup2, "Server 2 never came up"); + boolean isup1 = ClientBase.waitForServerUp("127.0.0.1:" + CLIENT_PORT_QP1, 30000); + boolean isup2 = ClientBase.waitForServerUp("127.0.0.1:" + CLIENT_PORT_QP2, 30000); + assertTrue(isup1, "Server 1 never came up"); + assertTrue(isup2, "Server 2 never came up"); - q1.shutdown(); - q2.shutdown(); + q1.shutdown(); + q2.shutdown(); - assertTrue(ClientBase.waitForServerDown("127.0.0.1:" + CLIENT_PORT_QP1, ClientBase.CONNECTION_TIMEOUT), - "waiting for server 1 down"); + assertTrue(ClientBase.waitForServerDown("127.0.0.1:" + CLIENT_PORT_QP1, ClientBase.CONNECTION_TIMEOUT), + "waiting for server 1 down"); - assertTrue(ClientBase.waitForServerDown("127.0.0.1:" + CLIENT_PORT_QP2, ClientBase.CONNECTION_TIMEOUT), - "waiting for server 2 down"); - } + assertTrue(ClientBase.waitForServerDown("127.0.0.1:" + CLIENT_PORT_QP2, ClientBase.CONNECTION_TIMEOUT), + "waiting for server 2 down"); assertTrue(BaseTestMetricsProvider.MetricsProviderCapturingLifecycle.configureCalled.get(), "metrics provider lifecycle error"); assertTrue(BaseTestMetricsProvider.MetricsProviderCapturingLifecycle.startCalled.get(), "metrics provider lifecycle error"); @@ -1292,38 +1290,36 @@ public void testMetricsProviderConfiguration() throws Exception { ClientBase.setupTestEnv(); BaseTestMetricsProvider.MetricsProviderWithConfiguration.httpPort.set(0); - try (LoggerTestTool loggerTestTool = new LoggerTestTool<>(QuorumPeerMainTest.class)) { - final int CLIENT_PORT_QP1 = PortAssignment.unique(); - final int CLIENT_PORT_QP2 = PortAssignment.unique(); + final int CLIENT_PORT_QP1 = PortAssignment.unique(); + final int CLIENT_PORT_QP2 = PortAssignment.unique(); - String quorumCfgSectionServer = "server.1=127.0.0.1:" + PortAssignment.unique() + ":" + PortAssignment.unique() + ";" + CLIENT_PORT_QP1 + "\n" - + "server.2=127.0.0.1:" + PortAssignment.unique() + ":" + PortAssignment.unique() + ";" + CLIENT_PORT_QP2 + "\n"; + String quorumCfgSectionServer = "server.1=127.0.0.1:" + PortAssignment.unique() + ":" + PortAssignment.unique() + ";" + CLIENT_PORT_QP1 + "\n" + + "server.2=127.0.0.1:" + PortAssignment.unique() + ":" + PortAssignment.unique() + ";" + CLIENT_PORT_QP2 + "\n"; - // server 1 boots with a MetricsProvider - String quorumCfgSectionServer1 = quorumCfgSectionServer - + "metricsProvider.className=" - + BaseTestMetricsProvider.MetricsProviderWithConfiguration.class.getName() - + "\n" + "metricsProvider.httpPort=1234"; + // server 1 boots with a MetricsProvider + String quorumCfgSectionServer1 = quorumCfgSectionServer + + "metricsProvider.className=" + + BaseTestMetricsProvider.MetricsProviderWithConfiguration.class.getName() + + "\n" + "metricsProvider.httpPort=1234"; - MainThread q1 = new MainThread(1, CLIENT_PORT_QP1, quorumCfgSectionServer1); - MainThread q2 = new MainThread(2, CLIENT_PORT_QP2, quorumCfgSectionServer); - q1.start(); - q2.start(); + MainThread q1 = new MainThread(1, CLIENT_PORT_QP1, quorumCfgSectionServer1); + MainThread q2 = new MainThread(2, CLIENT_PORT_QP2, quorumCfgSectionServer); + q1.start(); + q2.start(); - boolean isup1 = ClientBase.waitForServerUp("127.0.0.1:" + CLIENT_PORT_QP1, 30000); - boolean isup2 = ClientBase.waitForServerUp("127.0.0.1:" + CLIENT_PORT_QP2, 30000); - assertTrue(isup1, "Server 1 never came up"); - assertTrue(isup2, "Server 2 never came up"); + boolean isup1 = ClientBase.waitForServerUp("127.0.0.1:" + CLIENT_PORT_QP1, 30000); + boolean isup2 = ClientBase.waitForServerUp("127.0.0.1:" + CLIENT_PORT_QP2, 30000); + assertTrue(isup1, "Server 1 never came up"); + assertTrue(isup2, "Server 2 never came up"); - q1.shutdown(); - q2.shutdown(); + q1.shutdown(); + q2.shutdown(); - assertTrue(ClientBase.waitForServerDown("127.0.0.1:" + CLIENT_PORT_QP1, ClientBase.CONNECTION_TIMEOUT), - "waiting for server 1 down"); + assertTrue(ClientBase.waitForServerDown("127.0.0.1:" + CLIENT_PORT_QP1, ClientBase.CONNECTION_TIMEOUT), + "waiting for server 1 down"); - assertTrue(ClientBase.waitForServerDown("127.0.0.1:" + CLIENT_PORT_QP2, ClientBase.CONNECTION_TIMEOUT), - "waiting for server 2 down"); - } + assertTrue(ClientBase.waitForServerDown("127.0.0.1:" + CLIENT_PORT_QP2, ClientBase.CONNECTION_TIMEOUT), + "waiting for server 2 down"); assertEquals(1234, BaseTestMetricsProvider.MetricsProviderWithConfiguration.httpPort.get()); } @@ -1336,7 +1332,7 @@ public void testFaultyMetricsProviderOnStop() throws Exception { ClientBase.setupTestEnv(); BaseTestMetricsProvider.MetricsProviderCapturingLifecycle.reset(); - try (LoggerTestTool loggerTestTool = new LoggerTestTool<>("org.apache.zookeeper.server.quorum")) { + try (LoggerTestTool loggerTestTool = new LoggerTestTool("org.apache.zookeeper.server.quorum")) { ByteArrayOutputStream os = loggerTestTool.getOutputStream(); final int CLIENT_PORT_QP1 = PortAssignment.unique(); @@ -1392,7 +1388,7 @@ public void testFaultyMetricsProviderOnStop() throws Exception { public void testInvalidMetricsProvider() throws Exception { ClientBase.setupTestEnv(); - try (LoggerTestTool loggerTestTool = new LoggerTestTool("org.apache.zookeeper.server.quorum")) { + try (LoggerTestTool loggerTestTool = new LoggerTestTool("org.apache.zookeeper.server.quorum")) { ByteArrayOutputStream os = loggerTestTool.getOutputStream(); final int CLIENT_PORT_QP1 = PortAssignment.unique(); @@ -1434,7 +1430,7 @@ public void testInvalidMetricsProvider() throws Exception { public void testFaultyMetricsProviderOnStart() throws Exception { ClientBase.setupTestEnv(); - try (LoggerTestTool loggerTestTool = new LoggerTestTool<>("org.apache.zookeeper.server.quorum")) { + try (LoggerTestTool loggerTestTool = new LoggerTestTool("org.apache.zookeeper.server.quorum")) { ByteArrayOutputStream os = loggerTestTool.getOutputStream(); final int CLIENT_PORT_QP1 = PortAssignment.unique(); @@ -1477,7 +1473,7 @@ public void testFaultyMetricsProviderOnStart() throws Exception { public void testFaultyMetricsProviderOnConfigure() throws Exception { ClientBase.setupTestEnv(); - try (LoggerTestTool loggerTestTool = new LoggerTestTool<>("org.apache.zookeeper.server.quorum")) { + try (LoggerTestTool loggerTestTool = new LoggerTestTool("org.apache.zookeeper.server.quorum")) { ByteArrayOutputStream os = loggerTestTool.getOutputStream(); final int CLIENT_PORT_QP1 = PortAssignment.unique(); diff --git a/zookeeper-server/src/test/java/org/apache/zookeeper/test/LoggerTestTool.java b/zookeeper-server/src/test/java/org/apache/zookeeper/test/LoggerTestTool.java index 3642b6fe6b4..c95a0d32d5e 100644 --- a/zookeeper-server/src/test/java/org/apache/zookeeper/test/LoggerTestTool.java +++ b/zookeeper-server/src/test/java/org/apache/zookeeper/test/LoggerTestTool.java @@ -29,12 +29,12 @@ import java.io.ByteArrayOutputStream; import org.slf4j.LoggerFactory; -public class LoggerTestTool implements AutoCloseable { +public class LoggerTestTool implements AutoCloseable { private final ByteArrayOutputStream os; private Appender appender; private Logger qlogger; - public LoggerTestTool(Class cls) { + public LoggerTestTool(Class cls) { os = createLoggingStream(cls); } @@ -46,7 +46,7 @@ public ByteArrayOutputStream getOutputStream() { return os; } - private ByteArrayOutputStream createLoggingStream(Class cls) { + private ByteArrayOutputStream createLoggingStream(Class cls) { ByteArrayOutputStream os = new ByteArrayOutputStream(); appender = getConsoleAppender(os); qlogger = (ch.qos.logback.classic.Logger) LoggerFactory.getLogger(cls); diff --git a/zookeeper-server/src/test/java/org/apache/zookeeper/test/ReadOnlyModeTest.java b/zookeeper-server/src/test/java/org/apache/zookeeper/test/ReadOnlyModeTest.java index ef4cfa74f4c..c8f4ac42cc4 100644 --- a/zookeeper-server/src/test/java/org/apache/zookeeper/test/ReadOnlyModeTest.java +++ b/zookeeper-server/src/test/java/org/apache/zookeeper/test/ReadOnlyModeTest.java @@ -291,7 +291,7 @@ public void testSeekForRwServer() throws Exception { qu.enableLocalSession(true); qu.startQuorum(); - try (LoggerTestTool loggerTestTool = new LoggerTestTool<>("org.apache.zookeeper")) { + try (LoggerTestTool loggerTestTool = new LoggerTestTool("org.apache.zookeeper")) { ByteArrayOutputStream os = loggerTestTool.getOutputStream(); qu.shutdown(2); diff --git a/zookeeper-server/src/test/resources/logback.xml b/zookeeper-server/src/test/resources/logback.xml index 5d1d9aee438..a1e627724a5 100644 --- a/zookeeper-server/src/test/resources/logback.xml +++ b/zookeeper-server/src/test/resources/logback.xml @@ -21,91 +21,15 @@ --> - - - - - - - - - %d{ISO8601} [myid:%X{myid}] - %-5p [%t:%C{1}@%L] - %m%n - ${zookeeper.console.threshold} - - - - - - - - - - - - From 953d081e1661afa8f0929d0ccbf2c6323f02171b Mon Sep 17 00:00:00 2001 From: Andor Molnar Date: Tue, 18 Jan 2022 18:44:33 +0100 Subject: [PATCH 11/20] ZOOKEEPER-4427. Remove root logger config from shell scripts --- bin/zkCleanup.sh | 4 ++-- bin/zkCli.cmd | 2 +- bin/zkCli.sh | 2 +- bin/zkEnv.cmd | 1 - bin/zkEnv.sh | 5 ----- bin/zkServer.cmd | 2 +- bin/zkServer.sh | 8 ++++---- 7 files changed, 9 insertions(+), 15 deletions(-) diff --git a/bin/zkCleanup.sh b/bin/zkCleanup.sh index 45532410e96..b6fe0e75841 100755 --- a/bin/zkCleanup.sh +++ b/bin/zkCleanup.sh @@ -43,11 +43,11 @@ ZOO_LOG_FILE=zookeeper-$USER-cleanup-$HOSTNAME.log if [ "x$ZOODATALOGDIR" = "x" ] then -"$JAVA" "-Dzookeeper.log.dir=${ZOO_LOG_DIR}" "-Dzookeeper.root.logger=${ZOO_LOG4J_PROP}" "-Dzookeeper.log.file=${ZOO_LOG_FILE}" \ +"$JAVA" "-Dzookeeper.log.dir=${ZOO_LOG_DIR}" "-Dzookeeper.log.file=${ZOO_LOG_FILE}" \ -cp "$CLASSPATH" $JVMFLAGS \ org.apache.zookeeper.server.PurgeTxnLog "$ZOODATADIR" $* else -"$JAVA" "-Dzookeeper.log.dir=${ZOO_LOG_DIR}" "-Dzookeeper.root.logger=${ZOO_LOG4J_PROP}" "-Dzookeeper.log.file=${ZOO_LOG_FILE}" \ +"$JAVA" "-Dzookeeper.log.dir=${ZOO_LOG_DIR}" "-Dzookeeper.log.file=${ZOO_LOG_FILE}" \ -cp "$CLASSPATH" $JVMFLAGS \ org.apache.zookeeper.server.PurgeTxnLog "$ZOODATALOGDIR" "$ZOODATADIR" $* fi diff --git a/bin/zkCli.cmd b/bin/zkCli.cmd index 807ffaa9e5c..7a4391dd9ff 100644 --- a/bin/zkCli.cmd +++ b/bin/zkCli.cmd @@ -20,7 +20,7 @@ call "%~dp0zkEnv.cmd" set ZOO_LOG_FILE=zookeeper-%USERNAME%-cli-%COMPUTERNAME%.log set ZOOMAIN=org.apache.zookeeper.ZooKeeperMain -call %JAVA% "-Dzookeeper.log.dir=%ZOO_LOG_DIR%" "-Dzookeeper.root.logger=%ZOO_LOG4J_PROP%" "-Dzookeeper.log.file=%ZOO_LOG_FILE%" -cp "%CLASSPATH%" %ZOOMAIN% %* +call %JAVA% "-Dzookeeper.log.dir=%ZOO_LOG_DIR%" "-Dzookeeper.log.file=%ZOO_LOG_FILE%" -cp "%CLASSPATH%" %ZOOMAIN% %* endlocal diff --git a/bin/zkCli.sh b/bin/zkCli.sh index ecf9a4525a0..3810e7f6880 100755 --- a/bin/zkCli.sh +++ b/bin/zkCli.sh @@ -38,6 +38,6 @@ fi ZOO_LOG_FILE=zookeeper-$USER-cli-$HOSTNAME.log -"$JAVA" "-Dzookeeper.log.dir=${ZOO_LOG_DIR}" "-Dzookeeper.root.logger=${ZOO_LOG4J_PROP}" "-Dzookeeper.log.file=${ZOO_LOG_FILE}" \ +"$JAVA" "-Dzookeeper.log.dir=${ZOO_LOG_DIR}" "-Dzookeeper.log.file=${ZOO_LOG_FILE}" \ -cp "$CLASSPATH" $CLIENT_JVMFLAGS $JVMFLAGS \ org.apache.zookeeper.ZooKeeperMain "$@" diff --git a/bin/zkEnv.cmd b/bin/zkEnv.cmd index 68e231d7084..de9fc929b61 100644 --- a/bin/zkEnv.cmd +++ b/bin/zkEnv.cmd @@ -16,7 +16,6 @@ REM limitations under the License. set ZOOCFGDIR=%~dp0%..\conf set ZOO_LOG_DIR=%~dp0%..\logs -set ZOO_LOG4J_PROP=INFO,CONSOLE REM for sanity sake assume Java 1.6 REM see: http://java.sun.com/javase/6/docs/technotes/tools/windows/java.html diff --git a/bin/zkEnv.sh b/bin/zkEnv.sh index 8d93a50c7c0..0728c2e709f 100755 --- a/bin/zkEnv.sh +++ b/bin/zkEnv.sh @@ -70,11 +70,6 @@ then ZOO_LOG_DIR="$ZOOKEEPER_PREFIX/logs" fi -if [ "x${ZOO_LOG4J_PROP}" = "x" ] -then - ZOO_LOG4J_PROP="INFO,CONSOLE" -fi - if [[ -n "$JAVA_HOME" ]] && [[ -x "$JAVA_HOME/bin/java" ]]; then JAVA="$JAVA_HOME/bin/java" elif type -p java; then diff --git a/bin/zkServer.cmd b/bin/zkServer.cmd index b03a44386e7..07ac3a30244 100644 --- a/bin/zkServer.cmd +++ b/bin/zkServer.cmd @@ -21,6 +21,6 @@ set ZOOMAIN=org.apache.zookeeper.server.quorum.QuorumPeerMain set ZOO_LOG_FILE=zookeeper-%USERNAME%-server-%COMPUTERNAME%.log echo on -call %JAVA% "-Dzookeeper.log.dir=%ZOO_LOG_DIR%" "-Dzookeeper.root.logger=%ZOO_LOG4J_PROP%" "-Dzookeeper.log.file=%ZOO_LOG_FILE%" "-XX:+HeapDumpOnOutOfMemoryError" "-XX:OnOutOfMemoryError=cmd /c taskkill /pid %%%%p /t /f" -cp "%CLASSPATH%" %ZOOMAIN% "%ZOOCFG%" %* +call %JAVA% "-Dzookeeper.log.dir=%ZOO_LOG_DIR%" "-Dzookeeper.log.file=%ZOO_LOG_FILE%" "-XX:+HeapDumpOnOutOfMemoryError" "-XX:OnOutOfMemoryError=cmd /c taskkill /pid %%%%p /t /f" -cp "%CLASSPATH%" %ZOOMAIN% "%ZOOCFG%" %* endlocal diff --git a/bin/zkServer.sh b/bin/zkServer.sh index 1a3cd359c27..977c1bbf8c6 100755 --- a/bin/zkServer.sh +++ b/bin/zkServer.sh @@ -162,7 +162,7 @@ start) fi fi nohup "$JAVA" $ZOO_DATADIR_AUTOCREATE "-Dzookeeper.log.dir=${ZOO_LOG_DIR}" \ - "-Dzookeeper.log.file=${ZOO_LOG_FILE}" "-Dzookeeper.root.logger=${ZOO_LOG4J_PROP}" \ + "-Dzookeeper.log.file=${ZOO_LOG_FILE}" \ -XX:+HeapDumpOnOutOfMemoryError -XX:OnOutOfMemoryError='kill -9 %p' \ -cp "$CLASSPATH" $JVMFLAGS $ZOOMAIN "$ZOOCFG" > "$_ZOO_DAEMON_OUT" 2>&1 < /dev/null & if [ $? -eq 0 ] @@ -200,13 +200,13 @@ start-foreground) ZOO_CMD=("$JAVA") fi "${ZOO_CMD[@]}" $ZOO_DATADIR_AUTOCREATE "-Dzookeeper.log.dir=${ZOO_LOG_DIR}" \ - "-Dzookeeper.log.file=${ZOO_LOG_FILE}" "-Dzookeeper.root.logger=${ZOO_LOG4J_PROP}" \ + "-Dzookeeper.log.file=${ZOO_LOG_FILE}" \ -XX:+HeapDumpOnOutOfMemoryError -XX:OnOutOfMemoryError='kill -9 %p' \ -cp "$CLASSPATH" $JVMFLAGS $ZOOMAIN "$ZOOCFG" ;; print-cmd) echo "\"$JAVA\" $ZOO_DATADIR_AUTOCREATE -Dzookeeper.log.dir=\"${ZOO_LOG_DIR}\" \ - -Dzookeeper.log.file=\"${ZOO_LOG_FILE}\" -Dzookeeper.root.logger=\"${ZOO_LOG4J_PROP}\" \ + -Dzookeeper.log.file=\"${ZOO_LOG_FILE}\" \ -XX:+HeapDumpOnOutOfMemoryError -XX:OnOutOfMemoryError='kill -9 %p' \ -cp \"$CLASSPATH\" $JVMFLAGS $ZOOMAIN \"$ZOOCFG\" > \"$_ZOO_DAEMON_OUT\" 2>&1 < /dev/null" ;; @@ -287,7 +287,7 @@ status) fi fi echo "Client port found: $clientPort. Client address: $clientPortAddress. Client SSL: $isSSL." - STAT=`"$JAVA" "-Dzookeeper.log.dir=${ZOO_LOG_DIR}" "-Dzookeeper.root.logger=${ZOO_LOG4J_PROP}" "-Dzookeeper.log.file=${ZOO_LOG_FILE}" \ + STAT=`"$JAVA" "-Dzookeeper.log.dir=${ZOO_LOG_DIR}" "-Dzookeeper.log.file=${ZOO_LOG_FILE}" \ -cp "$CLASSPATH" $CLIENT_JVMFLAGS $JVMFLAGS org.apache.zookeeper.client.FourLetterWordMain \ $clientPortAddress $clientPort srvr $isSSL 2> /dev/null \ | $GREP Mode` From d485248f565e726d6b5019493622b8dec8378ce0 Mon Sep 17 00:00:00 2001 From: Andor Molnar Date: Tue, 18 Jan 2022 18:59:19 +0100 Subject: [PATCH 12/20] ZOOKEEPER-4427. Added (comment) logback JMX configurator to logback.xml --- conf/logback.xml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/conf/logback.xml b/conf/logback.xml index a36f4d425ff..84a53896954 100644 --- a/conf/logback.xml +++ b/conf/logback.xml @@ -20,6 +20,8 @@ Define some default values that can be overridden by system properties --> + + From 2ca6c20bd3fe890c3e9959880c8ce046d3df9274 Mon Sep 17 00:00:00 2001 From: Andor Molnar Date: Tue, 18 Jan 2022 19:01:04 +0100 Subject: [PATCH 13/20] ZOOKEEPER-4427. Remove log4j1 OWASP suppressions --- owaspSuppressions.xml | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/owaspSuppressions.xml b/owaspSuppressions.xml index 2565f0db148..5c4bc33bcf5 100644 --- a/owaspSuppressions.xml +++ b/owaspSuppressions.xml @@ -41,14 +41,4 @@ this writing --> CVE-2019-3826 - - - CVE-2019-17571 - - - - CVE-2020-9488 - From 9c65a5e6cdb271f0a90c90688aea1677dd90f4de Mon Sep 17 00:00:00 2001 From: Andor Molnar Date: Wed, 19 Jan 2022 20:19:14 +0100 Subject: [PATCH 14/20] ZOOKEEPER-4427. Docs changes --- .../main/resources/markdown/zookeeperAdmin.md | 38 +++++++------- .../resources/markdown/zookeeperAuditLogs.md | 49 ++++++++++++------- .../resources/markdown/zookeeperInternals.md | 9 ++-- .../main/resources/markdown/zookeeperJMX.md | 13 ++++- .../resources/markdown/zookeeperStarted.md | 9 ++-- .../main/resources/markdown/zookeeperTools.md | 3 +- 6 files changed, 69 insertions(+), 52 deletions(-) diff --git a/zookeeper-docs/src/main/resources/markdown/zookeeperAdmin.md b/zookeeper-docs/src/main/resources/markdown/zookeeperAdmin.md index 3671f1eea4c..040a78a1c92 100644 --- a/zookeeper-docs/src/main/resources/markdown/zookeeperAdmin.md +++ b/zookeeper-docs/src/main/resources/markdown/zookeeperAdmin.md @@ -1,5 +1,5 @@ + Change above configuration to customize the auditlog file, number of backups, max file size, custom audit logger etc. diff --git a/zookeeper-docs/src/main/resources/markdown/zookeeperInternals.md b/zookeeper-docs/src/main/resources/markdown/zookeeperInternals.md index 0505807e187..171f33ddb4c 100644 --- a/zookeeper-docs/src/main/resources/markdown/zookeeperInternals.md +++ b/zookeeper-docs/src/main/resources/markdown/zookeeperInternals.md @@ -1,5 +1,5 @@