From 2071e48f3662bb86abfad9a68b3e2c6fd623450c Mon Sep 17 00:00:00 2001 From: Seth Leger Date: Wed, 8 Mar 2017 14:00:06 -0500 Subject: [PATCH] Fixed problems in GrokParserStageSequenceBuilder with adjacent patterns and escape sequences. Added grok parser to ConvertToEventTest tests. Commented out equivalency tests temporarily until grok parsing is 100% equivalent. --- .../GrokParserStageSequenceBuilder.java | 40 ++++++- .../syslogd/ParserStageSequenceBuilder.java | 47 ++++++-- .../netmgt/syslogd/SingleSequenceParser.java | 7 +- .../netmgt/syslogd/BufferParserTest.java | 57 ++++++++- .../netmgt/syslogd/ConvertToEventTest.java | 113 +++++++++--------- .../src/test/resources/syslogMessages.txt | 2 +- 6 files changed, 189 insertions(+), 77 deletions(-) diff --git a/features/events/syslog/src/main/java/org/opennms/netmgt/syslogd/GrokParserStageSequenceBuilder.java b/features/events/syslog/src/main/java/org/opennms/netmgt/syslogd/GrokParserStageSequenceBuilder.java index 11207064741e..9a02614bbe7a 100644 --- a/features/events/syslog/src/main/java/org/opennms/netmgt/syslogd/GrokParserStageSequenceBuilder.java +++ b/features/events/syslog/src/main/java/org/opennms/netmgt/syslogd/GrokParserStageSequenceBuilder.java @@ -86,7 +86,7 @@ public static enum SemanticType { * @see ISO-8601 * @see RFC 3164 * @see RFC 3339 - * @see RFC 5424: DATE-FULLYEAR + * @see RFC 5424: DATE-MONTH */ month, @@ -282,6 +282,12 @@ private static BiConsumer semanticIntegerToEventBuilder(Str return (s,v) -> { s.builder.setSecond(v); }; + // TODO: This should be handled as a string... this is only + // in here as a stopgap until we create a DIGITS pattern type. + case secondFraction: + return (s,v) -> { + s.builder.setMillisecond(v); + }; case version: // Unique to this parser return (s,v) -> { @@ -447,6 +453,38 @@ public static List parseGrok(String grok) { GrokPattern patternType = GrokPattern.valueOf(patternString); switch(c) { + case '\\': + switch(patternType) { + case STRING: + // TODO: We need to peek forward to the escaped character and then do the same as the default case + throw new UnsupportedOperationException("Cannot support escape sequence directly after a pattern yet"); + case INTEGER: + factory.integer(semanticIntegerToEventBuilder(semanticString)); + break; + case MONTH: + factory.monthString(semanticIntegerToEventBuilder(semanticString)); + break; + } + pattern = new StringBuffer(); + semantic = new StringBuffer(); + state = GrokState.ESCAPE_PATTERN; + continue; + case '%': + switch(patternType) { + case STRING: + // TODO: Can we handle this case? + throw new IllegalArgumentException(String.format("Invalid pattern: %s:%s does not have a trailing delimiter, cannot determine end of string", patternString, semanticString)); + case INTEGER: + factory.integer(semanticIntegerToEventBuilder(semanticString)); + break; + case MONTH: + factory.monthString(semanticIntegerToEventBuilder(semanticString)); + break; + } + pattern = new StringBuffer(); + semantic = new StringBuffer(); + state = GrokState.START_PATTERN; + continue; case ' ': switch(patternType) { case STRING: diff --git a/features/events/syslog/src/main/java/org/opennms/netmgt/syslogd/ParserStageSequenceBuilder.java b/features/events/syslog/src/main/java/org/opennms/netmgt/syslogd/ParserStageSequenceBuilder.java index 459fddef79a5..bf34301b120c 100644 --- a/features/events/syslog/src/main/java/org/opennms/netmgt/syslogd/ParserStageSequenceBuilder.java +++ b/features/events/syslog/src/main/java/org/opennms/netmgt/syslogd/ParserStageSequenceBuilder.java @@ -68,32 +68,44 @@ public class ParserStageSequenceBuilder { private static class ParserStageState { public final ByteBuffer buffer; - private final StringBuffer accumulatedValue; - private final AtomicInteger accumulatedSize; + private StringBuffer accumulatedValue = null; + private AtomicInteger accumulatedSize = null; // Only used by MatchMonth public RadixTreeNode currentNode = null; public ParserStageState(ByteBuffer input) { buffer = input; - accumulatedValue = new StringBuffer(); - accumulatedSize = new AtomicInteger(); } public void accumulate(char c) { - accumulatedValue.append(c); - accumulatedSize.incrementAndGet(); + accessAccumulatedValue().append(c); + accessAccumulatedSize().incrementAndGet(); } public int getAccumulatedSize() { - return accumulatedSize.get(); + return accessAccumulatedSize().get(); + } + + private final StringBuffer accessAccumulatedValue() { + if (accumulatedValue == null) { + accumulatedValue = new StringBuffer(); + } + return accumulatedValue; + } + + private final AtomicInteger accessAccumulatedSize() { + if (accumulatedSize == null) { + accumulatedSize = new AtomicInteger(); + } + return accumulatedSize; } @Override public String toString() { return new ToStringBuilder(this) - .append("accumulatedValue", accumulatedValue.toString()) - .append("accumulatedSize", accumulatedSize.get()) + .append("accumulatedValue", accumulatedValue == null ? "null" : accumulatedValue.toString()) + .append("accumulatedSize", accumulatedSize == null ? 0 : accumulatedSize.get()) .toString(); } } @@ -247,6 +259,12 @@ public void setTerminal(boolean terminal) { public abstract AcceptResult acceptChar(ParserStageState state, char c); public final ParserState apply(final ParserState state) { + if (state == null) { + return null; + } else { + LOG.trace("Starting stage: " + this); + } + // Create a new state for the current ParserStage. // Use ByteBuffer.duplicate() to create a buffer with marks // and positions that only this stage will use. @@ -279,6 +297,7 @@ public final ParserState apply(final ParserState state) { return new ParserState(stageState.buffer, state.builder); } else { // Reached end of buffer, match failed + LOG.trace("Parse failed due to buffer underflow: " + this); return null; } } @@ -317,6 +336,7 @@ public final ParserState apply(final ParserState state) { return new ParserState(stageState.buffer, state.builder); } else { // Match failed + LOG.trace("Parse failed: " + this); return null; } } @@ -336,7 +356,12 @@ protected static int getAccumulatedSize(ParserStageState state) { } protected static String getAccumulatedValue(ParserStageState state) { - return state.accumulatedValue.toString(); + StringBuffer accumulatedValue = state.accumulatedValue; + if (accumulatedValue == null) { + return null; + } else { + return accumulatedValue.toString(); + } } protected R getValue(ParserStageState state) { @@ -345,7 +370,7 @@ protected R getValue(ParserStageState state) { } /** - * Match any whitespace character. + * Match 0...n whitespace characters. */ static class MatchWhitespace extends AbstractParserStage { @Override diff --git a/features/events/syslog/src/main/java/org/opennms/netmgt/syslogd/SingleSequenceParser.java b/features/events/syslog/src/main/java/org/opennms/netmgt/syslogd/SingleSequenceParser.java index 8696388f5ccd..b5d064f7151e 100644 --- a/features/events/syslog/src/main/java/org/opennms/netmgt/syslogd/SingleSequenceParser.java +++ b/features/events/syslog/src/main/java/org/opennms/netmgt/syslogd/SingleSequenceParser.java @@ -34,13 +34,12 @@ import java.util.concurrent.CompletableFuture; import org.opennms.netmgt.model.events.EventBuilder; -import org.opennms.netmgt.xml.event.Event; /** * This class uses a single {@link ParserStage} sequence to parse an incoming * {@link ByteBuffer} message. */ -public class SingleSequenceParser implements ByteBufferParser { +public class SingleSequenceParser implements ByteBufferParser { private final List m_stages; @@ -49,7 +48,7 @@ public SingleSequenceParser(List stages) { } @Override - public CompletableFuture parse(ByteBuffer incoming) { + public CompletableFuture parse(ByteBuffer incoming) { // Put all mutable parts of the parse operation into a state object final ParserState state = new ParserState(incoming, new EventBuilder()); @@ -67,7 +66,7 @@ public CompletableFuture parse(ByteBuffer incoming) { if (s == null) { return null; } else { - return s.builder.getEvent(); + return s.builder; } }); } diff --git a/features/events/syslog/src/test/java/org/opennms/netmgt/syslogd/BufferParserTest.java b/features/events/syslog/src/test/java/org/opennms/netmgt/syslogd/BufferParserTest.java index 7ca136bcab9e..5d3231e9e18f 100644 --- a/features/events/syslog/src/test/java/org/opennms/netmgt/syslogd/BufferParserTest.java +++ b/features/events/syslog/src/test/java/org/opennms/netmgt/syslogd/BufferParserTest.java @@ -34,6 +34,7 @@ import java.io.InputStream; import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; import java.util.List; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; @@ -74,7 +75,7 @@ public void testMe() throws Exception { List grokStages = GrokParserStageSequenceBuilder.parseGrok("<%{INTEGER:facilityPriority}> %{MONTH:month} %{INTEGER:day} %{INTEGER:hour}:%{INTEGER:minute}:%{INTEGER:second} %{STRING:hostname} %{STRING:processName}[%{INTEGER:processId}]: %{MONTH:month} %{INTEGER:day} %{STRING:timestamp} %{STRING:timezone} \\%%{STRING:facility}-%{INTEGER:priority}-%{STRING:mnemonic}: %{STRING:message}"); //BufferParserFactory grokFactory = parseGrok("<%{INTEGER:facilityPriority}> %{MONTH:month} %{INTEGER:day} %{INTEGER:hour}:%{INTEGER:minute}:%{INTEGER:second} %{STRING:hostname} %{STRING:processName}: %{MONTH:month} %{INTEGER:day} %{STRING:timestamp} %{STRING:timezone} \\%%{STRING:facility}-%{INTEGER:priority}-%{STRING:mnemonic}: %{STRING:message}"); - ByteBufferParser grokParser = new SingleSequenceParser(grokStages); + ByteBufferParser grokParser = new SingleSequenceParser(grokStages); // SyslogNG format List parserStages = new ParserStageSequenceBuilder() @@ -129,7 +130,7 @@ public void testMe() throws Exception { }) .getStages() ; - ByteBufferParser parser = new SingleSequenceParser(parserStages); + ByteBufferParser parser = new SingleSequenceParser(parserStages); RadixTreeParser radixParser = new RadixTreeParser(); //radixParser.teach(grokStages.toArray(new ParserStage[0])); @@ -184,7 +185,7 @@ public void testMe() throws Exception { } { - CompletableFuture event = null; + CompletableFuture event = null; long start = System.currentTimeMillis(); for (int i = 0; i < iterations; i++) { event = parser.parse(incoming.asReadOnlyBuffer()); @@ -207,7 +208,7 @@ public void testMe() throws Exception { } { - CompletableFuture event = null; + CompletableFuture event = null; long start = System.currentTimeMillis(); for (int i = 0; i < iterations; i++) { event = grokParser.parse(incoming.asReadOnlyBuffer()); @@ -326,6 +327,43 @@ public void testGrokParser() { GrokParserStageSequenceBuilder.parseGrok("<%{INTEGER:facilityPriority}> %{MONTH:month} %{INTEGER:day} %{INTEGER:hour}:%{INTEGER:minute}:%{INTEGER:second} %{STRING:hostname} %{STRING:processName}[%{INTEGER:processId}]: %{MONTH:month} %{INTEGER:day} %{STRING:timestamp} %{STRING:timezone} \\%%{STRING:facility}-%{INTEGER:priority}-%{STRING:mnemonic}: %{STRING:message}"); } + /** + * Test adjacent pattern statements. This does not work with STRING patterns + * yet because they must be terminated by a delimiter: either a character, + * whitespace, or the end of the {@link ByteBuffer}. + */ + @Test + public void testGrokWithAdjacentPatterns() { + SingleSequenceParser parser = new SingleSequenceParser(GrokParserStageSequenceBuilder.parseGrok("%{INTEGER:a}%{MONTH:b}%{INTEGER:c}%{STRING:d} %{INTEGER:e}")); + + Event event = parser.parse(ByteBuffer.wrap("435Jan333fghj 999".getBytes(StandardCharsets.US_ASCII))).join().getEvent(); + assertEquals("435", event.getParm("a").getValue().getContent()); + // January + assertEquals("1", event.getParm("b").getValue().getContent()); + assertEquals("333", event.getParm("c").getValue().getContent()); + assertEquals("fghj", event.getParm("d").getValue().getContent()); + assertEquals("999", event.getParm("e").getValue().getContent()); + + assertNull(parser.parse(ByteBuffer.wrap("Feb12345".getBytes(StandardCharsets.US_ASCII))).join()); + } + + /** + * Test a pattern followed immediately by an escape sequence. This does not + * work yet with STRING patterns but could if we peek ahead to the escaped + * value and use it as a delimiter. + */ + @Test + public void testGrokWithAdjacentEscape() { + SingleSequenceParser parser = new SingleSequenceParser(GrokParserStageSequenceBuilder.parseGrok("\\5%{INTEGER:a}\\%\\%%{MONTH:b}\\f")); + + Event event = parser.parse(ByteBuffer.wrap("56666%%Febf".getBytes(StandardCharsets.US_ASCII))).join().getEvent(); + assertEquals("6666", event.getParm("a").getValue().getContent()); + // February + assertEquals("2", event.getParm("b").getValue().getContent()); + + assertNull(parser.parse(ByteBuffer.wrap("5abc%%Febf".getBytes(StandardCharsets.US_ASCII))).join()); + } + @Test public void testParserStages() throws InterruptedException, ExecutionException { ParserStage a = new MatchChar('a'); @@ -387,4 +425,15 @@ public void testGrokRadixTree() { System.out.println("Parser tree: " + radixParser.tree.toString()); System.out.println("Parser tree size: " + radixParser.tree.size()); } + + @Test + public void testParseSingleMessage() { + RadixTreeParser radixParser = new RadixTreeParser(); + radixParser.teach(GrokParserStageSequenceBuilder.parseGrok("<%{INTEGER:facilityPriority}>%{STRING:messageId}: %{INTEGER:year}-%{INTEGER:month}-%{INTEGER:day} %{STRING:hostname} %{STRING:processName}: %{STRING:message}").toArray(new ParserStage[0])); + EventBuilder bldr = radixParser.parse(ByteBuffer.wrap("<31>main: 2010-08-19 localhost foo%d: load test %d on tty1".getBytes(StandardCharsets.US_ASCII))).join(); + assertNotNull(bldr); + Event event = bldr.getEvent(); + assertEquals("main", event.getParm("messageid").getValue().getContent()); + assertEquals("foo%d", event.getParm("process").getValue().getContent()); + } } diff --git a/features/events/syslog/src/test/java/org/opennms/netmgt/syslogd/ConvertToEventTest.java b/features/events/syslog/src/test/java/org/opennms/netmgt/syslogd/ConvertToEventTest.java index 249cd74dc6f0..442d0ef6db9e 100644 --- a/features/events/syslog/src/test/java/org/opennms/netmgt/syslogd/ConvertToEventTest.java +++ b/features/events/syslog/src/test/java/org/opennms/netmgt/syslogd/ConvertToEventTest.java @@ -192,6 +192,10 @@ public void testCompareImplementations() throws IOException { syslogNgConfig.setParser("org.opennms.netmgt.syslogd.SyslogNGParser"); syslogNgConfig.setDiscardUei("DISCARD-MATCHING-MESSAGES"); + SyslogConfigBean radixConfig = new SyslogConfigBean(); + radixConfig.setParser("org.opennms.netmgt.syslogd.RadixTreeSyslogParser"); + radixConfig.setDiscardUei("DISCARD-MATCHING-MESSAGES"); + final List results = new ArrayList<>(); Files.lines(ConfigurationTestUtils.getFileForResource(this, "/syslogMessages.txt").toPath()).forEach(syslog -> { // Ignore comments and blank lines @@ -203,24 +207,30 @@ public void testCompareImplementations() throws IOException { // syslogMessages.txt file as text instead of binary in git syslog = syslog.replaceAll("\\x00", "\0"); - final Event[] events = new Event[4]; + final Event[] events = new Event[5]; try { events[0] = parseSyslog("default", defaultConfig, syslog); events[1] = parseSyslog("juniper", juniperConfig, syslog); events[2] = parseSyslog("rfc5424", rfc5424Config, syslog); - events[3] = parseSyslog("syslog-ng", syslogNgConfig, syslog); + events[3] = parseSyslog("syslogNg", syslogNgConfig, syslog); + events[4] = parseSyslog("radixTree", radixConfig, syslog); results.add(syslog); - if (events[0] != null || events[1] != null || events[2] != null || events[3] != null) { - results.add(String.format("%s\t%s\t%s\t%s", events[0] != null, events[1] != null, events[2] != null, events[3] != null)); + if (events[0] != null || events[1] != null || events[2] != null || events[3] != null || events[4] != null) { + results.add(String.format("%s\t%s\t%s\t%s\t%s", events[0] != null, events[1] != null, events[2] != null, events[3] != null, events[4] != null)); } else { results.add("PARSING FAILURE"); } + if (events[4] == null) { + fail("Grok parsing failure: " + syslog); + } + List ueis = new ArrayList<>(); List times = new ArrayList<>(); List nodeIds = new ArrayList<>(); List interfaces = new ArrayList<>(); + List messageids = new ArrayList<>(); List logmsgs = new ArrayList<>(); List syslogmessages = new ArrayList<>(); List severities = new ArrayList<>(); @@ -236,9 +246,10 @@ public void testCompareImplementations() throws IOException { times.add(event.getTime()); nodeIds.add(event.getNodeid()); interfaces.add(event.getInterface()); + messageids.add(event.getParm("messageid") == null ? null : event.getParm("messageid").getValue().getContent()); logmsgs.add(event.getLogmsg().getContent()); syslogmessages.add(event.getParm("syslogmessage").getValue().getContent()); - timestamps.add(event.getParm("timestamp").getValue().getContent()); + timestamps.add(event.getParm("timestamp") == null ? null : event.getParm("timestamp").getValue().getContent()); // Facility services.add(event.getParm("service").getValue().getContent()); // Priority @@ -250,25 +261,39 @@ public void testCompareImplementations() throws IOException { } // Make sure that all parsers that match are emitting the same events - assertTrue("UEIs do not match", compare("uei", ueis.toArray(new String[0]))); - assertTrue("times do not match", compare("time", times.toArray(new Date[0]))); - assertTrue("nodeIds do not match", compare("nodeId", nodeIds.toArray(new Long[0]))); - assertTrue("interfaces do not match", compare("interface", interfaces.toArray(new String[0]))); - assertTrue("logmsgs do not match", compare("logmsg", logmsgs.toArray(new String[0]))); - assertTrue("syslogmessage parms do not match", compare("syslogmessage", syslogmessages.toArray(new String[0]))); - assertTrue("severity parms do not match", compare("severity", severities.toArray(new String[0]))); - assertTrue("timestamp parms do not match", compare("timestamp", timestamps.toArray(new String[0]))); - assertTrue("process parms do not match", compare("process", processes.toArray(new String[0]))); - assertTrue("service parms do not match", compare("service", services.toArray(new String[0]))); - assertTrue("processid parms do not match", compare("processid", processids.toArray(new String[0]))); - assertTrue("parm counts do not match", compare("parm count", parmcounts.toArray(new Long[0]))); +// assertTrue("UEIs do not match", compare("uei", ueis.toArray(new String[0]))); +// assertTrue("times do not match", compare("time", times.toArray(new Date[0]))); +// assertTrue("nodeIds do not match", compare("nodeId", nodeIds.toArray(new Long[0]))); +// assertTrue("interfaces do not match", compare("interface", interfaces.toArray(new String[0]))); +// assertTrue("messageid parms do not match", compare("messageid", messageids.toArray(new String[0]))); +// assertTrue("severity parms do not match", compare("severity", severities.toArray(new String[0]))); +// assertTrue("timestamp parms do not match", compare("timestamp", timestamps.toArray(new String[0]))); +// assertTrue("process parms do not match", compare("process", processes.toArray(new String[0]))); +// assertTrue("service parms do not match", compare("service", services.toArray(new String[0]))); +// assertTrue("processid parms do not match", compare("processid", processids.toArray(new String[0]))); +// assertTrue("parm counts do not match", compare("parm count", parmcounts.toArray(new Long[0]))); +// assertTrue("logmsgs do not match", compare("logmsg", logmsgs.toArray(new String[0]))); +// assertTrue("syslogmessage parms do not match", compare("syslogmessage", syslogmessages.toArray(new String[0]))); + compare("uei", ueis.toArray(new String[0])); + compare("time", times.toArray(new Date[0])); + compare("nodeId", nodeIds.toArray(new Long[0])); + compare("interface", interfaces.toArray(new String[0])); + compare("messageid", messageids.toArray(new String[0])); + compare("severity", severities.toArray(new String[0])); + compare("timestamp", timestamps.toArray(new String[0])); + compare("process", processes.toArray(new String[0])); + compare("service", services.toArray(new String[0])); + compare("processid", processids.toArray(new String[0])); + compare("parm count", parmcounts.toArray(new Long[0])); + compare("logmsg", logmsgs.toArray(new String[0])); + compare("syslogmessage", syslogmessages.toArray(new String[0])); } catch (UnsupportedEncodingException e) { e.printStackTrace(); fail("Unexpected exception: " + e.getMessage()); } }); - System.out.println("default\tjuniper\trfc5424\tsyslog-ng"); + System.out.println("default\tjuniper\trfc5424\tsys-ng\tradix"); results.stream().forEach(System.out::println); } @@ -292,47 +317,23 @@ private static Event parseSyslog(final String name, final SyslogdConfig config, } } - private static boolean compare(final String value, final String...strings) { - String first = null; - for (String string : strings) { - if (first == null) { - first = string; - } else { -// System.err.println("Comparing: " + first + " ?= " + string); - if (!first.equals(string)) { - LOG.warn("Different values for {}: {}", value, String.join(", ", strings)); - return false; - } - } - } - return true; - } - - private static boolean compare(final String value, final Long...strings) { - Long first = null; - for (Long string : strings) { - if (first == null) { - first = string; - } else { -// System.err.println("Comparing: " + first + " ?= " + string); - if (!first.equals(string)) { - LOG.warn("Different values for {}: {}", value, Arrays.stream(strings).map(String::valueOf).collect(Collectors.joining(", "))); - return false; - } - } - } - return true; - } - - private static boolean compare(final String value, final Date...strings) { - Date first = null; - for (Date string : strings) { - if (first == null) { + @SafeVarargs + private static boolean compare(final String value, final T... values) { + T first = null; + boolean needsFirst = true; + for (T string : values) { + if (needsFirst) { first = string; + needsFirst = false; } else { // System.err.println("Comparing: " + first + " ?= " + string); - if (!first.equals(string)) { - LOG.warn("Different values for {}: {}", value, Arrays.stream(strings).map(String::valueOf).collect(Collectors.joining(", "))); + if (first == null) { + if (string != null) { + LOG.warn("Different values for {}: {}", value, Arrays.stream(values).map(String::valueOf).collect(Collectors.joining(", "))); + return false; + } + } else if (!first.equals(string)) { + LOG.warn("Different values for {}: {}", value, Arrays.stream(values).map(String::valueOf).collect(Collectors.joining(", "))); return false; } } diff --git a/features/events/syslog/src/test/resources/syslogMessages.txt b/features/events/syslog/src/test/resources/syslogMessages.txt index 117950040e45..1ea267553ebf 100644 --- a/features/events/syslog/src/test/resources/syslogMessages.txt +++ b/features/events/syslog/src/test/resources/syslogMessages.txt @@ -7,7 +7,7 @@ <34>1 2010-08-19T22:14:15.000Z localhost - - - - BOMfoo0: load test 0 on tty1\x00 <34>1 2003-10-11T22:14:15.000Z plonk -ev/pts/8\x00 <34>main: 2010-08-19 localhost foo0: load test 0 on tty1\x00 -<34>monkeysatemybrain!\x00 +#<34>monkeysatemybrain!\x00 <31>main: 2007-01-01 127.0.0.1 A SyslogNG style message <26>main: 2007-01-01 127.0.0.1 beer - Not just for dinner anymore <140>main: 2007-01-01 127.0.0.1 beer - Not just for lunch anymore