Skip to content

Commit

Permalink
Fixed problems in GrokParserStageSequenceBuilder with adjacent
Browse files Browse the repository at this point in the history 
patterns and escape sequences. Added grok parser to ConvertToEventTest
tests. Commented out equivalency tests temporarily until grok parsing
is 100% equivalent.
  • Loading branch information
@soleger
soleger committed Mar 24, 2017
1 parent d5e581a commit 2071e48
Show file tree
Hide file tree
Showing 6 changed files with 189 additions and 77 deletions.
40 changes: 39 additions & 1 deletion ...vents/syslog/src/main/java/org/opennms/netmgt/syslogd/GrokParserStageSequenceBuilder.java
View file
Expand Up @@ -86,7 +86,7 @@ public static enum SemanticType {
* @see ISO-8601
* @see RFC 3164
* @see RFC 3339
* @see RFC 5424: DATE-FULLYEAR
* @see RFC 5424: DATE-MONTH
*/
month,

Expand Down Expand Up @@ -282,6 +282,12 @@ private static BiConsumer<ParserState,Integer> semanticIntegerToEventBuilder(Str
return (s,v) -> {
s.builder.setSecond(v);
};
// TODO: This should be handled as a string... this is only
// in here as a stopgap until we create a DIGITS pattern type.
case secondFraction:
return (s,v) -> {
s.builder.setMillisecond(v);
};
case version:
// Unique to this parser
return (s,v) -> {
Expand Down Expand Up @@ -447,6 +453,38 @@ public static List<ParserStage> parseGrok(String grok) {
GrokPattern patternType = GrokPattern.valueOf(patternString);

switch(c) {
case '\\':
switch(patternType) {
case STRING:
// TODO: We need to peek forward to the escaped character and then do the same as the default case
throw new UnsupportedOperationException("Cannot support escape sequence directly after a pattern yet");
case INTEGER:
factory.integer(semanticIntegerToEventBuilder(semanticString));
break;
case MONTH:
factory.monthString(semanticIntegerToEventBuilder(semanticString));
break;
}
pattern = new StringBuffer();
semantic = new StringBuffer();
state = GrokState.ESCAPE_PATTERN;
continue;
case '%':
switch(patternType) {
case STRING:
// TODO: Can we handle this case?
throw new IllegalArgumentException(String.format("Invalid pattern: %s:%s does not have a trailing delimiter, cannot determine end of string", patternString, semanticString));
case INTEGER:
factory.integer(semanticIntegerToEventBuilder(semanticString));
break;
case MONTH:
factory.monthString(semanticIntegerToEventBuilder(semanticString));
break;
}
pattern = new StringBuffer();
semantic = new StringBuffer();
state = GrokState.START_PATTERN;
continue;
case ' ':
switch(patternType) {
case STRING:
Expand Down
47 changes: 36 additions & 11 deletions ...es/events/syslog/src/main/java/org/opennms/netmgt/syslogd/ParserStageSequenceBuilder.java
View file
Expand Up @@ -68,32 +68,44 @@ public class ParserStageSequenceBuilder {
private static class ParserStageState {
public final ByteBuffer buffer;

private final StringBuffer accumulatedValue;
private final AtomicInteger accumulatedSize;
private StringBuffer accumulatedValue = null;
private AtomicInteger accumulatedSize = null;

// Only used by MatchMonth
public RadixTreeNode<CharacterWithValue> currentNode = null;

public ParserStageState(ByteBuffer input) {
buffer = input;
accumulatedValue = new StringBuffer();
accumulatedSize = new AtomicInteger();
}

public void accumulate(char c) {
accumulatedValue.append(c);
accumulatedSize.incrementAndGet();
accessAccumulatedValue().append(c);
accessAccumulatedSize().incrementAndGet();
}

public int getAccumulatedSize() {
return accumulatedSize.get();
return accessAccumulatedSize().get();
}

private final StringBuffer accessAccumulatedValue() {
if (accumulatedValue == null) {
accumulatedValue = new StringBuffer();
}
return accumulatedValue;
}

private final AtomicInteger accessAccumulatedSize() {
if (accumulatedSize == null) {
accumulatedSize = new AtomicInteger();
}
return accumulatedSize;
}

@Override
public String toString() {
return new ToStringBuilder(this)
.append("accumulatedValue", accumulatedValue.toString())
.append("accumulatedSize", accumulatedSize.get())
.append("accumulatedValue", accumulatedValue == null ? "null" : accumulatedValue.toString())
.append("accumulatedSize", accumulatedSize == null ? 0 : accumulatedSize.get())
.toString();
}
}
Expand Down Expand Up @@ -247,6 +259,12 @@ public void setTerminal(boolean terminal) {
public abstract AcceptResult acceptChar(ParserStageState state, char c);

public final ParserState apply(final ParserState state) {
if (state == null) {
return null;
} else {
LOG.trace("Starting stage: " + this);
}

// Create a new state for the current ParserStage.
// Use ByteBuffer.duplicate() to create a buffer with marks
// and positions that only this stage will use.
Expand Down Expand Up @@ -279,6 +297,7 @@ public final ParserState apply(final ParserState state) {
return new ParserState(stageState.buffer, state.builder);
} else {
// Reached end of buffer, match failed
LOG.trace("Parse failed due to buffer underflow: " + this);
return null;
}
}
Expand Down Expand Up @@ -317,6 +336,7 @@ public final ParserState apply(final ParserState state) {
return new ParserState(stageState.buffer, state.builder);
} else {
// Match failed
LOG.trace("Parse failed: " + this);
return null;
}
}
Expand All @@ -336,7 +356,12 @@ protected static int getAccumulatedSize(ParserStageState state) {
}

protected static String getAccumulatedValue(ParserStageState state) {
return state.accumulatedValue.toString();
StringBuffer accumulatedValue = state.accumulatedValue;
if (accumulatedValue == null) {
return null;
} else {
return accumulatedValue.toString();
}
}

protected R getValue(ParserStageState state) {
Expand All @@ -345,7 +370,7 @@ protected R getValue(ParserStageState state) {
}

/**
* Match any whitespace character.
* Match 0...n whitespace characters.
*/
static class MatchWhitespace extends AbstractParserStage<Void> {
@Override
Expand Down
7 changes: 3 additions & 4 deletions features/events/syslog/src/main/java/org/opennms/netmgt/syslogd/SingleSequenceParser.java
View file
Expand Up @@ -34,13 +34,12 @@
import java.util.concurrent.CompletableFuture;

import org.opennms.netmgt.model.events.EventBuilder;
import org.opennms.netmgt.xml.event.Event;

/**
* This class uses a single {@link ParserStage} sequence to parse an incoming
* {@link ByteBuffer} message.
*/
public class SingleSequenceParser implements ByteBufferParser<Event> {
public class SingleSequenceParser implements ByteBufferParser<EventBuilder> {

private final List<ParserStage> m_stages;

Expand All @@ -49,7 +48,7 @@ public SingleSequenceParser(List<ParserStage> stages) {
}

@Override
public CompletableFuture<Event> parse(ByteBuffer incoming) {
public CompletableFuture<EventBuilder> parse(ByteBuffer incoming) {

// Put all mutable parts of the parse operation into a state object
final ParserState state = new ParserState(incoming, new EventBuilder());
Expand All @@ -67,7 +66,7 @@ public CompletableFuture<Event> parse(ByteBuffer incoming) {
if (s == null) {
return null;
} else {
return s.builder.getEvent();
return s.builder;
}
});
}
Expand Down
57 changes: 53 additions & 4 deletions features/events/syslog/src/test/java/org/opennms/netmgt/syslogd/BufferParserTest.java
View file
Expand Up @@ -34,6 +34,7 @@

import java.io.InputStream;
import java.nio.ByteBuffer;
import java.nio.charset.StandardCharsets;
import java.util.List;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutionException;
Expand Down Expand Up @@ -74,7 +75,7 @@ public void testMe() throws Exception {

List<ParserStage> grokStages = GrokParserStageSequenceBuilder.parseGrok("<%{INTEGER:facilityPriority}> %{MONTH:month} %{INTEGER:day} %{INTEGER:hour}:%{INTEGER:minute}:%{INTEGER:second} %{STRING:hostname} %{STRING:processName}[%{INTEGER:processId}]: %{MONTH:month} %{INTEGER:day} %{STRING:timestamp} %{STRING:timezone} \\%%{STRING:facility}-%{INTEGER:priority}-%{STRING:mnemonic}: %{STRING:message}");
//BufferParserFactory grokFactory = parseGrok("<%{INTEGER:facilityPriority}> %{MONTH:month} %{INTEGER:day} %{INTEGER:hour}:%{INTEGER:minute}:%{INTEGER:second} %{STRING:hostname} %{STRING:processName}: %{MONTH:month} %{INTEGER:day} %{STRING:timestamp} %{STRING:timezone} \\%%{STRING:facility}-%{INTEGER:priority}-%{STRING:mnemonic}: %{STRING:message}");
ByteBufferParser<Event> grokParser = new SingleSequenceParser(grokStages);
ByteBufferParser<EventBuilder> grokParser = new SingleSequenceParser(grokStages);

// SyslogNG format
List<ParserStage> parserStages = new ParserStageSequenceBuilder()
Expand Down Expand Up @@ -129,7 +130,7 @@ public void testMe() throws Exception {
})
.getStages()
;
ByteBufferParser<Event> parser = new SingleSequenceParser(parserStages);
ByteBufferParser<EventBuilder> parser = new SingleSequenceParser(parserStages);

RadixTreeParser radixParser = new RadixTreeParser();
//radixParser.teach(grokStages.toArray(new ParserStage[0]));
Expand Down Expand Up @@ -184,7 +185,7 @@ public void testMe() throws Exception {
}

{
CompletableFuture<Event> event = null;
CompletableFuture<EventBuilder> event = null;
long start = System.currentTimeMillis();
for (int i = 0; i < iterations; i++) {
event = parser.parse(incoming.asReadOnlyBuffer());
Expand All @@ -207,7 +208,7 @@ public void testMe() throws Exception {
}

{
CompletableFuture<Event> event = null;
CompletableFuture<EventBuilder> event = null;
long start = System.currentTimeMillis();
for (int i = 0; i < iterations; i++) {
event = grokParser.parse(incoming.asReadOnlyBuffer());
Expand Down Expand Up @@ -326,6 +327,43 @@ public void testGrokParser() {
GrokParserStageSequenceBuilder.parseGrok("<%{INTEGER:facilityPriority}> %{MONTH:month} %{INTEGER:day} %{INTEGER:hour}:%{INTEGER:minute}:%{INTEGER:second} %{STRING:hostname} %{STRING:processName}[%{INTEGER:processId}]: %{MONTH:month} %{INTEGER:day} %{STRING:timestamp} %{STRING:timezone} \\%%{STRING:facility}-%{INTEGER:priority}-%{STRING:mnemonic}: %{STRING:message}");
}

/**
* Test adjacent pattern statements. This does not work with STRING patterns
* yet because they must be terminated by a delimiter: either a character,
* whitespace, or the end of the {@link ByteBuffer}.
*/
@Test
public void testGrokWithAdjacentPatterns() {
SingleSequenceParser parser = new SingleSequenceParser(GrokParserStageSequenceBuilder.parseGrok("%{INTEGER:a}%{MONTH:b}%{INTEGER:c}%{STRING:d} %{INTEGER:e}"));

Event event = parser.parse(ByteBuffer.wrap("435Jan333fghj 999".getBytes(StandardCharsets.US_ASCII))).join().getEvent();
assertEquals("435", event.getParm("a").getValue().getContent());
// January
assertEquals("1", event.getParm("b").getValue().getContent());
assertEquals("333", event.getParm("c").getValue().getContent());
assertEquals("fghj", event.getParm("d").getValue().getContent());
assertEquals("999", event.getParm("e").getValue().getContent());

assertNull(parser.parse(ByteBuffer.wrap("Feb12345".getBytes(StandardCharsets.US_ASCII))).join());
}

/**
* Test a pattern followed immediately by an escape sequence. This does not
* work yet with STRING patterns but could if we peek ahead to the escaped
* value and use it as a delimiter.
*/
@Test
public void testGrokWithAdjacentEscape() {
SingleSequenceParser parser = new SingleSequenceParser(GrokParserStageSequenceBuilder.parseGrok("\\5%{INTEGER:a}\\%\\%%{MONTH:b}\\f"));

Event event = parser.parse(ByteBuffer.wrap("56666%%Febf".getBytes(StandardCharsets.US_ASCII))).join().getEvent();
assertEquals("6666", event.getParm("a").getValue().getContent());
// February
assertEquals("2", event.getParm("b").getValue().getContent());

assertNull(parser.parse(ByteBuffer.wrap("5abc%%Febf".getBytes(StandardCharsets.US_ASCII))).join());
}

@Test
public void testParserStages() throws InterruptedException, ExecutionException {
ParserStage a = new MatchChar('a');
Expand Down Expand Up @@ -387,4 +425,15 @@ public void testGrokRadixTree() {
System.out.println("Parser tree: " + radixParser.tree.toString());
System.out.println("Parser tree size: " + radixParser.tree.size());
}

@Test
public void testParseSingleMessage() {
RadixTreeParser radixParser = new RadixTreeParser();
radixParser.teach(GrokParserStageSequenceBuilder.parseGrok("<%{INTEGER:facilityPriority}>%{STRING:messageId}: %{INTEGER:year}-%{INTEGER:month}-%{INTEGER:day} %{STRING:hostname} %{STRING:processName}: %{STRING:message}").toArray(new ParserStage[0]));
EventBuilder bldr = radixParser.parse(ByteBuffer.wrap("<31>main: 2010-08-19 localhost foo%d: load test %d on tty1".getBytes(StandardCharsets.US_ASCII))).join();
assertNotNull(bldr);
Event event = bldr.getEvent();
assertEquals("main", event.getParm("messageid").getValue().getContent());
assertEquals("foo%d", event.getParm("process").getValue().getContent());
}
}

0 comments on commit 2071e48

Please sign in to comment.