Skip to content

Commit

Permalink
Added NOSPACE pattern and employed it with messageId and processName.
Browse files Browse the repository at this point in the history
  • Loading branch information
@soleger
soleger committed Mar 24, 2017
1 parent db2f85d commit 66bf598
Show file tree
Hide file tree
Showing 3 changed files with 105 additions and 66 deletions.
51 changes: 45 additions & 6 deletions ...vents/syslog/src/main/java/org/opennms/netmgt/syslogd/GrokParserStageSequenceBuilder.java
View file
Expand Up @@ -34,9 +34,14 @@

import org.opennms.netmgt.model.events.EventBuilder;
import org.opennms.netmgt.syslogd.ParserStageSequenceBuilder.MatchInteger;
import org.opennms.netmgt.syslogd.ParserStageSequenceBuilder.MatchUntil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

public abstract class GrokParserStageSequenceBuilder {

private static final Logger LOG = LoggerFactory.getLogger(GrokParserStageSequenceBuilder.class);

private static enum GrokState {
TEXT,
ESCAPE_PATTERN,
Expand All @@ -47,9 +52,10 @@ private static enum GrokState {
}

private static enum GrokPattern {
STRING,
INT,
MONTH
MONTH,
NOSPACE,
STRING
}

/**
Expand Down Expand Up @@ -339,16 +345,28 @@ private static BiConsumer<ParserState,String> semanticStringToEventBuilder(Strin
case messageId:
// Unique to this parser
return (s,v) -> {
s.builder.setParam("messageid", v);
if ("-".equals(v.trim())) {
// Ignore
} else {
s.builder.setParam("messageid", v);
}
};
case processId:
// processId can be an integer or string
return (s,v) -> {
s.builder.setParam("processid", v);
if ("-".equals(v.trim())) {
// Ignore
} else {
s.builder.setParam("processid", v);
}
};
case processName:
return (s,v) -> {
s.builder.setParam("process", v);
if ("-".equals(v.trim())) {
// Ignore
} else {
s.builder.setParam("process", v);
}
};
case secondFraction:
return (s,v) -> {
Expand Down Expand Up @@ -455,9 +473,18 @@ public static List<ParserStage> parseGrok(String grok) {
switch(c) {
case '\\':
switch(patternType) {
case NOSPACE:
// TODO: We need to peek forward to the escaped character and then do the same as the default case
// factory.stringUntil(MatchUntil.WHITESPACE + c, semanticStringToEventBuilder(semanticString));
// factory.character(c);
// break;
throw new UnsupportedOperationException("Cannot support escape sequence directly after a NOSPACE pattern yet");
case STRING:
// TODO: We need to peek forward to the escaped character and then do the same as the default case
throw new UnsupportedOperationException("Cannot support escape sequence directly after a pattern yet");
// factory.stringUntil(String.valueOf(c), semanticStringToEventBuilder(semanticString));
// factory.character(c);
// break;
throw new UnsupportedOperationException("Cannot support escape sequence directly after a STRING pattern yet");
case INT:
factory.integer(semanticIntegerToEventBuilder(semanticString));
break;
Expand All @@ -471,6 +498,12 @@ public static List<ParserStage> parseGrok(String grok) {
continue;
case '%':
switch(patternType) {
case NOSPACE:
// This is probably not an intended behavior
LOG.warn("NOSPACE pattern followed immediately by another pattern will greedily consume until whitespace is encountered");
factory.stringUntilWhitespace(semanticStringToEventBuilder(semanticString));
factory.whitespace();
break;
case STRING:
// TODO: Can we handle this case?
throw new IllegalArgumentException(String.format("Invalid pattern: %s:%s does not have a trailing delimiter, cannot determine end of string", patternString, semanticString));
Expand All @@ -487,6 +520,7 @@ public static List<ParserStage> parseGrok(String grok) {
continue;
case ' ':
switch(patternType) {
case NOSPACE:
case STRING:
factory.stringUntilWhitespace(semanticStringToEventBuilder(semanticString));
factory.whitespace();
Expand All @@ -503,6 +537,10 @@ public static List<ParserStage> parseGrok(String grok) {
break;
default:
switch(patternType) {
case NOSPACE:
factory.stringUntil(MatchUntil.WHITESPACE + c, semanticStringToEventBuilder(semanticString));
factory.character(c);
break;
case STRING:
factory.stringUntil(String.valueOf(c), semanticStringToEventBuilder(semanticString));
factory.character(c);
Expand Down Expand Up @@ -532,6 +570,7 @@ public static List<ParserStage> parseGrok(String grok) {
GrokPattern patternType = GrokPattern.valueOf(patternString);

switch(patternType) {
case NOSPACE:
case STRING:
factory.terminal().string(semanticStringToEventBuilder(semanticString));
break;
Expand Down
60 changes: 30 additions & 30 deletions features/events/syslog/src/main/resources/org/opennms/netmgt/syslogd/grok-patterns.txt
View file
@@ -1,64 +1,64 @@
# RFC 3164 with messageId, time
<%{INT:facilityPriority}>%{STRING:messageId}: %{MONTH:month} %{INT:day} %{INT:hour}:%{INT:minute}:%{INT:second} %{STRING:hostname} %{STRING:processName}[%{INT:processId}]: %{STRING:message}
<%{INT:facilityPriority}>%{STRING:messageId}: %{MONTH:month} %{INT:day} %{INT:hour}:%{INT:minute}:%{INT:second} %{STRING:hostname} [%{INT:processId}]: %{STRING:message}
<%{INT:facilityPriority}>%{STRING:messageId}: %{MONTH:month} %{INT:day} %{INT:hour}:%{INT:minute}:%{INT:second} %{STRING:hostname} %{STRING:processName}: %{STRING:message}
<%{INT:facilityPriority}>%{STRING:messageId}: %{MONTH:month} %{INT:day} %{INT:hour}:%{INT:minute}:%{INT:second} %{STRING:hostname} %{STRING:message}
<%{INT:facilityPriority}>%{NOSPACE:messageId}: %{MONTH:month} %{INT:day} %{INT:hour}:%{INT:minute}:%{INT:second} %{STRING:hostname} %{NOSPACE:processName}[%{INT:processId}]: %{STRING:message}
<%{INT:facilityPriority}>%{NOSPACE:messageId}: %{MONTH:month} %{INT:day} %{INT:hour}:%{INT:minute}:%{INT:second} %{STRING:hostname} [%{INT:processId}]: %{STRING:message}
<%{INT:facilityPriority}>%{NOSPACE:messageId}: %{MONTH:month} %{INT:day} %{INT:hour}:%{INT:minute}:%{INT:second} %{STRING:hostname} %{NOSPACE:processName}: %{STRING:message}
<%{INT:facilityPriority}>%{NOSPACE:messageId}: %{MONTH:month} %{INT:day} %{INT:hour}:%{INT:minute}:%{INT:second} %{STRING:hostname} %{STRING:message}

# RFC 3164 with time
<%{INT:facilityPriority}> %{MONTH:month} %{INT:day} %{INT:hour}:%{INT:minute}:%{INT:second} %{STRING:hostname} %{STRING:processName}[%{INT:processId}]: %{STRING:message}
<%{INT:facilityPriority}> %{MONTH:month} %{INT:day} %{INT:hour}:%{INT:minute}:%{INT:second} %{STRING:hostname} %{NOSPACE:processName}[%{INT:processId}]: %{STRING:message}
<%{INT:facilityPriority}> %{MONTH:month} %{INT:day} %{INT:hour}:%{INT:minute}:%{INT:second} %{STRING:hostname} [%{INT:processId}]: %{STRING:message}
<%{INT:facilityPriority}> %{MONTH:month} %{INT:day} %{INT:hour}:%{INT:minute}:%{INT:second} %{STRING:hostname} %{STRING:processName}: %{STRING:message}
<%{INT:facilityPriority}> %{MONTH:month} %{INT:day} %{INT:hour}:%{INT:minute}:%{INT:second} %{STRING:hostname} %{NOSPACE:processName}: %{STRING:message}
<%{INT:facilityPriority}> %{MONTH:month} %{INT:day} %{INT:hour}:%{INT:minute}:%{INT:second} %{STRING:hostname} %{STRING:message}


# RFC 3164/ISO8601 date with time, secondFraction, timezone
<%{INT:facilityPriority}> %{INT:year}-%{INT:month}-%{INT:day}T%{INT:hour}:%{INT:minute}:%{INT:second}.%{INT:secondFraction}%{STRING:timezone} %{STRING:hostname} %{STRING:processName}[%{INT:processId}]: %{STRING:message}
<%{INT:facilityPriority}> %{INT:year}-%{INT:month}-%{INT:day}T%{INT:hour}:%{INT:minute}:%{INT:second}.%{INT:secondFraction}%{STRING:timezone} %{STRING:hostname} %{NOSPACE:processName}[%{INT:processId}]: %{STRING:message}
<%{INT:facilityPriority}> %{INT:year}-%{INT:month}-%{INT:day}T%{INT:hour}:%{INT:minute}:%{INT:second}.%{INT:secondFraction}%{STRING:timezone} %{STRING:hostname} [%{INT:processId}]: %{STRING:message}
<%{INT:facilityPriority}> %{INT:year}-%{INT:month}-%{INT:day}T%{INT:hour}:%{INT:minute}:%{INT:second}.%{INT:secondFraction}%{STRING:timezone} %{STRING:hostname} %{STRING:processName}: %{STRING:message}
<%{INT:facilityPriority}> %{INT:year}-%{INT:month}-%{INT:day}T%{INT:hour}:%{INT:minute}:%{INT:second}.%{INT:secondFraction}%{STRING:timezone} %{STRING:hostname} %{NOSPACE:processName}: %{STRING:message}
<%{INT:facilityPriority}> %{INT:year}-%{INT:month}-%{INT:day}T%{INT:hour}:%{INT:minute}:%{INT:second}.%{INT:secondFraction}%{STRING:timezone} %{STRING:hostname} %{STRING:message}

# RFC 3164/ISO8601 date with time, timezone
<%{INT:facilityPriority}> %{INT:year}-%{INT:month}-%{INT:day}T%{INT:hour}:%{INT:minute}:%{INT:second}%{STRING:timezone} %{STRING:hostname} %{STRING:processName}[%{INT:processId}]: %{STRING:message}
<%{INT:facilityPriority}> %{INT:year}-%{INT:month}-%{INT:day}T%{INT:hour}:%{INT:minute}:%{INT:second}%{STRING:timezone} %{STRING:hostname} %{NOSPACE:processName}[%{INT:processId}]: %{STRING:message}
<%{INT:facilityPriority}> %{INT:year}-%{INT:month}-%{INT:day}T%{INT:hour}:%{INT:minute}:%{INT:second}%{STRING:timezone} %{STRING:hostname} [%{INT:processId}]: %{STRING:message}
<%{INT:facilityPriority}> %{INT:year}-%{INT:month}-%{INT:day}T%{INT:hour}:%{INT:minute}:%{INT:second}%{STRING:timezone} %{STRING:hostname} %{STRING:processName}: %{STRING:message}
<%{INT:facilityPriority}> %{INT:year}-%{INT:month}-%{INT:day}T%{INT:hour}:%{INT:minute}:%{INT:second}%{STRING:timezone} %{STRING:hostname} %{NOSPACE:processName}: %{STRING:message}
<%{INT:facilityPriority}> %{INT:year}-%{INT:month}-%{INT:day}T%{INT:hour}:%{INT:minute}:%{INT:second}%{STRING:timezone} %{STRING:hostname} %{STRING:message}

# RFC 3164/ISO8601 date with time
<%{INT:facilityPriority}> %{INT:year}-%{INT:month}-%{INT:day}T%{INT:hour}:%{INT:minute}:%{INT:second} %{STRING:hostname} %{STRING:processName}[%{INT:processId}]: %{STRING:message}
<%{INT:facilityPriority}> %{INT:year}-%{INT:month}-%{INT:day}T%{INT:hour}:%{INT:minute}:%{INT:second} %{STRING:hostname} %{NOSPACE:processName}[%{INT:processId}]: %{STRING:message}
<%{INT:facilityPriority}> %{INT:year}-%{INT:month}-%{INT:day}T%{INT:hour}:%{INT:minute}:%{INT:second} %{STRING:hostname} [%{INT:processId}]: %{STRING:message}
<%{INT:facilityPriority}> %{INT:year}-%{INT:month}-%{INT:day}T%{INT:hour}:%{INT:minute}:%{INT:second} %{STRING:hostname} %{STRING:processName}: %{STRING:message}
<%{INT:facilityPriority}> %{INT:year}-%{INT:month}-%{INT:day}T%{INT:hour}:%{INT:minute}:%{INT:second} %{STRING:hostname} %{NOSPACE:processName}: %{STRING:message}
<%{INT:facilityPriority}> %{INT:year}-%{INT:month}-%{INT:day}T%{INT:hour}:%{INT:minute}:%{INT:second} %{STRING:hostname} %{STRING:message}

# RFC 3164/ISO8601 date
<%{INT:facilityPriority}> %{INT:year}-%{INT:month}-%{INT:day} %{STRING:hostname} %{STRING:processName}[%{INT:processId}]: %{STRING:message}
<%{INT:facilityPriority}> %{INT:year}-%{INT:month}-%{INT:day} %{STRING:hostname} %{NOSPACE:processName}[%{INT:processId}]: %{STRING:message}
<%{INT:facilityPriority}> %{INT:year}-%{INT:month}-%{INT:day} %{STRING:hostname} [%{INT:processId}]: %{STRING:message}
<%{INT:facilityPriority}> %{INT:year}-%{INT:month}-%{INT:day} %{STRING:hostname} %{STRING:processName}: %{STRING:message}
<%{INT:facilityPriority}> %{INT:year}-%{INT:month}-%{INT:day} %{STRING:hostname} %{NOSPACE:processName}: %{STRING:message}
<%{INT:facilityPriority}> %{INT:year}-%{INT:month}-%{INT:day} %{STRING:hostname} %{STRING:message}


# RFC 3164/ISO8601 date with messageId, time, secondFraction, timezone
<%{INT:facilityPriority}>%{STRING:messageId}: %{INT:year}-%{INT:month}-%{INT:day}T%{INT:hour}:%{INT:minute}:%{INT:second}.%{INT:secondFraction}%{STRING:timezone} %{STRING:hostname} %{STRING:processName}[%{INT:processId}]: %{STRING:message}
<%{INT:facilityPriority}>%{STRING:messageId}: %{INT:year}-%{INT:month}-%{INT:day}T%{INT:hour}:%{INT:minute}:%{INT:second}.%{INT:secondFraction}%{STRING:timezone} %{STRING:hostname} [%{INT:processId}]: %{STRING:message}
<%{INT:facilityPriority}>%{STRING:messageId}: %{INT:year}-%{INT:month}-%{INT:day}T%{INT:hour}:%{INT:minute}:%{INT:second}.%{INT:secondFraction}%{STRING:timezone} %{STRING:hostname} %{STRING:processName}: %{STRING:message}
<%{INT:facilityPriority}>%{STRING:messageId}: %{INT:year}-%{INT:month}-%{INT:day}T%{INT:hour}:%{INT:minute}:%{INT:second}.%{INT:secondFraction}%{STRING:timezone} %{STRING:hostname} %{STRING:message}
<%{INT:facilityPriority}>%{NOSPACE:messageId}: %{INT:year}-%{INT:month}-%{INT:day}T%{INT:hour}:%{INT:minute}:%{INT:second}.%{INT:secondFraction}%{STRING:timezone} %{STRING:hostname} %{NOSPACE:processName}[%{INT:processId}]: %{STRING:message}
<%{INT:facilityPriority}>%{NOSPACE:messageId}: %{INT:year}-%{INT:month}-%{INT:day}T%{INT:hour}:%{INT:minute}:%{INT:second}.%{INT:secondFraction}%{STRING:timezone} %{STRING:hostname} [%{INT:processId}]: %{STRING:message}
<%{INT:facilityPriority}>%{NOSPACE:messageId}: %{INT:year}-%{INT:month}-%{INT:day}T%{INT:hour}:%{INT:minute}:%{INT:second}.%{INT:secondFraction}%{STRING:timezone} %{STRING:hostname} %{NOSPACE:processName}: %{STRING:message}
<%{INT:facilityPriority}>%{NOSPACE:messageId}: %{INT:year}-%{INT:month}-%{INT:day}T%{INT:hour}:%{INT:minute}:%{INT:second}.%{INT:secondFraction}%{STRING:timezone} %{STRING:hostname} %{STRING:message}

# RFC 3164/ISO8601 date with messageId, time, timezone
<%{INT:facilityPriority}>%{STRING:messageId}: %{INT:year}-%{INT:month}-%{INT:day}T%{INT:hour}:%{INT:minute}:%{INT:second}%{STRING:timezone} %{STRING:hostname} %{STRING:processName}[%{INT:processId}]: %{STRING:message}
<%{INT:facilityPriority}>%{STRING:messageId}: %{INT:year}-%{INT:month}-%{INT:day}T%{INT:hour}:%{INT:minute}:%{INT:second}%{STRING:timezone} %{STRING:hostname} [%{INT:processId}]: %{STRING:message}
<%{INT:facilityPriority}>%{STRING:messageId}: %{INT:year}-%{INT:month}-%{INT:day}T%{INT:hour}:%{INT:minute}:%{INT:second}%{STRING:timezone} %{STRING:hostname} %{STRING:processName}: %{STRING:message}
<%{INT:facilityPriority}>%{STRING:messageId}: %{INT:year}-%{INT:month}-%{INT:day}T%{INT:hour}:%{INT:minute}:%{INT:second}%{STRING:timezone} %{STRING:hostname} %{STRING:message}
<%{INT:facilityPriority}>%{NOSPACE:messageId}: %{INT:year}-%{INT:month}-%{INT:day}T%{INT:hour}:%{INT:minute}:%{INT:second}%{STRING:timezone} %{STRING:hostname} %{NOSPACE:processName}[%{INT:processId}]: %{STRING:message}
<%{INT:facilityPriority}>%{NOSPACE:messageId}: %{INT:year}-%{INT:month}-%{INT:day}T%{INT:hour}:%{INT:minute}:%{INT:second}%{STRING:timezone} %{STRING:hostname} [%{INT:processId}]: %{STRING:message}
<%{INT:facilityPriority}>%{NOSPACE:messageId}: %{INT:year}-%{INT:month}-%{INT:day}T%{INT:hour}:%{INT:minute}:%{INT:second}%{STRING:timezone} %{STRING:hostname} %{NOSPACE:processName}: %{STRING:message}
<%{INT:facilityPriority}>%{NOSPACE:messageId}: %{INT:year}-%{INT:month}-%{INT:day}T%{INT:hour}:%{INT:minute}:%{INT:second}%{STRING:timezone} %{STRING:hostname} %{STRING:message}

# RFC 3164/ISO8601 date with messageId, time
<%{INT:facilityPriority}>%{STRING:messageId}: %{INT:year}-%{INT:month}-%{INT:day}T%{INT:hour}:%{INT:minute}:%{INT:second} %{STRING:hostname} %{STRING:processName}[%{INT:processId}]: %{STRING:message}
<%{INT:facilityPriority}>%{STRING:messageId}: %{INT:year}-%{INT:month}-%{INT:day}T%{INT:hour}:%{INT:minute}:%{INT:second} %{STRING:hostname} [%{INT:processId}]: %{STRING:message}
<%{INT:facilityPriority}>%{STRING:messageId}: %{INT:year}-%{INT:month}-%{INT:day}T%{INT:hour}:%{INT:minute}:%{INT:second} %{STRING:hostname} %{STRING:processName}: %{STRING:message}
<%{INT:facilityPriority}>%{STRING:messageId}: %{INT:year}-%{INT:month}-%{INT:day}T%{INT:hour}:%{INT:minute}:%{INT:second} %{STRING:hostname} %{STRING:message}
<%{INT:facilityPriority}>%{NOSPACE:messageId}: %{INT:year}-%{INT:month}-%{INT:day}T%{INT:hour}:%{INT:minute}:%{INT:second} %{STRING:hostname} %{NOSPACE:processName}[%{INT:processId}]: %{STRING:message}
<%{INT:facilityPriority}>%{NOSPACE:messageId}: %{INT:year}-%{INT:month}-%{INT:day}T%{INT:hour}:%{INT:minute}:%{INT:second} %{STRING:hostname} [%{INT:processId}]: %{STRING:message}
<%{INT:facilityPriority}>%{NOSPACE:messageId}: %{INT:year}-%{INT:month}-%{INT:day}T%{INT:hour}:%{INT:minute}:%{INT:second} %{STRING:hostname} %{NOSPACE:processName}: %{STRING:message}
<%{INT:facilityPriority}>%{NOSPACE:messageId}: %{INT:year}-%{INT:month}-%{INT:day}T%{INT:hour}:%{INT:minute}:%{INT:second} %{STRING:hostname} %{STRING:message}

# RFC 3164/ISO8601 date with messageId
<%{INT:facilityPriority}>%{STRING:messageId}: %{INT:year}-%{INT:month}-%{INT:day} %{STRING:hostname} %{STRING:processName}[%{INT:processId}]: %{STRING:message}
<%{INT:facilityPriority}>%{STRING:messageId}: %{INT:year}-%{INT:month}-%{INT:day} %{STRING:hostname} [%{INT:processId}]: %{STRING:message}
<%{INT:facilityPriority}>%{STRING:messageId}: %{INT:year}-%{INT:month}-%{INT:day} %{STRING:hostname} %{STRING:processName}: %{STRING:message}
<%{INT:facilityPriority}>%{STRING:messageId}: %{INT:year}-%{INT:month}-%{INT:day} %{STRING:hostname} %{STRING:message}
<%{INT:facilityPriority}>%{NOSPACE:messageId}: %{INT:year}-%{INT:month}-%{INT:day} %{STRING:hostname} %{NOSPACE:processName}[%{INT:processId}]: %{STRING:message}
<%{INT:facilityPriority}>%{NOSPACE:messageId}: %{INT:year}-%{INT:month}-%{INT:day} %{STRING:hostname} [%{INT:processId}]: %{STRING:message}
<%{INT:facilityPriority}>%{NOSPACE:messageId}: %{INT:year}-%{INT:month}-%{INT:day} %{STRING:hostname} %{NOSPACE:processName}: %{STRING:message}
<%{INT:facilityPriority}>%{NOSPACE:messageId}: %{INT:year}-%{INT:month}-%{INT:day} %{STRING:hostname} %{STRING:message}


# RFC 5424 with secondFraction
Expand Down

0 comments on commit 66bf598

Please sign in to comment.