Skip to content

Commit

Permalink
Add configuration setting to use CEF full names
Browse files Browse the repository at this point in the history
  • Loading branch information
Jochen Schalanda committed Sep 19, 2017
1 parent 421ec52 commit 06315eb
Show file tree
Hide file tree
Showing 9 changed files with 199 additions and 109 deletions.
35 changes: 29 additions & 6 deletions src/main/java/org/graylog/plugins/cef/codec/BaseCEFCodec.java
View file
Expand Up @@ -2,10 +2,12 @@

import com.google.inject.assistedinject.Assisted;
import com.google.inject.assistedinject.AssistedInject;
import org.graylog.plugins.cef.parser.CEFMapping;
import org.graylog.plugins.cef.parser.CEFMessage;
import org.graylog2.plugin.ResolvableInetSocketAddress;
import org.graylog2.plugin.configuration.Configuration;
import org.graylog2.plugin.configuration.ConfigurationRequest;
import org.graylog2.plugin.configuration.fields.BooleanField;
import org.graylog2.plugin.configuration.fields.ConfigurationField;
import org.graylog2.plugin.configuration.fields.TextField;
import org.graylog2.plugin.inputs.annotations.ConfigClass;
Expand All @@ -19,16 +21,22 @@
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import java.net.InetSocketAddress;
import java.util.Map;

import static com.google.common.base.Strings.isNullOrEmpty;

public abstract class BaseCEFCodec implements Codec {
private static final Logger LOG = LoggerFactory.getLogger(BaseCEFCodec.class);

private static final String CK_TIMEZONE = "timezone";
static final String CK_TIMEZONE = "timezone";
static final String CK_USE_FULL_NAMES = "use_full_names";

protected final Configuration configuration;
final DateTimeZone timezone;
final boolean useFullNames;

@AssistedInject
public BaseCEFCodec(@Assisted Configuration configuration) {
BaseCEFCodec(@Assisted Configuration configuration) {
this.configuration = configuration;

DateTimeZone timezone;
Expand All @@ -38,17 +46,20 @@ public BaseCEFCodec(@Assisted Configuration configuration) {
LOG.warn("Could not configure CEF input timezone. Falling back to local default. Please check the error message:", e);
timezone = DateTimeZone.getDefault();
}
this.timezone = timezone;
this.useFullNames = configuration.getBoolean(CK_USE_FULL_NAMES);
}

protected String buildMessageSummary(CEFMessage cef) {
return cef.deviceProduct() + ": [" + cef.deviceEventClassId() + ", " + cef.severity().text() + "] " + cef.name();
}

protected String decideSource(CEFMessage cef, RawMessage raw) {
if (cef.fields() != null && cef.fields().containsKey("dvc")) {
String dvc = (String) cef.fields().get("dvc");
if (!dvc.isEmpty()) {
return dvc;
final Map<String, Object> fields = cef.fields();
if (fields != null && !fields.isEmpty()) {
final String deviceAddress = (String) fields.getOrDefault(CEFMapping.dvc.getFullName(), fields.get(CEFMapping.dvc.getKeyName()));
if (!isNullOrEmpty(deviceAddress)) {
return deviceAddress;
}
}

Expand All @@ -70,6 +81,12 @@ public CodecAggregator getAggregator() {
return null;
}

@Nonnull
@Override
public Configuration getConfiguration() {
return configuration;
}

@ConfigClass
public static class Config implements Codec.Config {
@Override
Expand All @@ -83,6 +100,12 @@ public ConfigurationRequest getRequestedConfiguration() {
"Timezone of the timestamps in the CEF messages we'l receive. Set this to the local timezone if in doubt. (CEF messages do not include timezone information) Format example: +01:00 or America/Chicago",
ConfigurationField.Optional.NOT_OPTIONAL
));
cr.addField(new BooleanField(
CK_USE_FULL_NAMES,
"Use full field names",
false,
"Use full field names in CEF messages (as defined in the CEF specification)"
));

return cr;
}
Expand Down
46 changes: 2 additions & 44 deletions src/main/java/org/graylog/plugins/cef/codec/CEFCodec.java
View file
Expand Up @@ -25,24 +25,13 @@
public class CEFCodec extends BaseCEFCodec {
public static final String NAME = "CEF";

private static final Logger LOG = LoggerFactory.getLogger(CEFCodec.class);
private static final String CK_TIMEZONE = "timezone";

private final CEFParser parser;

@AssistedInject
public CEFCodec(@Assisted Configuration configuration) {
super(configuration);

DateTimeZone timezone;
try {
timezone = DateTimeZone.forID(configuration.getString(CK_TIMEZONE));
} catch (Exception e) {
LOG.warn("Could not configure CEF input timezone. Falling back to local default. Please check the error message:", e);
timezone = DateTimeZone.getDefault();
}

this.parser = new CEFParser();
this.parser = new CEFParser(useFullNames);
}

@Nullable
Expand Down Expand Up @@ -77,23 +66,11 @@ public Message decode(@Nonnull RawMessage rawMessage) {
}
}

@Nullable
@Override
public CodecAggregator getAggregator() {
return null;
}

@Override
public String getName() {
return NAME;
}

@Nonnull
@Override
public Configuration getConfiguration() {
return configuration;
}

@FactoryClass
public interface Factory extends Codec.Factory<CEFCodec> {
@Override
Expand All @@ -104,25 +81,6 @@ public interface Factory extends Codec.Factory<CEFCodec> {
}

@ConfigClass
public static class Config implements Codec.Config {
@Override
public ConfigurationRequest getRequestedConfiguration() {
ConfigurationRequest cr = new ConfigurationRequest();

cr.addField(new TextField(
CK_TIMEZONE,
"Timezone",
DateTimeZone.getDefault().getID(),
"Timezone of the timestamps in the CEF messages we'l receive. Set this to the local timezone if in doubt. (CEF messages do not include timezone information) Format example: +01:00 or America/Chicago",
ConfigurationField.Optional.NOT_OPTIONAL
));

return cr;
}

@Override
public void overrideDefaultValues(@Nonnull ConfigurationRequest cr) {
}
public static class Config extends BaseCEFCodec.Config {
}

}
46 changes: 2 additions & 44 deletions src/main/java/org/graylog/plugins/cef/codec/CEFSyslogCodec.java
View file
Expand Up @@ -25,24 +25,13 @@
public class CEFSyslogCodec extends BaseCEFCodec {
public static final String NAME = "CEF Syslog";

private static final Logger LOG = LoggerFactory.getLogger(CEFSyslogCodec.class);
private static final String CK_TIMEZONE = "timezone";

private final SyslogCEFParser parser;

@AssistedInject
public CEFSyslogCodec(@Assisted Configuration configuration) {
super(configuration);

DateTimeZone timezone;
try {
timezone = DateTimeZone.forID(configuration.getString(CK_TIMEZONE));
} catch (Exception e) {
LOG.warn("Could not configure CEF input timezone. Falling back to local default. Please check the error message:", e);
timezone = DateTimeZone.getDefault();
}

this.parser = new SyslogCEFParser(timezone);
this.parser = new SyslogCEFParser(timezone, useFullNames);
}

@Nullable
Expand Down Expand Up @@ -77,23 +66,11 @@ public Message decode(@Nonnull RawMessage rawMessage) {
}
}

@Nullable
@Override
public CodecAggregator getAggregator() {
return null;
}

@Override
public String getName() {
return NAME;
}

@Nonnull
@Override
public Configuration getConfiguration() {
return configuration;
}

@FactoryClass
public interface Factory extends Codec.Factory<CEFSyslogCodec> {
@Override
Expand All @@ -104,25 +81,6 @@ public interface Factory extends Codec.Factory<CEFSyslogCodec> {
}

@ConfigClass
public static class Config implements Codec.Config {
@Override
public ConfigurationRequest getRequestedConfiguration() {
ConfigurationRequest cr = new ConfigurationRequest();

cr.addField(new TextField(
CK_TIMEZONE,
"Timezone",
DateTimeZone.getDefault().getID(),
"Timezone of the timestamps in the CEF messages we'l receive. Set this to the local timezone if in doubt. (CEF messages do not include timezone information) Format example: +01:00 or America/Chicago",
ConfigurationField.Optional.NOT_OPTIONAL
));

return cr;
}

@Override
public void overrideDefaultValues(@Nonnull ConfigurationRequest cr) {
}
public static class Config extends BaseCEFCodec.Config {
}

}
21 changes: 16 additions & 5 deletions src/main/java/org/graylog/plugins/cef/parser/CEFParser.java
View file
Expand Up @@ -17,6 +17,13 @@ public class CEFParser {

private static final Pattern PATTERN = Pattern.compile("^\\s*CEF:(?<version>\\d+?)(?<!\\\\)\\|(?<deviceVendor>.+?)(?<!\\\\)\\|(?<deviceProduct>.+?)(?<!\\\\)\\|(?<deviceVersion>.+?)(?<!\\\\)\\|(?<deviceEventClassId>.+?)(?<!\\\\)\\|(?<name>.+?)(?<!\\\\)\\|(?<severity>.+?)(?<!\\\\)\\|(?<fields>.+?)(?:$|msg=(?<message>.+))", Pattern.DOTALL);
private static final Pattern EXTENSIONS_PATTERN = Pattern.compile("(?<key>\\w+)=(?<value>.*?(?=\\s*\\w+=|\\s*$))");
private static final String LABEL_SUFFIX = "Label";

private final boolean useFullNames;

public CEFParser(boolean useFullNames) {
this.useFullNames = useFullNames;
}

public CEFMessage.Builder parse(String x) throws ParserException {
final Matcher m = PATTERN.matcher(x);
Expand Down Expand Up @@ -78,20 +85,19 @@ Map<String, Object> parseExtensions(String x) {
final String key = field.getKey();

// Skip "labels"
if (key.endsWith("Label")) {
if (key.endsWith(LABEL_SUFFIX)) {
continue;
}

final String label = getLabel(key, allFields);
final CEFMapping fieldMapping = CEFMapping.forKeyName(key);
if (fieldMapping != null) {
try {
resultBuilder.put(label, fieldMapping.convert(field.getValue()));
resultBuilder.put(getLabel(fieldMapping, allFields), fieldMapping.convert(field.getValue()));
} catch (Exception e) {
LOG.warn("Could not transform CEF field [{}] according to standard. Skipping.", key, e);
}
} else {
resultBuilder.put(label, field.getValue());
resultBuilder.put(getLabel(key, allFields), field.getValue());
}
}

Expand All @@ -111,8 +117,13 @@ private String sanitizeFieldValue(String value) {
.replace("\\=", "=");
}

private String getLabel(CEFMapping mapping, Map<String, String> fields) {
final String labelName = mapping.getKeyName() + LABEL_SUFFIX;
return fields.getOrDefault(labelName, useFullNames ? mapping.getFullName() : mapping.getKeyName());
}

private String getLabel(String valueName, Map<String, String> fields) {
final String labelName = valueName + "Label";
final String labelName = valueName + LABEL_SUFFIX;
return fields.getOrDefault(labelName, valueName);
}
}
4 changes: 2 additions & 2 deletions src/main/java/org/graylog/plugins/cef/parser/SyslogCEFParser.java
View file
Expand Up @@ -21,8 +21,8 @@ public class SyslogCEFParser {
private final DateTimeZone timezone;
private final CEFParser parser;

public SyslogCEFParser(DateTimeZone timezone) {
this(timezone, new CEFParser());
public SyslogCEFParser(DateTimeZone timezone, boolean useFullNames) {
this(timezone, new CEFParser(useFullNames));
}

public SyslogCEFParser(DateTimeZone timezone, CEFParser parser) {
Expand Down
3 changes: 2 additions & 1 deletion src/main/java/org/graylog/plugins/cef/pipelines/rules/CEFParserFunction.java
View file
Expand Up @@ -33,7 +33,8 @@ public class CEFParserFunction extends AbstractFunction<CEFParserResult> {
@Inject
public CEFParserFunction(final MetricRegistry metricRegistry) {
this.parseTime = metricRegistry.timer(name(this.getClass(), "parseTime"));
this.parser = new CEFParser();
// TODO
this.parser = new CEFParser(false);
}

@Override
Expand Down

0 comments on commit 06315eb

Please sign in to comment.