Skip to content

Commit

Permalink
Check configuration parameters passed into the Logstash filter
Browse files Browse the repository at this point in the history
  • Loading branch information
@nielsbasjes
nielsbasjes committed Mar 10, 2019
1 parent e2a1f3e commit 61d3dec
Show file tree
Hide file tree
Showing 3 changed files with 235 additions and 4 deletions.
2 changes: 1 addition & 1 deletion analyzer/src/main/java/nl/basjes/parse/useragent/UserAgent.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -326,7 +326,7 @@ public void reset() {
}
}

static boolean isSystemField(String fieldname) {
public static boolean isSystemField(String fieldname) {
return SET_ALL_FIELDS.equals(fieldname) ||
SYNTAX_ERROR.equals(fieldname) ||
USERAGENT_FIELDNAME.equals(fieldname);
Expand Down
83 changes: 81 additions & 2 deletions ...logstash/src/main/java/org/logstash/filters/nl/basjes/parse/useragent/logstash/Yauaa.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,8 @@
import nl.basjes.parse.useragent.UserAgentAnalyzer;
import nl.basjes.parse.useragent.UserAgentAnalyzer.UserAgentAnalyzerBuilder;
import org.logstash.Event;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
//import org.slf4j.Logger;
//import org.slf4j.LoggerFactory;

Expand All @@ -38,13 +40,12 @@
@LogstashPlugin(name = "yauaa")
public class Yauaa implements Filter {

// private static final Logger LOG = LoggerFactory.getLogger(Yauaa.class);
private static final Logger LOG = LoggerFactory.getLogger(Yauaa.class);

private UserAgentAnalyzer userAgentAnalyzer;

private List<String> requestedFieldNames = new ArrayList<>();


public static final PluginConfigSpec<String> SOURCE_CONFIG =
Configuration.requiredStringSetting("source");

Expand All @@ -59,6 +60,8 @@ public Yauaa(Configuration config, Context context) {
sourceField = config.get(SOURCE_CONFIG);
outputFields = config.get(FIELDS_CONFIG);

checkConfiguration();

UserAgentAnalyzerBuilder<?, ?> userAgentAnalyzerBuilder =
UserAgentAnalyzer
.newBuilder()
Expand Down Expand Up @@ -96,4 +99,80 @@ public Collection<Event> filter(Collection<Event> events) {
public Collection<PluginConfigSpec<?>> configSchema() {
return Arrays.asList(SOURCE_CONFIG, FIELDS_CONFIG);
}

private void checkConfiguration() {
List<String> configProblems = new ArrayList<>();

UserAgentAnalyzer uaa = UserAgentAnalyzer
.newBuilder()
.delayInitialization()
.dropTests()
.hideMatcherLoadStats()
.build();

List<String> allFieldNames = uaa.getAllPossibleFieldNamesSorted();

if (sourceField == null) {
configProblems.add("The \"source\" has not been specified.\n");
} else {
if (sourceField.isEmpty()) {
configProblems.add("The \"source\" is empty.\n");
}
}

if (outputFields == null) {
configProblems.add("The list of needed \"fields\" has not been specified.\n");
} else {
if (outputFields.isEmpty()) {
configProblems.add("The list of needed \"fields\" is empty.\n");
}
for (String outputField: outputFields.keySet()) {
if (!allFieldNames.contains(outputField)) {
configProblems.add("The requested field \"" + outputField + "\" does not exist.\n");
}
}
}

if (configProblems.isEmpty()) {
return; // All is fine
}

StringBuilder errorMessage = new StringBuilder();

int maxNameLength = 0;
for (String field: allFieldNames) {
maxNameLength = Math.max(maxNameLength, field.length());
}

errorMessage.append("\nThe Yauaa filter config is invalid.\n");
errorMessage.append("The problems we found:\n");

configProblems.forEach(problem -> errorMessage.append("- ").append(problem).append('\n'));

errorMessage.append("\n");
errorMessage.append("Example of a generic valid config:\n");
errorMessage.append("\n");
errorMessage.append("filter {\n");
errorMessage.append(" yauaa {\n");
errorMessage.append(" source => \"useragent\"\n");
errorMessage.append(" fields => {\n");

for (String field: allFieldNames) {
if (!UserAgent.isSystemField(field)) {
errorMessage.append(" \"").append(field).append("\"");
for (int i = field.length(); i < maxNameLength; i++) {
errorMessage.append(' ');
}
errorMessage.append(" => \"userAgent").append(field).append("\"\n");
}
}
errorMessage.append(" }\n");
errorMessage.append(" }\n");
errorMessage.append("}\n");
errorMessage.append("\n");

LOG.error("{}", errorMessage);
throw new IllegalArgumentException(errorMessage.toString());
}

}
154 changes: 153 additions & 1 deletion ...tash/src/test/java/org/logstash/filters/nl/basjes/parse/useragent/logstash/TestYauaa.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,18 +21,26 @@
import co.elastic.logstash.api.Context;
import co.elastic.logstash.api.v0.Filter;
import org.junit.Assert;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;
import org.logstash.Event;

import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;

import static org.hamcrest.core.AllOf.allOf;
import static org.hamcrest.core.StringContains.containsString;

public class TestYauaa {

@Rule
public final transient ExpectedException expectedEx = ExpectedException.none();

@Test
public void testYauaaFilter() {
public void testNormalUse() {
String sourceField = "foo";

Map<String, String> fieldMappings = new HashMap<>();
Expand Down Expand Up @@ -61,4 +69,148 @@ public void testYauaaFilter() {
Assert.assertEquals("Chrome 48.0.2564.82", e.getField("ANV"));
}

@Test()
public void testBadConfigNothing() {
expectedEx.expect(IllegalArgumentException.class);
expectedEx.expectMessage(
allOf(
containsString("The \"source\" has not been specified."),
containsString("The list of needed \"fields\" has not been specified.")));

Map<String, Object> configMap = new HashMap<>();
Configuration config = new Configuration(configMap);
Context context = new Context();
Filter filter = new Yauaa(config, context);
}

@Test()
public void testBadConfigNoSource() {
expectedEx.expect(IllegalArgumentException.class);
expectedEx.expectMessage("The \"source\" has not been specified.");

Map<String, String> fieldMappings = new HashMap<>();
fieldMappings.put("DeviceClass", "DC");
fieldMappings.put("AgentNameVersion", "ANV");

Map<String, Object> configMap = new HashMap<>();
configMap.put("fields", fieldMappings);

Configuration config = new Configuration(configMap);

Context context = new Context();
Filter filter = new Yauaa(config, context);
}

@Test()
public void testBadConfigEmptySource() {
expectedEx.expect(IllegalArgumentException.class);
expectedEx.expectMessage("The \"source\" is empty.");

Map<String, String> fieldMappings = new HashMap<>();
fieldMappings.put("DeviceClass", "DC");
fieldMappings.put("AgentNameVersion", "ANV");

Map<String, Object> configMap = new HashMap<>();
configMap.put("source", ""); // EMPTY STRING
configMap.put("fields", fieldMappings);

Configuration config = new Configuration(configMap);

Context context = new Context();
Filter filter = new Yauaa(config, context);
}

@Test()
public void testBadConfigNoFields() {
expectedEx.expect(IllegalArgumentException.class);
expectedEx.expectMessage("The list of needed \"fields\" has not been specified.");

Map<String, Object> configMap = new HashMap<>();
configMap.put("source", "foo");

Configuration config = new Configuration(configMap);

Context context = new Context();
Filter filter = new Yauaa(config, context);
}

@Test()
public void testBadConfigFieldsEmpty() {
expectedEx.expect(IllegalArgumentException.class);
expectedEx.expectMessage("The list of needed \"fields\" is empty.");

Map<String, String> fieldMappings = new HashMap<>();

Map<String, Object> configMap = new HashMap<>();
configMap.put("source", "foo");
configMap.put("fields", fieldMappings);

Configuration config = new Configuration(configMap);

Context context = new Context();
Filter filter = new Yauaa(config, context);
}

@Test()
public void testBadConfigIllegalField() {
expectedEx.expect(IllegalArgumentException.class);
expectedEx.expectMessage("The requested field \"NoSuchField\" does not exist.");

Map<String, String> fieldMappings = new HashMap<>();
fieldMappings.put("NoSuchField", "NSF");
fieldMappings.put("AgentNameVersion", "ANV");

Map<String, Object> configMap = new HashMap<>();
configMap.put("source", "foo");
configMap.put("fields", fieldMappings);

Configuration config = new Configuration(configMap);

Context context = new Context();
Filter filter = new Yauaa(config, context);
}

@Test()
public void testBadConfigIllegalFieldNoSource() {
expectedEx.expect(IllegalArgumentException.class);
expectedEx.expectMessage(
allOf(
containsString("The \"source\" has not been specified."),
containsString("The requested field \"NoSuchField\" does not exist.")));

Map<String, String> fieldMappings = new HashMap<>();
fieldMappings.put("NoSuchField", "NSF");
fieldMappings.put("AgentNameVersion", "ANV");

Map<String, Object> configMap = new HashMap<>();
configMap.put("fields", fieldMappings);

Configuration config = new Configuration(configMap);

Context context = new Context();
Filter filter = new Yauaa(config, context);
}

@Test()
public void testBadConfigIllegalFieldEmptySource() {
expectedEx.expect(IllegalArgumentException.class);
expectedEx.expectMessage(
allOf(
containsString("The \"source\" is empty."),
containsString("The requested field \"NoSuchField\" does not exist.")));

Map<String, String> fieldMappings = new HashMap<>();
fieldMappings.put("NoSuchField", "NSF");
fieldMappings.put("AgentNameVersion", "ANV");

Map<String, Object> configMap = new HashMap<>();
configMap.put("source", "");
configMap.put("fields", fieldMappings);

Configuration config = new Configuration(configMap);

Context context = new Context();
Filter filter = new Yauaa(config, context);
}

}

0 comments on commit 61d3dec

Please sign in to comment.