Skip to content

Commit

Permalink
Make the GeoIpFilter configurable via the web interface
Browse files Browse the repository at this point in the history 
Reload the filter engine if the configuration changes.
  • Loading branch information
@bernd @edmundoa
bernd authored and edmundoa committed Feb 25, 2016
1 parent 6e0a200 commit 7c634ea
Show file tree
Hide file tree
Showing 7 changed files with 346 additions and 125 deletions.
12 changes: 0 additions & 12 deletions src/main/java/org/graylog/plugins/map/MapWidgetModule.java
View file
@@ -1,27 +1,15 @@
package org.graylog.plugins.map; package org.graylog.plugins.map;


import com.google.common.collect.Sets;
import org.graylog.plugins.map.config.MapWidgetConfiguration;
import org.graylog.plugins.map.geoip.filter.GeoIpResolverFilter; import org.graylog.plugins.map.geoip.filter.GeoIpResolverFilter;
import org.graylog.plugins.map.rest.MapDataResource; import org.graylog.plugins.map.rest.MapDataResource;
import org.graylog.plugins.map.widget.strategy.MapWidgetStrategy; import org.graylog.plugins.map.widget.strategy.MapWidgetStrategy;
import org.graylog2.plugin.PluginConfigBean;
import org.graylog2.plugin.PluginModule; import org.graylog2.plugin.PluginModule;


import java.util.Set;

public class MapWidgetModule extends PluginModule { public class MapWidgetModule extends PluginModule {

@Override
public Set<? extends PluginConfigBean> getConfigBeans() {
return Sets.newHashSet(new MapWidgetConfiguration());
}

@Override @Override
protected void configure() { protected void configure() {
addMessageFilter(GeoIpResolverFilter.class); addMessageFilter(GeoIpResolverFilter.class);
addWidgetStrategy(MapWidgetStrategy.class, MapWidgetStrategy.Factory.class); addWidgetStrategy(MapWidgetStrategy.class, MapWidgetStrategy.Factory.class);
addRestResource(MapDataResource.class); addRestResource(MapDataResource.class);
addConfigBeans();
} }
} }
64 changes: 64 additions & 0 deletions src/main/java/org/graylog/plugins/map/config/GeoIpResolverConfig.java
View file
@@ -0,0 +1,64 @@
package org.graylog.plugins.map.config;

import com.fasterxml.jackson.annotation.JsonAutoDetect;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.auto.value.AutoValue;

@JsonAutoDetect
@AutoValue
public abstract class GeoIpResolverConfig {
public enum DatabaseType {
GEOLITE2_CITY, GEOLITE2_COUNTRY
}

@JsonProperty("enabled")
public abstract boolean enabled();

@JsonProperty("db_type")
public abstract DatabaseType dbType();

@JsonProperty("db_path")
public abstract String dbPath();

@JsonProperty("run_before_extractors")
public abstract boolean runBeforeExtractors();

@JsonCreator
public static GeoIpResolverConfig create(@JsonProperty("enabled") boolean enabled,
@JsonProperty("db_type") DatabaseType dbType,
@JsonProperty("db_path") String dbPath,
@JsonProperty("run_before_extractors") boolean runBeforeExtractors) {
return builder()
.enabled(enabled)
.dbType(dbType)
.dbPath(dbPath)
.runBeforeExtractors(runBeforeExtractors)
.build();
}

public static GeoIpResolverConfig defaultConfig() {
return builder()
.enabled(false)
.dbType(DatabaseType.GEOLITE2_CITY)
.dbPath("/tmp/GeoLite2-City.mmdb")
.runBeforeExtractors(false)
.build();
}

public static Builder builder() {
return new AutoValue_GeoIpResolverConfig.Builder();
}

public abstract Builder toBuilder();

@AutoValue.Builder
public static abstract class Builder {
public abstract Builder enabled(boolean enabled);
public abstract Builder dbType(DatabaseType dbType);
public abstract Builder dbPath(String dbPath);
public abstract Builder runBeforeExtractors(boolean runBeforeExtractors);

public abstract GeoIpResolverConfig build();
}
}
28 changes: 0 additions & 28 deletions src/main/java/org/graylog/plugins/map/config/MapWidgetConfiguration.java
View file

This file was deleted.

179 changes: 115 additions & 64 deletions src/main/java/org/graylog/plugins/map/geoip/filter/GeoIpResolverFilter.java
View file
Expand Up @@ -3,10 +3,16 @@
import com.codahale.metrics.MetricRegistry; import com.codahale.metrics.MetricRegistry;
import com.codahale.metrics.Timer; import com.codahale.metrics.Timer;
import com.google.common.collect.Lists; import com.google.common.collect.Lists;
import com.google.common.eventbus.EventBus;
import com.google.common.eventbus.Subscribe;
import com.maxmind.geoip2.DatabaseReader; import com.maxmind.geoip2.DatabaseReader;
import com.maxmind.geoip2.model.CityResponse; import com.maxmind.geoip2.model.CityResponse;
import com.maxmind.geoip2.record.Location; import com.maxmind.geoip2.record.Location;
import org.graylog.plugins.map.config.GeoIpResolverConfig;
import org.graylog2.cluster.ClusterConfigChangedEvent;
import org.graylog2.events.ClusterEventBus;
import org.graylog2.plugin.Message; import org.graylog2.plugin.Message;
import org.graylog2.plugin.cluster.ClusterConfigService;
import org.graylog2.plugin.filters.MessageFilter; import org.graylog2.plugin.filters.MessageFilter;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
Expand All @@ -18,6 +24,9 @@
import java.net.InetAddress; import java.net.InetAddress;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicReference;
import java.util.regex.Matcher; import java.util.regex.Matcher;
import java.util.regex.Pattern; import java.util.regex.Pattern;


Expand All @@ -28,97 +37,139 @@ public class GeoIpResolverFilter implements MessageFilter {
private static final Logger LOG = LoggerFactory.getLogger(GeoIpResolverFilter.class); private static final Logger LOG = LoggerFactory.getLogger(GeoIpResolverFilter.class);
// TODO: Match also IPv6 addresses // TODO: Match also IPv6 addresses
private static final Pattern IP_PATTERN = Pattern.compile("(\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3})"); private static final Pattern IP_PATTERN = Pattern.compile("(\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3})");
private final ClusterConfigService clusterConfigService;
private final ScheduledExecutorService scheduler;
private final MetricRegistry metricRegistry;


private DatabaseReader databaseReader; private final AtomicReference<GeoIpResolverConfig> config;
private final boolean shouldRunBeforeExtractors; private final AtomicReference<FilterEngine> filterEngine;
private boolean enabled;

private final Timer resolveTime;


@Inject @Inject
public GeoIpResolverFilter(@Named("geoip_resolver_database") String geoIpDatabase, public GeoIpResolverFilter(ClusterConfigService clusterConfigService,
@Named("geoip_resolver_run_before_extractors") boolean shouldRunBeforeExtractors, @Named("daemonScheduler") ScheduledExecutorService scheduler,
@Named("geoip_resolver_enabled") boolean enabled, @ClusterEventBus EventBus clusterEventBus,
MetricRegistry metricRegistry) { MetricRegistry metricRegistry) {
try { this.clusterConfigService = clusterConfigService;
final File database = new File(geoIpDatabase); this.scheduler = scheduler;
this.databaseReader = new DatabaseReader.Builder(database).build(); this.metricRegistry = metricRegistry;
this.enabled = enabled; final GeoIpResolverConfig config = clusterConfigService.getOrDefault(GeoIpResolverConfig.class,
} catch (IOException e) { GeoIpResolverConfig.defaultConfig());
LOG.error("Could not open GeoIP database {}", geoIpDatabase, e);
this.enabled = false; this.config = new AtomicReference<>(config);
this.filterEngine = new AtomicReference<>(new FilterEngine(config, metricRegistry));

clusterEventBus.register(this);
}

@Subscribe
@SuppressWarnings("unused")
public void updateConfig(ClusterConfigChangedEvent event) {
if (!GeoIpResolverConfig.class.getCanonicalName().equals(event.type())) {
return;
} }


this.shouldRunBeforeExtractors = shouldRunBeforeExtractors; scheduler.schedule((Runnable) this::reload, 0, TimeUnit.SECONDS);
}

private void reload() {
final GeoIpResolverConfig newConfig = clusterConfigService.getOrDefault(GeoIpResolverConfig.class,
GeoIpResolverConfig.defaultConfig());


this.resolveTime = metricRegistry.timer(name(GeoIpResolverFilter.class, "resolveTime")); LOG.debug("Updating GeoIP filter engine - {}", newConfig);
config.set(newConfig);
filterEngine.set(new FilterEngine(newConfig, metricRegistry));
} }


@Override @Override
public boolean filter(Message message) { public boolean filter(Message message) {
if (!enabled) { return filterEngine.get().filter(message);
return false; }
}


for (Map.Entry<String, Object> field : message.getFields().entrySet()) { @Override
String key = field.getKey() + "_geolocation"; public String getName() {
final List coordinates = extractGeoLocationInformation(field.getValue()); return "GeoIP resolver";
if (coordinates.size() == 2) { }
// We will store the coordinates as a "lat,long" string
final String stringGeoPoint = coordinates.get(1) + "," + coordinates.get(0);
message.addField(key, stringGeoPoint);
}
}


return false; @Override
public int getPriority() {
// MAGIC NUMBER: 10 is the priority of the ExtractorFilter, we either run before or after it, depending on what the user wants.
return 10 - (config.get().runBeforeExtractors() ? 1 : -1);
} }


protected String getIpFromFieldValue(String fieldValue) { protected static class FilterEngine {
Matcher matcher = IP_PATTERN.matcher(fieldValue); private final Timer resolveTime;


if (matcher.find()) { private DatabaseReader databaseReader;
return matcher.group(1); private boolean enabled;
}


return null;
}


protected List<Double> extractGeoLocationInformation(Object fieldValue) { public FilterEngine(GeoIpResolverConfig config, MetricRegistry metricRegistry) {
final List<Double> coordinates = Lists.newArrayList(); this.resolveTime = metricRegistry.timer(name(GeoIpResolverFilter.class, "resolveTime"));


if (!(fieldValue instanceof String) || isNullOrEmpty((String) fieldValue)) { try {
return coordinates; final File database = new File(config.dbPath());
this.databaseReader = new DatabaseReader.Builder(database).build();
this.enabled = config.enabled();
} catch (IOException e) {
LOG.error("Could not open GeoIP database {}", config.dbPath(), e);
this.enabled = false;
}
} }


final String stringFieldValue = (String) fieldValue; public boolean filter(Message message) {
final String ip = this.getIpFromFieldValue(stringFieldValue); if (!enabled) {
if (isNullOrEmpty(ip)) { return false;
return coordinates; }

for (Map.Entry<String, Object> field : message.getFields().entrySet()) {
String key = field.getKey() + "_geolocation";
final List coordinates = extractGeoLocationInformation(field.getValue());
if (coordinates.size() == 2) {
// We will store the coordinates as a "lat,long" string
final String stringGeoPoint = coordinates.get(1) + "," + coordinates.get(0);
message.addField(key, stringGeoPoint);
}
}

return false;
} }


try { protected List<Double> extractGeoLocationInformation(Object fieldValue) {
try (Timer.Context ignored = resolveTime.time()) { final List<Double> coordinates = Lists.newArrayList();
final InetAddress ipAddress = InetAddress.getByName(ip);
final CityResponse response = databaseReader.city(ipAddress); if (!(fieldValue instanceof String) || isNullOrEmpty((String) fieldValue)) {
final Location location = response.getLocation(); return coordinates;
coordinates.add(location.getLongitude()); }
coordinates.add(location.getLatitude());
final String stringFieldValue = (String) fieldValue;
final String ip = this.getIpFromFieldValue(stringFieldValue);
if (isNullOrEmpty(ip)) {
return coordinates;
} }
} catch (Exception e) {
LOG.debug("Could not get location from IP {}", ip, e); try {
try (Timer.Context ignored = resolveTime.time()) {
final InetAddress ipAddress = InetAddress.getByName(ip);
final CityResponse response = databaseReader.city(ipAddress);
final Location location = response.getLocation();
coordinates.add(location.getLongitude());
coordinates.add(location.getLatitude());
}
} catch (Exception e) {
LOG.debug("Could not get location from IP {}", ip, e);
}

return coordinates;
} }


return coordinates; protected String getIpFromFieldValue(String fieldValue) {
} Matcher matcher = IP_PATTERN.matcher(fieldValue);


@Override if (matcher.find()) {
public String getName() { return matcher.group(1);
return "GeoIP resolver"; }
}


@Override return null;
public int getPriority() { }
// MAGIC NUMBER: 10 is the priority of the ExtractorFilter, we either run before or after it, depending on what the user wants.
return 10 - (shouldRunBeforeExtractors ? 1 : -1);
} }
} }

0 comments on commit 7c634ea

Please sign in to comment.