From 7c634eaacb6a077ad7daa660f3df769f4bca1091 Mon Sep 17 00:00:00 2001 From: Bernd Ahlers Date: Thu, 25 Feb 2016 17:10:47 +0100 Subject: [PATCH] Make the GeoIpFilter configurable via the web interface Reload the filter engine if the configuration changes. --- .../graylog/plugins/map/MapWidgetModule.java | 12 -- .../map/config/GeoIpResolverConfig.java | 64 +++++++ .../map/config/MapWidgetConfiguration.java | 28 --- .../map/geoip/filter/GeoIpResolverFilter.java | 179 +++++++++++------- .../geoip/filter/GeoIpResolverFilterTest.java | 28 +-- src/web/components/GeoIpResolverConfig.jsx | 153 +++++++++++++++ src/web/index.jsx | 7 + 7 files changed, 346 insertions(+), 125 deletions(-) create mode 100644 src/main/java/org/graylog/plugins/map/config/GeoIpResolverConfig.java delete mode 100644 src/main/java/org/graylog/plugins/map/config/MapWidgetConfiguration.java create mode 100644 src/web/components/GeoIpResolverConfig.jsx diff --git a/src/main/java/org/graylog/plugins/map/MapWidgetModule.java b/src/main/java/org/graylog/plugins/map/MapWidgetModule.java index 73c2d8bd70ad..7ed4214536c1 100644 --- a/src/main/java/org/graylog/plugins/map/MapWidgetModule.java +++ b/src/main/java/org/graylog/plugins/map/MapWidgetModule.java @@ -1,27 +1,15 @@ package org.graylog.plugins.map; -import com.google.common.collect.Sets; -import org.graylog.plugins.map.config.MapWidgetConfiguration; import org.graylog.plugins.map.geoip.filter.GeoIpResolverFilter; import org.graylog.plugins.map.rest.MapDataResource; import org.graylog.plugins.map.widget.strategy.MapWidgetStrategy; -import org.graylog2.plugin.PluginConfigBean; import org.graylog2.plugin.PluginModule; -import java.util.Set; - public class MapWidgetModule extends PluginModule { - - @Override - public Set getConfigBeans() { - return Sets.newHashSet(new MapWidgetConfiguration()); - } - @Override protected void configure() { addMessageFilter(GeoIpResolverFilter.class); addWidgetStrategy(MapWidgetStrategy.class, MapWidgetStrategy.Factory.class); addRestResource(MapDataResource.class); - addConfigBeans(); } } diff --git a/src/main/java/org/graylog/plugins/map/config/GeoIpResolverConfig.java b/src/main/java/org/graylog/plugins/map/config/GeoIpResolverConfig.java new file mode 100644 index 000000000000..fc49ae86eb66 --- /dev/null +++ b/src/main/java/org/graylog/plugins/map/config/GeoIpResolverConfig.java @@ -0,0 +1,64 @@ +package org.graylog.plugins.map.config; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.auto.value.AutoValue; + +@JsonAutoDetect +@AutoValue +public abstract class GeoIpResolverConfig { + public enum DatabaseType { + GEOLITE2_CITY, GEOLITE2_COUNTRY + } + + @JsonProperty("enabled") + public abstract boolean enabled(); + + @JsonProperty("db_type") + public abstract DatabaseType dbType(); + + @JsonProperty("db_path") + public abstract String dbPath(); + + @JsonProperty("run_before_extractors") + public abstract boolean runBeforeExtractors(); + + @JsonCreator + public static GeoIpResolverConfig create(@JsonProperty("enabled") boolean enabled, + @JsonProperty("db_type") DatabaseType dbType, + @JsonProperty("db_path") String dbPath, + @JsonProperty("run_before_extractors") boolean runBeforeExtractors) { + return builder() + .enabled(enabled) + .dbType(dbType) + .dbPath(dbPath) + .runBeforeExtractors(runBeforeExtractors) + .build(); + } + + public static GeoIpResolverConfig defaultConfig() { + return builder() + .enabled(false) + .dbType(DatabaseType.GEOLITE2_CITY) + .dbPath("/tmp/GeoLite2-City.mmdb") + .runBeforeExtractors(false) + .build(); + } + + public static Builder builder() { + return new AutoValue_GeoIpResolverConfig.Builder(); + } + + public abstract Builder toBuilder(); + + @AutoValue.Builder + public static abstract class Builder { + public abstract Builder enabled(boolean enabled); + public abstract Builder dbType(DatabaseType dbType); + public abstract Builder dbPath(String dbPath); + public abstract Builder runBeforeExtractors(boolean runBeforeExtractors); + + public abstract GeoIpResolverConfig build(); + } +} \ No newline at end of file diff --git a/src/main/java/org/graylog/plugins/map/config/MapWidgetConfiguration.java b/src/main/java/org/graylog/plugins/map/config/MapWidgetConfiguration.java deleted file mode 100644 index 7205c555f630..000000000000 --- a/src/main/java/org/graylog/plugins/map/config/MapWidgetConfiguration.java +++ /dev/null @@ -1,28 +0,0 @@ -package org.graylog.plugins.map.config; - -import com.github.joschi.jadconfig.Parameter; -import org.graylog2.plugin.PluginConfigBean; - -public class MapWidgetConfiguration implements PluginConfigBean { - @Parameter(value = "geoip_resolver_database") - private String geoIpDatabase = "/usr/local/share/GeoIp/GeoIP2-City.mmdb"; - - @Parameter(value = "geoip_resolver_run_before_extractors") - private boolean runBeforeExtractors = true; - - @Parameter(value = "geoip_resolver_enabled") - private boolean enabled = false; - - public boolean isRunBeforeExtractors() { - return runBeforeExtractors; - } - - public String getGeoIpDatabase() { - return geoIpDatabase; - } - - public boolean isEnabled() { - return enabled; - } -} - diff --git a/src/main/java/org/graylog/plugins/map/geoip/filter/GeoIpResolverFilter.java b/src/main/java/org/graylog/plugins/map/geoip/filter/GeoIpResolverFilter.java index 0546bcab0137..b7f62b87ffab 100644 --- a/src/main/java/org/graylog/plugins/map/geoip/filter/GeoIpResolverFilter.java +++ b/src/main/java/org/graylog/plugins/map/geoip/filter/GeoIpResolverFilter.java @@ -3,10 +3,16 @@ import com.codahale.metrics.MetricRegistry; import com.codahale.metrics.Timer; import com.google.common.collect.Lists; +import com.google.common.eventbus.EventBus; +import com.google.common.eventbus.Subscribe; import com.maxmind.geoip2.DatabaseReader; import com.maxmind.geoip2.model.CityResponse; import com.maxmind.geoip2.record.Location; +import org.graylog.plugins.map.config.GeoIpResolverConfig; +import org.graylog2.cluster.ClusterConfigChangedEvent; +import org.graylog2.events.ClusterEventBus; import org.graylog2.plugin.Message; +import org.graylog2.plugin.cluster.ClusterConfigService; import org.graylog2.plugin.filters.MessageFilter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -18,6 +24,9 @@ import java.net.InetAddress; import java.util.List; import java.util.Map; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -28,97 +37,139 @@ public class GeoIpResolverFilter implements MessageFilter { private static final Logger LOG = LoggerFactory.getLogger(GeoIpResolverFilter.class); // TODO: Match also IPv6 addresses private static final Pattern IP_PATTERN = Pattern.compile("(\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3})"); + private final ClusterConfigService clusterConfigService; + private final ScheduledExecutorService scheduler; + private final MetricRegistry metricRegistry; - private DatabaseReader databaseReader; - private final boolean shouldRunBeforeExtractors; - private boolean enabled; - - private final Timer resolveTime; + private final AtomicReference config; + private final AtomicReference filterEngine; @Inject - public GeoIpResolverFilter(@Named("geoip_resolver_database") String geoIpDatabase, - @Named("geoip_resolver_run_before_extractors") boolean shouldRunBeforeExtractors, - @Named("geoip_resolver_enabled") boolean enabled, + public GeoIpResolverFilter(ClusterConfigService clusterConfigService, + @Named("daemonScheduler") ScheduledExecutorService scheduler, + @ClusterEventBus EventBus clusterEventBus, MetricRegistry metricRegistry) { - try { - final File database = new File(geoIpDatabase); - this.databaseReader = new DatabaseReader.Builder(database).build(); - this.enabled = enabled; - } catch (IOException e) { - LOG.error("Could not open GeoIP database {}", geoIpDatabase, e); - this.enabled = false; + this.clusterConfigService = clusterConfigService; + this.scheduler = scheduler; + this.metricRegistry = metricRegistry; + final GeoIpResolverConfig config = clusterConfigService.getOrDefault(GeoIpResolverConfig.class, + GeoIpResolverConfig.defaultConfig()); + + this.config = new AtomicReference<>(config); + this.filterEngine = new AtomicReference<>(new FilterEngine(config, metricRegistry)); + + clusterEventBus.register(this); + } + + @Subscribe + @SuppressWarnings("unused") + public void updateConfig(ClusterConfigChangedEvent event) { + if (!GeoIpResolverConfig.class.getCanonicalName().equals(event.type())) { + return; } - this.shouldRunBeforeExtractors = shouldRunBeforeExtractors; + scheduler.schedule((Runnable) this::reload, 0, TimeUnit.SECONDS); + } + + private void reload() { + final GeoIpResolverConfig newConfig = clusterConfigService.getOrDefault(GeoIpResolverConfig.class, + GeoIpResolverConfig.defaultConfig()); - this.resolveTime = metricRegistry.timer(name(GeoIpResolverFilter.class, "resolveTime")); + LOG.debug("Updating GeoIP filter engine - {}", newConfig); + config.set(newConfig); + filterEngine.set(new FilterEngine(newConfig, metricRegistry)); } @Override public boolean filter(Message message) { - if (!enabled) { - return false; - } + return filterEngine.get().filter(message); + } - for (Map.Entry field : message.getFields().entrySet()) { - String key = field.getKey() + "_geolocation"; - final List coordinates = extractGeoLocationInformation(field.getValue()); - if (coordinates.size() == 2) { - // We will store the coordinates as a "lat,long" string - final String stringGeoPoint = coordinates.get(1) + "," + coordinates.get(0); - message.addField(key, stringGeoPoint); - } - } + @Override + public String getName() { + return "GeoIP resolver"; + } - return false; + @Override + public int getPriority() { + // MAGIC NUMBER: 10 is the priority of the ExtractorFilter, we either run before or after it, depending on what the user wants. + return 10 - (config.get().runBeforeExtractors() ? 1 : -1); } - protected String getIpFromFieldValue(String fieldValue) { - Matcher matcher = IP_PATTERN.matcher(fieldValue); + protected static class FilterEngine { + private final Timer resolveTime; - if (matcher.find()) { - return matcher.group(1); - } + private DatabaseReader databaseReader; + private boolean enabled; - return null; - } - protected List extractGeoLocationInformation(Object fieldValue) { - final List coordinates = Lists.newArrayList(); + public FilterEngine(GeoIpResolverConfig config, MetricRegistry metricRegistry) { + this.resolveTime = metricRegistry.timer(name(GeoIpResolverFilter.class, "resolveTime")); - if (!(fieldValue instanceof String) || isNullOrEmpty((String) fieldValue)) { - return coordinates; + try { + final File database = new File(config.dbPath()); + this.databaseReader = new DatabaseReader.Builder(database).build(); + this.enabled = config.enabled(); + } catch (IOException e) { + LOG.error("Could not open GeoIP database {}", config.dbPath(), e); + this.enabled = false; + } } - final String stringFieldValue = (String) fieldValue; - final String ip = this.getIpFromFieldValue(stringFieldValue); - if (isNullOrEmpty(ip)) { - return coordinates; + public boolean filter(Message message) { + if (!enabled) { + return false; + } + + for (Map.Entry field : message.getFields().entrySet()) { + String key = field.getKey() + "_geolocation"; + final List coordinates = extractGeoLocationInformation(field.getValue()); + if (coordinates.size() == 2) { + // We will store the coordinates as a "lat,long" string + final String stringGeoPoint = coordinates.get(1) + "," + coordinates.get(0); + message.addField(key, stringGeoPoint); + } + } + + return false; } - try { - try (Timer.Context ignored = resolveTime.time()) { - final InetAddress ipAddress = InetAddress.getByName(ip); - final CityResponse response = databaseReader.city(ipAddress); - final Location location = response.getLocation(); - coordinates.add(location.getLongitude()); - coordinates.add(location.getLatitude()); + protected List extractGeoLocationInformation(Object fieldValue) { + final List coordinates = Lists.newArrayList(); + + if (!(fieldValue instanceof String) || isNullOrEmpty((String) fieldValue)) { + return coordinates; + } + + final String stringFieldValue = (String) fieldValue; + final String ip = this.getIpFromFieldValue(stringFieldValue); + if (isNullOrEmpty(ip)) { + return coordinates; } - } catch (Exception e) { - LOG.debug("Could not get location from IP {}", ip, e); + + try { + try (Timer.Context ignored = resolveTime.time()) { + final InetAddress ipAddress = InetAddress.getByName(ip); + final CityResponse response = databaseReader.city(ipAddress); + final Location location = response.getLocation(); + coordinates.add(location.getLongitude()); + coordinates.add(location.getLatitude()); + } + } catch (Exception e) { + LOG.debug("Could not get location from IP {}", ip, e); + } + + return coordinates; } - return coordinates; - } + protected String getIpFromFieldValue(String fieldValue) { + Matcher matcher = IP_PATTERN.matcher(fieldValue); - @Override - public String getName() { - return "GeoIP resolver"; - } + if (matcher.find()) { + return matcher.group(1); + } - @Override - public int getPriority() { - // MAGIC NUMBER: 10 is the priority of the ExtractorFilter, we either run before or after it, depending on what the user wants. - return 10 - (shouldRunBeforeExtractors ? 1 : -1); + return null; + } } } diff --git a/src/test/java/org/graylog/plugins/map/geoip/filter/GeoIpResolverFilterTest.java b/src/test/java/org/graylog/plugins/map/geoip/filter/GeoIpResolverFilterTest.java index 30e0ffd3c417..d37f960b5056 100644 --- a/src/test/java/org/graylog/plugins/map/geoip/filter/GeoIpResolverFilterTest.java +++ b/src/test/java/org/graylog/plugins/map/geoip/filter/GeoIpResolverFilterTest.java @@ -4,6 +4,7 @@ import com.codahale.metrics.MetricRegistry; import com.eaio.uuid.UUID; import com.google.common.collect.Maps; +import org.graylog.plugins.map.config.GeoIpResolverConfig; import org.graylog2.plugin.Message; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; @@ -22,9 +23,11 @@ public class GeoIpResolverFilterTest { private MetricRegistry metricRegistry; + private GeoIpResolverConfig config; @BeforeMethod public void setUp() { + config = GeoIpResolverConfig.defaultConfig(); metricRegistry = new MetricRegistry(); } @@ -48,12 +51,7 @@ private String getTestDatabasePath() { @Test public void getIpFromFieldValue() throws Exception { - final GeoIpResolverFilter resolver = new GeoIpResolverFilter( - this.getTestDatabasePath(), - false, - true, - metricRegistry); - + final GeoIpResolverFilter.FilterEngine resolver = new GeoIpResolverFilter.FilterEngine(config, metricRegistry); final String ip = "127.0.0.1"; assertEquals(resolver.getIpFromFieldValue(ip), ip); @@ -63,11 +61,7 @@ public void getIpFromFieldValue() throws Exception { @Test public void extractGeoLocationInformation() throws Exception { - final GeoIpResolverFilter resolver = new GeoIpResolverFilter( - this.getTestDatabasePath(), - false, - true, - metricRegistry); + final GeoIpResolverFilter.FilterEngine resolver = new GeoIpResolverFilter.FilterEngine(config, metricRegistry); List coordinates = resolver.extractGeoLocationInformation("1.2.3.4"); assertEquals(coordinates.size(), 2, "Should extract geo location information from public addresses"); @@ -77,11 +71,7 @@ public void extractGeoLocationInformation() throws Exception { @Test public void disabledFilterTest() throws Exception { - final GeoIpResolverFilter resolver = new GeoIpResolverFilter( - this.getTestDatabasePath(), - false, - false, - metricRegistry); + final GeoIpResolverFilter.FilterEngine resolver = new GeoIpResolverFilter.FilterEngine(config, metricRegistry); final Map messageFields = Maps.newHashMap(); messageFields.put("_id", (new UUID()).toString()); @@ -98,11 +88,7 @@ public void disabledFilterTest() throws Exception { @Test public void filterResolvesIpGeoLocation() throws Exception { - final GeoIpResolverFilter resolver = new GeoIpResolverFilter( - this.getTestDatabasePath(), - false, - true, - metricRegistry); + final GeoIpResolverFilter.FilterEngine resolver = new GeoIpResolverFilter.FilterEngine(config, metricRegistry); final Map messageFields = Maps.newHashMap(); messageFields.put("_id", (new UUID()).toString()); diff --git a/src/web/components/GeoIpResolverConfig.jsx b/src/web/components/GeoIpResolverConfig.jsx new file mode 100644 index 000000000000..950e398aec3a --- /dev/null +++ b/src/web/components/GeoIpResolverConfig.jsx @@ -0,0 +1,153 @@ +import React from 'react'; +import { Input, Button } from 'react-bootstrap'; +import BootstrapModalForm from 'components/bootstrap/BootstrapModalForm'; +import { IfPermitted, Select } from 'components/common'; +import ObjectUtils from 'util/ObjectUtils'; + +import style from '!style!css!components/configurations/ConfigurationStyles.css'; + +const GeoIpResolverConfig = React.createClass({ + propTypes: { + config: React.PropTypes.object.isRequired, + updateConfig: React.PropTypes.func.isRequired, + }, + + getInitialState() { + return this._getStateFromProps(this.props); + }, + + componentWillReceiveProps(newProps) { + this.setState(this._getStateFromProps(newProps)); + }, + + _getStateFromProps(props) { + return { + config: { + enabled: this._getPropConfigValue(props, 'enabled', false), + db_type: this._getPropConfigValue(props, 'db_type', 'GEOLITE2_CITY'), + db_path: this._getPropConfigValue(props, 'db_path', '/tmp/GeoLite2-City.mmdb'), + run_before_extractors: this._getPropConfigValue(props, 'run_before_extractors', false), + }, + }; + }, + + _getPropConfigValue(props, field, defaultValue = null) { + return props.config ? props.config[field] || defaultValue : defaultValue; + }, + + _updateConfigField(field, value) { + const update = ObjectUtils.clone(this.state.config); + update[field] = value; + this.setState({config: update}); + }, + + _onCheckboxClick(field, ref) { + return () => { + this._updateConfigField(field, this.refs[ref].getChecked()); + }; + }, + + _onSelect(field) { + return (selection) => { + this._updateConfigField(field, selection); + }; + }, + + _onUpdate(field) { + return (e) => { + this._updateConfigField(field, e.target.value); + }; + }, + + _openModal() { + this.refs.geoIpConfigModal.open(); + }, + + _closeModal() { + this.refs.geoIpConfigModal.close(); + }, + + _resetConfig() { + // Reset to initial state when the modal is closed without saving. + this.setState(this.getInitialState()); + }, + + _saveConfig() { + this.props.updateConfig(this.state.config).then(() => { + this._closeModal(); + }); + }, + + _availableDatabaseTypes() { + return [ + {value: 'GEOLITE2_CITY', label: 'GeoLite2 City'}, + {value: 'GEOLITE2_COUNTRY', label: 'GeoLite2 Country'}, + ]; + }, + + _activeDatabaseType(type) { + return this._availableDatabaseTypes().filter((t) => t.value === type)[0].label; + }, + + render() { + return ( +
+

GeoIP Filter

+ +
+
Enabled:
+
{this.state.config.enabled === true ? 'yes' : 'no'}
+
DB type:
+
{this._activeDatabaseType(this.state.config.db_type)}
+
DB path:
+
{this.state.config.db_path}
+
Run before extractors:
+
{this.state.config.run_before_extractors === true ? 'yes' : 'no'}
+
+ + + + + + +
+ +
+ + + +
+ +
+ ); + }, +}); + +export default GeoIpResolverConfig; diff --git a/src/web/index.jsx b/src/web/index.jsx index 3023251e636a..4b6f832be41a 100644 --- a/src/web/index.jsx +++ b/src/web/index.jsx @@ -2,6 +2,7 @@ import packageJson from '../../package.json'; import { PluginManifest, PluginStore } from 'graylog-web-plugin/plugin'; import MapVisualization from 'components/MapVisualization'; import FieldAnalyzerMapComponent from 'components/FieldAnalyzerMapComponent'; +import GeoIpResolverConfig from 'components/GeoIpResolverConfig'; PluginStore.register(new PluginManifest(packageJson, { widgets: [ @@ -21,4 +22,10 @@ PluginStore.register(new PluginManifest(packageJson, { displayPriority: 100, }, ], + systemConfigurations: [ + { + component: GeoIpResolverConfig, + configType: 'org.graylog.plugins.map.config.GeoIpResolverConfig', + }, + ], }));