From 01296bd7a11d8f6afe7c0e2cb157214fbc6c349b Mon Sep 17 00:00:00 2001 From: kroepke Date: Tue, 2 May 2017 17:58:32 +0200 Subject: [PATCH 1/2] create data adapter for maxmind's geoip2 databases currently supported are the city and country databases the entire record, if found, is returned, leaving the caller in charge of extracting the useful information about the IP location currently no migration of configuration is done --- pom.xml | 6 + .../graylog/plugins/map/MapWidgetModule.java | 6 + .../plugins/map/config/DatabaseType.java | 5 + .../map/config/GeoIpResolverConfig.java | 6 +- .../plugins/map/geoip/MaxmindDataAdapter.java | 204 ++++++++++++++++++ .../adapter/MaxmindAdapterDocumentation.jsx | 63 ++++++ .../adapter/MaxmindAdapterFieldSet.jsx | 75 +++++++ .../adapter/MaxmindAdapterSummary.jsx | 26 +++ src/web/index.jsx | 12 ++ 9 files changed, 399 insertions(+), 4 deletions(-) create mode 100644 src/main/java/org/graylog/plugins/map/config/DatabaseType.java create mode 100644 src/main/java/org/graylog/plugins/map/geoip/MaxmindDataAdapter.java create mode 100644 src/web/components/adapter/MaxmindAdapterDocumentation.jsx create mode 100644 src/web/components/adapter/MaxmindAdapterFieldSet.jsx create mode 100644 src/web/components/adapter/MaxmindAdapterSummary.jsx diff --git a/pom.xml b/pom.xml index c457f4e..993d2c8 100644 --- a/pom.xml +++ b/pom.xml @@ -90,6 +90,12 @@ ${graylog.version} provided + + org.graylog.autovalue + auto-value-javabean + ${graylog.version} + provided + com.maxmind.geoip2 geoip2 diff --git a/src/main/java/org/graylog/plugins/map/MapWidgetModule.java b/src/main/java/org/graylog/plugins/map/MapWidgetModule.java index bd42933..3c6c9fe 100644 --- a/src/main/java/org/graylog/plugins/map/MapWidgetModule.java +++ b/src/main/java/org/graylog/plugins/map/MapWidgetModule.java @@ -16,6 +16,7 @@ */ package org.graylog.plugins.map; +import org.graylog.plugins.map.geoip.MaxmindDataAdapter; import org.graylog.plugins.map.geoip.processor.GeoIpProcessor; import org.graylog.plugins.map.rest.MapDataResource; import org.graylog.plugins.map.widget.strategy.MapWidgetStrategy; @@ -27,5 +28,10 @@ protected void configure() { addMessageProcessor(GeoIpProcessor.class, GeoIpProcessor.Descriptor.class); addWidgetStrategy(MapWidgetStrategy.class, MapWidgetStrategy.Factory.class); addRestResource(MapDataResource.class); + + installLookupDataAdapter(MaxmindDataAdapter.NAME, + MaxmindDataAdapter.class, + MaxmindDataAdapter.Factory.class, + MaxmindDataAdapter.Config.class); } } diff --git a/src/main/java/org/graylog/plugins/map/config/DatabaseType.java b/src/main/java/org/graylog/plugins/map/config/DatabaseType.java new file mode 100644 index 0000000..5e6c652 --- /dev/null +++ b/src/main/java/org/graylog/plugins/map/config/DatabaseType.java @@ -0,0 +1,5 @@ +package org.graylog.plugins.map.config; + +public enum DatabaseType { + MAXMIND_CITY, MAXMIND_COUNTRY +} diff --git a/src/main/java/org/graylog/plugins/map/config/GeoIpResolverConfig.java b/src/main/java/org/graylog/plugins/map/config/GeoIpResolverConfig.java index d9da849..59c52a2 100644 --- a/src/main/java/org/graylog/plugins/map/config/GeoIpResolverConfig.java +++ b/src/main/java/org/graylog/plugins/map/config/GeoIpResolverConfig.java @@ -16,19 +16,17 @@ */ package org.graylog.plugins.map.config; +import com.google.auto.value.AutoValue; + import com.fasterxml.jackson.annotation.JsonAutoDetect; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonProperty; -import com.google.auto.value.AutoValue; @JsonAutoDetect @JsonIgnoreProperties(ignoreUnknown = true) @AutoValue public abstract class GeoIpResolverConfig { - public enum DatabaseType { - MAXMIND_CITY, MAXMIND_COUNTRY - } @JsonProperty("enabled") public abstract boolean enabled(); diff --git a/src/main/java/org/graylog/plugins/map/geoip/MaxmindDataAdapter.java b/src/main/java/org/graylog/plugins/map/geoip/MaxmindDataAdapter.java new file mode 100644 index 0000000..f94e75d --- /dev/null +++ b/src/main/java/org/graylog/plugins/map/geoip/MaxmindDataAdapter.java @@ -0,0 +1,204 @@ +package org.graylog.plugins.map.geoip; + +import com.google.auto.value.AutoValue; +import com.google.common.collect.ImmutableMap; +import com.google.common.net.InetAddresses; +import com.google.inject.assistedinject.Assisted; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonTypeName; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.maxmind.geoip2.DatabaseReader; +import com.maxmind.geoip2.exception.AddressNotFoundException; +import com.maxmind.geoip2.model.CityResponse; +import com.maxmind.geoip2.model.CountryResponse; + +import org.graylog.autovalue.WithBeanGetter; +import org.graylog.plugins.map.config.DatabaseType; +import org.graylog2.plugin.lookup.LookupDataAdapter; +import org.graylog2.plugin.lookup.LookupDataAdapterConfiguration; +import org.graylog2.plugin.lookup.LookupResult; +import org.hibernate.validator.constraints.NotEmpty; +import org.slf4j.Logger; + +import java.net.InetAddress; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.concurrent.TimeUnit; + +import javax.annotation.Nullable; +import javax.inject.Inject; +import javax.validation.constraints.Min; +import javax.validation.constraints.NotNull; + +import static org.slf4j.LoggerFactory.getLogger; + +public class MaxmindDataAdapter extends LookupDataAdapter { + + private static final Logger LOG = getLogger(MaxmindDataAdapter.class); + + public static final String NAME = "maxmind_geoip"; + private final Config config; + private DatabaseReader databaseReader; + + @Inject + protected MaxmindDataAdapter(@Assisted LookupDataAdapterConfiguration config) { + super(config); + this.config = (Config) config; + } + + @Override + protected void doStart() throws Exception { + Path path = Paths.get(config.path()); + if (!Files.isReadable(path)) { + throw new IllegalArgumentException("Cannot read database file: " + config.path()); + } + this.databaseReader = new DatabaseReader.Builder(path.toFile()).build(); + } + + @Override + protected void doStop() throws Exception { + databaseReader.close(); + } + + @Override + protected LookupResult doGet(Object key) { + InetAddress addr; + if (key instanceof InetAddress) { + addr = (InetAddress) key; + } else { + // try to convert it somehow + try { + addr = InetAddresses.forString(key.toString()); + } catch (IllegalArgumentException e) { + LOG.warn("Unable to parse IP address, returning empty result."); + return LookupResult.empty(); + } + } + switch (config.dbType()) { + case MAXMIND_CITY: + try { + final CityResponse city = databaseReader.city(addr); + final ImmutableMap.Builder map = ImmutableMap.builder(); + map.put("city", city.getCity()); + map.put("continent", city.getContinent()); + map.put("country", city.getCountry()); + map.put("location", city.getLocation()); + map.put("postal", city.getPostal()); + map.put("registered_country", city.getRegisteredCountry()); + map.put("represented_country", city.getRepresentedCountry()); + map.put("subdivisions", city.getSubdivisions()); + map.put("traits", city.getTraits()); + return new LookupResult(map.build()); + } catch (AddressNotFoundException nfe) { + return LookupResult.empty(); + } catch (Exception e) { + LOG.warn("Unable too look up IP address, returning empty result.", e); + return LookupResult.empty(); + } + case MAXMIND_COUNTRY: + try { + final CountryResponse country = databaseReader.country(addr); + final ImmutableMap.Builder map = ImmutableMap.builder(); + map.put("continent", country.getContinent()); + map.put("country", country.getCountry()); + map.put("registered_country", country.getRegisteredCountry()); + map.put("represented_country", country.getRepresentedCountry()); + map.put("traits", country.getTraits()); + return new LookupResult(map.build()); + } catch (AddressNotFoundException nfe) { + return LookupResult.empty(); + } catch (Exception e) { + LOG.warn("Unable too look up IP address, returning empty result.", e); + return LookupResult.empty(); + } + } + + return LookupResult.empty(); + } + + @Override + public void set(Object key, Object value) { + throw new UnsupportedOperationException(); + } + + public interface Factory extends LookupDataAdapter.Factory { + @Override + MaxmindDataAdapter create(LookupDataAdapterConfiguration configuration); + + @Override + MaxmindDataAdapter.Descriptor getDescriptor(); + } + + public static class Descriptor extends LookupDataAdapter.Descriptor { + public Descriptor() { + super(NAME, MaxmindDataAdapter.Config.class); + } + + @Override + public MaxmindDataAdapter.Config defaultConfiguration() { + return MaxmindDataAdapter.Config.builder() + .type(NAME) + .checkInterval(1) + .checkIntervalUnit(TimeUnit.MINUTES) + .path("/etc/graylog/server/GeoLite2-City.mmdb") + .dbType(DatabaseType.MAXMIND_CITY) + .build(); + } + } + + @AutoValue + @WithBeanGetter + @JsonAutoDetect + @JsonDeserialize(builder = AutoValue_MaxmindDataAdapter_Config.Builder.class) + @JsonTypeName(NAME) + public static abstract class Config implements LookupDataAdapterConfiguration { + + @Override + @JsonProperty(TYPE_FIELD) + public abstract String type(); + + @JsonProperty("path") + @NotEmpty + public abstract String path(); + + @JsonProperty("database_type") + @NotNull + public abstract DatabaseType dbType(); + + @JsonProperty("check_interval") + @Min(0) + public abstract long checkInterval(); + + @Nullable + @JsonProperty("check_interval_unit") + public abstract TimeUnit checkIntervalUnit(); + + public static Config.Builder builder() { + return new AutoValue_MaxmindDataAdapter_Config.Builder(); + } + + @AutoValue.Builder + public abstract static class Builder { + @JsonProperty(TYPE_FIELD) + public abstract Config.Builder type(String type); + + @JsonProperty("path") + public abstract Config.Builder path(String path); + + @JsonProperty("database_type") + public abstract Builder dbType(DatabaseType dbType); + + @JsonProperty("check_interval") + public abstract Builder checkInterval(long checkInterval); + + @JsonProperty("check_interval_unit") + public abstract Builder checkIntervalUnit(@Nullable TimeUnit checkIntervalUnit); + + + public abstract Config build(); + } + } +} diff --git a/src/web/components/adapter/MaxmindAdapterDocumentation.jsx b/src/web/components/adapter/MaxmindAdapterDocumentation.jsx new file mode 100644 index 0000000..f6ccf3f --- /dev/null +++ b/src/web/components/adapter/MaxmindAdapterDocumentation.jsx @@ -0,0 +1,63 @@ +/* eslint-disable react/no-unescaped-entities */ +import React from 'react'; +import { Alert } from 'react-bootstrap'; + +const MaxmindAdapterDocumentation = React.createClass({ + render() { + const cityFields = `{ + "city": { "geoname_id": 5375480, "names": { "en": "Mountain View" } }, + "location": { + "accuracy_radius": 1000, + "average_income": null, + "latitude": 37.386, + "longitude": -122.0838, + "metro_code": 807, + "population_density": null, + "time_zone": "America/Los_Angeles" + }, + "postal": { "code": "94035" }, + "subdivisions": [ { "geoname_id": 5332921, "iso_code": "CA", "names": { "en": "California" } } ], + }`; + + const countryFields = `{ + "continent": { "code": "NA", "geoname_id": 6255149, "names": { "en": "North America" } }, + "country": { "geoname_id": 6252001, "iso_code": "US", "names": { "en": "United States" } }, + "registered_country": { "geoname_id": 6252001, "iso_code": "US", "names": { } }, + "represented_country": { "geoname_id": null, "iso_code": "US", "names": { } }, + "traits": { + "ip_address": "8.8.8.8", + "is_anonymous_proxy": false, + "is_legitimate_proxy": false, + "is_satellite_provider": false, + "isp": null, + "organization": null, + } + }`; + + return (
+

The GeoIP data adapter supports reading MaxMind's GeoIP2 databases.

+ + +

Limitations

+

Currently the city and country databases are supported.

+

For support of additional database types, please visit our support channels.

+ + +
+ +

Country database fields

+ + 
{countryFields}
+ +

City database fields

+ +

In addition to the fields provided by the country database, the city database also includes the following fields:

+ + 
{cityFields}
+ +

For a complete documentation of the fields, please see MaxMind's developer documentation

+
); + }, +}); + +export default MaxmindAdapterDocumentation; diff --git a/src/web/components/adapter/MaxmindAdapterFieldSet.jsx b/src/web/components/adapter/MaxmindAdapterFieldSet.jsx new file mode 100644 index 0000000..1ffade4 --- /dev/null +++ b/src/web/components/adapter/MaxmindAdapterFieldSet.jsx @@ -0,0 +1,75 @@ +import React, { PropTypes } from 'react'; +import ObjectUtils from 'util/ObjectUtils'; + +import { Input } from 'components/bootstrap'; +import { Select, TimeUnitInput } from 'components/common'; + +const MaxmindAdapterFieldSet = React.createClass({ + propTypes: { + config: PropTypes.object.isRequired, +// eslint-disable-next-line react/no-unused-prop-types + updateConfig: PropTypes.func.isRequired, + handleFormEvent: PropTypes.func.isRequired, + }, + + _update(value, unit, enabled, name) { + const config = ObjectUtils.clone(this.props.config); + config[name] = enabled ? value : 0; + config[`${name}_unit`] = unit; + this.props.updateConfig(config); + }, + + updateCheckInterval(value, unit, enabled) { + this._update(value, unit, enabled, 'check_interval'); + }, + + _onDbTypeSelect(id) { + const config = ObjectUtils.clone(this.props.config); + config.database_type = id; + this.props.updateConfig(config); + }, + + render() { + const config = this.props.config; + const databaseTypes = [ + { label: 'City database', value: 'MAXMIND_CITY' }, + { label: 'Country database', value: 'MAXMIND_COUNTRY' }, + ]; + return (
+ + +