create data adapter for maxmind's geoip2 databases

pom.xml

@@ -90,6 +90,12 @@
             <version>${graylog.version}</version>
             <scope>provided</scope>
         </dependency>
+        <dependency>
+            <groupId>org.graylog.autovalue</groupId>
+            <artifactId>auto-value-javabean</artifactId>
+            <version>${graylog.version}</version>
+            <scope>provided</scope>
+        </dependency>
         <dependency>
             <groupId>com.maxmind.geoip2</groupId>
             <artifactId>geoip2</artifactId>

src/main/java/org/graylog/plugins/map/MapWidgetModule.java

@@ -16,6 +16,7 @@
  */
 package org.graylog.plugins.map;
 
+import org.graylog.plugins.map.geoip.MaxmindDataAdapter;
 import org.graylog.plugins.map.geoip.processor.GeoIpProcessor;
 import org.graylog.plugins.map.rest.MapDataResource;
 import org.graylog.plugins.map.widget.strategy.MapWidgetStrategy;
@@ -27,5 +28,10 @@ protected void configure() {
         addMessageProcessor(GeoIpProcessor.class, GeoIpProcessor.Descriptor.class);
         addWidgetStrategy(MapWidgetStrategy.class, MapWidgetStrategy.Factory.class);
         addRestResource(MapDataResource.class);
+
+        installLookupDataAdapter(MaxmindDataAdapter.NAME,
+                MaxmindDataAdapter.class,
+                MaxmindDataAdapter.Factory.class,
+                MaxmindDataAdapter.Config.class);
     }
 }

src/main/java/org/graylog/plugins/map/config/DatabaseType.java

@@ -0,0 +1,5 @@
+package org.graylog.plugins.map.config;
+
+public enum DatabaseType {
+    MAXMIND_CITY, MAXMIND_COUNTRY
+}

src/main/java/org/graylog/plugins/map/config/GeoIpResolverConfig.java

@@ -16,19 +16,17 @@
  */
 package org.graylog.plugins.map.config;
 
+import com.google.auto.value.AutoValue;
+
 import com.fasterxml.jackson.annotation.JsonAutoDetect;
 import com.fasterxml.jackson.annotation.JsonCreator;
 import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
 import com.fasterxml.jackson.annotation.JsonProperty;
-import com.google.auto.value.AutoValue;
 
 @JsonAutoDetect
 @JsonIgnoreProperties(ignoreUnknown = true)
 @AutoValue
 public abstract class GeoIpResolverConfig {
-    public enum DatabaseType {
-        MAXMIND_CITY, MAXMIND_COUNTRY
-    }
 
     @JsonProperty("enabled")
     public abstract boolean enabled();

src/main/java/org/graylog/plugins/map/geoip/MaxmindDataAdapter.java

@@ -0,0 +1,252 @@
+package org.graylog.plugins.map.geoip;
+
+import com.google.auto.value.AutoValue;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.net.InetAddresses;
+import com.google.inject.assistedinject.Assisted;
+
+import com.fasterxml.jackson.annotation.JsonAutoDetect;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonTypeName;
+import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
+import com.maxmind.geoip2.DatabaseReader;
+import com.maxmind.geoip2.exception.AddressNotFoundException;
+import com.maxmind.geoip2.model.CityResponse;
+import com.maxmind.geoip2.model.CountryResponse;
+
+import org.graylog.autovalue.WithBeanGetter;
+import org.graylog.plugins.map.config.DatabaseType;
+import org.graylog2.plugin.lookup.LookupDataAdapter;
+import org.graylog2.plugin.lookup.LookupDataAdapterConfiguration;
+import org.graylog2.plugin.lookup.LookupResult;
+import org.graylog2.plugin.utilities.FileInfo;
+import org.hibernate.validator.constraints.NotEmpty;
+import org.joda.time.Duration;
+import org.slf4j.Logger;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.InetAddress;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
+
+import javax.annotation.Nullable;
+import javax.inject.Inject;
+import javax.inject.Named;
+import javax.validation.constraints.Min;
+import javax.validation.constraints.NotNull;
+
+import static org.slf4j.LoggerFactory.getLogger;
+
+public class MaxmindDataAdapter extends LookupDataAdapter {
+
+    private static final Logger LOG = getLogger(MaxmindDataAdapter.class);
+
+    public static final String NAME = "maxmind_geoip";
+    private final Config config;
+    private final AtomicReference<DatabaseReader> databaseReader = new AtomicReference<>();
+    private FileInfo fileInfo;
+
+    @Inject
+    protected MaxmindDataAdapter(@Assisted LookupDataAdapterConfiguration config,
+                                 @Named("daemonScheduler") ScheduledExecutorService scheduler) {
+        super(config, scheduler);
+        this.config = (Config) config;
+    }
+
+    @Override
+    protected void doStart() throws Exception {
+        Path path = Paths.get(config.path());
+        if (!Files.isReadable(path)) {
+            throw new IllegalArgumentException("Cannot read database file: " + config.path());
+        }
+        fileInfo = FileInfo.forPath(path);
+        this.databaseReader.set(loadReader(path.toFile()));
+    }
+
+    @Override
+    protected void doStop() throws Exception {
+        databaseReader.get().close();
+    }
+
+    @Override
+    protected Duration refreshInterval() {
+        if (config.checkIntervalUnit() == null || config.checkInterval() == 0) {
+            return Duration.ZERO;
+        }
+        //noinspection ConstantConditions
+        return Duration.millis(config.checkIntervalUnit().toMillis(config.checkInterval()));
+    }
+
+    @Override
+    protected void doRefresh() throws Exception {
+        try {
+            final FileInfo.Change databaseFileCheck = fileInfo.checkForChange();
+            if (!databaseFileCheck.isChanged()) {
+                return;
+            }
+
+            // file has different attributes, let's reload it
+            LOG.debug("MaxMind database file has changed, reloading it from {}", config.path());
+            final DatabaseReader oldReader = this.databaseReader.get();
+            try {
+                this.databaseReader.set(loadReader(Paths.get(config.path()).toFile()));
+                getLookupTable().cache().purge();
+                oldReader.close();
+                fileInfo = databaseFileCheck.fileInfo();
+            } catch (IOException e) {
+                LOG.warn("Unable to load changed database file, leaving old one intact. Error message: {}", e.getMessage());
+            }
+        } catch (IllegalArgumentException iae) {
+            LOG.error("Unable to refresh MaxMind database file: {}", iae.getMessage());
+        }
+    }
+
+    private DatabaseReader loadReader(File file) throws IOException {
+        return new DatabaseReader.Builder(file).build();
+    }
+
+    @Override
+    protected LookupResult doGet(Object key) {
+        InetAddress addr;
+        if (key instanceof InetAddress) {
+            addr = (InetAddress) key;
+        } else {
+            // try to convert it somehow
+            try {
+                addr = InetAddresses.forString(key.toString());
+            } catch (IllegalArgumentException e) {
+                LOG.warn("Unable to parse IP address, returning empty result.");
+                return LookupResult.empty();
+            }
+        }
+        final DatabaseReader reader = this.databaseReader.get();
+        switch (config.dbType()) {
+            case MAXMIND_CITY:
+                try {
+                    final CityResponse city = reader.city(addr);
+                    final ImmutableMap.Builder<Object, Object> map = ImmutableMap.builder();
+                    map.put("city", city.getCity());
+                    map.put("continent", city.getContinent());
+                    map.put("country", city.getCountry());
+                    map.put("location", city.getLocation());
+                    map.put("postal", city.getPostal());
+                    map.put("registered_country", city.getRegisteredCountry());
+                    map.put("represented_country", city.getRepresentedCountry());
+                    map.put("subdivisions", city.getSubdivisions());
+                    map.put("traits", city.getTraits());
+                    return new LookupResult(map.build());
+                } catch (AddressNotFoundException nfe) {
+                    return LookupResult.empty();
+                } catch (Exception e) {
+                    LOG.warn("Unable to look up IP address, returning empty result.", e);
+                    return LookupResult.empty();
+                }
+            case MAXMIND_COUNTRY:
+                try {
+                    final CountryResponse country = reader.country(addr);
+                    final ImmutableMap.Builder<Object, Object> map = ImmutableMap.builder();
+                    map.put("continent", country.getContinent());
+                    map.put("country", country.getCountry());
+                    map.put("registered_country", country.getRegisteredCountry());
+                    map.put("represented_country", country.getRepresentedCountry());
+                    map.put("traits", country.getTraits());
+                    return new LookupResult(map.build());
+                } catch (AddressNotFoundException nfe) {
+                    return LookupResult.empty();
+                } catch (Exception e) {
+                    LOG.warn("Unable to look up IP address, returning empty result.", e);
+                    return LookupResult.empty();
+                }
+        }
+
+        return LookupResult.empty();
+    }
+
+    @Override
+    public void set(Object key, Object value) {
+        throw new UnsupportedOperationException();
+    }
+
+    public interface Factory extends LookupDataAdapter.Factory<MaxmindDataAdapter> {
+        @Override
+        MaxmindDataAdapter create(LookupDataAdapterConfiguration configuration);
+
+        @Override
+        MaxmindDataAdapter.Descriptor getDescriptor();
+    }
+
+    public static class Descriptor extends LookupDataAdapter.Descriptor<MaxmindDataAdapter.Config> {
+        public Descriptor() {
+            super(NAME, MaxmindDataAdapter.Config.class);
+        }
+
+        @Override
+        public MaxmindDataAdapter.Config defaultConfiguration() {
+            return MaxmindDataAdapter.Config.builder()
+                    .type(NAME)
+                    .checkInterval(1)
+                    .checkIntervalUnit(TimeUnit.MINUTES)
+                    .path("/etc/graylog/server/GeoLite2-City.mmdb")
+                    .dbType(DatabaseType.MAXMIND_CITY)
+                    .build();
+        }
+    }
+
+    @AutoValue
+    @WithBeanGetter
+    @JsonAutoDetect
+    @JsonDeserialize(builder = AutoValue_MaxmindDataAdapter_Config.Builder.class)
+    @JsonTypeName(NAME)
+    public static abstract class Config implements LookupDataAdapterConfiguration {
+
+        @Override
+        @JsonProperty(TYPE_FIELD)
+        public abstract String type();
+
+        @JsonProperty("path")
+        @NotEmpty
+        public abstract String path();
+
+        @JsonProperty("database_type")
+        @NotNull
+        public abstract DatabaseType dbType();
+
+        @JsonProperty("check_interval")
+        @Min(0)
+        public abstract long checkInterval();
+
+        @Nullable
+        @JsonProperty("check_interval_unit")
+        public abstract TimeUnit checkIntervalUnit();
+
+        public static Config.Builder builder() {
+            return new AutoValue_MaxmindDataAdapter_Config.Builder();
+        }
+
+        @AutoValue.Builder
+        public abstract static class Builder {
+            @JsonProperty(TYPE_FIELD)
+            public abstract Config.Builder type(String type);
+
+            @JsonProperty("path")
+            public abstract Config.Builder path(String path);
+
+            @JsonProperty("database_type")
+            public abstract Builder dbType(DatabaseType dbType);
+
+            @JsonProperty("check_interval")
+            public abstract Builder checkInterval(long checkInterval);
+
+            @JsonProperty("check_interval_unit")
+            public abstract Builder checkIntervalUnit(@Nullable TimeUnit checkIntervalUnit);
+
+
+            public abstract Config build();
+        }
+    }
+}

src/web/components/adapter/MaxmindAdapterDocumentation.jsx

@@ -0,0 +1,63 @@
+/* eslint-disable react/no-unescaped-entities */
+import React from 'react';
+import { Alert } from 'react-bootstrap';
+
+const MaxmindAdapterDocumentation = React.createClass({
+  render() {
+    const cityFields = `{
+    "city": { "geoname_id": 5375480, "names": { "en": "Mountain View" } },
+    "location": {
+      "accuracy_radius": 1000,
+      "average_income": null,
+      "latitude": 37.386,
+      "longitude": -122.0838,
+      "metro_code": 807,
+      "population_density": null,
+      "time_zone": "America/Los_Angeles"
+    },
+    "postal": { "code": "94035" },
+    "subdivisions": [ { "geoname_id": 5332921, "iso_code": "CA", "names": { "en": "California" } } ],
+    }`;
+
+    const countryFields = `{
+    "continent": { "code": "NA", "geoname_id": 6255149, "names": { "en": "North America" } },
+    "country": { "geoname_id": 6252001, "iso_code": "US", "names": { "en": "United States" } },
+    "registered_country": { "geoname_id": 6252001, "iso_code": "US", "names": { } },
+    "represented_country": { "geoname_id": null, "iso_code": "US", "names": { } },
+    "traits": {
+      "ip_address": "8.8.8.8",
+      "is_anonymous_proxy": false,
+      "is_legitimate_proxy": false,
+      "is_satellite_provider": false,
+      "isp": null,
+      "organization": null,
+    }
+    }`;
+
+    return (<div>
+      <p>The GeoIP data adapter supports reading MaxMind's GeoIP2 databases.</p>
+
+      <Alert style={{ marginBottom: 10 }} bsStyle="info">
+        <h4 style={{ marginBottom: 10 }}>Limitations</h4>
+        <p>Currently the city and country databases are supported.</p>
+        <p>For support of additional database types, please visit our support channels.</p>
+      </Alert>
+
+      <hr />
+
+      <h3 style={{ marginBottom: 10 }}>Country database fields</h3>
+
+      <pre>{countryFields}</pre>
+
+      <h3 style={{ marginBottom: 10 }}>City database fields</h3>
+
+      <p>In addition to the fields provided by the country database, the city database also includes the following fields:</p>
+
+      <pre>{cityFields}</pre>
+
+      <p>For a complete documentation of the fields, please see MaxMind's <a href="http://maxmind.github.io/GeoIP2-java/" target="_blank" rel="noopener noreferrer">developer documentation</a></p>
+    </div>);
+  },
+});
+
+export default MaxmindAdapterDocumentation;