Skip to content

Commit

Permalink
Add support for the 'Enterprise' database to the geoip processor (#10…
Browse files Browse the repository at this point in the history 
…7377)
  • Loading branch information
@joegallo
joegallo committed Apr 11, 2024
1 parent 1d0e3cf commit 6ff3a26
Show file tree
Hide file tree
Showing 9 changed files with 311 additions and 4 deletions.
5 changes: 5 additions & 0 deletions docs/changelog/107377.yaml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
pr: 107377
summary: Add support for the 'Enterprise' database to the geoip processor
area: Ingest Node
type: enhancement
issues: []
4 changes: 4 additions & 0 deletions docs/reference/ingest/processors/geoip.asciidoc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,10 @@ in `properties`.
* If the GeoIP2 Anonymous IP database is used, then the following fields may be added under the `target_field`: `ip`,
`hosting_provider`, `tor_exit_node`, `anonymous_vpn`, `anonymous`, `public_proxy`, and `residential_proxy`. The fields actually added
depend on what has been found and which properties were configured in `properties`.
* If the GeoIP2 Enterprise database is used, then the following fields may be added under the `target_field`: `ip`,
`country_iso_code`, `country_name`, `continent_name`, `region_iso_code`, `region_name`, `city_name`, `timezone`, `location`, `asn`,
`organization_name`, `network`, `hosting_provider`, `tor_exit_node`, `anonymous_vpn`, `anonymous`, `public_proxy`, and `residential_proxy`.
The fields actually added depend on what has been found and which properties were configured in `properties`.


Here is an example that uses the default city database and adds the geographical information to the `geoip` field based on the `ip` field:
Expand Down
34 changes: 34 additions & 0 deletions modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/Database.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -74,12 +74,44 @@ enum Database {
Property.PUBLIC_PROXY,
Property.RESIDENTIAL_PROXY
)
),
Enterprise(
Set.of(
Property.IP,
Property.COUNTRY_ISO_CODE,
Property.COUNTRY_NAME,
Property.CONTINENT_NAME,
Property.REGION_ISO_CODE,
Property.REGION_NAME,
Property.CITY_NAME,
Property.TIMEZONE,
Property.LOCATION,
Property.ASN,
Property.ORGANIZATION_NAME,
Property.NETWORK,
Property.HOSTING_PROVIDER,
Property.TOR_EXIT_NODE,
Property.ANONYMOUS_VPN,
Property.ANONYMOUS,
Property.PUBLIC_PROXY,
Property.RESIDENTIAL_PROXY
),
Set.of(
Property.COUNTRY_ISO_CODE,
Property.COUNTRY_NAME,
Property.CONTINENT_NAME,
Property.REGION_ISO_CODE,
Property.REGION_NAME,
Property.CITY_NAME,
Property.LOCATION
)
);

private static final String CITY_DB_SUFFIX = "-City";
private static final String COUNTRY_DB_SUFFIX = "-Country";
private static final String ASN_DB_SUFFIX = "-ASN";
private static final String ANONYMOUS_IP_DB_SUFFIX = "-Anonymous-IP";
private static final String ENTERPRISE_DB_SUFFIX = "-Enterprise";

/**
* Parses the passed-in databaseType (presumably from the passed-in databaseFile) and return the Database instance that is
Expand All @@ -101,6 +133,8 @@ public static Database getDatabase(final String databaseType, final String datab
database = Database.Asn;
} else if (databaseType.endsWith(Database.ANONYMOUS_IP_DB_SUFFIX)) {
database = Database.AnonymousIp;
} else if (databaseType.endsWith(Database.ENTERPRISE_DB_SUFFIX)) {
database = Database.Enterprise;
}
}

Expand Down
7 changes: 7 additions & 0 deletions ...s/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/DatabaseReaderLazyLoader.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
import com.maxmind.geoip2.model.AsnResponse;
import com.maxmind.geoip2.model.CityResponse;
import com.maxmind.geoip2.model.CountryResponse;
import com.maxmind.geoip2.model.EnterpriseResponse;

import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
Expand Down Expand Up @@ -176,6 +177,12 @@ public AnonymousIpResponse getAnonymousIp(InetAddress ipAddress) {
return getResponse(ipAddress, DatabaseReader::tryAnonymousIp);
}

@Nullable
@Override
public EnterpriseResponse getEnterprise(InetAddress ipAddress) {
return getResponse(ipAddress, DatabaseReader::tryEnterprise);
}

boolean preLookup() {
return currentUsages.updateAndGet(current -> current < 0 ? current : current + 1) > 0;
}
Expand Down
4 changes: 4 additions & 0 deletions modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpDatabase.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
import com.maxmind.geoip2.model.AsnResponse;
import com.maxmind.geoip2.model.CityResponse;
import com.maxmind.geoip2.model.CountryResponse;
import com.maxmind.geoip2.model.EnterpriseResponse;

import org.elasticsearch.core.Nullable;

Expand Down Expand Up @@ -57,6 +58,9 @@ public interface GeoIpDatabase {
@Nullable
AnonymousIpResponse getAnonymousIp(InetAddress ipAddress);

@Nullable
EnterpriseResponse getEnterprise(InetAddress ipAddress);

/**
* Releases the current database object. Called after processing a single document. Databases should be closed or returned to a
* resource pool. No further interactions should be expected.
Expand Down
123 changes: 123 additions & 0 deletions modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpProcessor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
import com.maxmind.geoip2.model.AsnResponse;
import com.maxmind.geoip2.model.CityResponse;
import com.maxmind.geoip2.model.CountryResponse;
import com.maxmind.geoip2.model.EnterpriseResponse;
import com.maxmind.geoip2.record.City;
import com.maxmind.geoip2.record.Continent;
import com.maxmind.geoip2.record.Country;
Expand Down Expand Up @@ -174,6 +175,7 @@ private Map<String, Object> getGeoData(GeoIpDatabase geoIpDatabase, String ip) t
case Country -> retrieveCountryGeoData(geoIpDatabase, ipAddress);
case Asn -> retrieveAsnGeoData(geoIpDatabase, ipAddress);
case AnonymousIp -> retrieveAnonymousIpGeoData(geoIpDatabase, ipAddress);
case Enterprise -> retrieveEnterpriseGeoData(geoIpDatabase, ipAddress);
};
}

Expand Down Expand Up @@ -382,6 +384,127 @@ private Map<String, Object> retrieveAnonymousIpGeoData(GeoIpDatabase geoIpDataba
return geoData;
}

private Map<String, Object> retrieveEnterpriseGeoData(GeoIpDatabase geoIpDatabase, InetAddress ipAddress) {
EnterpriseResponse response = geoIpDatabase.getEnterprise(ipAddress);
if (response == null) {
return Map.of();
}

Country country = response.getCountry();
City city = response.getCity();
Location location = response.getLocation();
Continent continent = response.getContinent();
Subdivision subdivision = response.getMostSpecificSubdivision();

Long asn = response.getTraits().getAutonomousSystemNumber();
String organization_name = response.getTraits().getAutonomousSystemOrganization();
Network network = response.getTraits().getNetwork();

boolean isHostingProvider = response.getTraits().isHostingProvider();
boolean isTorExitNode = response.getTraits().isTorExitNode();
boolean isAnonymousVpn = response.getTraits().isAnonymousVpn();
boolean isAnonymous = response.getTraits().isAnonymous();
boolean isPublicProxy = response.getTraits().isPublicProxy();
boolean isResidentialProxy = response.getTraits().isResidentialProxy();

Map<String, Object> geoData = new HashMap<>();
for (Property property : this.properties) {
switch (property) {
case IP -> geoData.put("ip", NetworkAddress.format(ipAddress));
case COUNTRY_ISO_CODE -> {
String countryIsoCode = country.getIsoCode();
if (countryIsoCode != null) {
geoData.put("country_iso_code", countryIsoCode);
}
}
case COUNTRY_NAME -> {
String countryName = country.getName();
if (countryName != null) {
geoData.put("country_name", countryName);
}
}
case CONTINENT_NAME -> {
String continentName = continent.getName();
if (continentName != null) {
geoData.put("continent_name", continentName);
}
}
case REGION_ISO_CODE -> {
// ISO 3166-2 code for country subdivisions.
// See iso.org/iso-3166-country-codes.html
String countryIso = country.getIsoCode();
String subdivisionIso = subdivision.getIsoCode();
if (countryIso != null && subdivisionIso != null) {
String regionIsoCode = countryIso + "-" + subdivisionIso;
geoData.put("region_iso_code", regionIsoCode);
}
}
case REGION_NAME -> {
String subdivisionName = subdivision.getName();
if (subdivisionName != null) {
geoData.put("region_name", subdivisionName);
}
}
case CITY_NAME -> {
String cityName = city.getName();
if (cityName != null) {
geoData.put("city_name", cityName);
}
}
case TIMEZONE -> {
String locationTimeZone = location.getTimeZone();
if (locationTimeZone != null) {
geoData.put("timezone", locationTimeZone);
}
}
case LOCATION -> {
Double latitude = location.getLatitude();
Double longitude = location.getLongitude();
if (latitude != null && longitude != null) {
Map<String, Object> locationObject = new HashMap<>();
locationObject.put("lat", latitude);
locationObject.put("lon", longitude);
geoData.put("location", locationObject);
}
}
case ASN -> {
if (asn != null) {
geoData.put("asn", asn);
}
}
case ORGANIZATION_NAME -> {
if (organization_name != null) {
geoData.put("organization_name", organization_name);
}
}
case NETWORK -> {
if (network != null) {
geoData.put("network", network.toString());
}
}
case HOSTING_PROVIDER -> {
geoData.put("is_hosting_provider", isHostingProvider);
}
case TOR_EXIT_NODE -> {
geoData.put("is_tor_exit_node", isTorExitNode);
}
case ANONYMOUS_VPN -> {
geoData.put("is_anonymous_vpn", isAnonymousVpn);
}
case ANONYMOUS -> {
geoData.put("is_anonymous", isAnonymous);
}
case PUBLIC_PROXY -> {
geoData.put("is_public_proxy", isPublicProxy);
}
case RESIDENTIAL_PROXY -> {
geoData.put("is_residential_proxy", isResidentialProxy);
}
}
}
return geoData;
}

/**
* Retrieves and verifies a {@link GeoIpDatabase} instance for each execution of the {@link GeoIpProcessor}. Guards against missing
* custom databases, and ensures that database instances are of the proper type before use.
Expand Down
45 changes: 45 additions & 0 deletions modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -336,6 +336,51 @@ public void testAnonymmousIp() throws Exception {
assertThat(geoData.get("residential_proxy"), equalTo(true));
}

public void testEnterprise() throws Exception {
String ip = "2.125.160.216";
GeoIpProcessor processor = new GeoIpProcessor(
randomAlphaOfLength(10),
null,
"source_field",
loader("/GeoIP2-Enterprise-Test.mmdb"),
() -> true,
"target_field",
ALL_PROPERTIES,
false,
false,
"filename"
);

Map<String, Object> document = new HashMap<>();
document.put("source_field", ip);
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document);
processor.execute(ingestDocument);

assertThat(ingestDocument.getSourceAndMetadata().get("source_field"), equalTo(ip));
@SuppressWarnings("unchecked")
Map<String, Object> geoData = (Map<String, Object>) ingestDocument.getSourceAndMetadata().get("target_field");
assertThat(geoData.size(), equalTo(16));
assertThat(geoData.get("ip"), equalTo(ip));
assertThat(geoData.get("country_iso_code"), equalTo("GB"));
assertThat(geoData.get("country_name"), equalTo("United Kingdom"));
assertThat(geoData.get("continent_name"), equalTo("Europe"));
assertThat(geoData.get("region_iso_code"), equalTo("GB-WBK"));
assertThat(geoData.get("region_name"), equalTo("West Berkshire"));
assertThat(geoData.get("city_name"), equalTo("Boxford"));
assertThat(geoData.get("timezone"), equalTo("Europe/London"));
Map<String, Object> location = new HashMap<>();
location.put("lat", 51.75);
location.put("lon", -1.25);
assertThat(geoData.get("location"), equalTo(location));
assertThat(geoData.get("network"), equalTo("2.125.160.216/29"));
assertThat(geoData.get("is_hosting_provider"), equalTo(false));
assertThat(geoData.get("is_tor_exit_node"), equalTo(false));
assertThat(geoData.get("is_anonymous_vpn"), equalTo(false));
assertThat(geoData.get("is_anonymous"), equalTo(false));
assertThat(geoData.get("is_public_proxy"), equalTo(false));
assertThat(geoData.get("is_residential_proxy"), equalTo(false));
}

public void testAddressIsNotInTheDatabase() throws Exception {
GeoIpProcessor processor = new GeoIpProcessor(
randomAlphaOfLength(10),
Expand Down

0 comments on commit 6ff3a26

Please sign in to comment.