elastic · martijnvg · Oct 13, 2021 · Sep 28, 2021 · Oct 7, 2021 · Oct 7, 2021
diff --git a/docs/build.gradle b/docs/build.gradle
@@ -63,8 +63,6 @@ testClusters.matching { it.name == "integTest"}.configureEach {
   if (singleNode().testDistribution == DEFAULT) {
     setting 'xpack.license.self_generated.type', 'trial'
     setting 'indices.lifecycle.history_index_enabled', 'false'
-    setting 'ingest.geoip.downloader.enabled', 'false'
-    systemProperty 'es.geoip_v2_feature_flag_enabled', 'true'
     keystorePassword 'keystore-password'
   }
 
@@ -89,6 +87,9 @@ testClusters.matching { it.name == "integTest"}.configureEach {
   // Whitelist reindexing from the local node so we can test it.
   setting 'reindex.remote.whitelist', '127.0.0.1:*'
 
+  extraConfigFile 'ingest-geoip/GeoLite2-City.mmdb', file("${project.projectDir}/src/test/resources/GeoLite2-City.mmdb")
+  extraConfigFile 'ingest-geoip/GeoLite2-Country.mmdb', file("${project.projectDir}/src/test/resources/GeoLite2-Country.mmdb")
+
   // TODO: remove this once cname is prepended to transport.publish_address by default in 8.0
   systemProperty 'es.transport.cname_in_publish_address', 'true'
 
@@ -114,6 +115,10 @@ tasks.named("integTest").configure {
   }
 }
 
+tasks.named("forbiddenPatterns").configure {
+  exclude '**/*.mmdb'
+}
+
 tasks.named("buildRestTests").configure {
   docs = docsFileTree
 }

diff --git a/docs/reference/ingest/common-log-format-example.asciidoc b/docs/reference/ingest/common-log-format-example.asciidoc
@@ -175,7 +175,7 @@ PUT _index_template/my-data-stream-template
 ----
 POST my-data-stream/_doc?pipeline=my-pipeline
 {
-  "message": "212.87.37.154 - - [05/May/2099:16:21:15 +0000] \"GET /favicon.ico HTTP/1.1\" 200 3638 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.116 Safari/537.36\""
+  "message": "89.160.20.128 - - [05/May/2099:16:21:15 +0000] \"GET /favicon.ico HTTP/1.1\" 200 3638 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.116 Safari/537.36\""
 }
 ----
 // TEST[s/my-pipeline/my-pipeline&refresh=wait_for/]
@@ -216,21 +216,21 @@ The API returns:
             "version": "1.1"
           },
           "source": {
-            "ip": "212.87.37.154",
+            "ip": "89.160.20.128",
             "geo": {
-              "continent_name": "Europe",
-              "region_iso_code": "DE-BE",
-              "city_name": "Berlin",
-              "country_iso_code": "DE",
-              "country_name": "Germany",
-              "region_name": "Land Berlin",
-              "location": {
-                "lon": 13.4978,
-                "lat": 52.411
+              "continent_name" : "Europe",
+              "country_name" : "Sweden",
+              "country_iso_code" : "SE",
+              "city_name" : "Linköping",
+              "region_iso_code" : "SE-E",
+              "region_name" : "Östergötland County",
+              "location" : {
+                "lon" : 15.6167,
+                "lat" : 58.4167
               }
             }
           },
-          "message": "212.87.37.154 - - [05/May/2099:16:21:15 +0000] \"GET /favicon.ico HTTP/1.1\" 200 3638 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.116 Safari/537.36\"",
+          "message": "89.160.20.128 - - [05/May/2099:16:21:15 +0000] \"GET /favicon.ico HTTP/1.1\" 200 3638 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.116 Safari/537.36\"",
           "url": {
             "original": "/favicon.ico"
           },

diff --git a/docs/reference/ingest/processors/geoip.asciidoc b/docs/reference/ingest/processors/geoip.asciidoc
@@ -69,7 +69,7 @@ PUT _ingest/pipeline/geoip
 }
 PUT my-index-000001/_doc/my_id?pipeline=geoip
 {
-  "ip": "8.8.8.8"
+  "ip": "89.160.20.128"
 }
 GET my-index-000001/_doc/my_id
 --------------------------------------------------
@@ -86,12 +86,15 @@ Which returns:
   "_seq_no": 55,
   "_primary_term": 1,
   "_source": {
-    "ip": "8.8.8.8",
+    "ip": "89.160.20.128",
     "geoip": {
-      "continent_name": "North America",
-      "country_name": "United States",
-      "country_iso_code": "US",
-      "location": { "lat": 37.751, "lon": -97.822 }
+      "continent_name": "Europe",
+      "country_name": "Sweden",
+      "country_iso_code": "SE",
+      "city_name" : "Linköping",
+      "region_iso_code" : "SE-E",
+      "region_name" : "Östergötland County",
+      "location": { "lat": 58.4167, "lon": 15.6167 }
     }
   }
 }
@@ -119,7 +122,7 @@ PUT _ingest/pipeline/geoip
 }
 PUT my-index-000001/_doc/my_id?pipeline=geoip
 {
-  "ip": "8.8.8.8"
+  "ip": "89.160.20.128"
 }
 GET my-index-000001/_doc/my_id
 --------------------------------------------------
@@ -136,11 +139,11 @@ returns this:
   "_seq_no": 65,
   "_primary_term": 1,
   "_source": {
-    "ip": "8.8.8.8",
+    "ip": "89.160.20.128",
     "geo": {
-      "continent_name": "North America",
-      "country_name": "United States",
-      "country_iso_code": "US"
+      "continent_name": "Europe",
+      "country_name": "Sweden",
+      "country_iso_code": "SE"
     }
   }
 }
@@ -236,7 +239,7 @@ PUT _ingest/pipeline/geoip
 
 PUT my_ip_locations/_doc/1?refresh=true&pipeline=geoip
 {
-  "ip": "8.8.8.8"
+  "ip": "89.160.20.128"
 }
 
 GET /my_ip_locations/_search
@@ -250,8 +253,8 @@ GET /my_ip_locations/_search
         "geo_distance": {
           "distance": "1m",
           "geoip.location": {
-            "lon": -97.822,
-            "lat": 37.751
+            "lon": 15.6167,
+            "lat": 58.4167
           }
         }
       }
@@ -285,15 +288,18 @@ GET /my_ip_locations/_search
         "_score" : 1.0,
         "_source" : {
           "geoip" : {
-            "continent_name" : "North America",
-            "country_name" : "United States",
-            "country_iso_code" : "US",
+            "continent_name" : "Europe",
+            "country_name" : "Sweden",
+            "country_iso_code" : "SE",
+            "city_name" : "Linköping",
+            "region_iso_code" : "SE-E",
+            "region_name" : "Östergötland County",
             "location" : {
-              "lon" : -97.822,
-              "lat" : 37.751
+              "lon" : 15.6167,
+              "lat" : 58.4167
             }
           },
-          "ip" : "8.8.8.8"
+          "ip" : "89.160.20.128"
         }
       }
     ]

diff --git a/docs/reference/migration/migrate_8_0/ingest.asciidoc b/docs/reference/migration/migrate_8_0/ingest.asciidoc
@@ -18,4 +18,32 @@ Common Schema (ECS)] fields, regardless of the `ecs` value.
 To avoid deprecation warnings, remove the parameter from your ingest pipelines.
 If a pipeline specifies an `ecs` value, the value is ignored.
 ====
+
+.The default Maxmind geoip databases have been removed.
+[%collapsible]
+====
+*Details* +
+The default Maxmind geoip databases that shipped by default with Elasticsearch
+have been removed. These databases are out dated and stale and using these
+databases will likely result in incorrect geoip lookups.
+
+By default since 7.13, these pre-packaged geoip databases
+were used in case no database were specified in the config directory or before
+the geoip downloader downloaded the geoip databases. When the geoip database
+downloader completed downloading the new databases then these pre-packaged
+databases were no longer used.
+
+*Impact* +
+If the geoip downloader is disabled and no geoip databases are provided
+in the config directory of each ingest node then the geoip processor will
+no longer perform geoip lookups and tag these documents with the fact that
+the requested database is no longer available.
+
+After a cluster has been started and before the geoip downloader has completed
+downloading the most up to data databases, the geoip processor will not perform
+any geoip lookups and tag documents that the requested database is not available.
+After the geoip downloader has completed downloading the most up to data databases
+then the geoip processor will function as normal. The window of time that the
+geoip processor can't do geoip lookups after cluster startup should be very small.
+====
 //end::notable-breaking-changes[]
diff --git a/docs/src/test/resources/GeoLite2-City.mmdb b/docs/src/test/resources/GeoLite2-City.mmdb
diff --git a/docs/src/test/resources/GeoLite2-Country.mmdb b/docs/src/test/resources/GeoLite2-Country.mmdb
diff --git a/modules/ingest-geoip/build.gradle b/modules/ingest-geoip/build.gradle
@@ -13,7 +13,7 @@ apply plugin: 'elasticsearch.yaml-rest-compat-test'
 apply plugin: 'elasticsearch.internal-cluster-test'
 
 esplugin {
-  description 'Ingest processor that uses lookup geo data based on IP adresses using the MaxMind geo database'
+  description 'Ingest processor that uses lookup geo data based on IP addresses using the MaxMind geo database'
   classname 'org.elasticsearch.ingest.geoip.IngestGeoIpPlugin'
 }
 
@@ -57,14 +57,7 @@ tasks.named("internalClusterTest").configure {
   }
 }
 
-tasks.register("copyDefaultGeoIp2DatabaseFiles", Copy) {
-  from { zipTree(configurations.testCompileClasspath.files.find { it.name.contains('geolite2-databases') }) }
-  into "${project.buildDir}/ingest-geoip"
-  include "*.mmdb"
-}
-
 tasks.named("bundlePlugin").configure {
-  dependsOn("copyDefaultGeoIp2DatabaseFiles")
   from("${project.buildDir}/ingest-geoip") {
     into '/'
   }
@@ -107,3 +100,18 @@ tasks.named("dependencyLicenses").configure {
   mapping from: /maxmind-db.*/, to: 'maxmind-db-reader'
   ignoreFile 'elastic-geoip-database-service-agreement-LICENSE.txt'
 }
+
+testClusters.configureEach {
+  // Needed for database downloader, uses delete-by-query to cleanup old databases from org.elasticsearch.ingest.geoip database system index
+  module ':modules:reindex'
+  // Downloader is enabled by default, but in test clusters in build disabled by default,
+  // but in this module, the downloader should be enabled by default
+  systemProperty 'ingest.geoip.downloader.enabled.default', 'true'
+  if (useFixture) {
+    setting 'ingest.geoip.downloader.endpoint', { "${-> fixtureAddress()}" }
+  }
+}
+
+tasks.named("yamlRestTestV7CompatTransform").configure { task ->
+  task.skipTestsByFilePattern("**/ingest_geoip/20_geoip_processor.yml", "from 8.0 yaml rest tests use geoip test fixture and default geoip are no longer packaged. In 7.x yaml tests used default databases which makes tests results very different, so skipping these tests")
+}
diff --git a/.../qa/file-based-update/src/test/java/org/elasticsearch/ingest/geoip/UpdateDatabasesIT.java b/.../qa/file-based-update/src/test/java/org/elasticsearch/ingest/geoip/UpdateDatabasesIT.java
@@ -7,25 +7,25 @@
  */
 package org.elasticsearch.ingest.geoip;
 
-import org.apache.http.util.EntityUtils;
 import org.elasticsearch.client.Request;
-import org.elasticsearch.client.Response;
-import org.elasticsearch.core.PathUtils;
 import org.elasticsearch.common.settings.SecureString;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.core.PathUtils;
 import org.elasticsearch.xcontent.ObjectPath;
-import org.elasticsearch.common.xcontent.XContentHelper;
-import org.elasticsearch.xcontent.json.JsonXContent;
 import org.elasticsearch.test.rest.ESRestTestCase;
 
 import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.util.List;
 import java.util.Map;
 
+import static org.hamcrest.Matchers.either;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.notNullValue;
+import static org.hamcrest.Matchers.nullValue;
 
 public class UpdateDatabasesIT extends ESRestTestCase {
 
@@ -35,8 +35,14 @@ public void test() throws Exception {
         Request simulatePipelineRequest = new Request("POST", "/_ingest/pipeline/_simulate");
         simulatePipelineRequest.setJsonEntity(body);
         {
-            Map<String, Object> response = toMap(client().performRequest(simulatePipelineRequest));
-            assertThat(ObjectPath.eval("docs.0.doc._source.geoip.city_name", response), equalTo("Tumba"));
+            Map<String, Object> response = entityAsMap(client().performRequest(simulatePipelineRequest));
+            assertThat(ObjectPath.eval("docs.0.doc._source.tags.0", response), equalTo("_geoip_database_unavailable_GeoLite2-City.mmdb"));
+        }
+
+        // Ensure no config databases have been setup:
+        {
+            Map<?, ?> stats = getGeoIpStatsForSingleNode();
+            assertThat(stats, nullValue());
         }
 
         Path configPath = PathUtils.get(System.getProperty("tests.config.dir"));
@@ -46,14 +52,25 @@ public void test() throws Exception {
         Files.copy(UpdateDatabasesIT.class.getResourceAsStream("/GeoLite2-City-Test.mmdb"),
             ingestGeoipDatabaseDir.resolve("GeoLite2-City.mmdb"));
 
-        assertBusy(() -> {
-            Map<String, Object> response = toMap(client().performRequest(simulatePipelineRequest));
-            assertThat(ObjectPath.eval("docs.0.doc._source.geoip.city_name", response), equalTo("Linköping"));
-        });
+        // Ensure that a config database has been setup:
+        {
+            assertBusy(() -> {
+                Map<?, ?> stats = getGeoIpStatsForSingleNode();
+                assertThat(stats, notNullValue());
+                assertThat(stats.get("config_databases"), equalTo(List.of("GeoLite2-City.mmdb")));
+            });
+        }
+
+        Map<String, Object> response = entityAsMap(client().performRequest(simulatePipelineRequest));
+        assertThat(ObjectPath.eval("docs.0.doc._source.geoip.city_name", response), equalTo("Linköping"));
     }
 
-    private static Map<String, Object> toMap(Response response) throws IOException {
-        return XContentHelper.convertToMap(JsonXContent.jsonXContent, EntityUtils.toString(response.getEntity()), false);
+    private static Map<?, ?> getGeoIpStatsForSingleNode() throws IOException {
+        Request request = new Request("GET", "/_ingest/geoip/stats");
+        Map<String, Object> response = entityAsMap(client().performRequest(request));
+        Map<?, ?> nodes = (Map<?, ?>) response.get("nodes");
+        assertThat(nodes.size(), either(equalTo(0)).or(equalTo(1)));
+        return nodes.isEmpty() ? null : (Map<?, ?>) nodes.values().iterator().next();
     }
 
     @Override